From a50a8dee0074b71c867298abf8dd1ec5f641d3d6 Mon Sep 17 00:00:00 2001 From: Relayflow Date: Mon, 5 Oct 2026 10:57:22 +0000 Subject: [PATCH 01/34] fix(sdk): preserve runs across daemon read timeouts --- docs/SURFACE.md | 26 + evidence/run-read-timeout/baseline-setup.md | 17 + evidence/run-read-timeout/final-focused.log | 59 + .../run-read-timeout/full-suite-final.log | 1970 +++++++++++++++++ .../handshake-probe-baseline.mjs | 25 + evidence/run-read-timeout/handshake-probe.mjs | 25 + .../live-failure-probe-baseline.log | 45 + .../run-read-timeout/live-failure-probe.log | 45 + .../run-read-timeout/mutation-heartbeat.log | 53 + evidence/run-read-timeout/mutation-reader.log | 57 + .../mutation-root-parking.log | 70 + .../mutation-watch-cadence.log | 51 + evidence/run-read-timeout/mutations.py | 57 + evidence/run-read-timeout/resume-live.log | 15 + .../run-read-timeout/typecheck-final.json | 10 + evidence/run-read-timeout/verification.md | 336 +++ kernel/DAEMON-LIFECYCLE.md | 12 + packages/sdk/src/authored-flow-error.ts | 1 + packages/sdk/src/authored-node-entry.ts | 7 +- packages/sdk/src/authored-root.ts | 8 + packages/sdk/src/cli/direct-run.ts | 5 +- packages/sdk/src/cli/journal-timeout.ts | 30 + packages/sdk/src/cli/run.ts | 84 +- packages/sdk/src/cli/running-step.ts | 90 + packages/sdk/src/failure-kinds.ts | 1 + packages/sdk/src/index.ts | 2 +- packages/sdk/src/journal-client.ts | 46 +- packages/sdk/src/journal-read-policy.ts | 79 + packages/sdk/src/worker-lease.ts | 9 +- packages/sdk/tests/authored-root.test.ts | 40 +- packages/sdk/tests/classify-outcome.test.ts | 7 + packages/sdk/tests/cli.test.ts | 7 +- .../sdk/tests/direct-run-worker-lease.test.ts | 15 + packages/sdk/tests/heartbeat-timeout.test.ts | 24 + packages/sdk/tests/journal-client-loopback.ts | 54 +- .../tests/journal-client-read-timeout.test.ts | 141 ++ .../tests/read-timeout-resume-live.test.ts | 60 + .../sdk/tests/resume-worker-lease.test.ts | 15 + .../sdk/tests/run-daemon-unresponsive.test.ts | 21 + packages/sdk/tests/run-read-load-live.test.ts | 68 + packages/sdk/tests/running-step-watch.test.ts | 59 + packages/sdk/tests/worker-lease-sweep.test.ts | 14 +- 42 files changed, 3657 insertions(+), 103 deletions(-) create mode 100644 evidence/run-read-timeout/baseline-setup.md create mode 100644 evidence/run-read-timeout/final-focused.log create mode 100644 evidence/run-read-timeout/full-suite-final.log create mode 100644 evidence/run-read-timeout/handshake-probe-baseline.mjs create mode 100644 evidence/run-read-timeout/handshake-probe.mjs create mode 100644 evidence/run-read-timeout/live-failure-probe-baseline.log create mode 100644 evidence/run-read-timeout/live-failure-probe.log create mode 100644 evidence/run-read-timeout/mutation-heartbeat.log create mode 100644 evidence/run-read-timeout/mutation-reader.log create mode 100644 evidence/run-read-timeout/mutation-root-parking.log create mode 100644 evidence/run-read-timeout/mutation-watch-cadence.log create mode 100644 evidence/run-read-timeout/mutations.py create mode 100644 evidence/run-read-timeout/resume-live.log create mode 100644 evidence/run-read-timeout/typecheck-final.json create mode 100644 evidence/run-read-timeout/verification.md create mode 100644 packages/sdk/src/cli/journal-timeout.ts create mode 100644 packages/sdk/src/cli/running-step.ts create mode 100644 packages/sdk/src/journal-read-policy.ts create mode 100644 packages/sdk/tests/heartbeat-timeout.test.ts create mode 100644 packages/sdk/tests/journal-client-read-timeout.test.ts create mode 100644 packages/sdk/tests/read-timeout-resume-live.test.ts create mode 100644 packages/sdk/tests/run-daemon-unresponsive.test.ts create mode 100644 packages/sdk/tests/run-read-load-live.test.ts create mode 100644 packages/sdk/tests/running-step-watch.test.ts diff --git a/docs/SURFACE.md b/docs/SURFACE.md index 29c6887f3..9585f606a 100644 --- a/docs/SURFACE.md +++ b/docs/SURFACE.md @@ -1736,6 +1736,32 @@ typed `run_not_found` refusal. A dropped connection, request failure, or journal may already have changed and the CLI cannot honestly claim the resume was refused before a write. +A timed-out read is handled separately: flow execution clients (`flows run` +and `flows resume`, authored and declarative) retry only `run.get`, +`journal.read`, `stream.read`, and `subscription.inspect` within a 300-second +total budget. Reads use a separate session, one in flight per body client, +with increasing attempt bounds and jitter. Interactive clients keep the +single-shot 30-second default; writes are never retried by this policy. +Unretried timeouts retain `journal client: timed out after ms`. +Exhausted reads instead name the verb, attempts, elapsed time, total read +budget, and possible CPU load. + +After a read budget expires, the CLI probes a fresh connection. A responding +daemon reports `daemon_unresponsive`; a failed probe reports +`daemon_unreachable` (also used for initial attach failures). A failed probe +cannot prove the daemon is dead: its diagnostic names both unreachability and +CPU load. Both reports exit 1, carry the known run/root id, retain +`status: running`, and include `flows resume`. This parks the CLI execution +without manufacturing a terminal journal fact; it does not claim a journaled +human park or verify journal integrity. The root worker session closes so the +kernel can recover its attempt, preserving completed work for resume. + +Worker completion waits use `run.watch` pushes with a snapshot every two +seconds for the live lease deadline. A scoped watch session closes after each +wait because the protocol has no unwatch verb. A heartbeat timeout relinquishes +worker ownership as lease loss, leaving recovery to the kernel instead of +recording `worker_error` against the body. + A step that ran and failed is **not** one of those. It reports `step_failed` with `status: failed`, for every step type and for authored TypeScript flows as well as declarative ones. `protocol_error` is reserved for an outcome the diff --git a/evidence/run-read-timeout/baseline-setup.md b/evidence/run-read-timeout/baseline-setup.md new file mode 100644 index 000000000..aa9e8e1ac --- /dev/null +++ b/evidence/run-read-timeout/baseline-setup.md @@ -0,0 +1,17 @@ +The handshake comparison used an unmodified detached checkout of +`3c58ee16d10a9e2400db980f5bbafac84e437f20`. The two probe sources are copied +verbatim from the `/tmp` scripts named in their captured output. They use this +session's absolute checkout/build paths; adjust those paths when reproducing +on another machine. + +Setup used (from the implementation checkout): + +```sh +git worktree add --detach /tmp/relayflow-read-timeout-baseline 3c58ee16d10a9e2400db980f5bbafac84e437f20 +ln -s /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk/node_modules /tmp/relayflow-read-timeout-baseline/packages/sdk/node_modules +npm run build --prefix /tmp/relayflow-read-timeout-baseline/packages/sdk +``` + +The baseline probe loads SDK code from that checkout while keeping the daemon +binary and stub fixture path identical to the current-code probe. It compares +that one fixture handshake failure, not the full suite. diff --git a/evidence/run-read-timeout/final-focused.log b/evidence/run-read-timeout/final-focused.log new file mode 100644 index 000000000..ce9b87422 --- /dev/null +++ b/evidence/run-read-timeout/final-focused.log @@ -0,0 +1,59 @@ +$ cd packages/sdk && RELAYFLOWD_BIN=/home/daytona/.relayflows-toolchain/target/2962130851/debug/relayflowd npx vitest run tests/journal-client-read-timeout.test.ts tests/journal-client.test.ts tests/journal-client-completion.test.ts tests/journal-client-subscriptions.test.ts tests/running-step-watch.test.ts tests/heartbeat-timeout.test.ts tests/run-daemon-unresponsive.test.ts tests/authored-root.test.ts tests/classify-outcome.test.ts tests/cli.test.ts tests/direct-run-worker-lease.test.ts tests/resume-worker-lease.test.ts tests/worker-lease.test.ts tests/worker-lease-lost.test.ts tests/worker-lease-sweep.test.ts tests/run-read-load-live.test.ts tests/worker-lease-lost-live.test.ts tests/flow-executor-chain.test.ts tests/agent-transcript-live.test.ts tests/human-live.test.ts --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ✓ tests/cli.test.ts (71 tests) 6026ms + ✓ flows check CLI > binds a checked relative wrapper to the flow directory for worker execution 566ms + ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 488ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2053ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2045ms +(node:44939) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. +(Use `node --trace-warnings ...` to show where the warning was created) + ✓ tests/authored-root.test.ts (26 tests) 398ms + ✓ tests/journal-client.test.ts (17 tests) 89ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 10315ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 1022ms + ✓ flow executor LLM and output-binding chain > runs a dollar-budgeted authored Claude agent with the same default used by preflight 700ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1595ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 3382ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 779ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 1068ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 3394ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 875ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 818ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 851ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 848ms + ✓ tests/classify-outcome.test.ts (11 tests) 7422ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2009ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3005ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms + ✓ tests/human-live.test.ts (3 tests) 7854ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 4785ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1916ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 1152ms + ✓ tests/worker-lease.test.ts (7 tests) 20ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 27ms + ✓ tests/journal-client-read-timeout.test.ts (13 tests) 1104ms + ✓ a recovered read timeout does not become an authored callback failure 368ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 943ms + ✓ reports journal success after completion rejects with lease_conflict 327ms + ✓ reports journal success when a renewal rejects after completion landed 327ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2821ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2095ms + ✓ drains read and watch promises before an authored flow completes 725ms + ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms + ✓ tests/journal-client-completion.test.ts (6 tests) 102ms + ✓ tests/resume-worker-lease.test.ts (3 tests) 6ms + ✓ tests/direct-run-worker-lease.test.ts (3 tests) 10ms + ✓ tests/running-step-watch.test.ts (2 tests) 2122ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms + ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms + ✓ tests/run-daemon-unresponsive.test.ts (2 tests) 4ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms + + Test Files 20 passed (20) + Tests 210 passed (210) + Start at 10:41:25 + Duration 54.29s (transform 1.45s, setup 97ms, collect 8.09s, tests 42.69s, environment 3ms, prepare 1.05s) + +exit=0 diff --git a/evidence/run-read-timeout/full-suite-final.log b/evidence/run-read-timeout/full-suite-final.log new file mode 100644 index 000000000..0f4c71463 --- /dev/null +++ b/evidence/run-read-timeout/full-suite-final.log @@ -0,0 +1,1970 @@ +$ cd packages/sdk && RELAYFLOWD_BIN=/home/daytona/.relayflows-toolchain/target/2962130851/debug/relayflowd npm test + +> @relayflows/sdk@2.0.42 test +> sh scripts/test.sh + + +> @relayflows/sdk@2.0.42 test:prep +> ( cd ../../kernel && sh ../ops/cargo.sh build ) && ( [ ! -d ../../testdata/preflight ] || find ../../testdata/preflight -name '*-cli' -type f -exec chmod +x {} + ) + + Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.07s + +> @relayflows/sdk@2.0.42 typecheck +> tsc --noEmit && tsc -p tsconfig.type-tests.json + + +> @relayflows/sdk@2.0.42 build +> tsc && node scripts/make-cli-executable.mjs + + +> @relayflows/sdk@2.0.42 typecheck:tests +> tsc -p tsconfig.tests.json + + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + +stdout | tests/live-kernel.test.ts +LIVE_KERNEL relayflowd=/home/daytona/.relayflows-toolchain/target/2962130851/debug/relayflowd +LIVE_KERNEL flows=/home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk/dist/cli.js + + ✓ tests/preflight.test.ts (70 tests) 115ms + ✓ tests/cli.test.ts (71 tests) 5824ms + ✓ flows check CLI > binds a checked relative wrapper to the flow directory for worker execution 460ms + ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 413ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2038ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2045ms + ✓ tests/cloud-read.test.ts (46 tests) 55ms +stdout | tests/live-kernel.test.ts > surface resume after a real daemon kill > resumes a three-step run with each successful completion exactly once +LIVE_KERNEL kill -9 pid=44150 run=01M45TEQ6WQ827KKF8VV8WPXD1 while step=two state=Running + + ❯ tests/live-kernel.test.ts (32 tests | 8 failed) 57015ms + ✓ built flows CLI against live relayflowd > twenty-six-step reuses 25 durable completions after editing the failed final step 2821ms + ✓ built flows CLI against live relayflowd > runs rung (a), parks rung (b), and keeps JSON report-shaped 2765ms + ✓ built flows CLI against live relayflowd > allows a deterministic run to exceed the bounded request timeout 32494ms + ✓ built flows CLI against live relayflowd > follows a live worker dispatch through flows run 668ms + ✓ built flows CLI against live relayflowd > runs an agent CLI end to end through the SDK worker 466ms + ✓ built flows CLI against live relayflowd > f.agent lowers to a real agent step and dispatches through a live worker 578ms + ✓ built flows CLI against live relayflowd > can always get a parked run to a late-attaching worker 5588ms + ✓ built flows CLI against live relayflowd > reports a real manual-recovery NeedsHuman state as parked 485ms + × built flows CLI against live relayflowd > runs hn-monitor analyze-story end-to-end via a stub agent CLI (gate 2 clause 2 demo) 543ms + → expected { …(12) } to match object { output: { …(3) }, …(1) } +(22 matching properties omitted from actual) + × built flows CLI against live relayflowd > hn-monitor analyze-story FAILS verification when the CLI omits required schema fields 543ms + → expected { …(12) } to match object { …(3) } +(21 matching properties omitted from actual) + × built flows CLI against live relayflowd > agent step preserves the CliResult wrapper as output when the CLI emits non-JSON text 506ms + → expected null not to be null + × built flows CLI against live relayflowd > AgentWorker exposes wake_context to the CLI via RELAYFLOW_WAKE_CONTEXT env var (real analyzer prerequisite) 635ms + → Cannot read properties of null (reading 'story_title') + × built flows CLI against live relayflowd > AgentWorker leaves RELAYFLOW_WAKE_CONTEXT UNSET when the run has no wake_context (undefined-vs-null pin) 587ms + → Cannot read properties of null (reading 'env_present') + ✓ built flows CLI against live relayflowd > AgentWorker passes a declared model to an identified wrapper as RELAYFLOW_MODEL 589ms + ✓ built flows CLI against live relayflowd > AgentWorker refuses a nonconforming journal-submitted wrapper before exposing RELAYFLOW_MODEL 522ms + ✓ built flows CLI against live relayflowd > AgentWorker executes the raw claude adapter with its real model flag 462ms + ✓ built flows CLI against live relayflowd > AgentWorker executes the raw codex adapter with its real model flag 635ms + × built flows CLI against live relayflowd > AgentWorker leaves RELAYFLOW_MODEL UNSET when the step declares no model 1454ms + → Cannot read properties of null (reading 'story_title') + × built flows CLI against live relayflowd > hn-monitor analyze-story reaches done through the real Claude analyzer CLI 29ms + → LIVE_ANALYZER_UNAVAILABLE: "/home/daytona/.relayflow-v2-supervisor/durable/repository/testdata/preflight/analyze-story-claude-cli" does not identify as relayflows-agent-cli-v1 — failing because gate-2 acceptance requires the real analyzer to execute. Set RELAYFLOWS_ALLOW_ANALYZER_SKIP=1 only if this run is not gate evidence. + ✓ built flows CLI against live relayflowd > preflights before journaling and names an unreachable socket 904ms + ✓ built flows CLI against live relayflowd > starts exactly one daemon when two runs race for one empty data dir 569ms + ✓ subprocess_gate output capture against live relayflowd > journals the gate command output and reports both tails 1007ms + ✓ surface resume after a real daemon kill > resumes a three-step run with each successful completion exactly once 882ms + × a relayflow can be scheduled: tick source against live relayflowd > a tick spawns a real run whose step reports the SCHEDULED instant 495ms + → expected null to deeply equal { schedule_id: 'heartbeat-1m', …(3) } + ✓ tests/cloud-live.test.ts (55 tests) 2058ms + ✓ argv and wiring > routes both live invocations through runCli, and refuses a missing run id 2004ms + ❯ tests/hosted-extension-isolation.test.ts (22 tests | 14 failed) 350ms + × hosted extension capability isolation > executes the exact capability-only handler for a queued receipt 9ms + → bubblewrap is unavailable + × hosted extension capability isolation > executes the exact capability-only handler for a duplicate receipt 2ms + → bubblewrap is unavailable + × hosted extension capability isolation > launches through the captured process primitive after builtin export synchronization 5ms + → promise rejected "Error: bubblewrap is unavailable { code: '…' }" instead of resolving + × hosted extension capability isolation > ignores inherited launcher overrides and decodes manifests with the captured Buffer intrinsic 5ms + → promise rejected "Error: bubblewrap is unavailable { code: '…' }" instead of resolving + × hosted extension capability isolation > streams verified bytes when the live store is replaced and no writable staging path exists 248ms + → promise rejected "Error: Hosted extension sandbox exited wi… { code: '…' }" instead of resolving + × hosted extension capability isolation > mounts pinned private Surface bytes when the live package changes before launch 4ms + → promise rejected "Error: bubblewrap is unavailable { code: '…' }" instead of resolving + × hosted extension capability isolation > refuses oversized Surface files through the bounded descriptor reader 6ms + → expected Error: bubblewrap is unavailable { code: '…' } to match object { code: 'plugin_unsupported', …(1) } + × hosted extension capability isolation > shields verified Surface files before async settlement 4ms + → promise rejected "Error: bubblewrap is unavailable { code: '…' }" instead of resolving + × hosted extension capability isolation > preserves a typed host refusal while disclosing only a fixed marker to the child 2ms + → expected Error: bubblewrap is unavailable { code: '…' } to be Error: private Cloud policy detail { code: '…' } // Object.is equality + × hosted extension capability isolation > denies ambient credentials, host files, writes, network, subprocesses, and undeclared context verbs 7ms + → bubblewrap is unavailable + × hosted extension capability isolation > enforces OS address-space and data bounds on native Buffer allocation 2ms + → expected Error: bubblewrap is unavailable { code: '…' } to match object { code: 'plugin_unsupported', …(1) } + × hosted extension capability isolation > blocks extra handler fields and authority-bearing receipt fields at the parent port 2ms + → expected Error: bubblewrap is unavailable { code: '…' } to match object { code: 'plugin_event_unroutable' } + × hosted extension capability isolation > constructs adapter authority with the captured freeze intrinsic 1ms + → bubblewrap is unavailable + × hosted extension capability isolation > writes the Surface manifest and protocol without inherited toJSON behavior 2ms + → promise rejected "Error: bubblewrap is unavailable { code: '…' }" instead of resolving + ✓ tests/cloud-deploy.test.ts (117 tests) 2599ms + ✓ tests/cloud-sync.test.ts (40 tests) 736ms + ✓ tests/observer-link.test.ts (44 tests) 177ms + ✓ tests/authored-flow.test.ts (38 tests) 806ms + ✓ tests/plugin-extension.test.ts (94 tests) 464ms + ✓ tests/cloud-transcript-codex.test.ts (44 tests) 21ms + ✓ tests/worker-cli.test.ts (25 tests) 27087ms + ✓ registered CLI model defaults > passes the same priced Claude default to the real provider invocation 482ms + ✓ registered CLI model defaults > uses the explicitly supplied agent environment for the provider subprocess 427ms + ✓ registered CLI model defaults > dispatches canonical generic bytes with the preflight-proved adapter identity 493ms + ✓ direct transport lifecycle evidence > classifies only the exact Codex stdin lifecycle signature as retryable 410ms + ✓ direct transport lifecycle evidence > records a signal close separately from an ordinary nonzero exit 849ms + ✓ direct transport lifecycle evidence > records a spawn error code without treating a missing executable as transient 450ms + ✓ direct transport lifecycle evidence > journals classified lifecycle evidence and reports crashed instead of generic worker_error 610ms + ✓ step discovery environment > names the run, step, attempt and an absolute data dir for a direct agent spawn 582ms + ✓ step discovery environment > exports none of the four without a data dir, even when the worker inherited them 478ms + ✓ wrapper discovery environment > sets the four names from the dispatch and still refuses ambient values and other secrets 466ms + ✓ wrapper discovery environment > exports none of the four to a wrapper without a data dir, even when the worker inherited them 486ms + ✓ custom wrapper execution identity > passes an explicit safe environment at identification and execution 449ms + ✓ custom wrapper execution identity > refuses a wrapper symlink retarget before delivering private values 453ms + ✓ custom wrapper execution identity > bounds wrapper execution after acknowledgement 498ms + ✓ custom wrapper execution identity > bounds captured wrapper output 470ms + ✓ custom wrapper execution identity > refuses a duplicate execute protocol frame 432ms + ✓ custom wrapper execution bounds are reader-owned > resolves when a conforming wrapper leaks a stdio pipe to a background helper 697ms + ✓ custom wrapper execution bounds are reader-owned > resolves when the leaked helper inherits stderr only 673ms + ✓ custom wrapper execution bounds are reader-owned > resolves when a wrapper leaks a stdio pipe and exits before identifying 3447ms + ✓ custom wrapper execution bounds are reader-owned > journals a completionReason at the default bound when a wrapper leaks a stdio pipe 11463ms + ✓ custom wrapper execution bounds are reader-owned > accepts an execute token and an over-8KiB payload flushed in one write 434ms + ✓ custom wrapper execution bounds are reader-owned > accepts the same over-8KiB payload whether or not it coalesces with the execute token 1430ms + ✓ custom wrapper execution bounds are reader-owned > still bounds an un-terminated handshake buffer and names the bound 441ms + ✓ delivers the journaled memory pack to the real wrapper and excludes its charge from completion usage 462ms + ✓ tests/flow-extension-compose.test.ts (33 tests) 6663ms + ✓ composing flow extensions onto a base flow > composes two extensions in declaration order, and the order is the lockfile order 556ms + ✓ composing flow extensions onto a base flow > flows check reports the composition and keeps the composed triggers deliverable 1122ms + ✓ composing flow extensions onto a base flow > uses extension permissions for hosted deploy preflight and the deploy body 324ms +(node:46445) ExperimentalWarning: SQLite is an experimental feature and might change at any time +(Use `node --trace-warnings ...` to show where the warning was created) + ✓ tests/cli-status.test.ts (27 tests) 1171ms + ✓ flows status > resolves the run with no arguments from inside a worker-spawned agent 940ms + ✓ tests/run-state.test.ts (28 tests) 34ms + ✓ tests/cloud-run.test.ts (59 tests) 597ms + ✓ tests/relay-cli-surface.test.ts (87 tests) 42ms + ✓ tests/agent-transcript.test.ts (29 tests) 332ms +(node:46773) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. +(Use `node --trace-warnings ...` to show where the warning was created) + ✓ tests/authored-root.test.ts (26 tests) 400ms + ✓ tests/babysitter-native-extension.test.ts (41 tests | 1 skipped) 1473ms + ✓ tests/shipped-source-model-provenance.test.ts (1 test) 109230ms + ✓ shipped-source model provenance > does not treat mutable aliases or incomplete named agents as pinned 109229ms + ✓ tests/authored-completion-detail.test.ts (59 tests) 223ms + ✓ tests/shipped-source-worker-call-forms.test.ts (1 test) 17255ms + ✓ shipped-source worker call forms > fails closed for computed keys, assertions, binds, call, and apply 17254ms + ✓ tests/step-failure-diagnostic.test.ts (26 tests) 72ms + ✓ tests/stop-process-group.test.ts (11 tests) 14220ms + ✓ every stop reaches the process group, not just the direct child > exits the run after an execution-timeout stop 970ms + ✓ every stop reaches the process group, not just the direct child > exits the run after a protocol terminate stop 486ms + ✓ every stop reaches the process group, not just the direct child > kills a SIGTERM-deaf grandchild after a protocol terminate stop 1508ms + ✓ every stop reaches the process group, not just the direct child > kills a SIGTERM-deaf grandchild after an execution-timeout stop 1913ms + ✓ every stop reaches the process group, not just the direct child > holds the loop open long enough for the escalation to run 1102ms + ✓ every stop reaches the process group, not just the direct child > bounds forced-stop confirmation when a group remains unprovable 1006ms + ✓ a wrapper that exits with no execution deadline still drains > reports the wrapper result and reaps a grandchild holding its pipes 707ms + ✓ a wrapper that exits with no execution deadline still drains > reaps a SIGTERM-deaf grandchild holding its pipes 1780ms + ✓ a wrapper that exits with no execution deadline still drains > settles on its own deadline when an escaped holder withholds close 4427ms + ✓ tests/mcp.test.ts (34 tests) 13995ms + ✓ MCP preflight and transports > flows check refuses an undeclared server with exit 2 and no daemon 865ms + ✓ MCP preflight and transports > flows check reports a refusing server and leaves no PID 944ms + ✓ MCP preflight and transports > kills a SIGTERM-resistant silent child after a parent-owned handshake deadline 1313ms + ✓ MCP preflight and transports > reaps a SIGTERM-resistant descendant with inherit stdio before cleanup finishes 1061ms + ✓ MCP preflight and transports > reaps a SIGTERM-resistant descendant with ignore stdio before cleanup finishes 2021ms + ✓ MCP preflight and transports > rejects close when forced group death remains unprovable 2065ms + ✓ MCP preflight and transports > cancels force escalation when close follows an exited child 1179ms + ✓ MCP preflight and transports > waits for child close when the transport cannot own a process group 1055ms + ✓ MCP preflight and transports > cancels force escalation for an already-closed child without a process group 1177ms + ✓ MCP preflight and transports > reports malformed connection configuration as config_invalid 803ms + ✓ authored MCP effects against the real kernel > reports a dropped tool connection as a failed CLI run 647ms + ✓ tests/step-attempt-history.test.ts (23 tests) 27ms + ✓ tests/daemon-lifecycle.test.ts (42 tests) 41ms + ✓ tests/close-pr-flow.test.ts (35 tests) 21131ms + ✓ close-pr journaled repair loop > reads an existing Bugbot finding, repairs in the same worktree, pushes and re-verifies before merging 2505ms + ✓ close-pr journaled repair loop > opens a PR and feeds failed CI logs into the repair agent 2128ms + ✓ close-pr journaled repair loop > pins the current generated model for codex when no override is supplied 4998ms + ✓ close-pr journaled repair loop > pins the current generated model for claude when no override is supplied 3900ms + ✓ close-pr journaled repair loop > parks after exactly three nonconverging repairs, with accumulated blockers 2385ms + ✓ close-pr journaled repair loop > can converge on the third repair 2486ms + ✓ close-pr journaled repair loop > executes the deterministic commit and force-push steps against a local Git remote, including a no-op repair 2527ms + ❯ tests/authored-node-runtime.test.ts (16 tests | 16 skipped) 12ms + ✓ tests/artifact-gates.test.ts (27 tests) 242ms + ❯ tests/hosted-extension-protocol.test.ts (27 tests | 8 failed) 40086ms + × hosted extension hostile protocol > uses captured JSON intrinsics for the complete parent boundary 6ms + → bubblewrap is unavailable + × hosted extension hostile protocol > rejects an import-time different PR frame with zero adapter calls 5ms + → expected Error: bubblewrap is unavailable { code: '…' } to match object { code: 'plugin_event_unroutable' } + × hosted extension hostile protocol > rejects an import-time different delivery frame with zero adapter calls 2ms + → expected Error: bubblewrap is unavailable { code: '…' } to match object { code: 'plugin_event_unroutable' } + × hosted extension hostile protocol > rejects an import-time different event frame with zero adapter calls 2ms + → expected Error: bubblewrap is unavailable { code: '…' } to match object { code: 'plugin_event_unroutable' } + × hosted extension hostile protocol > rejects two forged calls after the authoritative first outcome settles 10004ms + → hostile child did not invoke the adapter + × hosted extension hostile protocol > waits for a pending adapter to reject after a forged child error 10004ms + → hostile child did not invoke the adapter + × hosted extension hostile protocol > waits for a pending adapter to resolve after a forged child error 10005ms + → hostile child did not invoke the adapter + × hosted extension hostile protocol > returns a typed adapter rejection even when the hostile child hangs 10005ms + → hostile child did not invoke the adapter + ✓ tests/hosted-base-snapshot.test.ts (18 tests) 1873ms + ✓ hosted base private snapshot > stops streaming project entries at the shared count limit 1797ms + ✓ tests/cloud-connect.test.ts (24 tests) 3023ms + ✓ hosted verbs connect before they submit > flows run --cloud submits once the prompt connected the integration 2120ms + ✓ tests/journal-client.test.ts (17 tests) 81ms + ✓ tests/authored-node-result.test.ts (45 tests) 20ms + ✓ tests/verb-field-lint.test.ts (99 tests) 389ms + ✓ tests/bundle.test.ts (26 tests) 9078ms + ✓ immutable bundles > returns exit 2 naming a byte-flipped payload and refuses to reuse corruption 406ms + ✓ immutable bundles > verifies with --verify in any position and answers --json with one object 799ms + ✓ immutable bundles > refuses --out with --verify rather than ignoring the destination 401ms + ✓ immutable bundles > builds and verifies the canonical YAML fixture through the compiled CLI 1253ms + ✓ immutable bundles > emits the ephemeral warning on CLI stderr and uses the default output directory 793ms + ✓ immutable bundles > refuses build-provable CLI resolution errors without environment probes 431ms + ✓ immutable bundles > builds a standalone TS fixture twice with identical executable hashes 2353ms + ✓ immutable bundles > refuses to label installed dependency drift with lockfile pins 384ms + ✓ immutable bundles > refuses invalid CLI arguments %j 396ms + ✓ immutable bundles > refuses invalid CLI arguments "--out" 396ms + ✓ immutable bundles > refuses invalid CLI arguments "--verify" 397ms + ✓ immutable bundles > refuses invalid CLI arguments "--verify" 394ms + ✓ immutable bundles > refuses invalid CLI arguments "--out" 529ms + ✓ tests/validate.test.ts (68 tests) 34ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 9665ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 842ms + ✓ flow executor LLM and output-binding chain > runs a dollar-budgeted authored Claude agent with the same default used by preflight 547ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1465ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 3225ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 761ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 1096ms + ✓ tests/agent-relay-transport.test.ts (17 tests) 2272ms + ✓ Relay completion at the journal boundary > does not complete at readiness and journals exact output, receipt, and priced accounting 1006ms + ✓ Relay completion at the journal boundary > aborts polling on rejected renewal and never writes a stale completion 1000ms + ✓ tests/shipped-source-flow-provenance-repairs.test.ts (1 test) 30332ms + ✓ shipped-source flow provenance repairs > audits repaired writer, alias, binding, and cyclic provenance forms 30331ms + ✓ tests/tick-source.test.ts (33 tests) 21ms + ✓ tests/authored-step-graph.test.ts (25 tests) 837ms + ✓ the authored step DAG > does not walk a long-running step's own polling chain to find its dependents' edges 337ms + ✓ tests/cloud-mirror.test.ts (14 tests) 33ms + ✓ tests/pr-review-post.test.ts (21 tests) 1982ms + ✓ tests/authored-flow-lifecycle-executor.test.ts (27 tests) 587ms + ✓ tests/step-failure-excerpt.test.ts (42 tests) 195ms + ✓ tests/authored-step-terminal-completion.test.ts (13 tests) 4878ms + ✓ readCompletedStepOutput on an adopted gated child whose gate has not started > refuses a producer success when the wait ends with the run still not terminal 2006ms + ✓ readCompletedStepOutput given a suspended known status (#441) > refuses a producer success while the child is still not terminal 2006ms +(node:52566) ExperimentalWarning: SQLite is an experimental feature and might change at any time +(Use `node --trace-warnings ...` to show where the warning was created) + ✓ tests/cloud-mirror-step.test.ts (17 tests) 12ms + ✓ tests/cli-replay.test.ts (38 tests) 1314ms + ✓ flows replay > --json is byte-identical across two CLI invocations (diff) 941ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 1753ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 560ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 528ms + ✓ tests/wrapper-execution-duration.test.ts (7 tests) 10822ms + ✓ keeps the handshake deadline independent of the removed execution deadline 10054ms + ✓ still lets a lease abort stop an unlimited wrapper before it produces output 511ms + ✓ tests/tick-runner.test.ts (22 tests) 2354ms + ✓ CLI argument parsing refuses coercion rather than accepting it > refuses --interval-ms fractional as an invocation error 393ms + ✓ CLI argument parsing refuses coercion rather than accepting it > refuses --interval-ms exponent notation as an invocation error 385ms + ✓ CLI argument parsing refuses coercion rather than accepting it > refuses --interval-ms hex as an invocation error 385ms + ✓ CLI argument parsing refuses coercion rather than accepting it > refuses --interval-ms trailing text as an invocation error 390ms + ✓ CLI argument parsing refuses coercion rather than accepting it > refuses --interval-ms empty as an invocation error 391ms + ✓ CLI argument parsing refuses coercion rather than accepting it > accepts an exact integer and proceeds past parsing 385ms + ✓ tests/authored-step-index.test.ts (17 tests) 16ms + ✓ tests/named-gate-diagnostics.test.ts (20 tests) 640ms + ✓ tests/authored-agent-artifacts.test.ts (5 tests) 325ms + ✓ tests/direct-input.test.ts (6 tests) 6022ms + ✓ direct .flow.ts input through the built CLI and live runtime > returns exit 3 for an authored human handoff and persists its outcome 592ms + ✓ direct .flow.ts input through the built CLI and live runtime > returns exit 1 for an authored step_failed verdict and persists its outcome 632ms + ✓ direct .flow.ts input through the built CLI and live runtime > executes inline and file JSON input through relayflowd 2421ms + ✓ direct .flow.ts input through the built CLI and live runtime > refuses missing and malformed input before contacting relayflowd 1517ms + ✓ direct .flow.ts input through the built CLI and live runtime > does not run the authored body before daemon availability 478ms + ✓ direct .flow.ts input through the built CLI and live runtime > refuses oversized file input before contacting relayflowd 381ms + ✓ tests/worker-transcript.test.ts (9 tests) 866ms + ✓ tests/gate-contract.test.ts (20 tests) 151ms + ❯ tests/canonical-software-factory.test.ts (25 tests | 25 failed) 152ms + × canonical software-factory metadata contract > opens the actual catalog flow with the ticket title and exactly one GitHub closing line 11ms + → expected an @relayflows/surface flow handle + × canonical software-factory metadata contract > writes a Linear closing reference that the GitHub integration links back 7ms + → expected an @relayflows/surface flow handle + × canonical software-factory metadata contract > accepts a Linear team key that carries digits 6ms + → expected an @relayflows/surface flow handle + × canonical software-factory metadata contract > stops before push when a Linear ticket has no linkable identifier () 6ms + → expected an @relayflows/surface flow handle + × canonical software-factory metadata contract > stops before push when a Linear ticket has no linkable identifier (not-an-issue) 7ms + → expected an @relayflows/surface flow handle + × canonical software-factory metadata contract > fails closed before push when the Linear closing reference is duplicated in the final body 7ms + → expected an @relayflows/surface flow handle + × canonical software-factory metadata contract > fails closed before push when the body carries a foreign closing reference 6ms + → expected an @relayflows/surface flow handle + × canonical software-factory metadata contract > does not mistake ordinary closing-verb prose for a reference 6ms + → expected an @relayflows/surface flow handle + × canonical software-factory metadata contract > normalizes whitespace and caps the title at 240 Unicode code points 7ms + → expected an @relayflows/surface flow handle + × canonical software-factory metadata contract > fails closed before push when GitHub identity is missing or the final body duplicates its closing line 0ms + → expected an @relayflows/surface flow handle + × canonical software-factory review scope > fails closed for explicit defects 6ms + → expected an @relayflows/surface flow handle + × canonical software-factory review scope > fails closed for no verdict 5ms + → expected an @relayflows/surface flow handle + × canonical software-factory review scope > fails closed for contradictory blocked and unverified 6ms + → expected an @relayflows/surface flow handle + × canonical software-factory review scope > fails closed for contradictory passed and unverified 5ms + → expected an @relayflows/surface flow handle + × canonical software-factory review scope > fails closed for contradictory passed and blocked 7ms + → expected an @relayflows/surface flow handle + × canonical software-factory review scope > fails closed for empty unverified 7ms + → expected an @relayflows/surface flow handle + × canonical software-factory review scope > distinguishes a missing verification prerequisite from a defect 6ms + → expected an @relayflows/surface flow handle + × canonical software-factory review scope > preserves the passed body without a scope marker or draft 6ms + → expected an @relayflows/surface flow handle + × canonical software-factory review scope > rejects a forged scope for local before publication 6ms + → expected an @relayflows/surface flow handle + × canonical software-factory review scope > rejects a forged scope for github before publication 5ms + → expected an @relayflows/surface flow handle + × canonical software-factory review scope > scopes the post-review hook refusal 6ms + → expected an @relayflows/surface flow handle + × canonical software-factory review scope > scopes the merge-gate hook refusal 6ms + → expected an @relayflows/surface flow handle + × canonical software-factory review scope > rejects malformed head "" before publication 6ms + → expected an @relayflows/surface flow handle + × canonical software-factory review scope > rejects malformed head "not-a-sha" before publication 7ms + → expected an @relayflows/surface flow handle + × canonical software-factory review scope > rejects malformed head "a'; touch injected; #" before publication 6ms + → expected an @relayflows/surface flow handle + ✓ tests/authored-human.test.ts (13 tests) 91ms + ✓ tests/wrapper-exit-drain.test.ts (10 tests) 3512ms + ✓ reports a signalled wrapper death while a pipe is held, with its output intact 379ms + ✓ lets a lease abort outrank a successful exit still being drained 537ms + ✓ does not let a later drain overwrite abort while group confirmation is pending 631ms + ✓ tests/cloud-schedule.test.ts (17 tests) 5275ms + ✓ schedule lowering > marks a non-grid cron as Cloud-only rather than approximating it, with a silence budget from its own cadence 2913ms + ✓ flows check prints declared schedules > shows the lowering for a fixed interval and the Cloud-only note for a real cron 1956ms + ✓ tests/authored-run-failure-evidence.test.ts (9 tests) 848ms + ✓ the child index after the process that wrote it is gone > still names every child, with its own run id, after a daemon restart 484ms + ✓ tests/worker-cli-result-exit.test.ts (7 tests) 34844ms + ✓ a Claude agent step completes on its result, not only on process exit > settles a hung, successful run within the grace and stops its whole tree 32619ms + ✓ a Claude agent step completes on its result, not only on process exit > maps an error result on a hung run to a failed exit 31619ms + ✓ a Claude agent step completes on its result, not only on process exit > leaves a hang before any result to the existing stops 32014ms + ✓ an agent tree does not outlive the process that spawned it > kills the agent group when the run process is terminated by SIGTERM 795ms + ✓ an agent tree does not outlive the process that spawned it > removes a wrapper alias on host exit after group death stays unprovable 1341ms + ✓ tests/cli-hn-monitor.test.ts (16 tests) 95ms + ✓ tests/spec-parity.test.ts (46 tests) 505ms + ✓ tests/authored-activity.test.ts (16 tests) 108ms + ✓ tests/run-projection.test.ts (13 tests) 12ms + ✓ tests/agent-artifacts-live.test.ts (5 tests) 4163ms + ✓ agent artifacts and gates through the built CLI, a real daemon and the local agent > journals the files the agent wrote, including under a dot-directory, and every artifact gate passes on that journal 1030ms + ✓ agent artifacts and gates through the built CLI, a real daemon and the local agent > fails the run when the artifact_exists gate names a file the agent did not write 803ms + ✓ agent artifacts and gates through the built CLI, a real daemon and the local agent > fails the run with the author reason when a predicate gate returns false, journaling the verdict 877ms + ✓ review follow-ups > applies a predicate gate on a helper step too, and journals its verdict 790ms + ✓ review follow-ups > records predicate verdicts on the root run so a resume reuses them instead of re-running the closure 662ms + ✓ tests/daemon-lifecycle-live.test.ts (9 tests) 6889ms + ✓ flows run against a data dir with no daemon (§6 test 7) > cold start spawns exactly one daemon, the run succeeds, and the daemon outlives the CLI 492ms + ✓ flows run against a data dir with no daemon (§6 test 7) > polls, bounded, for a daemon that holds the lock before it binds 1369ms + ✓ flows run against a data dir with no daemon (§6 test 7) > attaches to a serving daemon that has not published a connection file 505ms + ✓ flows run against a data dir with no daemon (§6 test 7) > a second run attaches to the daemon the first one started, spawning nothing 964ms + ✓ flows run against a data dir with no daemon (§6 test 7) > detects a stale connection file left by a hard kill and starts a fresh daemon 1010ms + ✓ concurrent invocations against one empty data dir (§6 test 15) > ends with exactly one daemon owning the socket, and both runs succeed 1203ms + ✓ refusals from a spawn that cannot produce a daemon > names relayflowd_not_found rather than falling through to PATH 413ms + ✓ refusals from a spawn that cannot produce a daemon > names daemon_start_failed and quotes the daemon log when startup dies 470ms + ✓ refusals from a spawn that cannot produce a daemon > refuses a daemon speaking another protocol version instead of binding over it 464ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 3517ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 852ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 806ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 885ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 973ms + ✓ tests/resume-local-agent.test.ts (8 tests) 13ms +(node:55014) Warning: Transcript tail for run-9/analyze attempt 1 (stdout) could not be written; the step continues without it: EACCES: permission denied, mkdir '/tmp/transcript-tail-1nBIzB/runs/run-9/steps' +(Use `node --trace-warnings ...` to show where the warning was created) + ✓ tests/transcript-tail.test.ts (11 tests) 1192ms + ✓ direct agent spawn > tees stdout and stderr into tail files that name the dispatch 527ms + ✓ direct agent spawn > completes the step when the tail directory cannot be created 384ms + ✓ tests/advisory-outcome.test.ts (23 tests) 72ms + ✓ tests/shipped-source-cubic-regressions.test.ts (5 tests) 11768ms + ✓ shipped-source adversarial provenance regressions > retains callable writes through member mutation, spreads, this, holes, and alternate receivers 6294ms + ✓ shipped-source adversarial provenance regressions > does not manufacture intrinsic writes or comma-left callables 2442ms + ✓ shipped-source adversarial provenance regressions > treats constructor and implicit-this receiver escapes as possible worker writes 666ms + ✓ shipped-source adversarial provenance regressions > maps pure helper parameters and refuses mutations or conditional assignments as proofs 1755ms + ✓ shipped-source adversarial provenance regressions > does not apply later alias assignments retroactively to copied values 610ms + ✓ tests/agent-cwd-live.test.ts (4 tests) 2928ms + ✓ agents drive two checkouts under one run root > runs each CLI in its declared directory, and the run is not refused as an unknown field 868ms + ✓ agents drive two checkouts under one run root > gates on an artifact path relative to the agent directory 779ms + ✓ a declaration this contract cannot honour is refused at its own edge > refuses a path that climbs out of the run root without starting a run at all 600ms + ✓ a declaration this contract cannot honour is refused at its own edge > fails the step with the reason when the directory is not there, rather than running somewhere else 680ms + ✓ tests/authored-helpers.test.ts (6 tests) 3363ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 1593ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 712ms + ✓ replays after SIGKILL before complete with the same token and one successful completion 741ms + ✓ tests/cloud-mirror-session.test.ts (12 tests) 14ms + ✓ tests/agent-cwd.test.ts (53 tests) 50ms + ✓ tests/pty-sidechannel.test.ts (14 tests) 8465ms + ✓ view attach preserves worker completion and marks only drive 970ms + ✓ drive attach preserves worker completion and marks only drive 460ms + ✓ passthrough attach preserves worker completion and marks only drive 907ms + ✓ none attach preserves worker completion and marks only drive 947ms + ✓ none subscriber lets an unattended CLI read EOF 506ms + ✓ view subscriber lets an unattended CLI read EOF 497ms + ✓ passthrough subscriber lets an unattended CLI read EOF 494ms + ✓ incomplete subscriber lets an unattended CLI read EOF 513ms + ✓ unattended Codex receives closed stdin before startup instead of entering its additional-input lifecycle 591ms + ✓ ends driven stdin when the last drive peer disconnects after spawn 730ms + ✓ rejects drive after EOF without marking human intervention 951ms + ✓ delivers all drive bytes in order across child stdin backpressure 866ms + ✓ tests/authored-flow-operation.test.ts (24 tests) 507ms + ✓ tests/budget-attribution.test.ts (10 tests) 9ms + ✓ tests/helper-reference.test.ts (30 tests) 22ms + ✓ tests/cloud-mirror-live.test.ts (3 tests) 2782ms + ✓ a local run on the Cloud dashboard > registers, publishes its steps and its log, and reports terminal last 547ms + ✓ a local run on the Cloud dashboard > sends nothing at all unless it is asked to 1358ms + ✓ a local run on the Cloud dashboard > publishes a composed authored flow as one connected dashboard DAG 835ms + ✓ tests/flow-requirements.test.ts (14 tests) 720ms + ✓ flows check prints REQUIRES > names the helper, the harness and the mcp server of an authored flow 508ms + ✓ tests/backlog-picker.test.ts (14 tests) 41ms + ✓ tests/plugin-store-bounds.test.ts (11 tests) 69ms + ✓ tests/classify-outcome.test.ts (11 tests) 7417ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2007ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3005ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2000ms + ✓ tests/authored-completion-recovery.test.ts (5 tests) 375ms + ✓ tests/backlog-picker-flow.test.ts (6 tests) 262ms + ✓ tests/hosted-extension-protocol-intrinsics.test.ts (6 tests) 13ms + ✓ tests/preflight-permissions-unenforced.test.ts (17 tests) 517ms + ✓ an authored TypeScript body is out of reach, and spec.ts says so > checks clean on a .flow.ts whose body declares permissions 473ms + ✓ tests/budget-unmetered-live.test.ts (4 tests) 1245ms + ✓ unmetered budget spend through the live kernel > runs an unpriced step under a dollar budget without tripping it, journaling unknown dollars 444ms + ❯ tests/stuck-run-triage.test.ts (22 tests | 22 failed) 74ms + × stuck-run-triage input validation > refuses an 8-character run-id prefix: Cloud has no prefix lookup 4ms + → expected [Function] to throw error matching /not full Cloud run ids: c649fe14/ but got 'expected an @relayflows/surface flow …' + × stuck-run-triage input validation > refuses the whole batch when any id is invalid, rather than dropping it 1ms + → expected [Function] to throw error matching /not full Cloud run ids: nope!/ but got 'expected an @relayflows/surface flow …' + × stuck-run-triage input validation > refuses an empty batch 0ms + → expected [Function] to throw error matching /needs runIds/ but got 'expected an @relayflows/surface flow …' + × stuck-run-triage input validation > refuses a batch too large for the edge step lease 0ms + → expected [Function] to throw error matching /exceeds the 8 that fit/ but got 'expected an @relayflows/surface flow …' + × stuck-run-triage input validation > accepts eight ids — the incident batch is inside the bound 3ms + → promise rejected "TypeError: expected an @relayflows/surfac…" instead of resolving + × stuck-run-triage apiUrl > refuses to send the Cloud bearer token to an unapproved origin 0ms + → expected [Function] to throw error matching /refusing to send the Cloud bearer to…/\ but got 'expected an @relayflows/surface flow …' + × stuck-run-triage apiUrl > refuses a non-URL apiUrl 1ms + → expected [Function] to throw error matching /is not a URL/ but got 'expected an @relayflows/surface flow …' + × stuck-run-triage apiUrl > allows an approved origin and uses it in the curl 0ms + → expected an @relayflows/surface flow handle + × stuck-run-triage apiUrl > defaults to production Cloud 0ms + → expected an @relayflows/surface flow handle + × stuck-run-triage apiUrl > never publishes a run record the fetch did not produce 0ms + → expected an @relayflows/surface flow handle + × stuck-run-triage edge collection > names the Worker on every wrangler invocation 0ms + → expected an @relayflows/surface flow handle + × stuck-run-triage edge collection > accepts caller-supplied Workers and rejects option-shaped ones 0ms + → expected an @relayflows/surface flow handle + × stuck-run-triage edge collection > falls back when GNU timeout is absent, as it is on macOS 0ms + → expected an @relayflows/surface flow handle + × stuck-run-triage edge collection > runs the tails concurrently so wall time does not scale with the batch 0ms + → expected an @relayflows/surface flow handle + × stuck-run-triage edge collection > records wrangler's own exit status rather than head's 0ms + → expected an @relayflows/surface flow handle + × stuck-run-triage shell text > parses under both sh and bash 0ms + → expected an @relayflows/surface flow handle + × stuck-run-triage shell text > collects tails with no GNU timeout on PATH, as on a stock macOS 58ms + → expected an @relayflows/surface flow handle + × stuck-run-triage agents > declares read-only permissions on every agent 0ms + → expected an @relayflows/surface flow handle + × stuck-run-triage agents > tells the forensics agents their evidence is untrusted 0ms + → expected an @relayflows/surface flow handle + × stuck-run-triage fan-out > refuses a duplicate run id: two tails would share one evidence file 1ms + → expected [Function] to throw error matching /duplicate runIds: c649fe14-0c2e-4e51-…/ but got 'expected an @relayflows/surface flow …' + × stuck-run-triage fan-out > refuses a duplicate Worker name for the same reason 0ms + → expected [Function] to throw error matching /duplicate workers: w-one/ but got 'expected an @relayflows/surface flow …' + × stuck-run-triage fan-out > bounds ids x workers, not just ids 0ms + → expected [Function] to throw error matching /24 concurrent tails, over the 16/ but got 'expected an @relayflows/surface flow …' + ✓ tests/hosted-extension-routing.test.ts (7 tests) 7ms + ✓ tests/webhook.test.ts (9 tests) 784ms + ✓ webhook ingress > checks TS declarations against flows.json without invoking handlers 675ms + ✓ tests/redact.test.ts (54 tests) 10ms + ✓ tests/cli-probe.test.ts (17 tests) 1830ms + ✓ tests/authored-advisory-run-live.test.ts (16 tests) 883ms + ✓ tests/webhook-live.test.ts (6 tests) 9746ms + ✓ executes and deduplicates 'app_mention' only for its provider and matching payload 1471ms + ✓ executes and deduplicates 'reaction_added' only for its provider and matching payload 1447ms + ✓ executes and deduplicates 'pull_request' only for its provider and matching payload 1448ms + ✓ flows serve-webhook writes JSON before the daemon starts, then journals and archives exactly once 1444ms + ✓ replays a dropped file after SIGKILL before spawn 478ms + ✓ resumes the same journal after SIGKILL after spawn and before acknowledgement 3457ms + ✓ tests/authored-parallel-agents.test.ts (8 tests) 8916ms + ✓ authored steps under local workers with capacity > runs more concurrent f.llm calls than the worker holds side by side, never more than its capacity 1086ms + ✓ authored steps under local workers with capacity > completes more concurrent f.agent calls than the worker holds: the overflow waits for a slot instead of parking 1473ms + ✓ authored steps under local workers with capacity > runs agents in distinct working directories side by side (the kernel carries cwd) 596ms + ✓ authored steps under local workers with capacity > serializes agents whose cwd is a symlink alias of the same directory 1014ms + ✓ authored steps under local workers with capacity > serializes agents whose cwd is a directory nested inside the other 1036ms + ✓ authored steps under local workers with capacity > never starts queued agents once the body has failed 1568ms + ✓ authored steps under local workers with capacity > never starts a queued agent when the agent holding the only slot fails 1570ms + ✓ authored steps under local workers with capacity > parks the overflow when the body is not told the capacity (the defect this closes) 573ms + ✓ tests/local-agent-live.test.ts (7 tests) 42312ms + ✓ built CLI local agent against a real daemon > dispatches through the wrapper and keeps --json stdout report-shaped 766ms + ✓ built CLI local agent against a real daemon > runs beyond the initial 30-second lease without a second invocation 35817ms + ✓ built CLI local agent against a real daemon > renders actual agent completion in text output 888ms + ✓ built CLI local agent against a real daemon > returns a failed run when the agent process fails 813ms + ✓ built CLI local agent against a real daemon > prints a remedy that runs the parked authored flow to completion 1608ms + ✓ built CLI local agent against a real daemon > prints a remedy that runs, for a run started from an input file too long to be one 1818ms + ✓ built CLI local agent against a real daemon > refuses a workspace it cannot pin before invoking the agent 600ms + ✓ tests/cli-watch.test.ts (10 tests) 16514ms + ✓ flows check --watch > rechecks syntax errors, clears once, and returns the last refusal on Ctrl-C 1335ms + ✓ flows check --watch > streams JSON lines without ANSI, recovers after atomic saves, and exits zero after repair 1867ms + ✓ flows check --watch > coalesces 20 concurrent saves into at most two rechecks 1870ms + ✓ flows check --watch > watches transitive relative use imports, cycles, and nearest config changes 2461ms + ✓ flows check --watch > refreshes the import graph and notices missing imports being created 2520ms + ✓ flows check --watch > reloads authored TypeScript instead of reusing the first imported definition 1804ms + ✓ flows check --watch > detects a nearer config appearing and falls back after it is deleted 1973ms + ✓ flows check --watch > keeps watching after the target is deleted and recreated 1905ms + ✓ flows check --watch > queues changes during a slow check without overlapping checks 775ms + ✓ tests/check-worker-surface.test.ts (11 tests) 80ms + ✓ tests/human-live.test.ts (3 tests) 6825ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 3963ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1759ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 1102ms +(node:58939) ExperimentalWarning: SQLite is an experimental feature and might change at any time +(Use `node --trace-warnings ...` to show where the warning was created) + ✓ tests/authored-status-detail.test.ts (20 tests) 124ms + ✓ tests/authored-step-failed.test.ts (10 tests) 42ms + ✓ tests/budget-preflight.test.ts (25 tests) 18ms + ✓ tests/authored-retried-child-resume-live.test.ts (2 tests) 1732ms + ✓ an authored step whose child run retried an attempt > resumes after a kill mid-step and reads the terminal completion, not the crashed one 1619ms + ✓ tests/provider-trigger-contract.test.ts (7 tests) 872ms + ✓ provider trigger contract > accepts one subscription from each of five providers and refuses a bogus event on any of them 510ms + ✓ provider trigger contract > fails `flows check` before deployment and passes once the event is real 357ms + ✓ tests/shipped-source-flow-header-provenance.test.ts (1 test) 2910ms + ✓ shipped-source flow header provenance > fails closed for unsafe authored and declarative headers 2910ms + ✓ tests/work-package-consumer.test.ts (13 tests) 111ms + ✓ tests/helpers-fanout.test.ts (96 tests) 159ms +(node:59280) ExperimentalWarning: SQLite is an experimental feature and might change at any time +(Use `node --trace-warnings ...` to show where the warning was created) + ✓ tests/authored-step-graph-live.test.ts (1 test) 944ms + ✓ the authored step DAG through the live kernel > carries labels and predecessors on every index record and journal step, ids unchanged 943ms + ✓ tests/named-gate-journal.test.ts (5 tests) 1760ms + ✓ a lowered subprocess_gate journals the gate command's streams > persists what the gate printed before a timeout killed it 828ms + ✓ a gate on an agent step > journals the command's streams and surfaces them in the authored failure 647ms + ✓ tests/generate-triggers.test.ts (7 tests) 1403ms + ✓ discovers new adapters, preserves exact event names, and prefers adapter-local mappings 404ms + ✓ tests/local-agent-remedy.test.ts (16 tests) 13ms + ✓ tests/yaml-local-agent-live.test.ts (9 tests) 6529ms + ✓ YAML --local-agent through the built CLI and real daemon > runs with the checked step CLI and model and journals done 601ms + ✓ YAML --local-agent through the built CLI and real daemon > runs with the checked named CLI and model and journals done 622ms + ✓ YAML --local-agent through the built CLI and real daemon > runs with the checked flow CLI and model and journals done 590ms + ✓ YAML --local-agent through the built CLI and real daemon > runs with the checked project CLI and model and journals done 582ms + ✓ YAML --local-agent through the built CLI and real daemon > still parks without --local-agent 539ms + ✓ YAML --local-agent through the built CLI and real daemon > resumes a parked spec run with --local-agent instead of parking identically again 1004ms + ✓ YAML --local-agent through the built CLI and real daemon > parks a second time with a different message when the attached worker is not eligible 1020ms + ✓ YAML --local-agent through the built CLI and real daemon > reports the agent process failure 852ms + ✓ YAML --local-agent through the built CLI and real daemon > preserves declared workspace surfaces that the local worker cannot pin 717ms + ✓ tests/yaml-helpers.test.ts (34 tests) 77ms + ✓ tests/direct-run-failure.test.ts (12 tests) 41ms + ✓ tests/authored-agent-permissions.test.ts (27 tests) 902ms + ✓ tests/authored-detail-live.test.ts (3 tests) 3737ms + ✓ carries done("step_failed", { detail }) into the report, the journal and flows status 2146ms + ✓ reports a detail a caller sliced through an emoji, instead of hanging on it 603ms + ✓ leaves a one-argument done("step_failed") reporting exactly as it always did 987ms + ✓ tests/live-event-activities.test.ts (2 tests) 202ms + ✓ tests/software-garden-babysitter-composition.test.ts (6 tests | 2 skipped) 43ms + ✓ tests/worker-lease.test.ts (7 tests) 18ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 25ms + ✓ tests/webhook-hardening.test.ts (11 tests) 58ms + ✓ tests/authored-probe-cache.test.ts (5 tests) 21050ms + ✓ authored run CLI probe cache > nine calls have no expired attempts at capacity 1 6998ms + ✓ authored run CLI probe cache > nine calls have no expired attempts at capacity 4 5396ms + ✓ authored run CLI probe cache > does not serialize nine starts behind nine probes, and probes again on a new run 7027ms + ✓ authored run CLI probe cache > caches probe failures while refusing all nine calls 397ms + ✓ authored run CLI probe cache > shares probe results across agent calls too 1232ms + ✓ tests/yaml-declared-streams-live.test.ts (8 tests) 5063ms + ✓ YAML declared streams through the built CLI and real daemon > runs with the checked step CLI and model and journals done 609ms + ✓ YAML declared streams through the built CLI and real daemon > runs with the checked named CLI and model and journals done 599ms + ✓ YAML declared streams through the built CLI and real daemon > runs with the checked flow CLI and model and journals done 606ms + ✓ YAML declared streams through the built CLI and real daemon > runs with the checked project CLI and model and journals done 587ms + ✓ YAML declared streams through the built CLI and real daemon > still parks without --local-agent 539ms + ✓ YAML declared streams through the built CLI and real daemon > reports the agent process failure 613ms + ✓ YAML declared streams through the built CLI and real daemon > resumes the journaled stream declarations after the input YAML changes 1001ms + ✓ YAML declared streams through the built CLI and real daemon > preserves declared workspace surfaces that the local worker cannot pin 508ms + ✓ tests/human-to.test.ts (8 tests) 13ms +(node:61719) ExperimentalWarning: SQLite is an experimental feature and might change at any time +(Use `node --trace-warnings ...` to show where the warning was created) + ✓ tests/authored-dispatch-live.test.ts (2 tests) 2417ms + ✓ durable child flow dispatch > runs a declared child in the parent DAG and replays it without repeating effects 1621ms + ✓ durable child flow dispatch > lowers a named gate attached to the dispatch receipt 794ms + ✓ tests/plugin-loader.test.ts (9 tests) 195ms + ✓ tests/worker-lease-drift-live.test.ts (3 tests) 36205ms + ✓ worker lease under clock skew between worker and daemon > renews before the daemon deadline when the worker clock runs 75s behind the daemon 36062ms + ✓ tests/deploy.test.ts (11 tests) 5316ms + ✓ flows deploy file buckets > publishes the full signed layout byte-for-byte and redeploys as a noop 824ms + ✓ flows deploy file buckets > answers --json with one object per outcome 830ms + ✓ flows deploy file buckets > reports a refusal as JSON under --json 398ms + ✓ flows deploy file buckets > refuses a missing local bundle before creating the bucket 394ms + ✓ flows deploy file buckets > refuses an unreachable bucket before copying 403ms + ✓ flows deploy file buckets > refuses an unwritable bucket 410ms + ✓ flows deploy file buckets > refuses local tampering of spec.canonical.json 405ms + ✓ flows deploy file buckets > refuses local tampering of identity.json 419ms + ✓ flows deploy file buckets > refuses asset bundles instead of using daemon-relative files 392ms + ✓ flows deploy file buckets > never labels a corrupt existing deployment as a noop 813ms + ✓ tests/bin.test.ts (12 tests) 5976ms + ✓ built flows binary > prints the installed SDK version for --version 383ms + ✓ built flows binary > prints the installed SDK version for -V 389ms + ✓ built flows binary > keeps version flags strict when extra arguments are supplied 377ms + ✓ built flows binary > prints the installed SDK version from the standalone binary for --version 973ms + ✓ built flows binary > prints the installed SDK version from the standalone binary for -V 987ms + ✓ built flows binary > refuses through a symlink to the built artifact 383ms + ✓ built flows binary > refuses through a symlinked directory component 400ms + ✓ built flows binary > classifies a signal-terminated auth probe as probe_failed 859ms + ✓ built flows binary > classifies an unavailable PATH resolver as probe_failed 420ms + ✓ built flows binary > does not describe a present non-executable CLI as missing 403ms + ✓ built flows binary > runs one auth probe for three steps sharing a flow CLI 399ms + ✓ tests/babysitter-catalog-export.test.ts (14 tests) 759ms + ✓ Babysitter catalog artifact export > CLI refuses an existing output and leaves no file on validation failure 606ms + ✓ tests/journal-client-read-timeout.test.ts (13 tests) 1065ms + ✓ a recovered read timeout does not become an authored callback failure 337ms + ✓ tests/communication.test.ts (10 tests) 13ms + ✓ tests/communication-review.test.ts (6 tests) 324ms + ✓ tests/transport-evidence-bounds.test.ts (7 tests) 11ms + ✓ tests/typed-output.test.ts (14 tests) 209ms + ✓ tests/shipped-source-models.test.ts (2 tests) 39331ms + ✓ first-party shipped source model pins > gives every TypeScript agent and LLM an explicit supported pair or a pinned named-agent declaration 39293ms + ✓ tests/agent-timeout-live.test.ts (3 tests) 4254ms + ✓ journals timeout, stops the process, runs a predicate gate and publishes under a budget header 1156ms + ✓ replays timeout after killing the root and daemon, without executing the agent again 1973ms + ✓ lets an author reject timeout through a predicate gate 1123ms + ✓ tests/effect-channel.test.ts (5 tests) 397ms + ✓ tests/worker-slots.test.ts (8 tests) 7ms + ✓ tests/step-lease.test.ts (36 tests) 66445ms + ✓ f.run leases against the live kernel > enforces 10000 ms for 'sleep 5; printf ok' 5065ms + ✓ f.run leases against the live kernel > enforces 40000 ms for 'sleep 31; printf ok' 31101ms + ✓ f.run leases against the live kernel > enforces 30000 ms for 'sleep 31; printf ok' 30086ms + ✓ tests/json-schema-bound.test.ts (71 tests) 2320ms + ✓ JSON Schema termination bound > walks a deep schema with an explicit stack rather than recursion 1896ms + ✓ tests/cloud-mirror-transport.test.ts (5 tests) 9ms + ✓ tests/mcp-lifecycle.test.ts (4 tests) 19ms + ✓ tests/communication-worker.test.ts (17 tests) 1708ms + ✓ tests/model-selection.test.ts (10 tests) 17ms + ✓ tests/f-memory.test.ts (7 tests) 781ms + ✓ tests/relayflowd-path.test.ts (10 tests) 5ms + ✓ tests/agent-artifacts.test.ts (9 tests) 20ms + ✓ tests/authored-plugin-effect.test.ts (6 tests) 65ms + ✓ tests/agent-timeout-worker.test.ts (9 tests) 2190ms + ✓ stops claude at its declared limit, preserving work and stopping descendants 523ms + ✓ stops codex at its declared limit, preserving work and stopping descendants 517ms + ✓ stops wrapper at its declared limit, preserving work and stopping descendants 536ms + ✓ tests/hosted-hardening.test.ts (11 tests) 15ms + ✓ tests/local-dev-ux.test.ts (8 tests) 14ms + ✓ tests/authored-preflight.test.ts (5 tests) 214ms + ✓ tests/deterministic-llm.test.ts (5 tests) 53ms + ✓ tests/relay-cli-surface-live.test.ts (3 tests) 416ms + ✓ tests/authored-declined.test.ts (13 tests) 51ms + ✓ tests/resume-failure.test.ts (2 tests) 5ms + ✓ tests/dependency-validation.test.ts (6 tests) 759ms + ✓ dependency validation > accepts a valid 10,000-step reverse chain through every direct public boundary 417ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 878ms + ✓ reports journal success after completion rejects with lease_conflict 315ms + ✓ tests/authored-hooks.test.ts (5 tests) 6ms + ✓ tests/input-binding.test.ts (12 tests) 201ms + ✓ tests/yaml-helper-effect.test.ts (4 tests) 83ms + ✓ tests/cloud-mirror-ledger.test.ts (7 tests) 1121ms + ✓ the mirror ledger > keeps the newest entries when the count cap bites 1109ms + ✓ tests/scope-preflight.test.ts (6 tests) 8ms + ✓ tests/run-from-digest.test.ts (6 tests) 4580ms + ✓ flows run digest input > submits the sealed canonical spec through the normal journal path without checkout 431ms + ✓ flows run digest input > uses a verified cache hit even after the bucket is removed 453ms + ✓ flows run digest input > resolves deploy.bucket from flows.json and honors explicit override 1216ms + ✓ flows run digest input > refuses an unconfigured bucket 809ms + ✓ flows run digest input > refuses tampered spec.canonical.json before creating run data 810ms + ✓ flows run digest input > refuses tampered identity.json before creating run data 859ms + ✓ tests/build-gate.test.ts (3 tests) 1201ms + ✓ flows build gates on flows check green (#318) > refuses a flow with an unresolvable named-agent CLI and leaves no artifacts 388ms + ✓ flows build gates on flows check green (#318) > --json emits one CheckReport object on stdout on refusal, exits 2, no artifacts 394ms + ✓ flows build gates on flows check green (#318) > builds the bundle on success (regression: gate must not block valid flows) 418ms + ✓ tests/scope-compiler.test.ts (25 tests) 11ms + ✓ tests/worker-cli-abort.test.ts (3 tests) 3748ms + ✓ stops claude and its process group when lease ownership is lost 1365ms + ✓ stops wrapper.mjs and its process group when lease ownership is lost 1350ms + ✓ fails closed when abort cannot prove the process group is gone 1033ms + ✓ tests/hn-poller.test.ts (6 tests) 7ms + ✓ tests/plugin-add.test.ts (7 tests) 1144ms + ✓ typechecks the augmented verb and rejects unknown namespaces 855ms + ✓ tests/authored-step-failed-exit.test.ts (3 tests) 9ms + ✓ tests/run-read-load-live.test.ts (2 tests) 3060ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2431ms + ✓ drains read and watch promises before an authored flow completes 628ms + ✓ tests/retried-step-failure.test.ts (1 test) 904ms + ✓ a retried step failing differently through the live kernel > reports both attempts and says the evidence differs 903ms + ✓ tests/dir-watcher-poller.test.ts (6 tests) 7ms + ✓ tests/subscription-report.test.ts (8 tests) 5ms + ✓ tests/worker-lease-sweep.test.ts (4 tests) 9ms + ✓ tests/model-pricing.test.ts (10 tests) 5ms + ✓ tests/authored-use-loader.test.ts (6 tests) 693ms + ✓ tests/yaml-helper-live.test.ts (1 test) 957ms + ✓ runs compiled YAML helpers through the built CLI and kernel effect journal 957ms + ✓ tests/provider-trigger-executor.test.ts (4 tests) 228ms + ✓ tests/journal-client-completion.test.ts (6 tests) 104ms + ✓ tests/transcript-tail-close.test.ts (2 tests) 1303ms + ✓ a stalled transcript-tail close > does not hold the spawn open past its bounded window 647ms + ✓ a stalled tail close beside a transcript that finished > still journals the transcript pointer 654ms + ✓ tests/wrapper-artifacts-cwd.test.ts (2 tests) 123ms + ✓ tests/hello-deterministic.test.ts (5 tests) 17ms + ✓ tests/transcript-exclusion-timeout.test.ts (1 test) 287ms + ✓ tests/cli-adapter.test.ts (4 tests) 5ms + ✓ tests/communication-mixed-resume.test.ts (1 test) 169ms + ✓ tests/work-package-validator.test.ts (7 tests) 5ms + ✓ tests/activity-preflight.test.ts (10 tests) 21ms + ✓ tests/authored-declined-live.test.ts (1 test) 1606ms + ✓ runs an input guard and resumes its completed declined root without repeated effects 1605ms + ✓ tests/cli-answer.test.ts (15 tests) 10ms + ✓ tests/bundle-preflight.test.ts (4 tests) 880ms + ✓ bundle execution preflight > ignores surrounding cache configuration on a verified cache hit 448ms + ✓ bundle execution preflight > uses the built alias for a nameless flow even in a digest-only cache directory 416ms + ✓ tests/agent-timeout-outcome.test.ts (6 tests) 18ms + ✓ tests/agent-relay-hardening.test.ts (12 tests) 16ms + ✓ tests/resume-worker-lease.test.ts (3 tests) 6ms + ✓ tests/communication-preflight.test.ts (13 tests) 31ms + ✓ tests/direct-run-worker-lease.test.ts (3 tests) 9ms + ↓ tests/real-cli-adapters.test.ts (3 tests | 3 skipped) + ✓ tests/memoization.test.ts (57 tests) 54ms + ✓ tests/shipped-source-call-array-forwarding.test.ts (1 test) 2781ms + ✓ shipped-source call-array forwarding > maps rest indexes and nested same-helper actuals for workers and flows 2780ms + ✓ tests/local-agent-environment.test.ts (8 tests) 6ms + ✓ tests/fs-descriptor.test.ts (1 test) 4ms + ✓ tests/running-step-watch.test.ts (2 tests) 2120ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms + ✓ tests/runtime-dependencies.test.ts (1 test) 210ms + ✓ tests/parse-json-output.test.ts (7 tests) 4ms + ✓ tests/reported-cost.test.ts (4 tests) 3ms + ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms + ✓ tests/authored-model-probe-kinds.test.ts (2 tests) 199ms + ✓ tests/communication-environment-preflight.test.ts (6 tests) 5ms + ✓ tests/budget-authored-live.test.ts (2 tests) 181ms + ✓ tests/slack-writeback.test.ts (1 test) 257ms + ✓ tests/authored-surface-authority.test.ts (2 tests) 19ms + ✓ tests/adapters/claude.test.ts (7 tests) 4ms + ✓ tests/worker-cli-cwd.test.ts (2 tests) 360ms + ✓ runAgentCli — cwd propagation (flows#357) > omits cwd when not provided (inherits parent cwd) 354ms + ✓ tests/adapters/codex.test.ts (7 tests) 5ms + ✓ tests/shipped-source-parameter-provenance.test.ts (1 test) 1630ms + ✓ shipped-source parameter provenance > keeps caller-supplied identifier and destructured keys unknown 1629ms + ✓ tests/slack-block-kit.test.ts (5 tests) 14ms + ✓ tests/agent-timeout.test.ts (23 tests) 35ms + ✓ tests/communication-history.test.ts (1 test) 3ms + ✓ tests/adapters/registry.test.ts (4 tests) 4ms + ✓ tests/authored-declined-report.test.ts (6 tests) 7ms + ✓ tests/promise-ancestry.test.ts (2 tests) 260ms + ✓ tests/agent-cwd-validation.test.ts (2 tests) 397ms + ✓ declarative agent cwd > is refused by `flows check` on a YAML flow before anything runs 394ms + ✓ tests/communication-refusal.test.ts (1 test) 14ms + ✓ tests/bundle-transport.test.ts (20 tests) 2491ms + ✓ digest references > accepts and deploys the build output for hello 463ms + ✓ digest references > accepts and deploys the build output for Hello 413ms + ✓ digest references > accepts and deploys the build output for hello.world 408ms + ✓ digest references > accepts and deploys the build output for hello_world 397ms + ✓ digest references > accepts and deploys the build output for 123 408ms + ✓ digest references > accepts and deploys the build output for A_b.c-1 399ms + ✓ tests/cli-cloud-mirror-flag.test.ts (4 tests) 3ms + ✓ tests/catalog-plugins.test.ts (2 tests) 4ms + ✓ tests/check-command-cwd.test.ts (1 test) 23ms + ✓ tests/communication-lazy.test.ts (1 test) 5ms + ✓ tests/cli-progress-wait.test.ts (2 tests) 4ms + ✓ tests/run-digest-live.test.ts (1 test) 924ms + ✓ executes a deployed digest on the real kernel after deleting the authoring tree 923ms + ✓ tests/run-daemon-unresponsive.test.ts (2 tests) 4ms + ✓ tests/placement.test.ts (54 tests) 18ms + ✓ tests/preflight-run-cache.test.ts (1 test) 5ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms + ✓ tests/canonical-tree.test.ts (1 test) 2ms + ✓ tests/local-agent-stream-selection.test.ts (1 test) 5ms + ✓ tests/run-digest.test.ts (4 tests) 1552ms + ✓ digest run configuration refusals > reports config_invalid before fetching or starting a run for {invalid json 397ms + ✓ digest run configuration refusals > reports config_invalid before fetching or starting a run for {"deploy":{}} 383ms + ✓ digest run configuration refusals > reports config_invalid before fetching or starting a run for {"deploy":{"bucket":123}} 380ms + ✓ digest run configuration refusals > reports config_invalid before fetching or starting a run for {"deploy":{"bucket":""}} 391ms + ✓ tests/communication-tools.test.ts (1 test) 64ms + ✓ tests/authored-admission.test.ts (2 tests) 3ms + ✓ tests/memory.test.ts (18 tests) 8ms + ✓ tests/worker-platform.test.ts (1 test) 3ms +$ /usr/local/share/nvm/versions/node/v25.6.0/bin/node scripts/pack-release.mjs surface /tmp/flows-cli-package-h2FUqF/tarballs +PACK_OK @relayflows/surface@2.0.42: package/dist/index.js, package/dist/index.d.ts, package/dist/runtime.js, package/dist/runtime.d.ts +$ /usr/local/share/nvm/versions/node/v25.6.0/bin/node scripts/pack-release.mjs sdk /tmp/flows-cli-package-h2FUqF/tarballs +PACK_OK @relayflows/sdk@2.0.42: package/dist/index.js, package/dist/index.d.ts, package/dist/cli.js +$ /usr/local/share/nvm/versions/node/v25.6.0/bin/node scripts/pack-release.mjs relayflows /tmp/flows-cli-package-h2FUqF/tarballs +PACK_OK relayflows@2.0.42: package/bin/flows.js +$ npm install --ignore-scripts --omit=optional --no-audit --no-fund /tmp/flows-cli-package-h2FUqF/tarballs/relayflows-2.0.42.tgz /tmp/flows-cli-package-h2FUqF/tarballs/relayflows-sdk-2.0.42.tgz /tmp/flows-cli-package-h2FUqF/tarballs/relayflows-surface-2.0.42.tgz + +added 150 packages in 5s +$ /tmp/flows-cli-package-h2FUqF/local/node_modules/.bin/flows check examples/dependency-upgrade-bot.flow.ts +REQUIRES claude (agent "upgrader") +CHECK PASSED examples/dependency-upgrade-bot.flow.ts +$ /usr/local/share/nvm/versions/node/v25.6.0/bin/node /tmp/flows-cli-package-h2FUqF/local/node_modules/relayflows/bin/flows.js check examples/dependency-upgrade-bot.flow.ts +REQUIRES claude (agent "upgrader") +CHECK PASSED examples/dependency-upgrade-bot.flow.ts +$ npm install -g --prefix /tmp/flows-cli-package-h2FUqF/prefix --ignore-scripts --omit=optional --no-audit --no-fund /tmp/flows-cli-package-h2FUqF/tarballs/relayflows-2.0.42.tgz /tmp/flows-cli-package-h2FUqF/tarballs/relayflows-sdk-2.0.42.tgz /tmp/flows-cli-package-h2FUqF/tarballs/relayflows-surface-2.0.42.tgz + +added 294 packages in 5s +$ npm install --ignore-scripts --omit=optional --no-audit --no-fund /tmp/flows-cli-package-h2FUqF/tarballs/relayflows-surface-2.0.42.tgz + +added 140 packages in 2s +$ /tmp/flows-cli-package-h2FUqF/prefix/bin/flows check examples/dependency-upgrade-bot.flow.ts +REQUIRES claude (agent "upgrader") +CHECK PASSED examples/dependency-upgrade-bot.flow.ts +$ /usr/local/share/nvm/versions/node/v25.6.0/bin/node /tmp/flows-cli-package-h2FUqF/prefix/lib/node_modules/relayflows/bin/flows.js check examples/dependency-upgrade-bot.flow.ts +REQUIRES claude (agent "upgrader") +CHECK PASSED examples/dependency-upgrade-bot.flow.ts +CLI_PACKAGE_OK: local and global installs + ✓ tests/cli-package-gate.test.ts (1 test) 19640ms + ✓ checks authored flows with locally and globally installed release tarballs 19639ms + ✓ tests/event-await-cli.test.ts (1 test) 13231ms + ✓ parks, restarts, and replays two event wakes through the actual CLI 13228ms + ✓ tests/authored-parallel-llm.test.ts (6 tests) 115215ms + ✓ parallel llm capacity 1 > deduplicates concurrent preflight probes 3257ms + ✓ parallel llm capacity 1 > completes nine calls without expired child leases during slow preflight 8921ms + ✓ parallel llm capacity 1 > keeps the durable root lease alive across two cold models 47937ms + ✓ parallel llm capacity 4 > deduplicates concurrent preflight probes 1356ms + ✓ parallel llm capacity 4 > completes nine calls without expired child leases during slow preflight 7251ms + ✓ parallel llm capacity 4 > keeps the durable root lease alive across two cold models 46493ms + ✓ tests/shipped-source-worker-invocations.test.ts (1 test) 756344ms + ✓ shipped-source worker invocation resolution > fails closed across direct, extracted, bound, mutable, and escaped callables 756343ms + +⎯⎯⎯⎯⎯⎯ Failed Suites 1 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/authored-node-runtime.test.ts [ tests/authored-node-runtime.test.ts ] +AssertionError: expected '1.3.6' to be '1.4.0' // Object.is equality + +Expected: "1.4.0" +Received: "1.3.6" + + ❯ tests/authored-node-runtime.test.ts:18:77 + 16| + 17| beforeAll(() => { + 18| expect(spawnSync(bun, ['--version'], { encoding: 'utf8' }).stdout.tr… + | ^ + 19| expect(existsSync(daemon), 'build the current kernel or set RELAYFLO… + 20| stage = mkdtempSync(join(tmpdir(), 'authored-standalone-build-')); + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/78]⎯ + +⎯⎯⎯⎯⎯⎯ Failed Tests 77 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/canonical-software-factory.test.ts > canonical software-factory metadata contract > opens the actual catalog flow with the ticket title and exactly one GitHub closing line +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:169:11 + ❯ runCanonical tests/canonical-software-factory.test.ts:73:22 + 71| }; + 72| + 73| const definition = getFlowDefinition(softwareFactory); + | ^ + 74| return definition.body(context as never, { issue, approver: 'khaliq'… + 75| root, + ❯ tests/canonical-software-factory.test.ts:89:26 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[2/78]⎯ + + FAIL tests/canonical-software-factory.test.ts > canonical software-factory metadata contract > writes a Linear closing reference that the GitHub integration links back +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:169:11 + ❯ runCanonical tests/canonical-software-factory.test.ts:73:22 + 71| }; + 72| + 73| const definition = getFlowDefinition(softwareFactory); + | ^ + 74| return definition.body(context as never, { issue, approver: 'khaliq'… + 75| root, + ❯ tests/canonical-software-factory.test.ts:106:26 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[3/78]⎯ + + FAIL tests/canonical-software-factory.test.ts > canonical software-factory metadata contract > accepts a Linear team key that carries digits +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:169:11 + ❯ runCanonical tests/canonical-software-factory.test.ts:73:22 + 71| }; + 72| + 73| const definition = getFlowDefinition(softwareFactory); + | ^ + 74| return definition.body(context as never, { issue, approver: 'khaliq'… + 75| root, + ❯ tests/canonical-software-factory.test.ts:115:26 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[4/78]⎯ + + FAIL tests/canonical-software-factory.test.ts > canonical software-factory metadata contract > stops before push when a Linear ticket has no linkable identifier () + FAIL tests/canonical-software-factory.test.ts > canonical software-factory metadata contract > stops before push when a Linear ticket has no linkable identifier (not-an-issue) +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:169:11 + ❯ runCanonical tests/canonical-software-factory.test.ts:73:22 + 71| }; + 72| + 73| const definition = getFlowDefinition(softwareFactory); + | ^ + 74| return definition.body(context as never, { issue, approver: 'khaliq'… + 75| root, + ❯ tests/canonical-software-factory.test.ts:126:28 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[5/78]⎯ + + FAIL tests/canonical-software-factory.test.ts > canonical software-factory metadata contract > fails closed before push when the Linear closing reference is duplicated in the final body +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:169:11 + ❯ runCanonical tests/canonical-software-factory.test.ts:73:22 + 71| }; + 72| + 73| const definition = getFlowDefinition(softwareFactory); + | ^ + 74| return definition.body(context as never, { issue, approver: 'khaliq'… + 75| root, + ❯ tests/canonical-software-factory.test.ts:138:26 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[6/78]⎯ + + FAIL tests/canonical-software-factory.test.ts > canonical software-factory metadata contract > fails closed before push when the body carries a foreign closing reference +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:169:11 + ❯ runCanonical tests/canonical-software-factory.test.ts:73:22 + 71| }; + 72| + 73| const definition = getFlowDefinition(softwareFactory); + | ^ + 74| return definition.body(context as never, { issue, approver: 'khaliq'… + 75| root, + ❯ tests/canonical-software-factory.test.ts:150:26 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[7/78]⎯ + + FAIL tests/canonical-software-factory.test.ts > canonical software-factory metadata contract > does not mistake ordinary closing-verb prose for a reference +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:169:11 + ❯ runCanonical tests/canonical-software-factory.test.ts:73:22 + 71| }; + 72| + 73| const definition = getFlowDefinition(softwareFactory); + | ^ + 74| return definition.body(context as never, { issue, approver: 'khaliq'… + 75| root, + ❯ tests/canonical-software-factory.test.ts:162:26 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[8/78]⎯ + + FAIL tests/canonical-software-factory.test.ts > canonical software-factory metadata contract > normalizes whitespace and caps the title at 240 Unicode code points +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:169:11 + ❯ runCanonical tests/canonical-software-factory.test.ts:73:22 + 71| }; + 72| + 73| const definition = getFlowDefinition(softwareFactory); + | ^ + 74| return definition.body(context as never, { issue, approver: 'khaliq'… + 75| root, + ❯ tests/canonical-software-factory.test.ts:171:26 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[9/78]⎯ + + FAIL tests/canonical-software-factory.test.ts > canonical software-factory metadata contract > fails closed before push when GitHub identity is missing or the final body duplicates its closing line +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:169:11 + ❯ tests/canonical-software-factory.test.ts:180:24 + 178| + 179| it('fails closed before push when GitHub identity is missing or the … + 180| const definition = getFlowDefinition(softwareFactory); + | ^ + 181| const commands: string[] = []; + 182| let completionReason = ''; + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[10/78]⎯ + + FAIL tests/canonical-software-factory.test.ts > canonical software-factory review scope > fails closed for explicit defects + FAIL tests/canonical-software-factory.test.ts > canonical software-factory review scope > fails closed for no verdict + FAIL tests/canonical-software-factory.test.ts > canonical software-factory review scope > fails closed for contradictory blocked and unverified + FAIL tests/canonical-software-factory.test.ts > canonical software-factory review scope > fails closed for contradictory passed and unverified + FAIL tests/canonical-software-factory.test.ts > canonical software-factory review scope > fails closed for contradictory passed and blocked + FAIL tests/canonical-software-factory.test.ts > canonical software-factory review scope > fails closed for empty unverified +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:169:11 + ❯ runCanonical tests/canonical-software-factory.test.ts:73:22 + 71| }; + 72| + 73| const definition = getFlowDefinition(softwareFactory); + | ^ + 74| return definition.body(context as never, { issue, approver: 'khaliq'… + 75| root, + ❯ tests/canonical-software-factory.test.ts:214:26 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[11/78]⎯ + + FAIL tests/canonical-software-factory.test.ts > canonical software-factory review scope > distinguishes a missing verification prerequisite from a defect +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:169:11 + ❯ runCanonical tests/canonical-software-factory.test.ts:73:22 + 71| }; + 72| + 73| const definition = getFlowDefinition(softwareFactory); + | ^ + 74| return definition.body(context as never, { issue, approver: 'khaliq'… + 75| root, + ❯ tests/canonical-software-factory.test.ts:224:26 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[12/78]⎯ + + FAIL tests/canonical-software-factory.test.ts > canonical software-factory review scope > preserves the passed body without a scope marker or draft +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:169:11 + ❯ runCanonical tests/canonical-software-factory.test.ts:73:22 + 71| }; + 72| + 73| const definition = getFlowDefinition(softwareFactory); + | ^ + 74| return definition.body(context as never, { issue, approver: 'khaliq'… + 75| root, + ❯ tests/canonical-software-factory.test.ts:235:26 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[13/78]⎯ + + FAIL tests/canonical-software-factory.test.ts > canonical software-factory review scope > rejects a forged scope for local before publication + FAIL tests/canonical-software-factory.test.ts > canonical software-factory review scope > rejects a forged scope for github before publication +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:169:11 + ❯ runCanonical tests/canonical-software-factory.test.ts:73:22 + 71| }; + 72| + 73| const definition = getFlowDefinition(softwareFactory); + | ^ + 74| return definition.body(context as never, { issue, approver: 'khaliq'… + 75| root, + ❯ tests/canonical-software-factory.test.ts:243:26 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[14/78]⎯ + + FAIL tests/canonical-software-factory.test.ts > canonical software-factory review scope > scopes the post-review hook refusal + FAIL tests/canonical-software-factory.test.ts > canonical software-factory review scope > scopes the merge-gate hook refusal +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:169:11 + ❯ runCanonical tests/canonical-software-factory.test.ts:73:22 + 71| }; + 72| + 73| const definition = getFlowDefinition(softwareFactory); + | ^ + 74| return definition.body(context as never, { issue, approver: 'khaliq'… + 75| root, + ❯ tests/canonical-software-factory.test.ts:250:26 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[15/78]⎯ + + FAIL tests/canonical-software-factory.test.ts > canonical software-factory review scope > rejects malformed head "" before publication + FAIL tests/canonical-software-factory.test.ts > canonical software-factory review scope > rejects malformed head "not-a-sha" before publication + FAIL tests/canonical-software-factory.test.ts > canonical software-factory review scope > rejects malformed head "a'; touch injected; #" before publication +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:169:11 + ❯ runCanonical tests/canonical-software-factory.test.ts:73:22 + 71| }; + 72| + 73| const definition = getFlowDefinition(softwareFactory); + | ^ + 74| return definition.body(context as never, { issue, approver: 'khaliq'… + 75| root, + ❯ tests/canonical-software-factory.test.ts:259:26 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[16/78]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > executes the exact capability-only handler for a queued receipt + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > executes the exact capability-only handler for a duplicate receipt +Error: bubblewrap is unavailable + ❯ unsupported src/hosted-extension-sandbox.ts:503:9 + 501| + 502| function unsupported(message: string): never { + 503| throw new PluginError('plugin_unsupported', message); + | ^ + 504| } + 505| + ❯ executable src/hosted-extension-sandbox.ts:484:18 + ❯ Module.runHostedExtensionSandbox src/hosted-extension-sandbox.ts:176:17 + ❯ Module.runVerifiedNativeExtensionSandbox src/hosted-extension-isolation.ts:209:16 + ❯ tests/hosted-extension-isolation.test.ts:221:26 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[17/78]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > launches through the captured process primitive after builtin export synchronization +AssertionError: promise rejected "Error: bubblewrap is unavailable { code: '…' }" instead of resolving + ❯ tests/hosted-extension-isolation.test.ts:283:11 + 281| input: descriptor(), + 282| babysitterTurn: { queue: async () => ({ receiptId: 'receipt-… + 283| })).resolves.toEqual({ completionReason: 'success', capability… + | ^ + 284| } finally { + 285| process.execPath = originalExecPath; + +Caused by: Error: bubblewrap is unavailable + ❯ unsupported src/hosted-extension-sandbox.ts:503:9 + ❯ executable src/hosted-extension-sandbox.ts:484:18 + ❯ Module.runHostedExtensionSandbox src/hosted-extension-sandbox.ts:176:17 + ❯ Module.runVerifiedNativeExtensionSandbox src/hosted-extension-isolation.ts:209:16 + ❯ tests/hosted-extension-isolation.test.ts:277:22 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ +Serialized Error: { code: 'plugin_unsupported' } +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[18/78]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > ignores inherited launcher overrides and decodes manifests with the captured Buffer intrinsic +AssertionError: promise rejected "Error: bubblewrap is unavailable { code: '…' }" instead of resolving + ❯ tests/hosted-extension-isolation.test.ts:373:11 + 371| input: descriptor('delivery-options'), + 372| babysitterTurn: { queue: async () => ({ receiptId: 'receipt-… + 373| })).resolves.toEqual({ completionReason: 'success', capability… + | ^ + 374| } finally { + 375| Buffer.prototype.toString = bufferToString; + +Caused by: Error: bubblewrap is unavailable + ❯ unsupported src/hosted-extension-sandbox.ts:503:9 + ❯ executable src/hosted-extension-sandbox.ts:484:18 + ❯ Module.runHostedExtensionSandbox src/hosted-extension-sandbox.ts:176:17 + ❯ Module.runVerifiedNativeExtensionSandbox src/hosted-extension-isolation.ts:209:16 + ❯ tests/hosted-extension-isolation.test.ts:367:22 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ +Serialized Error: { code: 'plugin_unsupported' } +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[19/78]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > streams verified bytes when the live store is replaced and no writable staging path exists +AssertionError: promise rejected "Error: Hosted extension sandbox exited wi… { code: '…' }" instead of resolving + ❯ tests/hosted-extension-isolation.test.ts:431:7 + 429| return { receiptId: 'receipt-1', status: 'queued' }; + 430| } }, + 431| })).resolves.toEqual({ completionReason: 'success', capabilityCall… + | ^ + 432| expect(calls).toBe(1); + 433| expect(readFileSync(join(installed.directory, 'babysitter.flow.ts'… + +Caused by: Error: Hosted extension sandbox exited without a valid completion (exit 1): /tmp/hosted-bwrap-wrapper-Bvf1hY/bwrap-wrapper:20 +if (child.error) throw child.error; + ^ + +Error: spawnSync /usr/bin/bwrap ENOENT + at Object.spawnSync (node:internal/child_process:1103:20) + at spawnSync (node:child_process:911:24) + at Object. (/tmp/hosted-bwrap-wrapper-Bvf1hY/bwrap-wrapper:19:15) + at Module._compile (node:internal/modules/cjs/loader:1809:14) + at Object..js (node:internal/modules/cjs/loader:1940:10) + at Module.load (node:internal/modules/cjs/loader:1530:32) + at Module._load (node:internal/modules/cjs/loader:1332:12) + at wrapModuleLoad (node:internal/modules/cjs/loader:255:19) + at Module.executeUserEntryPoint [as runMain] (node:internal/modules/run_main:154:5) + at node:internal/main/run_main_module:33:47 { + errno: -2, + code: 'ENOENT', + syscall: 'spawnSync /usr/bin/bwrap', + path: '/usr/bin/bwrap', + spawnargs: [ + '--unshare-all', + '--die-with-parent', + '--new-session', + '--clearenv', + '--cap-drop', + 'ALL', + '--dir', + '/usr', + '--ro-bind', + '/usr/lib', + '/usr/lib', + '--ro-bind', + '/usr/lib64', + '/usr/lib64', + '--ro-bind', + '/usr/lib', + '/lib', + '--ro-bind', + '/usr/lib64', + '/lib64', + '--proc', + '/proc', + '--dev', + '/dev', + '--tmpfs', + '/tmp', + '--dir', + '/runtime', + '--dir', + '/extension', + '--dir', + '/extension/node_modules', + '--dir', + '/extension/node_modules/@relayflows', + '--dir', + '/extension/node_modules/@relayflows/surface', + '--dir', + '/extension/node_modules/@relayflows/surface/dist', + '--dir', + '/extension/node_modules/@relayflows/surface/dist/helpers', + '--dir', + '/extension/node_modules/@relayflows/surface/dist/triggers', + '--dir', + '/extension/src', + '--perms', + '0500', + '--ro-bind-data', + '4', + '/runtime/node', + '--perms', + '0400', + '--ro-bind-data', + '5', + '/runtime/runner.mjs', + '--perms', + '0400', + '--ro-bind-data', + '6', + '/extension/node_modules/@relayflows/surface/package.json', + '--perms', + '0400', + '--ro-bind-data', + '7', + '/extension/node_modules/@relayflows/surface/index.js', + '--perms', + '0400', + '--ro-bind-data', + '8', + '/extension/node_modules/@relayflows/surface/runtime.js', + '--perms', + '0400', + '--ro-bind-data', + '9', + '/extension/node_modules/@relayflows/surface/dist/flow.js', + '--perms', + '0400', + '--ro-bind-data', + '10', + '/extension/node_modules/@relayflows/surface/dist/helpers/providers.js', + '--perms', + '0400', + '--ro-bind-data', + '11', + '/extension/node_modules/@relayflows/surface/dist/provider-trigger.js', + '--perms', + '0400', + '--ro-bind-data', + '12', + '/extension/node_modules/@relayflows/surface/dist/schedule.js', + '--perms', + '0400', + '--ro-bind-data', + '13', + '/extension/node_modules/@relayflows/surface/dist/triggers.js', + '--perms', + '0400', + '--ro-bind-data', + '14', + '/extension/node_modules/@relayflows/surface/dist/triggers/github.js', + '--perms', + ... 27 more items + ] +} + +Node.js v25.6.0 + + ❯ Object. ../../../../../../../tmp/hosted-bwrap-wrapper-Bvf1hY/bwrap-wrapper:19:15 + ❯ refuse src/hosted-extension-protocol.ts:135:21 + ❯ ChildProcess. src/hosted-extension-protocol.ts:234:21 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ +Serialized Error: { code: 'plugin_unsupported' } +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[20/78]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > mounts pinned private Surface bytes when the live package changes before launch +AssertionError: promise rejected "Error: bubblewrap is unavailable { code: '…' }" instead of resolving + ❯ tests/hosted-extension-isolation.test.ts:456:7 + 454| return { receiptId: 'receipt-1', status: 'queued' }; + 455| } }, + 456| })).resolves.toEqual({ completionReason: 'success', capabilityCall… + | ^ + 457| expect(calls).toBe(1); + 458| expect(readFileSync(join(surfaceRoot, 'dist/flow.js'), 'utf8')).to… + +Caused by: Error: bubblewrap is unavailable + ❯ unsupported src/hosted-extension-sandbox.ts:503:9 + ❯ executable src/hosted-extension-sandbox.ts:484:18 + ❯ Module.runHostedExtensionSandbox src/hosted-extension-sandbox.ts:176:17 + ❯ Module.runVerifiedNativeExtensionSandbox src/hosted-extension-isolation.ts:209:16 + ❯ tests/hosted-extension-isolation.test.ts:443:18 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ +Serialized Error: { code: 'plugin_unsupported' } +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[21/78]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > refuses oversized Surface files through the bounded descriptor reader +AssertionError: expected Error: bubblewrap is unavailable { code: '…' } to match object { code: 'plugin_unsupported', …(1) } + +- Expected ++ Received + +- Object { ++ PluginError { + "code": "plugin_unsupported", +- "message": StringContaining "cannot read pinned Surface runtime flow.js", + } + + ❯ tests/hosted-extension-isolation.test.ts:489:5 + 487| }); + 488| let calls = 0; + 489| await expect(runVerifiedNativeExtensionSandbox({ + | ^ + 490| artifact: await artifact(), + 491| manifest: validateFlowExtensionManifest(manifest()), + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[22/78]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > shields verified Surface files before async settlement +AssertionError: promise rejected "Error: bubblewrap is unavailable { code: '…' }" instead of resolving + ❯ tests/hosted-extension-isolation.test.ts:532:9 + 530| surfaceRoot, + 531| babysitterTurn: { queue: async () => ({ receiptId: 'receipt-1'… + 532| })).resolves.toEqual({ completionReason: 'success', capabilityCa… + | ^ + 533| } finally { + 534| if (previous === undefined) delete (Array.prototype as { then?: … + +Caused by: Error: bubblewrap is unavailable + ❯ unsupported src/hosted-extension-sandbox.ts:503:9 + ❯ executable src/hosted-extension-sandbox.ts:484:18 + ❯ Module.runHostedExtensionSandbox src/hosted-extension-sandbox.ts:176:17 + ❯ Module.runVerifiedNativeExtensionSandbox src/hosted-extension-isolation.ts:209:16 + ❯ tests/hosted-extension-isolation.test.ts:525:20 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ +Serialized Error: { code: 'plugin_unsupported' } +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[23/78]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > preserves a typed host refusal while disclosing only a fixed marker to the child +AssertionError: expected Error: bubblewrap is unavailable { code: '…' } to be Error: private Cloud policy detail { code: '…' } // Object.is equality + +- Expected ++ Received + +- [Error: private Cloud policy detail] ++ [Error: bubblewrap is unavailable] + + ❯ tests/hosted-extension-isolation.test.ts:560:5 + 558| provider: 'github', eventType: 'pull_request.labeled', deliveryI… + 559| }); + 560| await expect(runVerifiedNativeExtensionSandbox({ + | ^ + 561| artifact: await artifact(source), manifest: validateFlowExtensio… + 562| babysitterTurn: { queue: async () => { throw refusal; } }, + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[24/78]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > denies ambient credentials, host files, writes, network, subprocesses, and undeclared context verbs +Error: bubblewrap is unavailable + ❯ unsupported src/hosted-extension-sandbox.ts:503:9 + 501| + 502| function unsupported(message: string): never { + 503| throw new PluginError('plugin_unsupported', message); + | ^ + 504| } + 505| + ❯ executable src/hosted-extension-sandbox.ts:484:18 + ❯ Module.runHostedExtensionSandbox src/hosted-extension-sandbox.ts:176:17 + ❯ Module.runVerifiedNativeExtensionSandbox src/hosted-extension-isolation.ts:209:16 + ❯ tests/hosted-extension-isolation.test.ts:624:13 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[25/78]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > enforces OS address-space and data bounds on native Buffer allocation +AssertionError: expected Error: bubblewrap is unavailable { code: '…' } to match object { code: 'plugin_unsupported', …(1) } + +- Expected ++ Received + +- Object { ++ PluginError { + "code": "plugin_unsupported", +- "message": StringMatching /(?:Failed to allocate memory|Array buffer allocation failed)/u, + } + + ❯ tests/hosted-extension-isolation.test.ts:646:5 + 644| }); + 645| let calls = 0; + 646| await expect(runVerifiedNativeExtensionSandbox({ + | ^ + 647| artifact: installed, + 648| manifest: validateFlowExtensionManifest(manifest()), + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[26/78]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > blocks extra handler fields and authority-bearing receipt fields at the parent port +AssertionError: expected Error: bubblewrap is unavailable { code: '…' } to match object { code: 'plugin_event_unroutable' } + +- Expected ++ Received + +- Object { +- "code": "plugin_event_unroutable", ++ PluginError { ++ "code": "plugin_unsupported", + } + + ❯ tests/hosted-extension-isolation.test.ts:675:5 + 673| }); + 674| let calls = 0; + 675| await expect(runVerifiedNativeExtensionSandbox({ + | ^ + 676| artifact: await artifact(source), manifest: validateFlowExtensio… + 677| babysitterTurn: { queue: async () => { calls += 1; return { rece… + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[27/78]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > constructs adapter authority with the captured freeze intrinsic +Error: bubblewrap is unavailable + ❯ unsupported src/hosted-extension-sandbox.ts:503:9 + 501| + 502| function unsupported(message: string): never { + 503| throw new PluginError('plugin_unsupported', message); + | ^ + 504| } + 505| + ❯ executable src/hosted-extension-sandbox.ts:484:18 + ❯ Module.runHostedExtensionSandbox src/hosted-extension-sandbox.ts:176:17 + ❯ Module.runVerifiedNativeExtensionSandbox src/hosted-extension-isolation.ts:209:16 + ❯ tests/hosted-extension-isolation.test.ts:745:22 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[28/78]⎯ + + FAIL tests/hosted-extension-isolation.test.ts > hosted extension capability isolation > writes the Surface manifest and protocol without inherited toJSON behavior +AssertionError: promise rejected "Error: bubblewrap is unavailable { code: '…' }" instead of resolving + ❯ tests/hosted-extension-isolation.test.ts:788:11 + 786| babysitterTurn: { queue: async () => ({ receiptId: 'receipt-… + 787| timeoutMs: 3_000, + 788| })).resolves.toEqual({ completionReason: 'success', capability… + | ^ + 789| } finally { + 790| if (previous === undefined) delete (Object.prototype as { toJS… + +Caused by: Error: bubblewrap is unavailable + ❯ unsupported src/hosted-extension-sandbox.ts:503:9 + ❯ executable src/hosted-extension-sandbox.ts:484:18 + ❯ Module.runHostedExtensionSandbox src/hosted-extension-sandbox.ts:176:17 + ❯ Module.runVerifiedNativeExtensionSandbox src/hosted-extension-isolation.ts:209:16 + ❯ tests/hosted-extension-isolation.test.ts:781:22 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ +Serialized Error: { code: 'plugin_unsupported' } +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[29/78]⎯ + + FAIL tests/hosted-extension-protocol.test.ts > hosted extension hostile protocol > uses captured JSON intrinsics for the complete parent boundary +Error: bubblewrap is unavailable + ❯ unsupported src/hosted-extension-sandbox.ts:503:9 + 501| + 502| function unsupported(message: string): never { + 503| throw new PluginError('plugin_unsupported', message); + | ^ + 504| } + 505| + ❯ executable src/hosted-extension-sandbox.ts:484:18 + ❯ Module.runHostedExtensionSandbox src/hosted-extension-sandbox.ts:176:17 + ❯ Module.runVerifiedNativeExtensionSandbox src/hosted-extension-isolation.ts:209:16 + ❯ tests/hosted-extension-protocol.test.ts:326:24 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[30/78]⎯ + + FAIL tests/hosted-extension-protocol.test.ts > hosted extension hostile protocol > rejects an import-time different PR frame with zero adapter calls + FAIL tests/hosted-extension-protocol.test.ts > hosted extension hostile protocol > rejects an import-time different delivery frame with zero adapter calls + FAIL tests/hosted-extension-protocol.test.ts > hosted extension hostile protocol > rejects an import-time different event frame with zero adapter calls +AssertionError: expected Error: bubblewrap is unavailable { code: '…' } to match object { code: 'plugin_event_unroutable' } + +- Expected ++ Received + +- Object { +- "code": "plugin_event_unroutable", ++ PluginError { ++ "code": "plugin_unsupported", + } + + ❯ tests/hosted-extension-protocol.test.ts:447:5 + 445| ])('rejects an import-time %s frame with zero adapter calls', async … + 446| let calls = 0; + 447| await expect(runVerifiedNativeExtensionSandbox({ + | ^ + 448| artifact: await artifact(hostileImport([frame, { type: 'error', … + 449| manifest: validateFlowExtensionManifest(manifest()), dispatch: d… + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[31/78]⎯ + + FAIL tests/hosted-extension-protocol.test.ts > hosted extension hostile protocol > rejects two forged calls after the authoritative first outcome settles + FAIL tests/hosted-extension-protocol.test.ts > hosted extension hostile protocol > waits for a pending adapter to reject after a forged child error + FAIL tests/hosted-extension-protocol.test.ts > hosted extension hostile protocol > waits for a pending adapter to resolve after a forged child error + FAIL tests/hosted-extension-protocol.test.ts > hosted extension hostile protocol > returns a typed adapter rejection even when the hostile child hangs +Error: hostile child did not invoke the adapter + ❯ Timeout._onTimeout tests/hosted-extension-protocol.test.ts:135:45 + 133| async function waitForInvocation(invoked: Promise): Promise((resolve, reject) => { + 135| const timeout = setTimeout(() => reject(new Error('hostile child d… + | ^ + 136| void invoked.then(() => { clearTimeout(timeout); resolve(); }, rej… + 137| }); + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[32/78]⎯ + + FAIL tests/live-kernel.test.ts > built flows CLI against live relayflowd > runs hn-monitor analyze-story end-to-end via a stub agent CLI (gate 2 clause 2 demo) +AssertionError: expected { …(12) } to match object { output: { …(3) }, …(1) } +(22 matching properties omitted from actual) + +- Expected ++ Received + + Object { +- "output": Object { +- "reasoning": "stub agent runtime — deterministic output for gate-2 clause-2 demo", +- "relevance_score": 5, +- "story_title": "stub", +- }, ++ "output": null, + "verification": Object { +- "gate": "json_schema", +- "verdict": "pass", ++ "gate": "execution", ++ "verdict": "fail", + }, + } + + ❯ tests/live-kernel.test.ts:657:36 + 655| && (entry as { step_id?: string }).step_id === 'analyze-story', + 656| ) as { payload: { output: unknown; verification: unknown } } | und… + 657| expect(stepCompleted?.payload).toMatchObject({ + | ^ + 658| output: { + 659| story_title: 'stub', + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[33/78]⎯ + + FAIL tests/live-kernel.test.ts > built flows CLI against live relayflowd > hn-monitor analyze-story FAILS verification when the CLI omits required schema fields +AssertionError: expected { …(12) } to match object { …(3) } +(21 matching properties omitted from actual) + +- Expected ++ Received + + Object { +- "completionReason": "retries_exhausted", ++ "completionReason": "worker_error", + "output": null, + "verification": Object { +- "gate": "json_schema", ++ "gate": "execution", + "verdict": "fail", + }, + } + + ❯ tests/live-kernel.test.ts:752:36 + 750| // its verification record names the json_schema rejection. The re… + 751| // parsed value is nulled before the completion is persisted. + 752| expect(stepCompleted?.payload).toMatchObject({ + | ^ + 753| completionReason: 'retries_exhausted', + 754| output: null, + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[34/78]⎯ + + FAIL tests/live-kernel.test.ts > built flows CLI against live relayflowd > agent step preserves the CliResult wrapper as output when the CLI emits non-JSON text +AssertionError: expected null not to be null + ❯ tests/live-kernel.test.ts:823:24 + 821| // here (parseJsonOutput returned null on non-JSON stdout) and + 822| // these assertions would all fail. + 823| expect(output).not.toBeNull(); + | ^ + 824| expect(output.exit_code).toBe(0); + 825| expect(output.stdout_tail).toContain('looked at the story'); + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[35/78]⎯ + + FAIL tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker exposes wake_context to the CLI via RELAYFLOW_WAKE_CONTEXT env var (real analyzer prerequisite) +TypeError: Cannot read properties of null (reading 'story_title') + ❯ tests/live-kernel.test.ts:891:42 + 889| ) as { payload: { output: { story_title: string; reasoning: string… + 890| expect(stepCompleted).toBeDefined(); + 891| expect(stepCompleted!.payload.output.story_title).toBe(`echoed:${s… + | ^ + 892| expect(stepCompleted!.payload.output.reasoning).toContain(String(s… + 893| + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[36/78]⎯ + + FAIL tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker leaves RELAYFLOW_WAKE_CONTEXT UNSET when the run has no wake_context (undefined-vs-null pin) +TypeError: Cannot read properties of null (reading 'env_present') + ❯ tests/live-kernel.test.ts:958:38 + 956| ) as { payload: { output: { env_present: boolean } } } | undefined; + 957| expect(completed).toBeDefined(); + 958| expect(completed!.payload.output.env_present).toBe(false); + | ^ + 959| + 960| delete process.env.RELAYFLOW_WAKE_CONTEXT; + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[37/78]⎯ + + FAIL tests/live-kernel.test.ts > built flows CLI against live relayflowd > AgentWorker leaves RELAYFLOW_MODEL UNSET when the step declares no model +TypeError: Cannot read properties of null (reading 'story_title') + ❯ tests/live-kernel.test.ts:1194:38 + 1192| expect(completed).toBeDefined(); + 1193| // UNSET, not EMPTY and not the leaked parent value. + 1194| expect(completed!.payload.output.story_title).toBe('model:UNSET'); + | ^ + 1195| + 1196| delete process.env.RELAYFLOW_MODEL; + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[38/78]⎯ + + FAIL tests/live-kernel.test.ts > built flows CLI against live relayflowd > hn-monitor analyze-story reaches done through the real Claude analyzer CLI +Error: LIVE_ANALYZER_UNAVAILABLE: "/home/daytona/.relayflow-v2-supervisor/durable/repository/testdata/preflight/analyze-story-claude-cli" does not identify as relayflows-agent-cli-v1 — failing because gate-2 acceptance requires the real analyzer to execute. Set RELAYFLOWS_ALLOW_ANALYZER_SKIP=1 only if this run is not gate evidence. + ❯ tests/live-kernel.test.ts:1223:15 + 1221| const notice = `LIVE_ANALYZER_UNAVAILABLE: ${readiness.detail}`; + 1222| if (process.env['RELAYFLOWS_ALLOW_ANALYZER_SKIP'] !== '1') { + 1223| throw new Error( + | ^ + 1224| `${notice} — failing because gate-2 acceptance requires the … + 1225| + 'Set RELAYFLOWS_ALLOW_ANALYZER_SKIP=1 only if this run is … + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[39/78]⎯ + + FAIL tests/live-kernel.test.ts > a relayflow can be scheduled: tick source against live relayflowd > a tick spawns a real run whose step reports the SCHEDULED instant +AssertionError: expected null to deeply equal { schedule_id: 'heartbeat-1m', …(3) } + +- Expected: +Object { + "lag_ms": 43000, + "schedule_id": "heartbeat-1m", + "scheduled_for_ms": 1764000000000, + "slot": 29400000, +} + ++ Received: +null + + ❯ tests/live-kernel.test.ts:1739:39 + 1737| // The bound: the run reports the grid instant and its own lag, so… + 1738| // backfilled run can tell it is running for a slot from the past. + 1739| expect(completed!.payload.output).toEqual({ + | ^ + 1740| schedule_id: 'heartbeat-1m', + 1741| slot: 29_400_000, + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[40/78]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage input validation > refuses an 8-character run-id prefix: Cloud has no prefix lookup +AssertionError: expected [Function] to throw error matching /not full Cloud run ids: c649fe14/ but got 'expected an @relayflows/surface flow …' + +- Expected: +/not full Cloud run ids: c649fe14/ + ++ Received: +"expected an @relayflows/surface flow handle" + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[41/78]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage input validation > refuses the whole batch when any id is invalid, rather than dropping it +AssertionError: expected [Function] to throw error matching /not full Cloud run ids: nope!/ but got 'expected an @relayflows/surface flow …' + +- Expected: +/not full Cloud run ids: nope!/ + ++ Received: +"expected an @relayflows/surface flow handle" + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[42/78]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage input validation > refuses an empty batch +AssertionError: expected [Function] to throw error matching /needs runIds/ but got 'expected an @relayflows/surface flow …' + +- Expected: +/needs runIds/ + ++ Received: +"expected an @relayflows/surface flow handle" + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[43/78]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage input validation > refuses a batch too large for the edge step lease +AssertionError: expected [Function] to throw error matching /exceeds the 8 that fit/ but got 'expected an @relayflows/surface flow …' + +- Expected: +/exceeds the 8 that fit/ + ++ Received: +"expected an @relayflows/surface flow handle" + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[44/78]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage input validation > accepts eight ids — the incident batch is inside the bound +AssertionError: promise rejected "TypeError: expected an @relayflows/surfac…" instead of resolving + ❯ tests/stuck-run-triage.test.ts:62:40 + 60| it('accepts eight ids — the incident batch is inside the bound', asy… + 61| const ids = Array.from({ length: 8 }, (_, i) => `${ID_A.slice(0, -… + 62| await expect(drive({ runIds: ids })).resolves.toBeDefined(); + | ^ + 63| }); + 64| }); + +Caused by: TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:169:11 + ❯ drive tests/stuck-run-triage.test.ts:34:9 + ❯ tests/stuck-run-triage.test.ts:62:18 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[45/78]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage apiUrl > refuses to send the Cloud bearer token to an unapproved origin +AssertionError: expected [Function] to throw error matching /refusing to send the Cloud bearer to…/\ but got 'expected an @relayflows/surface flow …' + +- Expected: +/refusing to send the Cloud bearer token to https:\/\/evil\.example/ + ++ Received: +"expected an @relayflows/surface flow handle" + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[46/78]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage apiUrl > refuses a non-URL apiUrl +AssertionError: expected [Function] to throw error matching /is not a URL/ but got 'expected an @relayflows/surface flow …' + +- Expected: +/is not a URL/ + ++ Received: +"expected an @relayflows/surface flow handle" + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[47/78]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage apiUrl > allows an approved origin and uses it in the curl +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:169:11 + ❯ drive tests/stuck-run-triage.test.ts:34:9 + 32| done: () => {}, + 33| }; + 34| await getFlowDefinition(triage).body(f as never… + | ^ + 35| return rec; + 36| } + ❯ tests/stuck-run-triage.test.ts:77:23 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[48/78]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage apiUrl > defaults to production Cloud +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:169:11 + ❯ drive tests/stuck-run-triage.test.ts:34:9 + 32| done: () => {}, + 33| }; + 34| await getFlowDefinition(triage).body(f as never… + | ^ + 35| return rec; + 36| } + ❯ tests/stuck-run-triage.test.ts:82:23 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[49/78]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage apiUrl > never publishes a run record the fetch did not produce +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:169:11 + ❯ drive tests/stuck-run-triage.test.ts:34:9 + 32| done: () => {}, + 33| }; + 34| await getFlowDefinition(triage).body(f as never… + | ^ + 35| return rec; + 36| } + ❯ tests/stuck-run-triage.test.ts:89:34 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[50/78]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage edge collection > names the Worker on every wrangler invocation +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:169:11 + ❯ drive tests/stuck-run-triage.test.ts:34:9 + 32| done: () => {}, + 33| }; + 34| await getFlowDefinition(triage).body(f as never… + | ^ + 35| return rec; + 36| } + ❯ tests/stuck-run-triage.test.ts:98:33 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[51/78]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage edge collection > accepts caller-supplied Workers and rejects option-shaped ones +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:169:11 + ❯ drive tests/stuck-run-triage.test.ts:34:9 + 32| done: () => {}, + 33| }; + 34| await getFlowDefinition(triage).body(f as never… + | ^ + 35| return rec; + 36| } + ❯ tests/stuck-run-triage.test.ts:107:33 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[52/78]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage edge collection > falls back when GNU timeout is absent, as it is on macOS +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:169:11 + ❯ drive tests/stuck-run-triage.test.ts:34:9 + 32| done: () => {}, + 33| }; + 34| await getFlowDefinition(triage).body(f as never… + | ^ + 35| return rec; + 36| } + ❯ tests/stuck-run-triage.test.ts:115:33 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[53/78]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage edge collection > runs the tails concurrently so wall time does not scale with the batch +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:169:11 + ❯ drive tests/stuck-run-triage.test.ts:34:9 + 32| done: () => {}, + 33| }; + 34| await getFlowDefinition(triage).body(f as never… + | ^ + 35| return rec; + 36| } + ❯ tests/stuck-run-triage.test.ts:123:33 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[54/78]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage edge collection > records wrangler's own exit status rather than head's +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:169:11 + ❯ drive tests/stuck-run-triage.test.ts:34:9 + 32| done: () => {}, + 33| }; + 34| await getFlowDefinition(triage).body(f as never… + | ^ + 35| return rec; + 36| } + ❯ tests/stuck-run-triage.test.ts:129:33 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[55/78]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage shell text > parses under both sh and bash +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:169:11 + ❯ drive tests/stuck-run-triage.test.ts:34:9 + 32| done: () => {}, + 33| }; + 34| await getFlowDefinition(triage).body(f as never… + | ^ + 35| return rec; + 36| } + ❯ tests/stuck-run-triage.test.ts:137:23 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[56/78]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage shell text > collects tails with no GNU timeout on PATH, as on a stock macOS +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:169:11 + ❯ drive tests/stuck-run-triage.test.ts:34:9 + 32| done: () => {}, + 33| }; + 34| await getFlowDefinition(triage).body(f as never… + | ^ + 35| return rec; + 36| } + ❯ tests/stuck-run-triage.test.ts:157:33 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[57/78]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage agents > declares read-only permissions on every agent +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:169:11 + ❯ drive tests/stuck-run-triage.test.ts:34:9 + 32| done: () => {}, + 33| }; + 34| await getFlowDefinition(triage).body(f as never… + | ^ + 35| return rec; + 36| } + ❯ tests/stuck-run-triage.test.ts:176:23 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[58/78]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage agents > tells the forensics agents their evidence is untrusted +TypeError: expected an @relayflows/surface flow handle + ❯ Module.getFlowDefinition node_modules/@relayflows/surface/src/flow.ts:169:11 + ❯ drive tests/stuck-run-triage.test.ts:34:9 + 32| done: () => {}, + 33| }; + 34| await getFlowDefinition(triage).body(f as never… + | ^ + 35| return rec; + 36| } + ❯ tests/stuck-run-triage.test.ts:182:23 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[59/78]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage fan-out > refuses a duplicate run id: two tails would share one evidence file +AssertionError: expected [Function] to throw error matching /duplicate runIds: c649fe14-0c2e-4e51-…/ but got 'expected an @relayflows/surface flow …' + +- Expected: +/duplicate runIds: c649fe14-0c2e-4e51-9a6a-4f0d1b0f77aa/ + ++ Received: +"expected an @relayflows/surface flow handle" + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[60/78]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage fan-out > refuses a duplicate Worker name for the same reason +AssertionError: expected [Function] to throw error matching /duplicate workers: w-one/ but got 'expected an @relayflows/surface flow …' + +- Expected: +/duplicate workers: w-one/ + ++ Received: +"expected an @relayflows/surface flow handle" + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[61/78]⎯ + + FAIL tests/stuck-run-triage.test.ts > stuck-run-triage fan-out > bounds ids x workers, not just ids +AssertionError: expected [Function] to throw error matching /24 concurrent tails, over the 16/ but got 'expected an @relayflows/surface flow …' + +- Expected: +/24 concurrent tails, over the 16/ + ++ Received: +"expected an @relayflows/surface flow handle" + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[62/78]⎯ + +⎯⎯⎯⎯⎯⎯ Unhandled Errors ⎯⎯⎯⎯⎯⎯ + +Vitest caught 1 unhandled error during the test run. +This might cause false positive tests. Resolve unhandled errors to make sure your tests are not affected. + +⎯⎯⎯⎯ Unhandled Rejection ⎯⎯⎯⎯⎯ +Error: bubblewrap is unavailable + ❯ unsupported src/hosted-extension-sandbox.ts:503:9 + 501| + 502| function unsupported(message: string): never { + 503| throw new PluginError('plugin_unsupported', message); + | ^ + 504| } + 505| + ❯ executable src/hosted-extension-sandbox.ts:484:18 + ❯ Module.runHostedExtensionSandbox src/hosted-extension-sandbox.ts:176:17 + ❯ Module.runVerifiedNativeExtensionSandbox src/hosted-extension-isolation.ts:209:16 + ❯ tests/hosted-extension-protocol.test.ts:471:17 + ❯ node_modules/@vitest/runner/dist/index.js:533:5 + ❯ runTest node_modules/@vitest/runner/dist/index.js:1056:11 + ❯ runSuite node_modules/@vitest/runner/dist/index.js:1205:15 + ❯ runSuite node_modules/@vitest/runner/dist/index.js:1205:15 + ❯ runFiles node_modules/@vitest/runner/dist/index.js:1262:5 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ +Serialized Error: { code: 'plugin_unsupported' } +This error originated in "tests/hosted-extension-protocol.test.ts" test file. It doesn't mean the error was thrown inside the file itself, but while it was running. +The latest test that might've caused the error is "rejects two forged calls after the authoritative first outcome settles". It might mean one of the following: +- The error was thrown, while Vitest was running this test. +- If the error occurred after the test had been completed, this was the last documented test before it was thrown. +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯ + + Test Files 6 failed | 248 passed | 1 skipped (255) + Tests 77 failed | 3774 passed | 22 skipped (3873) + Errors 1 error + Start at 10:40:22 + Duration 757.43s (transform 4.18s, setup 1.09s, collect 64.50s, tests 1765.01s, environment 32ms, prepare 10.44s) + +exit=1 diff --git a/evidence/run-read-timeout/handshake-probe-baseline.mjs b/evidence/run-read-timeout/handshake-probe-baseline.mjs new file mode 100644 index 000000000..eb71885c3 --- /dev/null +++ b/evidence/run-read-timeout/handshake-probe-baseline.mjs @@ -0,0 +1,25 @@ +import {mkdtempSync,readFileSync,rmSync} from 'node:fs'; +import {join} from 'node:path'; +import {tmpdir} from 'node:os'; +import {spawn} from 'node:child_process'; +const root='/home/daytona/.relayflow-v2-supervisor/durable/repository'; +const {JournalClient}=await import('/tmp/relayflow-read-timeout-baseline'+'/packages/sdk/dist/journal-client.js'); +const {AgentWorker}=await import('/tmp/relayflow-read-timeout-baseline'+'/packages/sdk/dist/worker.js'); +const {socketPathFor}=await import('/tmp/relayflow-read-timeout-baseline'+'/packages/sdk/dist/daemon-connection.js'); +const data=mkdtempSync(join(tmpdir(),'rf-read-probe-')); +const daemon=spawn('/home/daytona/.relayflows-toolchain/target/2962130851/debug/relayflowd',['--data-dir',data,'serve'],{stdio:'ignore'}); +let client,worker; +try { + for(let i=0;i<100;i++) { + client=new JournalClient(socketPathFor(data)); + try {await client.connect();await client.hello('probe');break;} catch {client.close();await new Promise(r=>setTimeout(r,20));} + } + worker=new AgentWorker(client,{workerId:'probe',pins:{workspace:[{surface:'repo',revision_id:'rev-a'}],streams:[]}}); + await worker.attach(); + const spec=JSON.parse(readFileSync(root+'/testdata/hn-monitor.spec.canonical.json','utf8')); + for(const step of spec.steps) if(step.id==='analyze-story')step.cli=root+'/testdata/preflight/analyze-story-stub-cli'; + const outcome=await client.eventSubmit(spec,{type:'hn.story_posted',payload:{id:42000042,type:'story'}}); + for(let i=0;i<100;i++){if((await client.runGet(outcome.run.run_id)).steps['analyze-story'].state==='done')break;await new Promise(r=>setTimeout(r,50));} + const {entries}=await client.journalRead(outcome.run.run_id,1,1000); + console.log(JSON.stringify(entries.filter(e=>e.entry_type==='step.completed'),null,2)); +}finally{await worker?.close();client?.close();daemon.kill('SIGTERM');await new Promise(r=>daemon.once('exit',r));rmSync(data,{recursive:true,force:true});} diff --git a/evidence/run-read-timeout/handshake-probe.mjs b/evidence/run-read-timeout/handshake-probe.mjs new file mode 100644 index 000000000..60241ad88 --- /dev/null +++ b/evidence/run-read-timeout/handshake-probe.mjs @@ -0,0 +1,25 @@ +import {mkdtempSync,readFileSync,rmSync} from 'node:fs'; +import {join} from 'node:path'; +import {tmpdir} from 'node:os'; +import {spawn} from 'node:child_process'; +const root='/home/daytona/.relayflow-v2-supervisor/durable/repository'; +const {JournalClient}=await import(root+'/packages/sdk/dist/journal-client.js'); +const {AgentWorker}=await import(root+'/packages/sdk/dist/worker.js'); +const {socketPathFor}=await import(root+'/packages/sdk/dist/daemon-connection.js'); +const data=mkdtempSync(join(tmpdir(),'rf-read-probe-')); +const daemon=spawn('/home/daytona/.relayflows-toolchain/target/2962130851/debug/relayflowd',['--data-dir',data,'serve'],{stdio:'ignore'}); +let client,worker; +try { + for(let i=0;i<100;i++) { + client=new JournalClient(socketPathFor(data)); + try {await client.connect();await client.hello('probe');break;} catch {client.close();await new Promise(r=>setTimeout(r,20));} + } + worker=new AgentWorker(client,{workerId:'probe',pins:{workspace:[{surface:'repo',revision_id:'rev-a'}],streams:[]}}); + await worker.attach(); + const spec=JSON.parse(readFileSync(root+'/testdata/hn-monitor.spec.canonical.json','utf8')); + for(const step of spec.steps) if(step.id==='analyze-story')step.cli=root+'/testdata/preflight/analyze-story-stub-cli'; + const outcome=await client.eventSubmit(spec,{type:'hn.story_posted',payload:{id:42000042,type:'story'}}); + for(let i=0;i<100;i++){if((await client.runGet(outcome.run.run_id)).steps['analyze-story'].state==='done')break;await new Promise(r=>setTimeout(r,50));} + const {entries}=await client.journalRead(outcome.run.run_id,1,1000); + console.log(JSON.stringify(entries.filter(e=>e.entry_type==='step.completed'),null,2)); +}finally{await worker?.close();client?.close();daemon.kill('SIGTERM');await new Promise(r=>daemon.once('exit',r));rmSync(data,{recursive:true,force:true});} diff --git a/evidence/run-read-timeout/live-failure-probe-baseline.log b/evidence/run-read-timeout/live-failure-probe-baseline.log new file mode 100644 index 000000000..947cc0382 --- /dev/null +++ b/evidence/run-read-timeout/live-failure-probe-baseline.log @@ -0,0 +1,45 @@ +$ node /tmp/relayflow-read-probe-baseline.mjs +[ + { + "at_ms": 1791197583957, + "attempt": 1, + "entry_type": "step.completed", + "payload": { + "budget": { + "dollars": "0", + "dollars_unmetered": true, + "tokens_in": 0, + "tokens_out": 0 + }, + "completed_by": "probe", + "completionReason": "worker_error", + "disposition": "step_done", + "effects": [], + "end_pins": { + "streams": [], + "workspace": [] + }, + "input_hash": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + "next_attempt_at_ms": null, + "output": null, + "spend": { + "dollars": 0, + "dollars_unmetered": true, + "tokens_input": 0, + "tokens_output": 0, + "wallclock_ms": 888 + }, + "step_spec_hash": "12d9633edd0ffd882a0b1586fc347ce36c84d32656fe82f7e0b8d9d97cecc087", + "verification": { + "detail": "{\"artifacts\":[],\"exit_code\":null,\"stderr_tail\":\"CLI \\\"/home/daytona/.relayflow-v2-supervisor/durable/repository/testdata/preflight/analyze-story-stub-cli\\\" exited before completing the relayflows-agent-cli-v1 same-process handshake.\",\"stdout_tail\":\"\"}", + "gate": "execution", + "verdict": "fail" + } + }, + "run_id": "01M45V45PHWH45AD8128KRDAR7", + "segment_id": 1, + "seq": 7, + "step_id": "analyze-story" + } +] +exit=0 diff --git a/evidence/run-read-timeout/live-failure-probe.log b/evidence/run-read-timeout/live-failure-probe.log new file mode 100644 index 000000000..7d969c681 --- /dev/null +++ b/evidence/run-read-timeout/live-failure-probe.log @@ -0,0 +1,45 @@ +$ node /tmp/relayflow-read-probe.mjs +[ + { + "at_ms": 1791197554045, + "attempt": 1, + "entry_type": "step.completed", + "payload": { + "budget": { + "dollars": "0", + "dollars_unmetered": true, + "tokens_in": 0, + "tokens_out": 0 + }, + "completed_by": "probe", + "completionReason": "worker_error", + "disposition": "step_done", + "effects": [], + "end_pins": { + "streams": [], + "workspace": [] + }, + "input_hash": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + "next_attempt_at_ms": null, + "output": null, + "spend": { + "dollars": 0, + "dollars_unmetered": true, + "tokens_input": 0, + "tokens_output": 0, + "wallclock_ms": 723 + }, + "step_spec_hash": "12d9633edd0ffd882a0b1586fc347ce36c84d32656fe82f7e0b8d9d97cecc087", + "verification": { + "detail": "{\"artifacts\":[],\"exit_code\":null,\"stderr_tail\":\"CLI \\\"/home/daytona/.relayflow-v2-supervisor/durable/repository/testdata/preflight/analyze-story-stub-cli\\\" exited before completing the relayflows-agent-cli-v1 same-process handshake.\",\"stdout_tail\":\"\"}", + "gate": "execution", + "verdict": "fail" + } + }, + "run_id": "01M45V38MZYZ0F87JWZVAJBZ6F", + "segment_id": 1, + "seq": 7, + "step_id": "analyze-story" + } +] +exit=0 diff --git a/evidence/run-read-timeout/mutation-heartbeat.log b/evidence/run-read-timeout/mutation-heartbeat.log new file mode 100644 index 000000000..e0770eaaf --- /dev/null +++ b/evidence/run-read-timeout/mutation-heartbeat.log @@ -0,0 +1,53 @@ +Source: packages/sdk/src/worker-lease.ts +Original SHA256: 44c13f9f61ac13cb8c0f872f4c22b1c212e020d05880ed080ea7205ba6007213 +Replaced: + if (error instanceof JournalRequestTimeoutError && error.verb === 'step.heartbeat') { + throw new WorkerLeaseLostError('renewal_expired', error.message, { cause: error }); + } + +With: + + +REVERTED +$ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartbeat timeout' --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 20ms + × a heartbeat timeout is lease loss rather than a worker body failure 20ms + → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError + +⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/heartbeat-timeout.test.ts > a heartbeat timeout is lease loss rather than a worker body failure +AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError + ❯ tests/heartbeat-timeout.test.ts:16:5 + 14| try { + 15| await client.connect(); + 16| await expect(withWorkerLease(client, { + | ^ + 17| run_id: 'run', step_id: 'step', attempt: 1, step_type: 'agent', … + 18| lease_id: 'lease', lease_deadline_ms: Date.now() + 30_000, lease… + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ + + Test Files 1 failed (1) + Tests 1 failed (1) + Start at 10:38:23 + Duration 509ms (transform 171ms, setup 14ms, collect 315ms, tests 20ms, environment 0ms, prepare 45ms) + +exit=1 + +RESTORED SHA256: 44c13f9f61ac13cb8c0f872f4c22b1c212e020d05880ed080ea7205ba6007213 +$ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartbeat timeout' --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms + + Test Files 1 passed (1) + Tests 1 passed (1) + Start at 10:38:24 + Duration 491ms (transform 169ms, setup 15ms, collect 300ms, tests 15ms, environment 0ms, prepare 43ms) + +exit=0 diff --git a/evidence/run-read-timeout/mutation-reader.log b/evidence/run-read-timeout/mutation-reader.log new file mode 100644 index 000000000..546fe785c --- /dev/null +++ b/evidence/run-read-timeout/mutation-reader.log @@ -0,0 +1,57 @@ +Source: packages/sdk/src/journal-client.ts +Original SHA256: 9a390f44f9a1375f0e13be91786f24e848bba7fa07b93b28b119c7ca1d24027c +Replaced: +(reader ?? this).requestOnce(verb, params, remaining) +With: +this.requestOnce(verb, params, remaining) + +REVERTED +$ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'serves a bounded read' --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ❯ tests/journal-client-read-timeout.test.ts (13 tests | 1 failed | 12 skipped) 111ms + × serves a bounded read while an unbounded command is in flight 110ms + → expected true to be false // Object.is equality + +⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/journal-client-read-timeout.test.ts > serves a bounded read while an unbounded command is in flight +AssertionError: expected true to be false // Object.is equality + +- Expected ++ Received + +- false ++ true + + ❯ tests/journal-client-read-timeout.test.ts:36:23 + 34| await inFlight; + 35| expect(await client.runGet('run')).toMatchObject({ status: 'running'… + 36| expect(commandDone).toBe(false); + | ^ + 37| await command; + 38| }); + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ + + Test Files 1 failed (1) + Tests 1 failed | 12 skipped (13) + Start at 10:38:14 + Duration 588ms (transform 169ms, setup 15ms, collect 306ms, tests 111ms, environment 0ms, prepare 44ms) + +exit=1 + +RESTORED SHA256: 9a390f44f9a1375f0e13be91786f24e848bba7fa07b93b28b119c7ca1d24027c +$ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'serves a bounded read' --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ✓ tests/journal-client-read-timeout.test.ts (13 tests | 12 skipped) 108ms + + Test Files 1 passed (1) + Tests 1 passed | 12 skipped (13) + Start at 10:38:15 + Duration 586ms (transform 171ms, setup 15ms, collect 307ms, tests 108ms, environment 0ms, prepare 42ms) + +exit=0 diff --git a/evidence/run-read-timeout/mutation-root-parking.log b/evidence/run-read-timeout/mutation-root-parking.log new file mode 100644 index 000000000..a8d7556f2 --- /dev/null +++ b/evidence/run-read-timeout/mutation-root-parking.log @@ -0,0 +1,70 @@ +Source: packages/sdk/src/authored-root.ts +Original SHA256: ef090c86e849b5b4d77700672b60f9a42979e6649e6207a800a1e7561ca6be1e +Replaced: + if ((error instanceof JournalRequestTimeoutError && (READ_ONLY_VERBS.has(error.verb) || error.verb === 'run.watch')) + || (error instanceof AuthoredFlowExecutionError && error.code === 'daemon_unresponsive')) { + const parked = new AuthoredFlowExecutionError('daemon_unresponsive', + `${error.message}. The run remains resumable. Continue with: ${resumeCommand(dispatch.run_id, options.dataDir, options.localAgentStream !== undefined)}.`); + parked.rootRunId = dispatch.run_id; + throw parked; + } + +With: + + +REVERTED +$ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the root resumable after a read timeout' --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ❯ tests/authored-root.test.ts (25 tests | 1 failed | 24 skipped) 22ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch 21ms + → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } +(6 matching properties omitted from actual) + +⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/authored-root.test.ts > durable authored root > leaves the root resumable after a read timeout without waiting for redispatch +AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } +(6 matching properties omitted from actual) + +- Expected ++ Received + +- Object { +- "code": "daemon_unresponsive", +- "message": StringContaining "flows resume", +- "rootRunId": "root-run", +- } ++ [JournalRequestTimeoutError: journal client: run.get timed out after 10ms] + + ❯ tests/authored-root.test.ts:411:5 + 409| const loaded = await fixture(false, 0, async () => { throw new Jou… + 410| const journal = new RootJournal(); + 411| await expect(executeDurableAuthoredFlow(loaded, journal as unknown… + | ^ + 412| { dataDir: '/unused', admissionKey: 'read-timeout' })).rejects.t… + 413| code: 'daemon_unresponsive', rootRunId: 'root-run', message: e… + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ + + Test Files 1 failed (1) + Tests 1 failed | 24 skipped (25) + Start at 10:38:25 + Duration 1.36s (transform 704ms, setup 18ms, collect 1.16s, tests 22ms, environment 0ms, prepare 46ms) + +exit=1 + +RESTORED SHA256: ef090c86e849b5b4d77700672b60f9a42979e6649e6207a800a1e7561ca6be1e +$ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the root resumable after a read timeout' --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ✓ tests/authored-root.test.ts (25 tests | 24 skipped) 18ms + + Test Files 1 passed (1) + Tests 1 passed | 24 skipped (25) + Start at 10:38:27 + Duration 1.32s (transform 674ms, setup 15ms, collect 1.13s, tests 18ms, environment 0ms, prepare 45ms) + +exit=0 diff --git a/evidence/run-read-timeout/mutation-watch-cadence.log b/evidence/run-read-timeout/mutation-watch-cadence.log new file mode 100644 index 000000000..c10f7873f --- /dev/null +++ b/evidence/run-read-timeout/mutation-watch-cadence.log @@ -0,0 +1,51 @@ +Source: packages/sdk/src/cli/running-step.ts +Original SHA256: 0f8b6b9e160ebb047b5a83ad48ad41e2c201d5ba905c27c3a872acd3b9b288c2 +Replaced: +const LEASE_POLL_MS = 2_000; +With: +const LEASE_POLL_MS = 50; + +REVERTED +$ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pushes for completion' --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ❯ tests/running-step-watch.test.ts (2 tests | 1 failed | 1 skipped) 2110ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2109ms + → expected 41 to be less than or equal to 1 + +⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/running-step-watch.test.ts > uses pushes for completion with lease-cadence reads and releases its watcher +AssertionError: expected 41 to be less than or equal to 1 + ❯ tests/running-step-watch.test.ts:37:19 + 35| await waitForRunningStep(client, 'run', { id: 'step', type: 'agent… + 36| expect(performance.now() - start).toBeGreaterThan(2000); + 37| expect(reads).toBeLessThanOrEqual(1); + | ^ + 38| await sleep(10); + 39| expect(watchClosed).toBe(true); + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ + + Test Files 1 failed (1) + Tests 1 failed | 1 skipped (2) + Start at 10:38:16 + Duration 2.60s (transform 179ms, setup 16ms, collect 313ms, tests 2.11s, environment 0ms, prepare 43ms) + +exit=1 + +RESTORED SHA256: 0f8b6b9e160ebb047b5a83ad48ad41e2c201d5ba905c27c3a872acd3b9b288c2 +$ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pushes for completion' --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ✓ tests/running-step-watch.test.ts (2 tests | 1 skipped) 2120ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2119ms + + Test Files 1 passed (1) + Tests 1 passed | 1 skipped (2) + Start at 10:38:20 + Duration 2.61s (transform 186ms, setup 15ms, collect 322ms, tests 2.12s, environment 0ms, prepare 42ms) + +exit=0 diff --git a/evidence/run-read-timeout/mutations.py b/evidence/run-read-timeout/mutations.py new file mode 100644 index 000000000..5bcea4a9f --- /dev/null +++ b/evidence/run-read-timeout/mutations.py @@ -0,0 +1,57 @@ +"""Reproduce the four reviewed-plan mutations; always restore source bytes.""" +from pathlib import Path +import hashlib +import shlex +import subprocess + +ROOT = Path(__file__).resolve().parents[2] +SDK = ROOT / 'packages/sdk' +CASES = [ + ('reader', 'packages/sdk/src/journal-client.ts', + '(reader ?? this).requestOnce(verb, params, remaining)', + 'this.requestOnce(verb, params, remaining)', + 'tests/journal-client-read-timeout.test.ts', 'serves a bounded read'), + ('watch-cadence', 'packages/sdk/src/cli/running-step.ts', + 'const LEASE_POLL_MS = 2_000;', 'const LEASE_POLL_MS = 50;', + 'tests/running-step-watch.test.ts', 'uses pushes for completion'), + ('heartbeat', 'packages/sdk/src/worker-lease.ts', + """ if (error instanceof JournalRequestTimeoutError && error.verb === 'step.heartbeat') { + throw new WorkerLeaseLostError('renewal_expired', error.message, { cause: error }); + } +""", '', 'tests/heartbeat-timeout.test.ts', 'a heartbeat timeout'), + ('root-parking', 'packages/sdk/src/authored-root.ts', + """ if ((error instanceof JournalRequestTimeoutError && (READ_ONLY_VERBS.has(error.verb) || error.verb === 'run.watch')) + || (error instanceof AuthoredFlowExecutionError && error.code === 'daemon_unresponsive')) { + const parked = new AuthoredFlowExecutionError('daemon_unresponsive', + `${error.message}. The run remains resumable. Continue with: ${resumeCommand(dispatch.run_id, options.dataDir, options.localAgentStream !== undefined)}.`); + parked.rootRunId = dispatch.run_id; + throw parked; + } +""", '', 'tests/authored-root.test.ts', 'leaves the root resumable after a read timeout'), +] + +for name, source, old, new, test, pattern in CASES: + path = ROOT / source + original = path.read_bytes() + assert original.count(old.encode()) == 1, (name, 'mutation must match once') + digest = hashlib.sha256(original).hexdigest() + command = ['npx', 'vitest', 'run', test, '-t', pattern, '--maxWorkers=1', '--minWorkers=1'] + log = ROOT / 'evidence/run-read-timeout' / f'mutation-{name}.log' + with log.open('w') as output: + output.write(f'Source: {source}\nOriginal SHA256: {digest}\nReplaced:\n{old}\nWith:\n{new}\n') + try: + path.write_bytes(original.replace(old.encode(), new.encode())) + output.write('\nREVERTED\n$ cd packages/sdk && ' + shlex.join(command) + '\n') + output.flush() + failed = subprocess.run(command, cwd=SDK, stdout=output, stderr=subprocess.STDOUT) + output.write(f'exit={failed.returncode}\n') + finally: + path.write_bytes(original) + assert path.read_bytes() == original + output.write(f'\nRESTORED SHA256: {hashlib.sha256(path.read_bytes()).hexdigest()}\n') + output.write('$ cd packages/sdk && ' + shlex.join(command) + '\n') + output.flush() + passed = subprocess.run(command, cwd=SDK, stdout=output, stderr=subprocess.STDOUT) + output.write(f'exit={passed.returncode}\n') + print(f'{name}: reverted exit={failed.returncode}; restored exit={passed.returncode}; {log.relative_to(ROOT)}', flush=True) + assert failed.returncode == 1 and passed.returncode == 0, name diff --git a/evidence/run-read-timeout/resume-live.log b/evidence/run-read-timeout/resume-live.log new file mode 100644 index 000000000..1fe792d96 --- /dev/null +++ b/evidence/run-read-timeout/resume-live.log @@ -0,0 +1,15 @@ +$ cd packages/sdk && npx vitest run tests/read-timeout-resume-live.test.ts +$ cd packages/sdk && npx vitest run tests/read-timeout-resume-live.test.ts + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ✓ tests/read-timeout-resume-live.test.ts (1 test) 1512ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 1511ms + + Test Files 1 passed (1) + Tests 1 passed (1) + Start at 10:49:12 + Duration 3.20s (transform 891ms, setup 54ms, collect 1.39s, tests 1.51s, environment 0ms, prepare 43ms) + +exit=0 +exit=0 diff --git a/evidence/run-read-timeout/typecheck-final.json b/evidence/run-read-timeout/typecheck-final.json new file mode 100644 index 000000000..2c0063d8c --- /dev/null +++ b/evidence/run-read-timeout/typecheck-final.json @@ -0,0 +1,10 @@ +{ + "command": "cd packages/sdk && npx tsc --noEmit && npx tsc -p tsconfig.tests.json", + "result": { + "chunk_id": "10aa44", + "wall_time_seconds": 0.000001521, + "exit_code": 0, + "original_token_count": 0, + "output": "" + } +} diff --git a/evidence/run-read-timeout/verification.md b/evidence/run-read-timeout/verification.md new file mode 100644 index 000000000..6d6f5b028 --- /dev/null +++ b/evidence/run-read-timeout/verification.md @@ -0,0 +1,336 @@ +Captured verification. Commands and output below are verbatim; the full-suite log is linked from summary.md. + +## final-focused.log + +```text +$ cd packages/sdk && RELAYFLOWD_BIN=/home/daytona/.relayflows-toolchain/target/2962130851/debug/relayflowd npx vitest run tests/journal-client-read-timeout.test.ts tests/journal-client.test.ts tests/journal-client-completion.test.ts tests/journal-client-subscriptions.test.ts tests/running-step-watch.test.ts tests/heartbeat-timeout.test.ts tests/run-daemon-unresponsive.test.ts tests/authored-root.test.ts tests/classify-outcome.test.ts tests/cli.test.ts tests/direct-run-worker-lease.test.ts tests/resume-worker-lease.test.ts tests/worker-lease.test.ts tests/worker-lease-lost.test.ts tests/worker-lease-sweep.test.ts tests/run-read-load-live.test.ts tests/worker-lease-lost-live.test.ts tests/flow-executor-chain.test.ts tests/agent-transcript-live.test.ts tests/human-live.test.ts --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ✓ tests/cli.test.ts (71 tests) 6026ms + ✓ flows check CLI > binds a checked relative wrapper to the flow directory for worker execution 566ms + ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 488ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2053ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2045ms +(node:44939) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. +(Use `node --trace-warnings ...` to show where the warning was created) + ✓ tests/authored-root.test.ts (26 tests) 398ms + ✓ tests/journal-client.test.ts (17 tests) 89ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 10315ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 1022ms + ✓ flow executor LLM and output-binding chain > runs a dollar-budgeted authored Claude agent with the same default used by preflight 700ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1595ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 3382ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 779ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 1068ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 3394ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 875ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 818ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 851ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 848ms + ✓ tests/classify-outcome.test.ts (11 tests) 7422ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2009ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3005ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms + ✓ tests/human-live.test.ts (3 tests) 7854ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 4785ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1916ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 1152ms + ✓ tests/worker-lease.test.ts (7 tests) 20ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 27ms + ✓ tests/journal-client-read-timeout.test.ts (13 tests) 1104ms + ✓ a recovered read timeout does not become an authored callback failure 368ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 943ms + ✓ reports journal success after completion rejects with lease_conflict 327ms + ✓ reports journal success when a renewal rejects after completion landed 327ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2821ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2095ms + ✓ drains read and watch promises before an authored flow completes 725ms + ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms + ✓ tests/journal-client-completion.test.ts (6 tests) 102ms + ✓ tests/resume-worker-lease.test.ts (3 tests) 6ms + ✓ tests/direct-run-worker-lease.test.ts (3 tests) 10ms + ✓ tests/running-step-watch.test.ts (2 tests) 2122ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms + ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms + ✓ tests/run-daemon-unresponsive.test.ts (2 tests) 4ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms + + Test Files 20 passed (20) + Tests 210 passed (210) + Start at 10:41:25 + Duration 54.29s (transform 1.45s, setup 97ms, collect 8.09s, tests 42.69s, environment 3ms, prepare 1.05s) + +exit=0 +``` + +## resume-live.log + +```text +$ cd packages/sdk && npx vitest run tests/read-timeout-resume-live.test.ts +$ cd packages/sdk && npx vitest run tests/read-timeout-resume-live.test.ts + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ✓ tests/read-timeout-resume-live.test.ts (1 test) 1512ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 1511ms + + Test Files 1 passed (1) + Tests 1 passed (1) + Start at 10:49:12 + Duration 3.20s (transform 891ms, setup 54ms, collect 1.39s, tests 1.51s, environment 0ms, prepare 43ms) + +exit=0 +exit=0 +``` + +## mutation-reader.log + +```text +Source: packages/sdk/src/journal-client.ts +Original SHA256: 9a390f44f9a1375f0e13be91786f24e848bba7fa07b93b28b119c7ca1d24027c +Replaced: +(reader ?? this).requestOnce(verb, params, remaining) +With: +this.requestOnce(verb, params, remaining) + +REVERTED +$ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'serves a bounded read' --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ❯ tests/journal-client-read-timeout.test.ts (13 tests | 1 failed | 12 skipped) 111ms + × serves a bounded read while an unbounded command is in flight 110ms + → expected true to be false // Object.is equality + +⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/journal-client-read-timeout.test.ts > serves a bounded read while an unbounded command is in flight +AssertionError: expected true to be false // Object.is equality + +- Expected ++ Received + +- false ++ true + + ❯ tests/journal-client-read-timeout.test.ts:36:23 + 34| await inFlight; + 35| expect(await client.runGet('run')).toMatchObject({ status: 'running'… + 36| expect(commandDone).toBe(false); + | ^ + 37| await command; + 38| }); + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ + + Test Files 1 failed (1) + Tests 1 failed | 12 skipped (13) + Start at 10:38:14 + Duration 588ms (transform 169ms, setup 15ms, collect 306ms, tests 111ms, environment 0ms, prepare 44ms) + +exit=1 + +RESTORED SHA256: 9a390f44f9a1375f0e13be91786f24e848bba7fa07b93b28b119c7ca1d24027c +$ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'serves a bounded read' --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ✓ tests/journal-client-read-timeout.test.ts (13 tests | 12 skipped) 108ms + + Test Files 1 passed (1) + Tests 1 passed | 12 skipped (13) + Start at 10:38:15 + Duration 586ms (transform 171ms, setup 15ms, collect 307ms, tests 108ms, environment 0ms, prepare 42ms) + +exit=0 +``` + +## mutation-watch-cadence.log + +```text +Source: packages/sdk/src/cli/running-step.ts +Original SHA256: 0f8b6b9e160ebb047b5a83ad48ad41e2c201d5ba905c27c3a872acd3b9b288c2 +Replaced: +const LEASE_POLL_MS = 2_000; +With: +const LEASE_POLL_MS = 50; + +REVERTED +$ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pushes for completion' --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ❯ tests/running-step-watch.test.ts (2 tests | 1 failed | 1 skipped) 2110ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2109ms + → expected 41 to be less than or equal to 1 + +⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/running-step-watch.test.ts > uses pushes for completion with lease-cadence reads and releases its watcher +AssertionError: expected 41 to be less than or equal to 1 + ❯ tests/running-step-watch.test.ts:37:19 + 35| await waitForRunningStep(client, 'run', { id: 'step', type: 'agent… + 36| expect(performance.now() - start).toBeGreaterThan(2000); + 37| expect(reads).toBeLessThanOrEqual(1); + | ^ + 38| await sleep(10); + 39| expect(watchClosed).toBe(true); + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ + + Test Files 1 failed (1) + Tests 1 failed | 1 skipped (2) + Start at 10:38:16 + Duration 2.60s (transform 179ms, setup 16ms, collect 313ms, tests 2.11s, environment 0ms, prepare 43ms) + +exit=1 + +RESTORED SHA256: 0f8b6b9e160ebb047b5a83ad48ad41e2c201d5ba905c27c3a872acd3b9b288c2 +$ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pushes for completion' --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ✓ tests/running-step-watch.test.ts (2 tests | 1 skipped) 2120ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2119ms + + Test Files 1 passed (1) + Tests 1 passed | 1 skipped (2) + Start at 10:38:20 + Duration 2.61s (transform 186ms, setup 15ms, collect 322ms, tests 2.12s, environment 0ms, prepare 42ms) + +exit=0 +``` + +## mutation-heartbeat.log + +```text +Source: packages/sdk/src/worker-lease.ts +Original SHA256: 44c13f9f61ac13cb8c0f872f4c22b1c212e020d05880ed080ea7205ba6007213 +Replaced: + if (error instanceof JournalRequestTimeoutError && error.verb === 'step.heartbeat') { + throw new WorkerLeaseLostError('renewal_expired', error.message, { cause: error }); + } + +With: + + +REVERTED +$ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartbeat timeout' --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 20ms + × a heartbeat timeout is lease loss rather than a worker body failure 20ms + → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError + +⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/heartbeat-timeout.test.ts > a heartbeat timeout is lease loss rather than a worker body failure +AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError + ❯ tests/heartbeat-timeout.test.ts:16:5 + 14| try { + 15| await client.connect(); + 16| await expect(withWorkerLease(client, { + | ^ + 17| run_id: 'run', step_id: 'step', attempt: 1, step_type: 'agent', … + 18| lease_id: 'lease', lease_deadline_ms: Date.now() + 30_000, lease… + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ + + Test Files 1 failed (1) + Tests 1 failed (1) + Start at 10:38:23 + Duration 509ms (transform 171ms, setup 14ms, collect 315ms, tests 20ms, environment 0ms, prepare 45ms) + +exit=1 + +RESTORED SHA256: 44c13f9f61ac13cb8c0f872f4c22b1c212e020d05880ed080ea7205ba6007213 +$ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartbeat timeout' --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms + + Test Files 1 passed (1) + Tests 1 passed (1) + Start at 10:38:24 + Duration 491ms (transform 169ms, setup 15ms, collect 300ms, tests 15ms, environment 0ms, prepare 43ms) + +exit=0 +``` + +## mutation-root-parking.log + +```text +Source: packages/sdk/src/authored-root.ts +Original SHA256: ef090c86e849b5b4d77700672b60f9a42979e6649e6207a800a1e7561ca6be1e +Replaced: + if ((error instanceof JournalRequestTimeoutError && (READ_ONLY_VERBS.has(error.verb) || error.verb === 'run.watch')) + || (error instanceof AuthoredFlowExecutionError && error.code === 'daemon_unresponsive')) { + const parked = new AuthoredFlowExecutionError('daemon_unresponsive', + `${error.message}. The run remains resumable. Continue with: ${resumeCommand(dispatch.run_id, options.dataDir, options.localAgentStream !== undefined)}.`); + parked.rootRunId = dispatch.run_id; + throw parked; + } + +With: + + +REVERTED +$ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the root resumable after a read timeout' --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ❯ tests/authored-root.test.ts (25 tests | 1 failed | 24 skipped) 22ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch 21ms + → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } +(6 matching properties omitted from actual) + +⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/authored-root.test.ts > durable authored root > leaves the root resumable after a read timeout without waiting for redispatch +AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } +(6 matching properties omitted from actual) + +- Expected ++ Received + +- Object { +- "code": "daemon_unresponsive", +- "message": StringContaining "flows resume", +- "rootRunId": "root-run", +- } ++ [JournalRequestTimeoutError: journal client: run.get timed out after 10ms] + + ❯ tests/authored-root.test.ts:411:5 + 409| const loaded = await fixture(false, 0, async () => { throw new Jou… + 410| const journal = new RootJournal(); + 411| await expect(executeDurableAuthoredFlow(loaded, journal as unknown… + | ^ + 412| { dataDir: '/unused', admissionKey: 'read-timeout' })).rejects.t… + 413| code: 'daemon_unresponsive', rootRunId: 'root-run', message: e… + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ + + Test Files 1 failed (1) + Tests 1 failed | 24 skipped (25) + Start at 10:38:25 + Duration 1.36s (transform 704ms, setup 18ms, collect 1.16s, tests 22ms, environment 0ms, prepare 46ms) + +exit=1 + +RESTORED SHA256: ef090c86e849b5b4d77700672b60f9a42979e6649e6207a800a1e7561ca6be1e +$ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the root resumable after a read timeout' --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + + ✓ tests/authored-root.test.ts (25 tests | 24 skipped) 18ms + + Test Files 1 passed (1) + Tests 1 passed | 24 skipped (25) + Start at 10:38:27 + Duration 1.32s (transform 674ms, setup 15ms, collect 1.13s, tests 18ms, environment 0ms, prepare 45ms) + +exit=0 +``` diff --git a/kernel/DAEMON-LIFECYCLE.md b/kernel/DAEMON-LIFECYCLE.md index 3d739fab2..257db138d 100644 --- a/kernel/DAEMON-LIFECYCLE.md +++ b/kernel/DAEMON-LIFECYCLE.md @@ -559,3 +559,15 @@ SDK (`npm test` in `packages/sdk`): Test 15 is the one that matters most and is the hardest to fake: it must spawn real processes against a real temp data dir, because the property under test is enforced by `flock(2)`, not by any code we could stub. + +## Reads during long-running commands + +The daemon handles frames sequentially per connection; `run.start`, +`run.resume`, and `step.complete` can drive deterministic commands before +replying. The SDK's flow execution clients therefore send read-only requests +on an unconditional lazy reader session, serialized and retried within a +bounded budget. Worker registrations, leases, writes, and watches retain their +own session ordering. Watch pushes come from the hub and can arrive while a +command is in flight. A read timeout is not a terminal run fact: the CLI probes +a fresh connection and reports a resumable interruption (see `docs/SURFACE.md` +§5). No daemon protocol or runtime release is required for this policy. diff --git a/packages/sdk/src/authored-flow-error.ts b/packages/sdk/src/authored-flow-error.ts index d43aeeb96..e2ec5828a 100644 --- a/packages/sdk/src/authored-flow-error.ts +++ b/packages/sdk/src/authored-flow-error.ts @@ -35,6 +35,7 @@ export type AuthoredFlowExecutionErrorCode = * re-dispatch the root to it. The run is not failed: completed steps are * journaled and `flows resume ` continues it. */ + | 'daemon_unresponsive' | 'root_lease_lost' | 'unsupported_completion' | 'unsupported_gate' diff --git a/packages/sdk/src/authored-node-entry.ts b/packages/sdk/src/authored-node-entry.ts index ac6f6fdab..b4f585230 100644 --- a/packages/sdk/src/authored-node-entry.ts +++ b/packages/sdk/src/authored-node-entry.ts @@ -1,3 +1,4 @@ +import { FLOW_READ_BUDGET_MS, JournalRequestTimeoutError, READ_ONLY_VERBS } from './journal-read-policy.js'; import { Worker } from 'node:worker_threads'; import { createHash, createHmac } from 'node:crypto'; import { readFileSync, writeSync } from 'node:fs'; @@ -74,7 +75,7 @@ try { const loaded = await loadPinnedAuthoredSource(request.metadata, true); if (request.localAgentStream !== request.metadata.localAgentStream) throw new Error('authored root local agent surface mismatch'); if (request.workerCapacity !== undefined && !isAgentCapacity(request.workerCapacity)) throw new Error('invalid authored worker capacity'); - client = new JournalClient(request.socketPath); + client = new JournalClient(request.socketPath, { readBudgetMs: FLOW_READ_BUDGET_MS }); await client.connect(); await client.hello('flows-authored-node'); const result = await executeAuthoredFlow(loaded.handle, client, request.metadata.inputPresent ? request.metadata.input : undefined, { @@ -99,6 +100,10 @@ try { // `parkCause` travels for the same reason: it is the difference between // "attach a worker" and "a human has to recover this", and this process is // the only one that saw the child's classification. + // Preserve a read interruption across the Node/Bun error frame so the + // lease-owning parent can leave the root resumable. + ...(error instanceof JournalRequestTimeoutError && (READ_ONLY_VERBS.has(error.verb) || error.verb === 'run.watch') + ? { code: 'daemon_unresponsive' } : {}), ...(error instanceof AuthoredFlowExecutionError ? { code: error.code, completionReason: error.completionReason, runId: error.runId, details: error.details, diff --git a/packages/sdk/src/authored-root.ts b/packages/sdk/src/authored-root.ts index 1929fd381..974703326 100644 --- a/packages/sdk/src/authored-root.ts +++ b/packages/sdk/src/authored-root.ts @@ -1,3 +1,4 @@ +import { JournalRequestTimeoutError, READ_ONLY_VERBS } from './journal-read-policy.js'; import { loadPinnedAuthoredSource } from './authored-source-authority.js'; import { assertAuthoredRuntimeAvailable, runAuthoredInNode } from './authored-node-runner.js'; import { createHash, randomUUID } from 'node:crypto'; @@ -368,6 +369,13 @@ async function driveRoot( // A lost lease is owned by the kernel's retry path, not by this attempt's // terminalization or suspension handling. if (isLeaseLost(error)) throw error; + if ((error instanceof JournalRequestTimeoutError && (READ_ONLY_VERBS.has(error.verb) || error.verb === 'run.watch')) + || (error instanceof AuthoredFlowExecutionError && error.code === 'daemon_unresponsive')) { + const parked = new AuthoredFlowExecutionError('daemon_unresponsive', + `${error.message}. The run remains resumable. Continue with: ${resumeCommand(dispatch.run_id, options.dataDir, options.localAgentStream !== undefined)}.`); + parked.rootRunId = dispatch.run_id; + throw parked; + } if (error instanceof AuthoredFlowExecutionError && error.code === 'subscription_suspended' && error.suspension !== undefined) { diff --git a/packages/sdk/src/cli/direct-run.ts b/packages/sdk/src/cli/direct-run.ts index 904573700..7109ff599 100644 --- a/packages/sdk/src/cli/direct-run.ts +++ b/packages/sdk/src/cli/direct-run.ts @@ -1,3 +1,5 @@ +import { daemonUnresponsiveReport, isReadInterruption } from './journal-timeout.js'; +import { FLOW_READ_BUDGET_MS } from '../journal-read-policy.js'; import { onWorkerFailure } from '../worker-lease.js'; import { McpStepError } from '../authored-mcp.js'; import { randomUUID } from 'node:crypto'; @@ -67,7 +69,7 @@ export async function runDirectFlow( } const socketPath = socketFor(dataDir); const base: RunReport = { ...emptyReport('run'), path }; - const client = new JournalClient(socketPath); + const client = new JournalClient(socketPath, { readBudgetMs: FLOW_READ_BUDGET_MS }); const connected = await connect(client, 'run', dataDir, base, options); if (connected !== undefined) return connected; @@ -212,6 +214,7 @@ export async function runDirectFlow( if (error instanceof AuthoredFlowExecutionError && (error.code === 'step_failed' || error.code === 'gate_failed')) { return authoredStepFailure('run', base, socketPath, error); } + if (isReadInterruption(error)) return daemonUnresponsiveReport('run', base, socketPath, error, base.rootRunId, options, dataDir); if (error instanceof AuthoredFlowExecutionError && error.code === 'root_lease_lost') { return rootLeaseLostReport('run', base, socketPath, error); } diff --git a/packages/sdk/src/cli/journal-timeout.ts b/packages/sdk/src/cli/journal-timeout.ts new file mode 100644 index 000000000..8edce8af7 --- /dev/null +++ b/packages/sdk/src/cli/journal-timeout.ts @@ -0,0 +1,30 @@ +import { probeSocket } from '../daemon-connection.js'; +import { AuthoredFlowExecutionError } from '../authored-flow-error.js'; +import { resumeCommand } from '../authored-human.js'; +import { JournalRequestTimeoutError, READ_ONLY_VERBS } from '../journal-read-policy.js'; +import type { CheckReport } from './check.js'; +import type { RunExecution, RunReport, RunLifecycleOptions, RunCommand } from './run.js'; + +export function isReadInterruption(error: unknown): boolean { + return (error instanceof JournalRequestTimeoutError + && (READ_ONLY_VERBS.has(error.verb) || error.verb === 'run.watch')) + || (error instanceof AuthoredFlowExecutionError && error.code === 'daemon_unresponsive'); +} + +export async function daemonUnresponsiveReport(command: RunCommand, + base: CheckReport | RunReport, socketPath: string, error: unknown, fallbackRunId?: string, + options: RunLifecycleOptions = {}, dataDir?: string): Promise { + const runId = error instanceof AuthoredFlowExecutionError ? error.rootRunId ?? fallbackRunId : fallbackRunId; + const reachable = (await probeSocket(socketPath)).reachable; + const kind = reachable ? 'daemon_unresponsive' : 'daemon_unreachable'; + const detail = reachable ? 'relayflowd answers a fresh connection but the run read timed out; CPU load may be delaying it.' + : 'relayflowd did not answer a fresh connection; it may be unreachable or delayed by CPU load.'; + return { exitCode: 1, report: { + ...base, command, ok: false, socketPath, status: 'running', + ...(runId === undefined ? {} : { runId, rootRunId: runId }), + diagnostics: [...base.diagnostics, { severity: 'failure', kind, + message: `${detail} ${error instanceof Error ? error.message : String(error)}` + + (runId === undefined ? '' : ` The run remains resumable. Continue with: ${resumeCommand(runId, dataDir ?? options.dataDir, options.localAgent === true)}.`), + }], + } }; +} diff --git a/packages/sdk/src/cli/run.ts b/packages/sdk/src/cli/run.ts index 5ec492868..abc8d54f5 100644 --- a/packages/sdk/src/cli/run.ts +++ b/packages/sdk/src/cli/run.ts @@ -1,3 +1,7 @@ +import { waitForRunningStep, type RunningStep } from './running-step.js'; +export { LEASE_SWEEP_GRACE_MS } from './running-step.js'; +import { daemonUnresponsiveReport, isReadInterruption } from './journal-timeout.js'; +import { FLOW_READ_BUDGET_MS } from '../journal-read-policy.js'; import { onWorkerFailure } from '../worker-lease.js'; import { communicationInstruction } from '../communication/spec.js'; import { checkCommunicationEnvironment, CommunicationEnvironmentError } from '../communication/preflight.js'; @@ -203,7 +207,7 @@ async function executeCheckedFlow( catch (error) { return { exitCode: 2, report: { ...base, diagnostics: [...base.diagnostics, { severity: 'refusal', kind: 'probe_failed', message: error instanceof Error ? error.message : 'Communication environment could not be checked.' }] } }; } } - const client = new JournalClient(socketPath); + const client = new JournalClient(socketPath, { readBudgetMs: FLOW_READ_BUDGET_MS }); const connected = await connect(client, 'run', dataDir, base, options); if (connected !== undefined) return connected; @@ -225,6 +229,7 @@ async function executeCheckedFlow( } if (options.onJournalEntry !== undefined) client.on('entry', options.onJournalEntry); const outcome = await startWatched(client, spec, options); + base.runId = outcome.run_id; // `run.start { watch: true }` is an event stream, not the source of the // root identity. A short deterministic run can finish before its first // watched entry is delivered (and an older daemon may refuse `watch`), so @@ -239,6 +244,7 @@ async function executeCheckedFlow( } return execution; } catch (error) { + if (isReadInterruption(error)) return daemonUnresponsiveReport('run', base, socketPath, error, base.runId, options, dataDir); if (error instanceof CommunicationEnvironmentError) return { exitCode: 2, report: { ...base, diagnostics: [...base.diagnostics, { severity: 'refusal', kind: 'probe_failed', message: error.message }] } }; if (error instanceof JournalProtocolError && ( @@ -289,7 +295,7 @@ export async function resumeFlow( const agentEnvironment = localAgentEnvironment(); const socketPath = socketFor(dataDir); const base = emptyReport('resume'); - const client = new JournalClient(socketPath); + const client = new JournalClient(socketPath, { readBudgetMs: FLOW_READ_BUDGET_MS }); const connected = await connect(client, 'resume', dataDir, base, options); if (connected !== undefined) return connected; @@ -408,6 +414,7 @@ export async function resumeFlow( if (error instanceof AuthoredFlowExecutionError && (error.code === 'step_failed' || error.code === 'gate_failed')) { return authoredStepFailure('resume', base, socketPath, error, runId); } + if (isReadInterruption(error)) return daemonUnresponsiveReport('resume', base, socketPath, error, runId, options, dataDir); if (error instanceof AuthoredFlowExecutionError && error.code === 'root_lease_lost') { return rootLeaseLostReport('resume', base, socketPath, error, runId); } @@ -822,6 +829,19 @@ export async function connect( /// needs a stubbed client rather than a real run -- the integration test that /// found it reproduced the bug roughly 1 time in 12. export async function classifyOutcome( + client: JournalClient, command: RunCommand, outcome: RunOutcome, + base: CheckReport | RunReport, socketPath: string, options: RunLifecycleOptions, +): Promise { + try { return await classifyOutcomeInner(client, command, outcome, base, socketPath, options); } + catch (error) { + // Authored child calls have no CLI socket path: let their durable root + // handle the interruption, rather than turn it into a child step failure. + if (socketPath !== '' && isReadInterruption(error)) return daemonUnresponsiveReport(command, base, socketPath, error, outcome.run_id, options); + throw error; + } +} + +async function classifyOutcomeInner( client: JournalClient, command: RunCommand, outcome: RunOutcome, @@ -1021,10 +1041,6 @@ interface OutOfBandInspection { backoffStep?: ParkedStep; } -interface RunningStep extends ParkedStep { - leaseDeadlineMs: number; -} - // Bound on re-polling a run that reports `running` with no identifiable step. // 40 x 50ms = 2s, far longer than the sub-second window observed in #179, and // short enough that a genuinely stuck run still reports rather than hangs. @@ -1071,63 +1087,7 @@ async function inspectOutOfBandStep( }; } -// Match kernel/relayflowd/src/server/client.rs: allow the lease sweep to dispatch a retry. -export const LEASE_SWEEP_GRACE_MS = 5_000; -// How often to re-ask the daemon about a step whose lease looks expired here. const EXPIRED_LEASE_POLL_MS = 250; -// A deadline unchanged for a whole lease (relayflowd LEASE_RENEWAL_MS, 30s) -// plus the sweep grace was renewed by nobody: the attempt is dead and the -// daemon never swept it. Fail rather than hang. -const STALE_LEASE_MS = 30_000 + LEASE_SWEEP_GRACE_MS; - -async function waitForRunningStep( - client: JournalClient, - runId: string, - runningStep: RunningStep, - options: RunLifecycleOptions, -): Promise { - let leaseDeadlineMs = runningStep.leaseDeadlineMs; - if (!Number.isFinite(leaseDeadlineMs)) { - throw new Error(`running step "${runningStep.id}" omitted lease_deadline_ms`); - } - options.onWait?.({ - runId, - stepId: runningStep.id, - stepType: runningStep.type, - leaseDeadlineMs, - }); - // When the lease deadline last changed, on this process's monotonic clock. - // A live worker renews every ~10s, moving the deadline; a dead lease the - // daemon has not swept keeps it. That -- not a comparison of the daemon's - // deadline with our wall clock -- is what separates the two under skew. - let deadlineSeenAt = performance.now(); - while (true) { - throwIfCanceled(options.signal, runningStep.id); - const remainingMs = leaseDeadlineMs + LEASE_SWEEP_GRACE_MS - Date.now(); - if (performance.now() - deadlineSeenAt > STALE_LEASE_MS) { - throw new Error( - `worker lease for step "${runningStep.id}" expired at ${leaseDeadlineMs} without completion`, - ); - } - // Past the deadline by THIS clock alone is not an expiry: the daemon may - // still hold the lease (skew) or be sweeping it into a retry. Keep - // following the step, at a slower poll, until the daemon says otherwise. - await delay(remainingMs <= 0 ? EXPIRED_LEASE_POLL_MS : Math.min(50, remainingMs), options.signal); - const snapshot = await client.runGet(runId); - const step = snapshot.steps[runningStep.id]; - if (step?.state !== 'running') return; - if (step.lease_deadline_ms !== undefined && step.lease_deadline_ms !== leaseDeadlineMs) { - leaseDeadlineMs = step.lease_deadline_ms; - deadlineSeenAt = performance.now(); - options.onWait?.({ - runId, - stepId: runningStep.id, - stepType: runningStep.type, - leaseDeadlineMs, - }); - } - } -} export function protocolFailure( command: RunCommand, diff --git a/packages/sdk/src/cli/running-step.ts b/packages/sdk/src/cli/running-step.ts new file mode 100644 index 000000000..7f666d917 --- /dev/null +++ b/packages/sdk/src/cli/running-step.ts @@ -0,0 +1,90 @@ +import { JournalProtocolError, type JournalClient } from '../journal-client.js'; +import { AuthoredFlowExecutionError } from '../authored-flow-error.js'; +import type { JournalEvent } from '../journal-reader.js'; +import type { RunLifecycleOptions, ParkedStep } from './run.js'; + +export const LEASE_SWEEP_GRACE_MS = 5_000; +const LEASE_POLL_MS = 2_000; +const STALE_LEASE_MS = 30_000 + LEASE_SWEEP_GRACE_MS; +export interface RunningStep extends ParkedStep { leaseDeadlineMs: number } + +/** Watch completion; snapshots only refresh the non-journaled live lease deadline. */ +export async function waitForRunningStep(client: JournalClient, runId: string, + runningStep: RunningStep, options: RunLifecycleOptions): Promise { + let leaseDeadlineMs = runningStep.leaseDeadlineMs; + if (!Number.isFinite(leaseDeadlineMs)) { + throw new Error(`running step "${runningStep.id}" omitted lease_deadline_ms`); + } + const notify = () => options.onWait?.({ runId, stepId: runningStep.id, + stepType: runningStep.type, leaseDeadlineMs }); + notify(); + let deadlineSeenAt = performance.now(); + // The protocol has no unwatch. Closing this scoped, non-worker session removes + // the server watcher, even on cancellation or a read/registration failure. + const watch = client.createPeer(); + let settled = false; + let ready = false; + let finish: () => void = () => {}; + let cancel: () => void = () => {}; + const completed = new Promise((resolve, reject) => { + finish = resolve; + cancel = () => reject(new Error(`worker wait for step "${runningStep.id}" was canceled`)); + }); + // The subscription handshake itself may be pending when cancellation arrives. + void completed.catch(() => {}); + const onEntry = (entry: JournalEvent) => { + if (entry.run_id !== runId) return; + if (entry.entry_type === 'run.completed') settled = true; + else if (entry.step_id === runningStep.id) { + if (entry.entry_type === 'step.attempt.started') settled = false; + if (entry.entry_type === 'step.completed' || entry.entry_type === 'wait.human') settled = true; + } + // Fold the entire replay before deciding: an older attempt may have failed. + if (ready && settled) finish(); + }; + watch.on('entry', onEntry); + options.signal?.addEventListener('abort', cancel, { once: true }); + if (options.signal?.aborted) cancel(); + try { + await Promise.race([watch.connect().then(() => watch.hello('flows-step-watch')).catch(error => { + if (error instanceof JournalProtocolError) throw error; + throw new AuthoredFlowExecutionError('daemon_unresponsive', + `could not establish the completion watch: ${error instanceof Error ? error.message : String(error)}`); + }).then(() => watch.runWatch(runId)), completed]); + ready = true; + if (settled) return; + while (true) { + let timer: ReturnType | undefined; + let releaseTimer: () => void = () => {}; + try { + const elapsed = await Promise.race([ + new Promise(resolve => { releaseTimer = () => resolve(false); timer = setTimeout(() => resolve(true), LEASE_POLL_MS); }), + completed.then(() => false), + ]); + if (!elapsed) return; + } finally { clearTimeout(timer); releaseTimer(); } + // Drain the read before leaving the authored promise scope. A raced but + // unresolved read would look like unawaited derived work to that scope. + const snapshot = await client.runGet(runId).catch(error => { + if (settled) return undefined; + throw error; + }); + if (settled || snapshot === undefined || snapshot.steps[runningStep.id]?.state !== 'running') return; + const deadline = snapshot.steps[runningStep.id]?.lease_deadline_ms; + if (deadline !== undefined && deadline !== leaseDeadlineMs) { + leaseDeadlineMs = deadline; + deadlineSeenAt = performance.now(); + notify(); + } else if (performance.now() - deadlineSeenAt > STALE_LEASE_MS) { + // A stalled sweep/renewal is not evidence that the body itself failed. + throw new AuthoredFlowExecutionError('daemon_unresponsive', + `worker lease for step "${runningStep.id}" expired at ${leaseDeadlineMs} without completion; relayflowd's lease sweep may be delayed by CPU load`); + } + } + } finally { + finish(); + options.signal?.removeEventListener('abort', cancel); + watch.off('entry', onEntry); + watch.close(); + } +} diff --git a/packages/sdk/src/failure-kinds.ts b/packages/sdk/src/failure-kinds.ts index 7a05c98a2..09cc78fda 100644 --- a/packages/sdk/src/failure-kinds.ts +++ b/packages/sdk/src/failure-kinds.ts @@ -149,6 +149,7 @@ export const RUN_FAILURE_KINDS = [ * reach this process. The run is NOT failed (`status: running`): its * completed steps are journaled and `flows resume ` continues it. */ + 'daemon_unresponsive', 'root_lease_lost', ] as const; diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index 50909fa8b..d69f03a0d 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -217,7 +217,7 @@ export type { } from './protocol.js'; export { JOURNAL_WRITE_FAILED, PROTOCOL_VERSION } from './protocol.js'; -export { JournalClient, type JournalClientOptions } from './journal-client.js'; +export { JournalClient, JournalRequestTimeoutError, type JournalClientOptions } from './journal-client.js'; export { AgentWorker, type AgentWorkerOptions } from './worker.js'; export { diff --git a/packages/sdk/src/journal-client.ts b/packages/sdk/src/journal-client.ts index 9fe998eed..112a9bde2 100644 --- a/packages/sdk/src/journal-client.ts +++ b/packages/sdk/src/journal-client.ts @@ -9,6 +9,8 @@ // stream plumbing. The client's framing and failure behavior is covered by a // loopback double in tests. +import { JournalReadPolicy, JournalRequestTimeoutError, READ_ONLY_VERBS } from './journal-read-policy.js'; +export { JournalRequestTimeoutError } from './journal-read-policy.js'; import { EventEmitter } from 'node:events'; export { walkJournal, JournalReadError, type JournalEvent, type JournalReadFailure } from './journal-reader.js'; import { randomUUID } from 'node:crypto'; @@ -27,6 +29,8 @@ import { import type { KernelRunSpec, StepType } from './spec.js'; export interface JournalClientOptions { + /** Opt-in total budget for read-only requests; interactive clients stay single-shot. */ + readBudgetMs?: number; /** Override the timeout for bounded protocol requests (ms). Default 30000. */ requestTimeoutMs?: number; /** @@ -61,9 +65,14 @@ export class JournalProtocolError extends Error { } export class JournalClient extends EventEmitter { + private readonly reads = new JournalReadPolicy(); + private reader: JournalClient | undefined; + private readerReady: Promise | undefined; + private readonly readBudgetMs?: number; /** Additive request capabilities the daemon advertised at `hello`; none until then. */ private features: ReadonlySet = new Set(); private socket: Socket | null = null; + private connectingSocket: Socket | undefined; /** Why the connection ended, so a later "not connected" names its cause rather than hiding it. */ private disconnectCause: Error | undefined; private buffer = ''; @@ -76,6 +85,7 @@ export class JournalClient extends EventEmitter { options: JournalClientOptions = {}, ) { super(); + this.readBudgetMs = options.readBudgetMs; this.requestTimeoutMs = options.requestTimeoutMs ?? 30_000; this.connectTimeoutMs = options.connectTimeoutMs ?? 2_000; } @@ -92,6 +102,7 @@ export class JournalClient extends EventEmitter { return new Promise((resolve, reject) => { if (this.socket) return resolve(); const socket = createConnection({ path: this.socketPath }); + this.connectingSocket = socket; const timer = setTimeout(() => { socket.removeAllListeners(); socket.destroy(); @@ -112,10 +123,13 @@ export class JournalClient extends EventEmitter { socket.on('data', (chunk) => this.onData(chunk)); socket.on('close', () => { const closed = new Error('journal client: connection closed'); + this.reads.close(); + this.reader?.close(closed); this.disconnectCause ??= closed; this.failAll(closed); }); this.disconnectCause = undefined; + this.connectingSocket = undefined; this.socket = socket; resolve(); }); @@ -131,6 +145,9 @@ export class JournalClient extends EventEmitter { const closed = cause === undefined ? new Error('journal client: closed by caller') : new Error(`journal client: closed after ${cause instanceof Error ? cause.message : String(cause)}`, { cause }); + this.reads.close(); + this.reader?.close(cause); + this.connectingSocket?.destroy(closed); this.disconnectCause ??= closed; this.failAll(closed); this.socket?.destroy(); @@ -181,7 +198,31 @@ export class JournalClient extends EventEmitter { this.pending.clear(); } - private request( + private async readSession(): Promise { + if (this.readerReady === undefined) { + const reader = this.createPeer(); + this.reader = reader; + this.readerReady = reader.connect().then(() => reader.hello('flows-reader')).then(() => reader) + .catch(() => { reader.close(); return undefined; }); + } + return this.readerReady; + } + + private request(verb: V, params: VerbContract[V]['params'], + timeoutMs: number | null = this.requestTimeoutMs): Promise { + if (this.socket && !this.socket.destroyed && this.readBudgetMs !== undefined && timeoutMs !== null && READ_ONLY_VERBS.has(verb)) { + return this.reads.read(verb, timeoutMs, this.readBudgetMs, async bound => { + const started = performance.now(); + const reader = await this.readSession(); + const remaining = bound - (performance.now() - started); + if (remaining <= 0) throw new JournalRequestTimeoutError(verb, bound); + return (reader ?? this).requestOnce(verb, params, remaining); + }); + } + return this.requestOnce(verb, params, timeoutMs); + } + + private requestOnce( verb: V, params: VerbContract[V]['params'], timeoutMs: number | null = this.requestTimeoutMs, @@ -196,10 +237,11 @@ export class JournalClient extends EventEmitter { return; } const id = randomUUID(); + const started = performance.now(); const frame: Request = { id, verb: verb as string, params }; const timer = timeoutMs === null ? undefined : setTimeout(() => { this.pending.delete(id); - reject(new Error(`journal client: ${verb} timed out after ${timeoutMs}ms`)); + reject(new JournalRequestTimeoutError(verb, timeoutMs, 1, performance.now() - started)); }, timeoutMs); this.pending.set(id, { resolve: resolve as (v: unknown) => void, reject, timer }); this.socket.write(JSON.stringify(frame) + '\n', (err) => { diff --git a/packages/sdk/src/journal-read-policy.ts b/packages/sdk/src/journal-read-policy.ts new file mode 100644 index 000000000..76f666a70 --- /dev/null +++ b/packages/sdk/src/journal-read-policy.ts @@ -0,0 +1,79 @@ +import { setMaxListeners } from 'node:events'; +import { setTimeout as sleep } from 'node:timers/promises'; + +export const READ_ONLY_VERBS = new Set(['run.get', 'journal.read', 'stream.read', 'subscription.inspect']); +export const FLOW_READ_BUDGET_MS = 300_000; + +export class JournalRequestTimeoutError extends Error { + constructor( + readonly verb: string, + readonly timeoutMs: number, + readonly attempts = 1, + readonly elapsedMs = timeoutMs, + readonly readBudgetMs?: number, + ) { + super(readBudgetMs === undefined + ? `journal client: ${verb} timed out after ${timeoutMs}ms` + : `journal client: ${verb} timed out after ${attempts} attempts in ${Math.round(elapsedMs)}ms (read budget ${readBudgetMs}ms); relayflowd may be delayed by CPU load`); + this.name = 'JournalRequestTimeoutError'; + } +} + +/** Serial admission bounds read amplification, including retries, per body client. */ +export class JournalReadPolicy { + private tail: Promise = Promise.resolve(); + private readonly closed = new AbortController(); + + constructor() { + // Concurrent children share this cancellation signal; each waiter removes its listener. + setMaxListeners(0, this.closed.signal); + } + + close(): void { this.closed.abort(new Error('journal client: closed by caller')); } + + async read(verb: string, timeoutMs: number, budgetMs: number, + request: (timeoutMs: number) => Promise): Promise { + const started = performance.now(); + const deadline = started + budgetMs; + let attempts = 0; + const exhausted = () => new JournalRequestTimeoutError(verb, timeoutMs, attempts, + performance.now() - started, budgetMs); + const signal = this.closed.signal; + let timer: ReturnType | undefined; + let onClose: () => void = () => {}; + let deliver: (value: T) => void = () => {}; + let fail: (error: unknown) => void = () => {}; + const response = new Promise((resolve, reject) => { + deliver = resolve; + fail = reject; + timer = setTimeout(() => reject(exhausted()), budgetMs); + onClose = () => reject(signal.reason); + signal.addEventListener('abort', onClose, { once: true }); + if (signal.aborted) onClose(); + }); + const job = this.tail.then(async () => { + while (true) { + signal.throwIfAborted(); + const remaining = deadline - performance.now(); + if (remaining <= 0) throw exhausted(); + attempts += 1; + try { + return await request(Math.min(remaining, timeoutMs * 2 ** Math.max(0, attempts - 2))); + } catch (error) { + if (!(error instanceof JournalRequestTimeoutError)) throw error; + const left = deadline - performance.now(); + if (left <= 0) throw exhausted(); + // Jitter plus escalating bounds keep a slow daemon from collecting a retry storm. + await sleep(Math.min(left, Math.min(1000, timeoutMs / 4) * (0.5 + Math.random())), undefined, { signal }); + } + } + }); + this.tail = job.catch(() => {}); + void job.then(deliver, fail); + try { return await response; } + finally { + clearTimeout(timer); + signal.removeEventListener('abort', onClose); + } + } +} diff --git a/packages/sdk/src/worker-lease.ts b/packages/sdk/src/worker-lease.ts index 8396f165a..9475b5428 100644 --- a/packages/sdk/src/worker-lease.ts +++ b/packages/sdk/src/worker-lease.ts @@ -1,4 +1,4 @@ -import { JournalProtocolError, type JournalClient } from './journal-client.js'; +import { JournalProtocolError, JournalRequestTimeoutError, type JournalClient } from './journal-client.js'; import type { StepDispatchEvent } from './protocol.js'; export class WorkerLeaseLostError extends Error { @@ -72,7 +72,12 @@ export async function withWorkerLease( const sentAt = performance.now(); const result = await untilAborted(client.stepHeartbeat( dispatch.run_id, dispatch.step_id, dispatch.attempt, dispatch.lease_id, - ), controller.signal); + ), controller.signal).catch(error => { + if (error instanceof JournalRequestTimeoutError && error.verb === 'step.heartbeat') { + throw new WorkerLeaseLostError('renewal_expired', error.message, { cause: error }); + } + throw error; + }); controller.signal.throwIfAborted(); // A response handled after local expiry cannot revive ownership, even // if its future deadline was issued before this event loop stalled. diff --git a/packages/sdk/tests/authored-root.test.ts b/packages/sdk/tests/authored-root.test.ts index 589c64fb7..2aa0eed0c 100644 --- a/packages/sdk/tests/authored-root.test.ts +++ b/packages/sdk/tests/authored-root.test.ts @@ -12,9 +12,9 @@ import { resumeDurableAuthoredFlow, type AuthoredRootMetadata, } from '../src/authored-root.js'; -import { JournalProtocolError, type JournalClient } from '../src/journal-client.js'; +import { JournalProtocolError, JournalRequestTimeoutError, type JournalClient } from '../src/journal-client.js'; import type { RunOutcome, StepDispatchEvent } from '../src/protocol.js'; -import { AuthoredHumanParked } from '../src/authored-flow-error.js'; +import { AuthoredFlowExecutionError, AuthoredHumanParked } from '../src/authored-flow-error.js'; vi.mock('../src/authored-flow-loader.js', async importOriginal => ({ ...await importOriginal(), @@ -383,6 +383,42 @@ describe('durable authored root', () => { expect(journal.peer.completions).toEqual([{ attempt: 2, reason: 'success' }]); }); + it('does not terminalize a root whose heartbeat times out', async () => { + const loaded = await fixture(); + const journal = new RootJournal(); + const controller = new AbortController(); + journal.peer.stepHeartbeat = async () => { + controller.abort(); + throw new JournalRequestTimeoutError('step.heartbeat', 10); + }; + await expect(executeDurableAuthoredFlow(loaded, journal as unknown as JournalClient, undefined, + { dataDir: '/unused', admissionKey: 'heartbeat-timeout', lifecycle: { signal: controller.signal } })) + .rejects.toThrow(); + expect(journal.peer.completions).toEqual([]); + }); + + it('still terminalizes a timed-out mutation as a body failure', async () => { + const loaded = await fixture(false, 0, async () => { throw new JournalRequestTimeoutError('stream.append', 10); }); + const journal = new RootJournal(); + await expect(executeDurableAuthoredFlow(loaded, journal as unknown as JournalClient, undefined, + { dataDir: '/unused', admissionKey: 'write-timeout' })).rejects.toMatchObject({ verb: 'stream.append' }); + expect(journal.peer.completions).toEqual([{ attempt: 1, reason: 'worker_error' }]); + }); + + it.each([false, true])('leaves the root resumable after a read timeout without waiting for redispatch (Node frame=%s)', async nodeFrame => { + const timeout = new JournalRequestTimeoutError('run.get', 10); + const loaded = await fixture(false, 0, async () => { + throw nodeFrame ? new AuthoredFlowExecutionError('daemon_unresponsive', timeout.message) : timeout; + }); + const journal = new RootJournal(); + await expect(executeDurableAuthoredFlow(loaded, journal as unknown as JournalClient, undefined, + { dataDir: '/unused', admissionKey: 'read-timeout' })).rejects.toMatchObject({ + code: 'daemon_unresponsive', rootRunId: 'root-run', message: expect.stringContaining('flows resume'), + }); + expect(journal.peer.completions).toEqual([]); + expect(journal.resumeCalls).toBe(1); + }); + it('terminalizes a returned body failure without replaying semantic side effects', async () => { const loaded = await fixture(true); const journal = new RootJournal(); diff --git a/packages/sdk/tests/classify-outcome.test.ts b/packages/sdk/tests/classify-outcome.test.ts index 47cb65562..d46da85ac 100644 --- a/packages/sdk/tests/classify-outcome.test.ts +++ b/packages/sdk/tests/classify-outcome.test.ts @@ -1,3 +1,4 @@ +import { EventEmitter } from 'node:events'; import { describe, expect, it } from 'vitest'; import { classifyOutcome } from '../src/cli/run.js'; @@ -163,6 +164,9 @@ describe('the remedy on a worker park', () => { const until = Date.now() + 3_000; let resumes = 0; const client = { + createPeer: () => Object.assign(new EventEmitter(), { + connect: async () => {}, hello: async () => {}, runWatch: async () => {}, close: () => {}, + }), runGet: async () => (Date.now() < until ? backoff : completed), runResume: async (): Promise => { resumes += 1; @@ -245,6 +249,9 @@ describe('the remedy on a worker park', () => { budget: { tokens_in: 0, tokens_out: 0, dollars: '0' }, }; const client = { + createPeer: () => Object.assign(new EventEmitter(), { + connect: async () => {}, hello: async () => {}, runWatch: async () => {}, close: () => {}, + }), runGet: async () => (Date.now() < until ? running : { ...running, status: 'completed', steps: { answer: { type: 'agent', state: 'done' } } }), runResume: async (): Promise => diff --git a/packages/sdk/tests/cli.test.ts b/packages/sdk/tests/cli.test.ts index c8b6138ad..e9d9c86be 100644 --- a/packages/sdk/tests/cli.test.ts +++ b/packages/sdk/tests/cli.test.ts @@ -1010,6 +1010,7 @@ describe('flows run/resume CLI over the journal protocol', () => { let snapshots = 0; await startCliLoopback(dataDir, { hello: sendOk, + 'run.watch': ctx => sendResult(ctx, {}), 'run.start': (ctx) => sendResult(ctx, { run_id: 'run-worker', status: 'parked', @@ -1059,6 +1060,7 @@ describe('flows run/resume CLI over the journal protocol', () => { let snapshots = 0; await startCliLoopback(dataDir, { hello: sendOk, + 'run.watch': ctx => sendResult(ctx, {}), 'run.start': (ctx) => sendResult(ctx, { run_id: 'run-stale-worker', status: 'parked', @@ -1066,7 +1068,7 @@ describe('flows run/resume CLI over the journal protocol', () => { completed_steps: 1, }), 'run.get': (ctx) => { - const running = snapshots++ < 4; + const running = snapshots++ < 1; sendResult(ctx, { run_id: 'run-stale-worker', status: running ? 'running' : 'completed', @@ -1095,13 +1097,14 @@ describe('flows run/resume CLI over the journal protocol', () => { expect(output.stderr.join('\n')).toContain('WAITING [worker_lease]'); expect(output.stderr.join('\n')).toContain(`until ${leaseDeadlineMs}`); expect(output.stderr.join('\n')).not.toContain('worker lease for step "answer" expired'); - expect(snapshots).toBeGreaterThan(4); + expect(snapshots).toBeGreaterThan(1); }); it('allows a caller to cancel a worker-lease wait', async () => { const dataDir = temporaryProject('flows-run-cancel-'); await startCliLoopback(dataDir, { hello: sendOk, + 'run.watch': ctx => sendResult(ctx, {}), 'run.start': (ctx) => sendResult(ctx, { run_id: 'run-cancel', status: 'parked', diff --git a/packages/sdk/tests/direct-run-worker-lease.test.ts b/packages/sdk/tests/direct-run-worker-lease.test.ts index 2b1bbe763..f0d2e9eeb 100644 --- a/packages/sdk/tests/direct-run-worker-lease.test.ts +++ b/packages/sdk/tests/direct-run-worker-lease.test.ts @@ -1,3 +1,4 @@ +import { AuthoredFlowExecutionError } from '../src/authored-flow-error.js'; import { afterEach, expect, it, vi } from 'vitest'; import { runDirectFlow } from '../src/cli/direct-run.js'; import { JournalClient, JournalProtocolError } from '../src/journal-client.js'; @@ -32,3 +33,17 @@ it.each([false, true])('direct run handles LLM errors with leaseLost=%s', async expect(result.exitCode).toBe(1); expect(JSON.stringify(result.report)).toContain(leaseLost ? 'connection closed' : 'cli exploded'); }); + +it('reports a read interruption without losing the resumable root', async () => { + vi.spyOn(JournalClient.prototype, 'connect').mockResolvedValue(); + vi.spyOn(JournalClient.prototype, 'hello').mockResolvedValue({} as never); + vi.spyOn(LlmWorker.prototype, 'attach').mockResolvedValue(); + const error = new AuthoredFlowExecutionError('daemon_unresponsive', 'read timed out under CPU load'); + error.rootRunId = 'saved-root'; + vi.mocked(executeDurableAuthoredFlow).mockRejectedValueOnce(error); + const result = await runDirectFlow('flow.ts', '{}', '/unused'); + expect(result).toMatchObject({ exitCode: 1, report: { + command: 'run', runId: 'saved-root', rootRunId: 'saved-root', status: 'running', + diagnostics: [{ kind: 'daemon_unresponsive', message: expect.stringContaining('flows resume --data-dir /unused saved-root') }], + } }); +}); diff --git a/packages/sdk/tests/heartbeat-timeout.test.ts b/packages/sdk/tests/heartbeat-timeout.test.ts new file mode 100644 index 000000000..e4db308b7 --- /dev/null +++ b/packages/sdk/tests/heartbeat-timeout.test.ts @@ -0,0 +1,24 @@ +import { once } from 'node:events'; +import { expect, it } from 'vitest'; +import { JournalClient } from '../src/journal-client.js'; +import { withWorkerLease, WorkerLeaseLostError } from '../src/worker-lease.js'; +import { sendOk, sockPath, startLoopback } from './journal-client-loopback.js'; + +it('a heartbeat timeout is lease loss rather than a worker body failure', async () => { + const path = sockPath(); + let heartbeats = 0; + const server = startLoopback(path, { hello: sendOk, 'step.heartbeat': () => { heartbeats++; } }); + await once(server, 'listening'); + const client = new JournalClient(path, { requestTimeoutMs: 10 }); + let executed = false; + try { + await client.connect(); + await expect(withWorkerLease(client, { + run_id: 'run', step_id: 'step', attempt: 1, step_type: 'agent', spec: {}, + lease_id: 'lease', lease_deadline_ms: Date.now() + 30_000, lease_ttl_ms: 30_000, + idempotency_key: 'key', pins: { workspace: [], streams: [] }, + }, async () => { executed = true; })).rejects.toBeInstanceOf(WorkerLeaseLostError); + expect(executed).toBe(false); + expect(heartbeats).toBe(1); + } finally { client.close(); await new Promise(resolve => server.close(() => resolve())); } +}); diff --git a/packages/sdk/tests/journal-client-loopback.ts b/packages/sdk/tests/journal-client-loopback.ts index caa434a74..ef35a709b 100644 --- a/packages/sdk/tests/journal-client-loopback.ts +++ b/packages/sdk/tests/journal-client-loopback.ts @@ -24,33 +24,34 @@ export interface FrameCtx { } export interface LoopbackHandlers { - 'subscription.inspect'?: (ctx: FrameCtx, params: Record) => void; - 'subscription.deliver'?: (ctx: FrameCtx, params: Record) => void; - 'subscription.fence_overflow'?: (ctx: FrameCtx, params: Record) => void; - hello?: (ctx: FrameCtx) => void; - 'run.start'?: (ctx: FrameCtx, params: Record) => void; - 'run.resume'?: (ctx: FrameCtx, params: Record) => void; - 'run.cancel'?: (ctx: FrameCtx, params: Record) => void; - 'run.get'?: (ctx: FrameCtx, params: Record) => void; - 'run.watch'?: (ctx: FrameCtx, params: Record) => void; - 'worker.attach'?: (ctx: FrameCtx, params: Record) => void; - 'step.heartbeat'?: (ctx: FrameCtx, params: Record) => void; - 'effect.record'?: (ctx: FrameCtx, params: Record) => void; - 'effect.confirm'?: (ctx: FrameCtx, params: Record) => void; - 'step.complete'?: (ctx: FrameCtx, params: Record) => void; - 'step.wait'?: (ctx: FrameCtx, params: Record) => void; - 'event.emit'?: (ctx: FrameCtx, params: Record) => void; - 'subscription.open'?: (ctx: FrameCtx, params: Record) => void; - 'subscription.next'?: (ctx: FrameCtx, params: Record) => void; - 'subscription.close'?: (ctx: FrameCtx, params: Record) => void; - 'journal.read'?: (ctx: FrameCtx, params: Record) => void; - 'stream.append'?: (ctx: FrameCtx, params: Record) => void; - 'stream.read'?: (ctx: FrameCtx, params: Record) => void; + 'subscription.inspect'?: (ctx: FrameCtx, params: Record) => void | Promise; + 'subscription.deliver'?: (ctx: FrameCtx, params: Record) => void | Promise; + 'subscription.fence_overflow'?: (ctx: FrameCtx, params: Record) => void | Promise; + hello?: (ctx: FrameCtx) => void | Promise; + 'run.start'?: (ctx: FrameCtx, params: Record) => void | Promise; + 'run.resume'?: (ctx: FrameCtx, params: Record) => void | Promise; + 'run.cancel'?: (ctx: FrameCtx, params: Record) => void | Promise; + 'run.get'?: (ctx: FrameCtx, params: Record) => void | Promise; + 'run.watch'?: (ctx: FrameCtx, params: Record) => void | Promise; + 'worker.attach'?: (ctx: FrameCtx, params: Record) => void | Promise; + 'step.heartbeat'?: (ctx: FrameCtx, params: Record) => void | Promise; + 'effect.record'?: (ctx: FrameCtx, params: Record) => void | Promise; + 'effect.confirm'?: (ctx: FrameCtx, params: Record) => void | Promise; + 'step.complete'?: (ctx: FrameCtx, params: Record) => void | Promise; + 'step.wait'?: (ctx: FrameCtx, params: Record) => void | Promise; + 'event.emit'?: (ctx: FrameCtx, params: Record) => void | Promise; + 'subscription.open'?: (ctx: FrameCtx, params: Record) => void | Promise; + 'subscription.next'?: (ctx: FrameCtx, params: Record) => void | Promise; + 'subscription.close'?: (ctx: FrameCtx, params: Record) => void | Promise; + 'journal.read'?: (ctx: FrameCtx, params: Record) => void | Promise; + 'stream.append'?: (ctx: FrameCtx, params: Record) => void | Promise; + 'stream.read'?: (ctx: FrameCtx, params: Record) => void | Promise; } -export function startLoopback(path: string, handlers: LoopbackHandlers): Server { +export function startLoopback(path: string, handlers: LoopbackHandlers, options: { serialize?: boolean } = {}): Server { const server = createServer((socket) => { let buffer = ''; + let queue = Promise.resolve(); const send = (obj: unknown): void => { socket.write(JSON.stringify(obj) + '\n'); }; @@ -62,11 +63,14 @@ export function startLoopback(path: string, handlers: LoopbackHandlers): Server buffer = buffer.slice(nl + 1); if (line.length === 0) continue; const req = JSON.parse(line) as { id: string; verb: keyof LoopbackHandlers; params: Record }; - const handler = handlers[req.verb] as ((ctx: FrameCtx, params: Record) => void) | undefined; + const handler = handlers[req.verb] as ((ctx: FrameCtx, params: Record) => void | Promise) | undefined; if (handler === undefined) { send({ id: req.id, ok: false, error: { code: 'unknown_verb', message: req.verb } }); } else { - handler({ id: req.id, socket, send }, req.params); + const handle = () => handler({ id: req.id, socket, send }, req.params); + // server.rs handles one frame at a time per connection. + if (options.serialize) queue = queue.then(handle); + else void handle(); } } }); diff --git a/packages/sdk/tests/journal-client-read-timeout.test.ts b/packages/sdk/tests/journal-client-read-timeout.test.ts new file mode 100644 index 000000000..2efeac22b --- /dev/null +++ b/packages/sdk/tests/journal-client-read-timeout.test.ts @@ -0,0 +1,141 @@ +import { once } from 'node:events'; +import { setTimeout as sleep } from 'node:timers/promises'; +import type { Server } from 'node:net'; +import { afterEach, expect, it } from 'vitest'; +import { JournalClient, JournalRequestTimeoutError } from '../src/journal-client.js'; +import { HELLO_SPEC, sendOk, sendResult, sockPath, startLoopback, type LoopbackHandlers } from './journal-client-loopback.js'; + +const clients: JournalClient[] = []; +const servers: Server[] = []; +afterEach(async () => { + for (const client of clients.splice(0)) client.close(); + await Promise.all(servers.splice(0).map(server => new Promise(resolve => server.close(() => resolve())))); +}); +async function setup(handlers: LoopbackHandlers, budget: number | undefined = 1000) { + const path = sockPath(); + const server = startLoopback(path, { hello: sendOk, ...handlers }, { serialize: true }); + servers.push(server); + await once(server, 'listening'); + const client = new JournalClient(path, { requestTimeoutMs: 50, readBudgetMs: budget }); + clients.push(client); + await client.connect(); + return client; +} + +it('serves a bounded read while an unbounded command is in flight', async () => { + let started!: () => void; + const inFlight = new Promise(resolve => { started = resolve; }); + const client = await setup({ + 'run.start': async ctx => { started(); await sleep(100); sendResult(ctx, { status: 'completed' }); }, + 'run.get': ctx => sendResult(ctx, { status: 'running' }), + }); + let commandDone = false; + const command = client.runStart(HELLO_SPEC).then(result => { commandDone = true; return result; }); + await inFlight; + expect(await client.runGet('run')).toMatchObject({ status: 'running' }); + expect(commandDone).toBe(false); + await command; +}); + +it('retries a read with a fresh id and ignores the late first reply', async () => { + const ids: string[] = []; + let first: Parameters[0]; + const client = await setup({ 'run.get': ctx => { + ids.push(ctx.id); + if (ids.length === 1) first = ctx; + else { sendResult(first, { status: 'stale' }); sendResult(ctx, { status: 'completed' }); } + } }); + expect(await client.runGet('run')).toMatchObject({ status: 'completed' }); + expect(new Set(ids).size).toBe(2); +}); + +it('never retries mutating verbs', async () => { + let writes = 0; + const client = await setup({ 'stream.append': () => { writes += 1; } }); + await expect(client.streamAppend('run', 'stream', {})).rejects.toMatchObject({ + verb: 'stream.append', attempts: 1, message: 'journal client: stream.append timed out after 50ms', + }); + expect(writes).toBe(1); +}); + +it('exhausts a total read budget with a typed diagnostic', async () => { + const client = await setup({ 'run.get': () => {} }, 250); + const error = await client.runGet('run').catch(error => error); + expect(error).toBeInstanceOf(JournalRequestTimeoutError); + expect(error.attempts).toBeGreaterThan(1); + expect(error.message).toContain('read budget 250ms'); + expect(error.message).toContain('CPU load'); +}); + +it('keeps interactive reads single-shot by default', async () => { + let reads = 0; + // Explicitly omit the opt-in. + const client = await setup({ 'run.get': () => { reads += 1; } }, 0); + client.close(); + const interactive = new JournalClient(client.socketPath, { requestTimeoutMs: 10 }); + clients.push(interactive); + await interactive.connect(); + await expect(interactive.runGet('run')).rejects.toMatchObject({ attempts: 1, readBudgetMs: undefined }); + expect(reads).toBe(1); +}); + +it('serializes concurrent reads and stops queued reads on close', async () => { + let reads = 0; + let received!: () => void; + const entered = new Promise(resolve => { received = resolve; }); + const client = await setup({ 'run.get': () => { reads += 1; received(); } }); + const results = Promise.allSettled(Array.from({ length: 8 }, () => client.runGet('run'))); + await entered; + client.close(); + expect((await results).every(result => result.status === 'rejected')).toBe(true); + expect(reads).toBe(1); +}); + +it('does not retry protocol rejections', async () => { + let reads = 0; + const client = await setup({ 'run.get': ctx => { + reads += 1; + ctx.send({ id: ctx.id, ok: false, error: { code: 'run_not_found', message: 'absent' } }); + } }); + await expect(client.runGet('absent')).rejects.toMatchObject({ code: 'run_not_found' }); + expect(reads).toBe(1); +}); + +it('a recovered read timeout does not become an authored callback failure', async () => { + const { flow } = await import('@relayflows/surface'); + const { executeAuthoredFlow } = await import('../src/authored-flow-executor.js'); + let reads = 0; + const client = await setup({ + 'run.start': (ctx, params) => sendResult(ctx, { run_id: (params['spec'] as { steps: Array<{ id: string }> }).steps[0]!.id, status: 'completed', completion_reason: 'success', completed_steps: 1 }), + 'journal.read': (ctx, params) => { + if (++reads === 1) return; + sendResult(ctx, { entries: [{ seq: 1, run_id: params['run_id'], step_id: params['run_id'], entry_type: 'step.completed', + payload: { completionReason: 'success', output: { stdout_tail: 'ok', exit_code: 0 } } }] }); + }, + }); + const result = await executeAuthoredFlow(flow('retried-read', async f => { + await f.run('echo ok'); + f.done('success'); + }), client, undefined); + expect(result.completionReason).toBe('success'); + expect(reads).toBe(3); +}); + +it('falls back to the primary connection if reader setup is refused', async () => { + const client = await setup({ + hello: ctx => ctx.send({ id: ctx.id, ok: false, error: { code: 'busy', message: 'reader unavailable' } }), + 'run.get': ctx => sendResult(ctx, { status: 'completed' }), + }); + expect(await client.runGet('run')).toMatchObject({ status: 'completed' }); +}); + +it.each(['run.get', 'journal.read', 'stream.read', 'subscription.inspect'] as const)('retries only the allowlisted read %s', async verb => { + let attempts = 0; + const client = await setup({ [verb]: ctx => { if (++attempts === 2) sendResult(ctx, {}); } }); + const result = verb === 'run.get' ? client.runGet('run') + : verb === 'journal.read' ? client.journalRead('run', 1) + : verb === 'stream.read' ? client.streamRead('run', 'stream', 0) + : client.subscriptionInspect({ run_id: 'run', subscription_id: 'subscription' }); + await result; + expect(attempts).toBe(2); +}); diff --git a/packages/sdk/tests/read-timeout-resume-live.test.ts b/packages/sdk/tests/read-timeout-resume-live.test.ts new file mode 100644 index 000000000..677309874 --- /dev/null +++ b/packages/sdk/tests/read-timeout-resume-live.test.ts @@ -0,0 +1,60 @@ +import { readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { expect, it } from 'vitest'; +import { JournalClient } from '../src/journal-client.js'; +import { loadAuthoredFlow } from '../src/authored-flow-loader.js'; +import { executeDurableAuthoredFlow, resumeDurableAuthoredFlow } from '../src/authored-root.js'; +import { LlmWorker } from '../src/llm-worker.js'; +import { chainFixture, shellWord } from './flow-chain-fixture.js'; + +it('parks an unreadable authored root and resumes without repeating its journaled effect', async () => { + const fixture = chainFixture(); + const journal = await fixture.connect(); + const { pid } = JSON.parse(readFileSync(join(fixture.data, 'connection.json'), 'utf8')); + const worker = new LlmWorker(journal, 'park-read-worker'); + const client = new JournalClient(journal.socketPath, { requestTimeoutMs: 100, readBudgetMs: 500 }); + const effects = join(fixture.root, 'effects'); + writeFileSync(fixture.flowPath, `import {flow} from '@relayflows/surface'; +export default flow('park-read', async f => { + await f.run(${JSON.stringify(`echo saved >> ${shellWord(effects)}`)}); + await f.llm('hello', {model:'test-model', output:{type:'object'}}); + await f.run(${JSON.stringify(`echo after >> ${shellWord(effects)}`)}); + f.done('success'); +}); +`); + const get = client.runGet.bind(client); + let interrupted = false; + // Suspend service exactly when the body first reads its running child. The + // request and retry timers are real; no synthetic rejection is injected. + client.runGet = runId => { + if (!interrupted) { interrupted = true; process.kill(pid, 'SIGSTOP'); } + return get(runId); + }; + let rootRunId = ''; + try { + await client.connect(); + await worker.attach(); + const loaded = await loadAuthoredFlow(fixture.flowPath); + try { + await expect(executeDurableAuthoredFlow(loaded, client, undefined, { + dataDir: fixture.data, admissionKey: 'park-read', onAdmitted: id => { rootRunId = id; }, + })).rejects.toMatchObject({ code: 'daemon_unresponsive', rootRunId: expect.any(String) }); + } finally { process.kill(pid, 'SIGCONT'); client.runGet = get; } + expect(interrupted).toBe(true); + expect(readFileSync(effects, 'utf8')).toBe('saved\n'); + const entries = (await journal.journalRead(rootRunId, 1, 1000)).entries as Array<{ + entry_type: string; step_id: string; payload: { completionReason?: string }; + }>; + expect(entries.some(entry => entry.step_id === 'authored-root' + && entry.entry_type === 'step.completed' && entry.payload.completionReason === 'worker_error')).toBe(false); + expect(await resumeDurableAuthoredFlow(rootRunId, client, { dataDir: fixture.data })) + .toMatchObject({ rootRunId, completionReason: 'success' }); + expect(readFileSync(effects, 'utf8')).toBe('saved\nafter\n'); + expect(await journal.runGet(rootRunId)).toMatchObject({ status: 'completed' }); + } finally { + process.kill(pid, 'SIGCONT'); + client.close(); + await worker.close(); + await fixture.close(); + } +}, 30_000); diff --git a/packages/sdk/tests/resume-worker-lease.test.ts b/packages/sdk/tests/resume-worker-lease.test.ts index 91a5b304c..102f2e259 100644 --- a/packages/sdk/tests/resume-worker-lease.test.ts +++ b/packages/sdk/tests/resume-worker-lease.test.ts @@ -1,3 +1,4 @@ +import { AuthoredFlowExecutionError } from '../src/authored-flow-error.js'; import { afterEach, expect, it, vi } from 'vitest'; import { JournalClient, JournalProtocolError } from '../src/journal-client.js'; import { LlmWorker } from '../src/llm-worker.js'; @@ -34,3 +35,17 @@ it.each([false, true])('resume handles LLM errors with leaseLost=%s', async leas expect(JSON.stringify(result.report)).toContain(leaseLost ? 'connection closed' : 'cli exploded'); expect(warning).toHaveBeenCalledTimes(leaseLost ? 1 : 0); }); + +it('reports a read interruption without losing the resumable root', async () => { + vi.spyOn(JournalClient.prototype, 'connect').mockResolvedValue(); + vi.spyOn(JournalClient.prototype, 'hello').mockResolvedValue({} as never); + vi.spyOn(LlmWorker.prototype, 'attach').mockResolvedValue(); + const error = new AuthoredFlowExecutionError('daemon_unresponsive', 'read timed out under CPU load'); + error.rootRunId = 'saved-root'; + vi.mocked(resumeDurableAuthoredFlow).mockRejectedValueOnce(error); + const result = await resumeFlow('saved-root', '/unused', { localAgent: true }); + expect(result).toMatchObject({ exitCode: 1, report: { + command: 'resume', runId: 'saved-root', rootRunId: 'saved-root', status: 'running', + diagnostics: [{ kind: 'daemon_unresponsive', message: expect.stringContaining('flows resume --data-dir /unused --local-agent saved-root') }], + } }); +}); diff --git a/packages/sdk/tests/run-daemon-unresponsive.test.ts b/packages/sdk/tests/run-daemon-unresponsive.test.ts new file mode 100644 index 000000000..641f1f228 --- /dev/null +++ b/packages/sdk/tests/run-daemon-unresponsive.test.ts @@ -0,0 +1,21 @@ +import { afterEach, expect, it, vi } from 'vitest'; +import { classifyOutcome } from '../src/cli/run.js'; +import { JournalRequestTimeoutError, type JournalClient } from '../src/journal-client.js'; +import { probeSocket } from '../src/daemon-connection.js'; +vi.mock('../src/daemon-connection.js', async original => ({ + ...await original(), probeSocket: vi.fn(), +})); +afterEach(() => vi.clearAllMocks()); +it.each([true, false])('reports an unreadable declarative run as resumable (reachable=%s)', async reachable => { + vi.mocked(probeSocket).mockResolvedValue({ reachable }); + const client = { runGet: async () => { throw new JournalRequestTimeoutError('run.get', 10, 3, 100, 100); } }; + const result = await classifyOutcome(client as unknown as JournalClient, 'run', { + run_id: 'saved-run', status: 'parked', completion_reason: null, completed_steps: 30, + }, { command: 'run', ok: true, diagnostics: [] } as never, '/socket', {}); + expect(result).toMatchObject({ exitCode: 1, report: { + runId: 'saved-run', rootRunId: 'saved-run', status: 'running', + diagnostics: [{ kind: reachable ? 'daemon_unresponsive' : 'daemon_unreachable', + message: expect.stringContaining('flows resume saved-run') }], + } }); + expect(probeSocket).toHaveBeenCalledWith('/socket'); +}); diff --git a/packages/sdk/tests/run-read-load-live.test.ts b/packages/sdk/tests/run-read-load-live.test.ts new file mode 100644 index 000000000..5b7e6de02 --- /dev/null +++ b/packages/sdk/tests/run-read-load-live.test.ts @@ -0,0 +1,68 @@ +import { execFileSync } from 'node:child_process'; +import { availableParallelism } from 'node:os'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { expect, it } from 'vitest'; +import { toKernelSpec } from '../src/compile.js'; +import { JournalClient } from '../src/journal-client.js'; +import { chainFixture, shellWord } from './flow-chain-fixture.js'; + +it('completes a CPU-saturating deterministic flow with reads in flight and preserves its journal', async () => { + const fixture = chainFixture(); + const primary = await fixture.connect(); + const client = new JournalClient(primary.socketPath, { requestTimeoutMs: 100, readBudgetMs: 15_000 }); + let processors = availableParallelism(); + if (process.platform === 'linux') { + // Saturate the daemon's entire CPU allocation without starving unrelated + // test fixtures whose synthetic worker leases last only 30 milliseconds. + const cpu = readFileSync('/proc/self/status', 'utf8').match(/^Cpus_allowed_list:\s*(\d+)/m)![1]!; + const { pid } = JSON.parse(readFileSync(join(fixture.data, 'connection.json'), 'utf8')); + execFileSync('taskset', ['-apc', cpu, String(pid)]); + processors = 1; + } + const stamp = join(fixture.root, 'effects'); + let started!: (runId: string) => void; + const admitted = new Promise(resolve => { started = resolve; }); + client.on('entry', entry => { if (entry.entry_type === 'run.spawned') started(entry.run_id); }); + // One runnable CPU burner per available processor, launched by the flow itself. + // The shell trap owns all burners even if the deterministic step is terminated. + const command = `pids=''; trap 'kill $pids 2>/dev/null; wait' EXIT; ` + + `i=0; while [ "$i" -lt ${processors} ]; do yes >/dev/null & pids="$pids $!"; i=$((i+1)); done; sleep 2`; + try { + await client.connect(); + await client.hello('load-regression'); + const execution = client.runStart(toKernelSpec({ version: '0.1.0', name: 'read-under-load', steps: [ + { id: 'saved', type: 'deterministic', command: `echo saved >> ${shellWord(stamp)}` }, + { id: 'cpu', type: 'deterministic', command, dependsOn: ['saved'] }, + { id: 'after', type: 'deterministic', command: 'echo survived', dependsOn: ['cpu'] }, + ] }), undefined, undefined, true); + const runId = await admitted; + const reads = Promise.all(Array.from({ length: 8 }, () => client.runGet(runId))); + const [outcome, snapshots] = await Promise.all([execution, reads]); + expect(outcome).toMatchObject({ status: 'completed', completion_reason: 'success', completed_steps: 3 }); + expect(snapshots.every(snapshot => snapshot.run_id === runId)).toBe(true); + const entries = (await client.journalRead(runId, 1, 1000)).entries as Array<{ entry_type: string }>; + expect(entries.filter(entry => entry.entry_type === 'step.completed')).toHaveLength(3); + expect(await client.runResume(runId)).toMatchObject({ status: 'completed', completion_reason: 'success' }); + expect(readFileSync(stamp, 'utf8')).toBe('saved\n'); + } finally { client.close(); await fixture.close(); } +}, 30_000); + +it('drains read and watch promises before an authored flow completes', async () => { + const { flow } = await import('@relayflows/surface'); + const { executeAuthoredFlow } = await import('../src/authored-flow-executor.js'); + const { LlmWorker } = await import('../src/llm-worker.js'); + const fixture = chainFixture(); + const workerClient = await fixture.connect(); + const worker = new LlmWorker(workerClient, 'read-policy-worker'); + const client = new JournalClient(workerClient.socketPath, { readBudgetMs: 5_000 }); + try { + await client.connect(); + await worker.attach(); + const result = await executeAuthoredFlow(flow('read-policy-scope', async f => { + await f.llm('hello', { model: 'test-model', output: { type: 'object' } }); + f.done('success'); + }), client, undefined, { flowPath: fixture.flowPath }); + expect(result.completionReason).toBe('success'); + } finally { await worker.close(); client.close(); await fixture.close(); } +}, 30_000); diff --git a/packages/sdk/tests/running-step-watch.test.ts b/packages/sdk/tests/running-step-watch.test.ts new file mode 100644 index 000000000..b55bf635f --- /dev/null +++ b/packages/sdk/tests/running-step-watch.test.ts @@ -0,0 +1,59 @@ +import { once } from 'node:events'; +import { setTimeout as sleep } from 'node:timers/promises'; +import { expect, it } from 'vitest'; +import { JournalClient } from '../src/journal-client.js'; +import { waitForRunningStep } from '../src/cli/running-step.js'; +import { sendOk, sendResult, sockPath, startLoopback } from './journal-client-loopback.js'; + +it('uses pushes for completion with lease-cadence reads and releases its watcher', async () => { + let reads = 0; + let watchClosed = false; + const path = sockPath(); + const server = startLoopback(path, { + hello: sendOk, + 'run.watch': (ctx, params) => { + const entry = (run: string, type: string) => ctx.send({ event: 'entry', data: { + run_id: run, step_id: 'step', entry_type: type, + } }); + // An older failed attempt in replay must not finish the current one. + entry('run', 'step.completed'); + entry('run', 'step.attempt.started'); + sendResult(ctx, {}); + entry('other-run', 'step.completed'); + setTimeout(() => entry(String(params['run_id']), 'step.completed'), 2100); + ctx.socket.once('close', () => { watchClosed = true; }); + }, + 'run.get': ctx => { reads++; sendResult(ctx, { steps: { step: { + state: 'running', lease_deadline_ms: Date.now() + 30_000, + } } }); }, + }); + await once(server, 'listening'); + const client = new JournalClient(path); + try { + await client.connect(); + const start = performance.now(); + await waitForRunningStep(client, 'run', { id: 'step', type: 'agent', leaseDeadlineMs: Date.now() + 30_000 }, {}); + expect(performance.now() - start).toBeGreaterThan(2000); + expect(reads).toBeLessThanOrEqual(1); + await sleep(10); + expect(watchClosed).toBe(true); + } finally { client.close(); await new Promise(resolve => server.close(() => resolve())); } +}); + +it('cancels a pending watch registration and removes its connection', async () => { + const path = sockPath(); + let registered!: () => void; + const registration = new Promise(resolve => { registered = resolve; }); + const server = startLoopback(path, { hello: sendOk, 'run.watch': () => registered() }); + await once(server, 'listening'); + const client = new JournalClient(path); + const controller = new AbortController(); + try { + await client.connect(); + const waiting = waitForRunningStep(client, 'run', { id: 'step', type: 'agent', leaseDeadlineMs: Date.now() + 30_000 }, { signal: controller.signal }); + const rejected = expect(waiting).rejects.toThrow('was canceled'); + await registration; + controller.abort(); + await rejected; + } finally { client.close(); await new Promise(resolve => server.close(() => resolve())); } +}); diff --git a/packages/sdk/tests/worker-lease-sweep.test.ts b/packages/sdk/tests/worker-lease-sweep.test.ts index 1cc66b931..c3348a308 100644 --- a/packages/sdk/tests/worker-lease-sweep.test.ts +++ b/packages/sdk/tests/worker-lease-sweep.test.ts @@ -1,3 +1,4 @@ +import { EventEmitter } from 'node:events'; import { afterEach, expect, it, vi } from 'vitest'; import { classifyOutcome, emptyReport } from '../src/cli/run.js'; import type { JournalClient } from '../src/journal-client.js'; @@ -14,10 +15,13 @@ function fixture() { }); const parked = { run_id: 'run', status: 'parked' as const, completion_reason: null, completed_steps: 0 }; const client = { + createPeer: () => Object.assign(new EventEmitter(), { + connect: async () => {}, hello: async () => {}, runWatch: async () => {}, close: () => {}, + }), runGet: vi.fn().mockResolvedValue(snapshot('running')), runResume: vi.fn().mockResolvedValue(parked), }; - const run = () => classifyOutcome(client as unknown as JournalClient, 'run', parked, emptyReport('run'), '/unused', {}); + const run = () => classifyOutcome(client as unknown as JournalClient, 'run', parked, emptyReport('run'), '', {}); return { client, snapshot, run }; } @@ -31,14 +35,14 @@ it('waits through an expired snapshot until the kernel retries and completes', a .mockResolvedValue({ run_id: 'run', status: 'completed', completion_reason: 'success', completed_steps: 1 }); const execution = run(); const assertion = expect(execution).resolves.toMatchObject({ exitCode: 0, report: { completionReason: 'success' } }); - await Promise.all([assertion, vi.advanceTimersByTimeAsync(1_000)]); + await Promise.all([assertion, vi.advanceTimersByTimeAsync(4_000)]); expect(client.runResume).toHaveBeenCalledTimes(2); }); it('still fails when the kernel never resolves an expired lease after sweep grace', async () => { const { run } = fixture(); const assertion = expect(run()).rejects.toThrow('without completion'); - await vi.advanceTimersByTimeAsync(35_500); + await vi.advanceTimersByTimeAsync(36_500); await assertion; }); @@ -46,7 +50,7 @@ it('bounds an unchanged lease when the CLI clock is behind the daemon', async () const { client, snapshot, run } = fixture(); client.runGet.mockResolvedValue(snapshot('running', Date.now() + 75_000)); const assertion = expect(run()).rejects.toThrow('without completion'); - await vi.advanceTimersByTimeAsync(35_500); + await vi.advanceTimersByTimeAsync(36_500); await assertion; }); @@ -69,5 +73,5 @@ it('follows a live step whose renewed deadline looks expired on a skewed CLI clo client.runGet.mockResolvedValue(snapshot('completed')); client.runResume.mockResolvedValue({ run_id: 'run', status: 'completed', completion_reason: 'success', completed_steps: 1 }); const assertion = expect(execution).resolves.toMatchObject({ exitCode: 0, report: { completionReason: 'success' } }); - await Promise.all([assertion, vi.advanceTimersByTimeAsync(1_000)]); + await Promise.all([assertion, vi.advanceTimersByTimeAsync(4_000)]); }); From 0b8f93f7dc513ab16480ca0a7bf1b79d03793209 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Wed, 7 Oct 2026 23:21:03 -0700 Subject: [PATCH 02/34] fix(sdk): park on a lost read session and let cancellation interrupt snapshot reads P1: the dedicated reader can disconnect while the primary and root worker sessions stay healthy. Its transport failures now raise a typed JournalReadInterruptedError, drop the cached reader so the next attempt reconnects, and are retried within the read budget like timeouts. An exhausted interruption is a read interruption everywhere a timeout is (isReadInterruptionError): the authored root parks resumable instead of terminalizing worker_error, the Node bridge carries daemon_unresponsive, and the CLI report names the interrupted session. Protocol refusals and primary-session/write failures keep their meaning. P2: runGet accepts an AbortSignal that settles the read at once, removes the pending request and drains its queued attempt; waitForRunningStep passes the caller's signal and checks it before interpreting a snapshot. The loopback double ignores writes to a client that already hung up. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: b8262468-ea26-4a6e-bee9-d193832e77dc --- packages/sdk/src/authored-node-entry.ts | 4 +- packages/sdk/src/authored-root.ts | 4 +- packages/sdk/src/cli/journal-timeout.ts | 10 ++-- packages/sdk/src/cli/running-step.ts | 17 ++++-- packages/sdk/src/journal-client.ts | 59 +++++++++++++++---- packages/sdk/src/journal-read-policy.ts | 46 +++++++++++++-- packages/sdk/tests/authored-root.test.ts | 13 ++++ packages/sdk/tests/journal-client-loopback.ts | 5 +- .../tests/journal-client-read-timeout.test.ts | 47 +++++++++++++++ .../sdk/tests/run-daemon-unresponsive.test.ts | 15 +++++ packages/sdk/tests/running-step-watch.test.ts | 25 ++++++++ 11 files changed, 213 insertions(+), 32 deletions(-) diff --git a/packages/sdk/src/authored-node-entry.ts b/packages/sdk/src/authored-node-entry.ts index b4f585230..978f59088 100644 --- a/packages/sdk/src/authored-node-entry.ts +++ b/packages/sdk/src/authored-node-entry.ts @@ -1,4 +1,4 @@ -import { FLOW_READ_BUDGET_MS, JournalRequestTimeoutError, READ_ONLY_VERBS } from './journal-read-policy.js'; +import { FLOW_READ_BUDGET_MS, isReadInterruptionError } from './journal-read-policy.js'; import { Worker } from 'node:worker_threads'; import { createHash, createHmac } from 'node:crypto'; import { readFileSync, writeSync } from 'node:fs'; @@ -102,7 +102,7 @@ try { // the only one that saw the child's classification. // Preserve a read interruption across the Node/Bun error frame so the // lease-owning parent can leave the root resumable. - ...(error instanceof JournalRequestTimeoutError && (READ_ONLY_VERBS.has(error.verb) || error.verb === 'run.watch') + ...(isReadInterruptionError(error) ? { code: 'daemon_unresponsive' } : {}), ...(error instanceof AuthoredFlowExecutionError ? { code: error.code, completionReason: error.completionReason, runId: error.runId, diff --git a/packages/sdk/src/authored-root.ts b/packages/sdk/src/authored-root.ts index 974703326..d3e3bc8d5 100644 --- a/packages/sdk/src/authored-root.ts +++ b/packages/sdk/src/authored-root.ts @@ -1,4 +1,4 @@ -import { JournalRequestTimeoutError, READ_ONLY_VERBS } from './journal-read-policy.js'; +import { isReadInterruptionError } from './journal-read-policy.js'; import { loadPinnedAuthoredSource } from './authored-source-authority.js'; import { assertAuthoredRuntimeAvailable, runAuthoredInNode } from './authored-node-runner.js'; import { createHash, randomUUID } from 'node:crypto'; @@ -369,7 +369,7 @@ async function driveRoot( // A lost lease is owned by the kernel's retry path, not by this attempt's // terminalization or suspension handling. if (isLeaseLost(error)) throw error; - if ((error instanceof JournalRequestTimeoutError && (READ_ONLY_VERBS.has(error.verb) || error.verb === 'run.watch')) + if (isReadInterruptionError(error) || (error instanceof AuthoredFlowExecutionError && error.code === 'daemon_unresponsive')) { const parked = new AuthoredFlowExecutionError('daemon_unresponsive', `${error.message}. The run remains resumable. Continue with: ${resumeCommand(dispatch.run_id, options.dataDir, options.localAgentStream !== undefined)}.`); diff --git a/packages/sdk/src/cli/journal-timeout.ts b/packages/sdk/src/cli/journal-timeout.ts index 8edce8af7..1afbb6959 100644 --- a/packages/sdk/src/cli/journal-timeout.ts +++ b/packages/sdk/src/cli/journal-timeout.ts @@ -1,13 +1,12 @@ import { probeSocket } from '../daemon-connection.js'; import { AuthoredFlowExecutionError } from '../authored-flow-error.js'; import { resumeCommand } from '../authored-human.js'; -import { JournalRequestTimeoutError, READ_ONLY_VERBS } from '../journal-read-policy.js'; +import { isReadInterruptionError, JournalReadInterruptedError } from '../journal-read-policy.js'; import type { CheckReport } from './check.js'; import type { RunExecution, RunReport, RunLifecycleOptions, RunCommand } from './run.js'; export function isReadInterruption(error: unknown): boolean { - return (error instanceof JournalRequestTimeoutError - && (READ_ONLY_VERBS.has(error.verb) || error.verb === 'run.watch')) + return isReadInterruptionError(error) || (error instanceof AuthoredFlowExecutionError && error.code === 'daemon_unresponsive'); } @@ -17,7 +16,10 @@ export async function daemonUnresponsiveReport(command: RunCommand, const runId = error instanceof AuthoredFlowExecutionError ? error.rootRunId ?? fallbackRunId : fallbackRunId; const reachable = (await probeSocket(socketPath)).reachable; const kind = reachable ? 'daemon_unresponsive' : 'daemon_unreachable'; - const detail = reachable ? 'relayflowd answers a fresh connection but the run read timed out; CPU load may be delaying it.' + const detail = error instanceof JournalReadInterruptedError + ? (reachable ? 'relayflowd answers a fresh connection but the run read session kept being interrupted.' + : 'relayflowd did not answer a fresh connection after the run read session was interrupted; it may be unreachable or delayed by CPU load.') + : reachable ? 'relayflowd answers a fresh connection but the run read timed out; CPU load may be delaying it.' : 'relayflowd did not answer a fresh connection; it may be unreachable or delayed by CPU load.'; return { exitCode: 1, report: { ...base, command, ok: false, socketPath, status: 'running', diff --git a/packages/sdk/src/cli/running-step.ts b/packages/sdk/src/cli/running-step.ts index 7f666d917..3621c3ef8 100644 --- a/packages/sdk/src/cli/running-step.ts +++ b/packages/sdk/src/cli/running-step.ts @@ -26,9 +26,10 @@ export async function waitForRunningStep(client: JournalClient, runId: string, let ready = false; let finish: () => void = () => {}; let cancel: () => void = () => {}; + const canceled = () => new Error(`worker wait for step "${runningStep.id}" was canceled`); const completed = new Promise((resolve, reject) => { finish = resolve; - cancel = () => reject(new Error(`worker wait for step "${runningStep.id}" was canceled`)); + cancel = () => reject(canceled()); }); // The subscription handshake itself may be pending when cancellation arrives. void completed.catch(() => {}); @@ -64,11 +65,15 @@ export async function waitForRunningStep(client: JournalClient, runId: string, if (!elapsed) return; } finally { clearTimeout(timer); releaseTimer(); } // Drain the read before leaving the authored promise scope. A raced but - // unresolved read would look like unawaited derived work to that scope. - const snapshot = await client.runGet(runId).catch(error => { - if (settled) return undefined; - throw error; - }); + // unresolved read would look like unawaited derived work to that scope, + // so cancellation aborts the read itself and it settles at once. + const snapshot = await client.runGet(runId, options.signal === undefined ? {} : { signal: options.signal }) + .catch(error => { + if (options.signal?.aborted) throw canceled(); + if (settled) return undefined; + throw error; + }); + if (options.signal?.aborted) throw canceled(); if (settled || snapshot === undefined || snapshot.steps[runningStep.id]?.state !== 'running') return; const deadline = snapshot.steps[runningStep.id]?.lease_deadline_ms; if (deadline !== undefined && deadline !== leaseDeadlineMs) { diff --git a/packages/sdk/src/journal-client.ts b/packages/sdk/src/journal-client.ts index 112a9bde2..19affe969 100644 --- a/packages/sdk/src/journal-client.ts +++ b/packages/sdk/src/journal-client.ts @@ -9,8 +9,8 @@ // stream plumbing. The client's framing and failure behavior is covered by a // loopback double in tests. -import { JournalReadPolicy, JournalRequestTimeoutError, READ_ONLY_VERBS } from './journal-read-policy.js'; -export { JournalRequestTimeoutError } from './journal-read-policy.js'; +import { JournalReadInterruptedError, JournalReadPolicy, JournalRequestTimeoutError, READ_ONLY_VERBS } from './journal-read-policy.js'; +export { JournalReadInterruptedError, JournalRequestTimeoutError } from './journal-read-policy.js'; import { EventEmitter } from 'node:events'; export { walkJournal, JournalReadError, type JournalEvent, type JournalReadFailure } from './journal-reader.js'; import { randomUUID } from 'node:crypto'; @@ -208,26 +208,50 @@ export class JournalClient extends EventEmitter { return this.readerReady; } + /** Forget a reader whose transport failed so the next attempt reconnects one. */ + private dropReader(reader: JournalClient): void { + if (this.reader !== reader) return; + this.reader = undefined; + this.readerReady = undefined; + reader.close(); + } + private request(verb: V, params: VerbContract[V]['params'], - timeoutMs: number | null = this.requestTimeoutMs): Promise { + timeoutMs: number | null = this.requestTimeoutMs, signal?: AbortSignal): Promise { if (this.socket && !this.socket.destroyed && this.readBudgetMs !== undefined && timeoutMs !== null && READ_ONLY_VERBS.has(verb)) { - return this.reads.read(verb, timeoutMs, this.readBudgetMs, async bound => { + return this.reads.read(verb, timeoutMs, this.readBudgetMs, async (bound, attemptSignal) => { const started = performance.now(); const reader = await this.readSession(); + attemptSignal.throwIfAborted(); const remaining = bound - (performance.now() - started); if (remaining <= 0) throw new JournalRequestTimeoutError(verb, bound); - return (reader ?? this).requestOnce(verb, params, remaining); - }); + // The primary session's own failures keep their meaning: it also carries writes. + if (reader === undefined) return this.requestOnce(verb, params, remaining, attemptSignal); + try { + return await reader.requestOnce(verb, params, remaining, attemptSignal); + } catch (error) { + if (attemptSignal.aborted || error instanceof JournalRequestTimeoutError + || error instanceof JournalProtocolError) throw error; + // Only the dedicated reader's transport failed; reconnect within the budget. + this.dropReader(reader); + throw new JournalReadInterruptedError(verb, 1, performance.now() - started, undefined, { cause: error }); + } + }, signal); } - return this.requestOnce(verb, params, timeoutMs); + return this.requestOnce(verb, params, timeoutMs, signal); } private requestOnce( verb: V, params: VerbContract[V]['params'], timeoutMs: number | null = this.requestTimeoutMs, + signal?: AbortSignal, ): Promise { return new Promise((resolve, reject) => { + if (signal?.aborted) { + reject(signal.reason); + return; + } if (!this.socket || this.socket.destroyed) { const cause = this.disconnectCause; reject(new Error( @@ -239,11 +263,24 @@ export class JournalClient extends EventEmitter { const id = randomUUID(); const started = performance.now(); const frame: Request = { id, verb: verb as string, params }; + // Cancellation forgets the request; a late reply is ignored like a timed-out one. + const onAbort = () => { + const p = this.pending.get(id); + if (p === undefined) return; + if (p.timer !== undefined) clearTimeout(p.timer); + this.pending.delete(id); + p.reject(signal!.reason); + }; + const settle = (fn: (arg: A) => void) => (arg: A) => { + signal?.removeEventListener('abort', onAbort); + fn(arg); + }; const timer = timeoutMs === null ? undefined : setTimeout(() => { this.pending.delete(id); - reject(new JournalRequestTimeoutError(verb, timeoutMs, 1, performance.now() - started)); + settle(reject)(new JournalRequestTimeoutError(verb, timeoutMs, 1, performance.now() - started)); }, timeoutMs); - this.pending.set(id, { resolve: resolve as (v: unknown) => void, reject, timer }); + this.pending.set(id, { resolve: settle(resolve as (v: unknown) => void), reject: settle(reject), timer }); + signal?.addEventListener('abort', onAbort, { once: true }); this.socket.write(JSON.stringify(frame) + '\n', (err) => { if (err) { const p = this.pending.get(id); @@ -297,8 +334,8 @@ export class JournalClient extends EventEmitter { } /** Snapshot for legibility. */ - runGet(runId: string): Promise { - return this.request('run.get', { run_id: runId }); + runGet(runId: string, options: { signal?: AbortSignal } = {}): Promise { + return this.request('run.get', { run_id: runId }, this.requestTimeoutMs, options.signal); } /** diff --git a/packages/sdk/src/journal-read-policy.ts b/packages/sdk/src/journal-read-policy.ts index 76f666a70..c7f8f4526 100644 --- a/packages/sdk/src/journal-read-policy.ts +++ b/packages/sdk/src/journal-read-policy.ts @@ -19,6 +19,35 @@ export class JournalRequestTimeoutError extends Error { } } +/** + * The dedicated read session lost its transport (closed, reset, not connected) + * while the primary session may still be healthy. Like a timeout, this says + * nothing about the run itself: retry within the budget, then leave the run + * resumable. Protocol refusals and write failures are never this error. + */ +export class JournalReadInterruptedError extends Error { + constructor( + readonly verb: string, + readonly attempts = 1, + readonly elapsedMs = 0, + readonly readBudgetMs?: number, + options?: ErrorOptions, + ) { + const cause = options?.cause instanceof Error ? `: ${options.cause.message}` : ''; + super(readBudgetMs === undefined + ? `journal client: ${verb} read session was interrupted${cause}` + : `journal client: ${verb} read session was interrupted after ${attempts} attempts in ${Math.round(elapsedMs)}ms (read budget ${readBudgetMs}ms)${cause}`, + options); + this.name = 'JournalReadInterruptedError'; + } +} + +/** A read that could not be answered — by timeout or lost read transport — not a run failure. */ +export function isReadInterruptionError(error: unknown): error is JournalRequestTimeoutError | JournalReadInterruptedError { + return (error instanceof JournalRequestTimeoutError && (READ_ONLY_VERBS.has(error.verb) || error.verb === 'run.watch')) + || error instanceof JournalReadInterruptedError; +} + /** Serial admission bounds read amplification, including retries, per body client. */ export class JournalReadPolicy { private tail: Promise = Promise.resolve(); @@ -32,13 +61,16 @@ export class JournalReadPolicy { close(): void { this.closed.abort(new Error('journal client: closed by caller')); } async read(verb: string, timeoutMs: number, budgetMs: number, - request: (timeoutMs: number) => Promise): Promise { + request: (timeoutMs: number, signal: AbortSignal) => Promise, caller?: AbortSignal): Promise { const started = performance.now(); const deadline = started + budgetMs; let attempts = 0; - const exhausted = () => new JournalRequestTimeoutError(verb, timeoutMs, attempts, - performance.now() - started, budgetMs); - const signal = this.closed.signal; + let last: unknown; + const exhausted = () => last instanceof JournalReadInterruptedError + ? new JournalReadInterruptedError(verb, attempts, performance.now() - started, budgetMs, { cause: last.cause }) + : new JournalRequestTimeoutError(verb, timeoutMs, attempts, performance.now() - started, budgetMs); + // A caller's cancellation settles this read and drains its queued attempt. + const signal = caller === undefined ? this.closed.signal : AbortSignal.any([this.closed.signal, caller]); let timer: ReturnType | undefined; let onClose: () => void = () => {}; let deliver: (value: T) => void = () => {}; @@ -58,9 +90,11 @@ export class JournalReadPolicy { if (remaining <= 0) throw exhausted(); attempts += 1; try { - return await request(Math.min(remaining, timeoutMs * 2 ** Math.max(0, attempts - 2))); + return await request(Math.min(remaining, timeoutMs * 2 ** Math.max(0, attempts - 2)), signal); } catch (error) { - if (!(error instanceof JournalRequestTimeoutError)) throw error; + signal.throwIfAborted(); + if (!(error instanceof JournalRequestTimeoutError || error instanceof JournalReadInterruptedError)) throw error; + last = error; const left = deadline - performance.now(); if (left <= 0) throw exhausted(); // Jitter plus escalating bounds keep a slow daemon from collecting a retry storm. diff --git a/packages/sdk/tests/authored-root.test.ts b/packages/sdk/tests/authored-root.test.ts index 2aa0eed0c..7acad6a38 100644 --- a/packages/sdk/tests/authored-root.test.ts +++ b/packages/sdk/tests/authored-root.test.ts @@ -15,6 +15,7 @@ import { import { JournalProtocolError, JournalRequestTimeoutError, type JournalClient } from '../src/journal-client.js'; import type { RunOutcome, StepDispatchEvent } from '../src/protocol.js'; import { AuthoredFlowExecutionError, AuthoredHumanParked } from '../src/authored-flow-error.js'; +import { JournalReadInterruptedError } from '../src/journal-read-policy.js'; vi.mock('../src/authored-flow-loader.js', async importOriginal => ({ ...await importOriginal(), @@ -419,6 +420,18 @@ describe('durable authored root', () => { expect(journal.resumeCalls).toBe(1); }); + it('leaves the root resumable when the read session disconnects while the root worker is healthy', async () => { + const interrupted = new JournalReadInterruptedError('run.get', 2, 40, 300_000, + { cause: new Error('journal client: connection closed') }); + const loaded = await fixture(false, 0, async () => { throw interrupted; }); + const journal = new RootJournal(); + await expect(executeDurableAuthoredFlow(loaded, journal as unknown as JournalClient, undefined, + { dataDir: '/unused', admissionKey: 'read-disconnect' })).rejects.toMatchObject({ + code: 'daemon_unresponsive', rootRunId: 'root-run', message: expect.stringContaining('flows resume'), + }); + expect(journal.peer.completions).toEqual([]); + }); + it('terminalizes a returned body failure without replaying semantic side effects', async () => { const loaded = await fixture(true); const journal = new RootJournal(); diff --git a/packages/sdk/tests/journal-client-loopback.ts b/packages/sdk/tests/journal-client-loopback.ts index ef35a709b..0ef0d2f0a 100644 --- a/packages/sdk/tests/journal-client-loopback.ts +++ b/packages/sdk/tests/journal-client-loopback.ts @@ -52,8 +52,11 @@ export function startLoopback(path: string, handlers: LoopbackHandlers, options: const server = createServer((socket) => { let buffer = ''; let queue = Promise.resolve(); + // A client may hang up before a reply lands (a dropped reader, a closed + // session); like the daemon, the double ignores the failed write. + socket.on('error', () => {}); const send = (obj: unknown): void => { - socket.write(JSON.stringify(obj) + '\n'); + if (!socket.destroyed) socket.write(JSON.stringify(obj) + '\n'); }; socket.on('data', (chunk) => { buffer += chunk.toString('utf8'); diff --git a/packages/sdk/tests/journal-client-read-timeout.test.ts b/packages/sdk/tests/journal-client-read-timeout.test.ts index 2efeac22b..95f81ea40 100644 --- a/packages/sdk/tests/journal-client-read-timeout.test.ts +++ b/packages/sdk/tests/journal-client-read-timeout.test.ts @@ -3,6 +3,7 @@ import { setTimeout as sleep } from 'node:timers/promises'; import type { Server } from 'node:net'; import { afterEach, expect, it } from 'vitest'; import { JournalClient, JournalRequestTimeoutError } from '../src/journal-client.js'; +import { JournalReadInterruptedError, isReadInterruptionError } from '../src/journal-read-policy.js'; import { HELLO_SPEC, sendOk, sendResult, sockPath, startLoopback, type LoopbackHandlers } from './journal-client-loopback.js'; const clients: JournalClient[] = []; @@ -139,3 +140,49 @@ it.each(['run.get', 'journal.read', 'stream.read', 'subscription.inspect'] as co await result; expect(attempts).toBe(2); }); + +it('reconnects the reader after it disconnects within the read budget', async () => { + let reads = 0; + const client = await setup({ + 'run.get': ctx => { + reads += 1; + if (reads === 1) return; // time out first + if (reads === 2) { ctx.socket.destroy(); return; } // then lose the reader transport + sendResult(ctx, { status: 'completed' }); + }, + 'stream.append': ctx => sendResult(ctx, { seq: 1 }), + }); + expect(await client.runGet('run')).toMatchObject({ status: 'completed' }); + expect(reads).toBe(3); + // The primary session that carries writes was never involved. + await expect(client.streamAppend('run', 'stream', {})).resolves.toBeDefined(); +}); + +it('surfaces an exhausted reader disconnect as a typed read interruption, not a body failure', async () => { + const client = await setup({ 'run.get': ctx => { ctx.socket.destroy(); } }, 250); + const error = await client.runGet('run').catch(error => error); + expect(error).toBeInstanceOf(JournalReadInterruptedError); + expect(isReadInterruptionError(error)).toBe(true); + expect(error.verb).toBe('run.get'); + expect(error.attempts).toBeGreaterThan(1); + expect(error.message).toContain('read budget 250ms'); +}); + +it('a canceled read settles promptly and frees the serial read queue', async () => { + let reads = 0; + let entered!: () => void; + const started = new Promise(resolve => { entered = resolve; }); + const client = await setup({ 'run.get': ctx => { + reads += 1; + if (reads === 1) { entered(); return; } // never answers + sendResult(ctx, { status: 'completed' }); + } }, 60_000); + const controller = new AbortController(); + const pending = client.runGet('run', { signal: controller.signal }); + await started; + const begun = performance.now(); + controller.abort(new Error('caller canceled')); + await expect(pending).rejects.toThrow('caller canceled'); + expect(performance.now() - begun).toBeLessThan(40); + expect(await client.runGet('run')).toMatchObject({ status: 'completed' }); +}); diff --git a/packages/sdk/tests/run-daemon-unresponsive.test.ts b/packages/sdk/tests/run-daemon-unresponsive.test.ts index 641f1f228..3ba6e0d66 100644 --- a/packages/sdk/tests/run-daemon-unresponsive.test.ts +++ b/packages/sdk/tests/run-daemon-unresponsive.test.ts @@ -2,6 +2,7 @@ import { afterEach, expect, it, vi } from 'vitest'; import { classifyOutcome } from '../src/cli/run.js'; import { JournalRequestTimeoutError, type JournalClient } from '../src/journal-client.js'; import { probeSocket } from '../src/daemon-connection.js'; +import { JournalReadInterruptedError } from '../src/journal-read-policy.js'; vi.mock('../src/daemon-connection.js', async original => ({ ...await original(), probeSocket: vi.fn(), })); @@ -19,3 +20,17 @@ it.each([true, false])('reports an unreadable declarative run as resumable (reac } }); expect(probeSocket).toHaveBeenCalledWith('/socket'); }); + +it('reports an interrupted read session as resumable rather than a failed run', async () => { + vi.mocked(probeSocket).mockResolvedValue({ reachable: true }); + const client = { runGet: async () => { + throw new JournalReadInterruptedError('run.get', 3, 100, 100, { cause: new Error('journal client: connection closed') }); + } }; + const result = await classifyOutcome(client as unknown as JournalClient, 'run', { + run_id: 'saved-run', status: 'parked', completion_reason: null, completed_steps: 30, + }, { command: 'run', ok: true, diagnostics: [] } as never, '/socket', {}); + expect(result).toMatchObject({ exitCode: 1, report: { + runId: 'saved-run', status: 'running', + diagnostics: [{ kind: 'daemon_unresponsive', message: expect.stringContaining('flows resume saved-run') }], + } }); +}); diff --git a/packages/sdk/tests/running-step-watch.test.ts b/packages/sdk/tests/running-step-watch.test.ts index b55bf635f..fda2b458a 100644 --- a/packages/sdk/tests/running-step-watch.test.ts +++ b/packages/sdk/tests/running-step-watch.test.ts @@ -57,3 +57,28 @@ it('cancels a pending watch registration and removes its connection', async () = await rejected; } finally { client.close(); await new Promise(resolve => server.close(() => resolve())); } }); + +it('cancels promptly while a lease snapshot read is in flight', async () => { + const path = sockPath(); + let reading!: () => void; + const readStarted = new Promise(resolve => { reading = resolve; }); + const server = startLoopback(path, { + hello: sendOk, + 'run.watch': ctx => sendResult(ctx, {}), + // The daemon never answers the snapshot: only cancellation can end the wait. + 'run.get': () => reading(), + }); + await once(server, 'listening'); + const client = new JournalClient(path, { readBudgetMs: 300_000 }); + const controller = new AbortController(); + try { + await client.connect(); + const waiting = waitForRunningStep(client, 'run', { id: 'step', type: 'agent', leaseDeadlineMs: Date.now() + 30_000 }, { signal: controller.signal }); + const outcome = waiting.then(() => 'returned', error => error); + await readStarted; + controller.abort(); + const settled = await Promise.race([outcome, sleep(200).then(() => 'still waiting')]); + expect(settled).toBeInstanceOf(Error); + expect((settled as Error).message).toContain('was canceled'); + } finally { client.close(); await new Promise(resolve => server.close(() => resolve())); } +}); From f194b8639b89b7de46269eede66081b97650a137 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Wed, 7 Oct 2026 23:37:52 -0700 Subject: [PATCH 03/34] fix(sdk): retry a reader session that failed to set up; regenerate mutation evidence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A connect or hello that failed under load was cached as "no reader" for the client's life, sending every later read to the primary session — behind its unbounded commands. Transient setup failures now drop the cached session and surface as a retryable read interruption, so the next attempt opens a fresh reader; a protocol refusal still falls back to the primary. runWatch now documents the daemon contract the completion wait relies on: the replay is written before the response, so it is folded before `ready`. The mutation harness is updated for the current source (reader, reader-reconnect, reader-setup-retry, watch-cadence, snapshot-cancel, heartbeat, root-parking) and its logs and verification.md regenerated; each mutation fails and each restoration passes. A cancellation-during-handshake test guards against unhandled rejections. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: b8262468-ea26-4a6e-bee9-d193832e77dc --- .../run-read-timeout/mutation-heartbeat.log | 14 +- evidence/run-read-timeout/mutation-reader.log | 42 +- .../mutation-root-parking.log | 104 +++-- .../mutation-watch-cadence.log | 28 +- evidence/run-read-timeout/mutations.py | 25 +- evidence/run-read-timeout/verification.md | 380 +++++++++++++++--- packages/sdk/src/journal-client.ts | 24 +- .../tests/journal-client-read-timeout.test.ts | 17 + packages/sdk/tests/running-step-watch.test.ts | 26 ++ 9 files changed, 517 insertions(+), 143 deletions(-) diff --git a/evidence/run-read-timeout/mutation-heartbeat.log b/evidence/run-read-timeout/mutation-heartbeat.log index e0770eaaf..67af0f235 100644 --- a/evidence/run-read-timeout/mutation-heartbeat.log +++ b/evidence/run-read-timeout/mutation-heartbeat.log @@ -11,7 +11,7 @@ With: REVERTED $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartbeat timeout' --maxWorkers=1 --minWorkers=1 - RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 20ms × a heartbeat timeout is lease loss rather than a worker body failure 20ms @@ -33,21 +33,21 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 10:38:23 - Duration 509ms (transform 171ms, setup 14ms, collect 315ms, tests 20ms, environment 0ms, prepare 45ms) + Start at 23:37:33 + Duration 464ms (transform 162ms, setup 16ms, collect 283ms, tests 20ms, environment 0ms, prepare 41ms) exit=1 RESTORED SHA256: 44c13f9f61ac13cb8c0f872f4c22b1c212e020d05880ed080ea7205ba6007213 $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartbeat timeout' --maxWorkers=1 --minWorkers=1 - RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 10:38:24 - Duration 491ms (transform 169ms, setup 15ms, collect 300ms, tests 15ms, environment 0ms, prepare 43ms) + Start at 23:37:33 + Duration 440ms (transform 163ms, setup 15ms, collect 272ms, tests 16ms, environment 0ms, prepare 37ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader.log b/evidence/run-read-timeout/mutation-reader.log index 546fe785c..fecd14783 100644 --- a/evidence/run-read-timeout/mutation-reader.log +++ b/evidence/run-read-timeout/mutation-reader.log @@ -1,17 +1,17 @@ Source: packages/sdk/src/journal-client.ts -Original SHA256: 9a390f44f9a1375f0e13be91786f24e848bba7fa07b93b28b119c7ca1d24027c +Original SHA256: e840ea971905def84fdecf0c3826430f058e0a88cf5650627fe0ca4fc4faa299 Replaced: -(reader ?? this).requestOnce(verb, params, remaining) +return await reader.requestOnce(verb, params, remaining, attemptSignal); With: -this.requestOnce(verb, params, remaining) +return await this.requestOnce(verb, params, remaining, attemptSignal); REVERTED $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'serves a bounded read' --maxWorkers=1 --minWorkers=1 - RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (13 tests | 1 failed | 12 skipped) 111ms - × serves a bounded read while an unbounded command is in flight 110ms + ❯ tests/journal-client-read-timeout.test.ts (17 tests | 1 failed | 16 skipped) 113ms + × serves a bounded read while an unbounded command is in flight 112ms → expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -25,33 +25,33 @@ AssertionError: expected true to be false // Object.is equality - false + true - ❯ tests/journal-client-read-timeout.test.ts:36:23 - 34| await inFlight; - 35| expect(await client.runGet('run')).toMatchObject({ status: 'running'… - 36| expect(commandDone).toBe(false); + ❯ tests/journal-client-read-timeout.test.ts:37:23 + 35| await inFlight; + 36| expect(await client.runGet('run')).toMatchObject({ status: 'running'… + 37| expect(commandDone).toBe(false); | ^ - 37| await command; - 38| }); + 38| await command; + 39| }); ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 12 skipped (13) - Start at 10:38:14 - Duration 588ms (transform 169ms, setup 15ms, collect 306ms, tests 111ms, environment 0ms, prepare 44ms) + Tests 1 failed | 16 skipped (17) + Start at 23:37:15 + Duration 572ms (transform 170ms, setup 17ms, collect 295ms, tests 113ms, environment 0ms, prepare 42ms) exit=1 -RESTORED SHA256: 9a390f44f9a1375f0e13be91786f24e848bba7fa07b93b28b119c7ca1d24027c +RESTORED SHA256: e840ea971905def84fdecf0c3826430f058e0a88cf5650627fe0ca4fc4faa299 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'serves a bounded read' --maxWorkers=1 --minWorkers=1 - RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (13 tests | 12 skipped) 108ms + ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 107ms Test Files 1 passed (1) - Tests 1 passed | 12 skipped (13) - Start at 10:38:15 - Duration 586ms (transform 171ms, setup 15ms, collect 307ms, tests 108ms, environment 0ms, prepare 42ms) + Tests 1 passed | 16 skipped (17) + Start at 23:37:16 + Duration 556ms (transform 169ms, setup 17ms, collect 293ms, tests 107ms, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-root-parking.log b/evidence/run-read-timeout/mutation-root-parking.log index a8d7556f2..7b3d68745 100644 --- a/evidence/run-read-timeout/mutation-root-parking.log +++ b/evidence/run-read-timeout/mutation-root-parking.log @@ -1,7 +1,7 @@ Source: packages/sdk/src/authored-root.ts -Original SHA256: ef090c86e849b5b4d77700672b60f9a42979e6649e6207a800a1e7561ca6be1e +Original SHA256: 999d3eca1236966ca84d0c4a07694a7449bf0a35dd1e7fac455ab0f510804f1a Replaced: - if ((error instanceof JournalRequestTimeoutError && (READ_ONLY_VERBS.has(error.verb) || error.verb === 'run.watch')) + if (isReadInterruptionError(error) || (error instanceof AuthoredFlowExecutionError && error.code === 'daemon_unresponsive')) { const parked = new AuthoredFlowExecutionError('daemon_unresponsive', `${error.message}. The run remains resumable. Continue with: ${resumeCommand(dispatch.run_id, options.dataDir, options.localAgentStream !== undefined)}.`); @@ -13,18 +13,24 @@ With: REVERTED -$ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the root resumable after a read timeout' --maxWorkers=1 --minWorkers=1 +$ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the root resumable' --maxWorkers=1 --minWorkers=1 - RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (25 tests | 1 failed | 24 skipped) 22ms - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch 21ms + ❯ tests/authored-root.test.ts (27 tests | 3 failed | 24 skipped) 31ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 22ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms + → expected AuthoredFlowExecutionError: daemon_unresp… { …(7) } to match object { code: 'daemon_unresponsive', …(2) } +(6 matching properties omitted from actual) + × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 4ms + → expected JournalReadInterruptedError: journal clie… { …(4) } to match object { code: 'daemon_unresponsive', …(2) } +(5 matching properties omitted from actual) -⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ +⎯⎯⎯⎯⎯⎯⎯ Failed Tests 3 ⎯⎯⎯⎯⎯⎯⎯ - FAIL tests/authored-root.test.ts > durable authored root > leaves the root resumable after a read timeout without waiting for redispatch + FAIL tests/authored-root.test.ts > durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) @@ -38,33 +44,81 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } - } + [JournalRequestTimeoutError: journal client: run.get timed out after 10ms] - ❯ tests/authored-root.test.ts:411:5 - 409| const loaded = await fixture(false, 0, async () => { throw new Jou… - 410| const journal = new RootJournal(); - 411| await expect(executeDurableAuthoredFlow(loaded, journal as unknown… + ❯ tests/authored-root.test.ts:415:5 + 413| }); + 414| const journal = new RootJournal(); + 415| await expect(executeDurableAuthoredFlow(loaded, journal as unknown… + | ^ + 416| { dataDir: '/unused', admissionKey: 'read-timeout' })).rejects.t… + 417| code: 'daemon_unresponsive', rootRunId: 'root-run', message: e… + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/3]⎯ + + FAIL tests/authored-root.test.ts > durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) +AssertionError: expected AuthoredFlowExecutionError: daemon_unresp… { …(7) } to match object { code: 'daemon_unresponsive', …(2) } +(6 matching properties omitted from actual) + +- Expected ++ Received + +- Object { ++ AuthoredFlowExecutionError { + "code": "daemon_unresponsive", +- "message": StringContaining "flows resume", + "rootRunId": "root-run", + } + + ❯ tests/authored-root.test.ts:415:5 + 413| }); + 414| const journal = new RootJournal(); + 415| await expect(executeDurableAuthoredFlow(loaded, journal as unknown… + | ^ + 416| { dataDir: '/unused', admissionKey: 'read-timeout' })).rejects.t… + 417| code: 'daemon_unresponsive', rootRunId: 'root-run', message: e… + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[2/3]⎯ + + FAIL tests/authored-root.test.ts > durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy +AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } to match object { code: 'daemon_unresponsive', …(2) } +(5 matching properties omitted from actual) + +- Expected ++ Received + +- Object { +- "code": "daemon_unresponsive", +- "message": StringContaining "flows resume", +- "rootRunId": "root-run", +- } ++ [JournalReadInterruptedError: journal client: run.get read session was interrupted after 2 attempts in 40ms (read budget 300000ms): journal client: connection closed] + + ❯ tests/authored-root.test.ts:428:5 + 426| const loaded = await fixture(false, 0, async () => { throw interru… + 427| const journal = new RootJournal(); + 428| await expect(executeDurableAuthoredFlow(loaded, journal as unknown… | ^ - 412| { dataDir: '/unused', admissionKey: 'read-timeout' })).rejects.t… - 413| code: 'daemon_unresponsive', rootRunId: 'root-run', message: e… + 429| { dataDir: '/unused', admissionKey: 'read-disconnect' })).reject… + 430| code: 'daemon_unresponsive', rootRunId: 'root-run', message: e… -⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[3/3]⎯ Test Files 1 failed (1) - Tests 1 failed | 24 skipped (25) - Start at 10:38:25 - Duration 1.36s (transform 704ms, setup 18ms, collect 1.16s, tests 22ms, environment 0ms, prepare 46ms) + Tests 3 failed | 24 skipped (27) + Start at 23:37:34 + Duration 1.28s (transform 656ms, setup 16ms, collect 1.08s, tests 31ms, environment 0ms, prepare 43ms) exit=1 -RESTORED SHA256: ef090c86e849b5b4d77700672b60f9a42979e6649e6207a800a1e7561ca6be1e -$ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the root resumable after a read timeout' --maxWorkers=1 --minWorkers=1 +RESTORED SHA256: 999d3eca1236966ca84d0c4a07694a7449bf0a35dd1e7fac455ab0f510804f1a +$ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the root resumable' --maxWorkers=1 --minWorkers=1 - RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (25 tests | 24 skipped) 18ms + ✓ tests/authored-root.test.ts (27 tests | 24 skipped) 25ms Test Files 1 passed (1) - Tests 1 passed | 24 skipped (25) - Start at 10:38:27 - Duration 1.32s (transform 674ms, setup 15ms, collect 1.13s, tests 18ms, environment 0ms, prepare 45ms) + Tests 3 passed | 24 skipped (27) + Start at 23:37:36 + Duration 1.24s (transform 647ms, setup 16ms, collect 1.06s, tests 25ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-watch-cadence.log b/evidence/run-read-timeout/mutation-watch-cadence.log index c10f7873f..3975b1b05 100644 --- a/evidence/run-read-timeout/mutation-watch-cadence.log +++ b/evidence/run-read-timeout/mutation-watch-cadence.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/cli/running-step.ts -Original SHA256: 0f8b6b9e160ebb047b5a83ad48ad41e2c201d5ba905c27c3a872acd3b9b288c2 +Original SHA256: a57da3dce86c9724d83bd3547a2d9186e05e7ea10409b743f89aaa25bf461bcd Replaced: const LEASE_POLL_MS = 2_000; With: @@ -8,10 +8,10 @@ const LEASE_POLL_MS = 50; REVERTED $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pushes for completion' --maxWorkers=1 --minWorkers=1 - RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (2 tests | 1 failed | 1 skipped) 2110ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2109ms + ❯ tests/running-step-watch.test.ts (4 tests | 1 failed | 3 skipped) 2110ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2110ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -29,23 +29,23 @@ AssertionError: expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 1 skipped (2) - Start at 10:38:16 - Duration 2.60s (transform 179ms, setup 16ms, collect 313ms, tests 2.11s, environment 0ms, prepare 43ms) + Tests 1 failed | 3 skipped (4) + Start at 23:37:21 + Duration 2.57s (transform 168ms, setup 15ms, collect 294ms, tests 2.11s, environment 0ms, prepare 40ms) exit=1 -RESTORED SHA256: 0f8b6b9e160ebb047b5a83ad48ad41e2c201d5ba905c27c3a872acd3b9b288c2 +RESTORED SHA256: a57da3dce86c9724d83bd3547a2d9186e05e7ea10409b743f89aaa25bf461bcd $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pushes for completion' --maxWorkers=1 --minWorkers=1 - RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (2 tests | 1 skipped) 2120ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2119ms + ✓ tests/running-step-watch.test.ts (4 tests | 3 skipped) 2117ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2116ms Test Files 1 passed (1) - Tests 1 passed | 1 skipped (2) - Start at 10:38:20 - Duration 2.61s (transform 186ms, setup 15ms, collect 322ms, tests 2.12s, environment 0ms, prepare 42ms) + Tests 1 passed | 3 skipped (4) + Start at 23:37:24 + Duration 2.57s (transform 165ms, setup 17ms, collect 291ms, tests 2.12s, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutations.py b/evidence/run-read-timeout/mutations.py index 5bcea4a9f..8efcbfdf1 100644 --- a/evidence/run-read-timeout/mutations.py +++ b/evidence/run-read-timeout/mutations.py @@ -1,4 +1,4 @@ -"""Reproduce the four reviewed-plan mutations; always restore source bytes.""" +"""Reproduce the reviewed mutations against the current source; always restore source bytes.""" from pathlib import Path import hashlib import shlex @@ -8,26 +8,41 @@ SDK = ROOT / 'packages/sdk' CASES = [ ('reader', 'packages/sdk/src/journal-client.ts', - '(reader ?? this).requestOnce(verb, params, remaining)', - 'this.requestOnce(verb, params, remaining)', + 'return await reader.requestOnce(verb, params, remaining, attemptSignal);', + 'return await this.requestOnce(verb, params, remaining, attemptSignal);', 'tests/journal-client-read-timeout.test.ts', 'serves a bounded read'), + ('reader-reconnect', 'packages/sdk/src/journal-client.ts', + """ this.dropReader(reader); +""", '', 'tests/journal-client-read-timeout.test.ts', 'reconnects the reader after it disconnects'), + ('reader-setup-retry', 'packages/sdk/src/journal-client.ts', + """ if (this.reader === reader) { + this.reader = undefined; + this.readerReady = undefined; + } + throw error; +""", """ return undefined; +""", 'tests/journal-client-read-timeout.test.ts', 'retries a reader setup that timed out'), ('watch-cadence', 'packages/sdk/src/cli/running-step.ts', 'const LEASE_POLL_MS = 2_000;', 'const LEASE_POLL_MS = 50;', 'tests/running-step-watch.test.ts', 'uses pushes for completion'), + ('snapshot-cancel', 'packages/sdk/src/cli/running-step.ts', + 'client.runGet(runId, options.signal === undefined ? {} : { signal: options.signal })', + 'client.runGet(runId)', + 'tests/running-step-watch.test.ts', 'cancels promptly while a lease snapshot read is in flight'), ('heartbeat', 'packages/sdk/src/worker-lease.ts', """ if (error instanceof JournalRequestTimeoutError && error.verb === 'step.heartbeat') { throw new WorkerLeaseLostError('renewal_expired', error.message, { cause: error }); } """, '', 'tests/heartbeat-timeout.test.ts', 'a heartbeat timeout'), ('root-parking', 'packages/sdk/src/authored-root.ts', - """ if ((error instanceof JournalRequestTimeoutError && (READ_ONLY_VERBS.has(error.verb) || error.verb === 'run.watch')) + """ if (isReadInterruptionError(error) || (error instanceof AuthoredFlowExecutionError && error.code === 'daemon_unresponsive')) { const parked = new AuthoredFlowExecutionError('daemon_unresponsive', `${error.message}. The run remains resumable. Continue with: ${resumeCommand(dispatch.run_id, options.dataDir, options.localAgentStream !== undefined)}.`); parked.rootRunId = dispatch.run_id; throw parked; } -""", '', 'tests/authored-root.test.ts', 'leaves the root resumable after a read timeout'), +""", '', 'tests/authored-root.test.ts', 'leaves the root resumable'), ] for name, source, old, new, test, pattern in CASES: diff --git a/evidence/run-read-timeout/verification.md b/evidence/run-read-timeout/verification.md index 6d6f5b028..b40a84f3d 100644 --- a/evidence/run-read-timeout/verification.md +++ b/evidence/run-read-timeout/verification.md @@ -84,23 +84,25 @@ exit=0 exit=0 ``` +The mutation sections below were regenerated by `python3 evidence/run-read-timeout/mutations.py` (run from `packages/sdk`) against the review-fix source; each mutation fails its test and the restored source passes. + ## mutation-reader.log ```text Source: packages/sdk/src/journal-client.ts -Original SHA256: 9a390f44f9a1375f0e13be91786f24e848bba7fa07b93b28b119c7ca1d24027c +Original SHA256: e840ea971905def84fdecf0c3826430f058e0a88cf5650627fe0ca4fc4faa299 Replaced: -(reader ?? this).requestOnce(verb, params, remaining) +return await reader.requestOnce(verb, params, remaining, attemptSignal); With: -this.requestOnce(verb, params, remaining) +return await this.requestOnce(verb, params, remaining, attemptSignal); REVERTED $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'serves a bounded read' --maxWorkers=1 --minWorkers=1 - RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (13 tests | 1 failed | 12 skipped) 111ms - × serves a bounded read while an unbounded command is in flight 110ms + ❯ tests/journal-client-read-timeout.test.ts (17 tests | 1 failed | 16 skipped) 113ms + × serves a bounded read while an unbounded command is in flight 112ms → expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -114,34 +116,168 @@ AssertionError: expected true to be false // Object.is equality - false + true - ❯ tests/journal-client-read-timeout.test.ts:36:23 - 34| await inFlight; - 35| expect(await client.runGet('run')).toMatchObject({ status: 'running'… - 36| expect(commandDone).toBe(false); + ❯ tests/journal-client-read-timeout.test.ts:37:23 + 35| await inFlight; + 36| expect(await client.runGet('run')).toMatchObject({ status: 'running'… + 37| expect(commandDone).toBe(false); | ^ - 37| await command; - 38| }); + 38| await command; + 39| }); ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 12 skipped (13) - Start at 10:38:14 - Duration 588ms (transform 169ms, setup 15ms, collect 306ms, tests 111ms, environment 0ms, prepare 44ms) + Tests 1 failed | 16 skipped (17) + Start at 23:37:15 + Duration 572ms (transform 170ms, setup 17ms, collect 295ms, tests 113ms, environment 0ms, prepare 42ms) exit=1 -RESTORED SHA256: 9a390f44f9a1375f0e13be91786f24e848bba7fa07b93b28b119c7ca1d24027c +RESTORED SHA256: e840ea971905def84fdecf0c3826430f058e0a88cf5650627fe0ca4fc4faa299 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'serves a bounded read' --maxWorkers=1 --minWorkers=1 - RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk + + ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 107ms + + Test Files 1 passed (1) + Tests 1 passed | 16 skipped (17) + Start at 23:37:16 + Duration 556ms (transform 169ms, setup 17ms, collect 293ms, tests 107ms, environment 0ms, prepare 40ms) + +exit=0 +``` + +## mutation-reader-reconnect.log + +```text +Source: packages/sdk/src/journal-client.ts +Original SHA256: e840ea971905def84fdecf0c3826430f058e0a88cf5650627fe0ca4fc4faa299 +Replaced: + this.dropReader(reader); + +With: + + +REVERTED +$ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'reconnects the reader after it disconnects' --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk + + ❯ tests/journal-client-read-timeout.test.ts (17 tests | 1 failed | 16 skipped) 1007ms + × reconnects the reader after it disconnects within the read budget 1006ms + → journal client: run.get read session was interrupted after 79 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + +⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget +JournalReadInterruptedError: journal client: run.get read session was interrupted after 79 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ exhausted src/journal-read-policy.ts:70:9 + 68| let last: unknown; + 69| const exhausted = () => last instanceof JournalReadInterruptedError + 70| ? new JournalReadInterruptedError(verb, attempts, performance.no… + | ^ + 71| : new JournalRequestTimeoutError(verb, timeoutMs, attempts, perf… + 72| // A caller's cancellation settles this read and drains its queued… + ❯ Timeout. src/journal-read-policy.ts:81:39 + +Caused by: Error: journal client: not connected (run.get): journal client: connection closed + ❯ src/journal-client.ts:270:16 + ❯ JournalClient.requestOnce src/journal-client.ts:263:12 + ❯ src/journal-client.ts:245:31 + ❯ src/journal-read-policy.ts:93:18 + +Caused by: Error: journal client: connection closed + ❯ Socket. src/journal-client.ts:125:26 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ + + Test Files 1 failed (1) + Tests 1 failed | 16 skipped (17) + Start at 23:37:16 + Duration 1.45s (transform 170ms, setup 18ms, collect 289ms, tests 1.01s, environment 0ms, prepare 42ms) + +exit=1 + +RESTORED SHA256: e840ea971905def84fdecf0c3826430f058e0a88cf5650627fe0ca4fc4faa299 +$ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'reconnects the reader after it disconnects' --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (13 tests | 12 skipped) 108ms + ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 76ms Test Files 1 passed (1) - Tests 1 passed | 12 skipped (13) - Start at 10:38:15 - Duration 586ms (transform 171ms, setup 15ms, collect 307ms, tests 108ms, environment 0ms, prepare 42ms) + Tests 1 passed | 16 skipped (17) + Start at 23:37:18 + Duration 532ms (transform 166ms, setup 16ms, collect 295ms, tests 76ms, environment 0ms, prepare 41ms) + +exit=0 +``` + +## mutation-reader-setup-retry.log + +```text +Source: packages/sdk/src/journal-client.ts +Original SHA256: e840ea971905def84fdecf0c3826430f058e0a88cf5650627fe0ca4fc4faa299 +Replaced: + if (this.reader === reader) { + this.reader = undefined; + this.readerReady = undefined; + } + throw error; + +With: + return undefined; + + +REVERTED +$ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'retries a reader setup that timed out' --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk + + ❯ tests/journal-client-read-timeout.test.ts (17 tests | 1 failed | 16 skipped) 72ms + × retries a reader setup that timed out instead of reading on the primary for good 71ms + → expected 1 to be 2 // Object.is equality + +⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/journal-client-read-timeout.test.ts > retries a reader setup that timed out instead of reading on the primary for good +AssertionError: expected 1 to be 2 // Object.is equality + +- Expected ++ Received + +- 2 ++ 1 + + ❯ tests/journal-client-read-timeout.test.ts:202:18 + 200| expect(await client.runGet('run')).toMatchObject({ status: 'complete… + 201| expect(await client.runGet('run')).toMatchObject({ status: 'complete… + 202| expect(hellos).toBe(2); + | ^ + 203| expect(served).toHaveLength(2); + 204| expect(served.every(socket => socket !== primary)).toBe(true); + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ + + Test Files 1 failed (1) + Tests 1 failed | 16 skipped (17) + Start at 23:37:19 + Duration 495ms (transform 161ms, setup 17ms, collect 270ms, tests 72ms, environment 0ms, prepare 41ms) + +exit=1 + +RESTORED SHA256: e840ea971905def84fdecf0c3826430f058e0a88cf5650627fe0ca4fc4faa299 +$ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'retries a reader setup that timed out' --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk + + ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 75ms + + Test Files 1 passed (1) + Tests 1 passed | 16 skipped (17) + Start at 23:37:20 + Duration 513ms (transform 164ms, setup 17ms, collect 287ms, tests 75ms, environment 0ms, prepare 40ms) exit=0 ``` @@ -150,7 +286,7 @@ exit=0 ```text Source: packages/sdk/src/cli/running-step.ts -Original SHA256: 0f8b6b9e160ebb047b5a83ad48ad41e2c201d5ba905c27c3a872acd3b9b288c2 +Original SHA256: a57da3dce86c9724d83bd3547a2d9186e05e7ea10409b743f89aaa25bf461bcd Replaced: const LEASE_POLL_MS = 2_000; With: @@ -159,10 +295,10 @@ const LEASE_POLL_MS = 50; REVERTED $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pushes for completion' --maxWorkers=1 --minWorkers=1 - RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (2 tests | 1 failed | 1 skipped) 2110ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2109ms + ❯ tests/running-step-watch.test.ts (4 tests | 1 failed | 3 skipped) 2110ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2110ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -180,24 +316,80 @@ AssertionError: expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 1 skipped (2) - Start at 10:38:16 - Duration 2.60s (transform 179ms, setup 16ms, collect 313ms, tests 2.11s, environment 0ms, prepare 43ms) + Tests 1 failed | 3 skipped (4) + Start at 23:37:21 + Duration 2.57s (transform 168ms, setup 15ms, collect 294ms, tests 2.11s, environment 0ms, prepare 40ms) exit=1 -RESTORED SHA256: 0f8b6b9e160ebb047b5a83ad48ad41e2c201d5ba905c27c3a872acd3b9b288c2 +RESTORED SHA256: a57da3dce86c9724d83bd3547a2d9186e05e7ea10409b743f89aaa25bf461bcd $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pushes for completion' --maxWorkers=1 --minWorkers=1 - RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk + + ✓ tests/running-step-watch.test.ts (4 tests | 3 skipped) 2117ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2116ms + + Test Files 1 passed (1) + Tests 1 passed | 3 skipped (4) + Start at 23:37:24 + Duration 2.57s (transform 165ms, setup 17ms, collect 291ms, tests 2.12s, environment 0ms, prepare 41ms) + +exit=0 +``` + +## mutation-snapshot-cancel.log + +```text +Source: packages/sdk/src/cli/running-step.ts +Original SHA256: a57da3dce86c9724d83bd3547a2d9186e05e7ea10409b743f89aaa25bf461bcd +Replaced: +client.runGet(runId, options.signal === undefined ? {} : { signal: options.signal }) +With: +client.runGet(runId) - ✓ tests/running-step-watch.test.ts (2 tests | 1 skipped) 2120ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2119ms +REVERTED +$ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels promptly while a lease snapshot read is in flight' --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk + + ❯ tests/running-step-watch.test.ts (4 tests | 1 failed | 3 skipped) 2215ms + × cancels promptly while a lease snapshot read is in flight 2214ms + → expected 'still waiting' to be an instance of Error + +⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/running-step-watch.test.ts > cancels promptly while a lease snapshot read is in flight +AssertionError: expected 'still waiting' to be an instance of Error + ❯ tests/running-step-watch.test.ts:81:21 + 79| controller.abort(); + 80| const settled = await Promise.race([outcome, sleep(200).then(() =>… + 81| expect(settled).toBeInstanceOf(Error); + | ^ + 82| expect((settled as Error).message).toContain('was canceled'); + 83| } finally { client.close(); await new Promise(resolve => serve… + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ + + Test Files 1 failed (1) + Tests 1 failed | 3 skipped (4) + Start at 23:37:27 + Duration 2.65s (transform 164ms, setup 16ms, collect 281ms, tests 2.21s, environment 0ms, prepare 40ms) + +exit=1 + +RESTORED SHA256: a57da3dce86c9724d83bd3547a2d9186e05e7ea10409b743f89aaa25bf461bcd +$ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels promptly while a lease snapshot read is in flight' --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk + + ✓ tests/running-step-watch.test.ts (4 tests | 3 skipped) 2010ms + ✓ cancels promptly while a lease snapshot read is in flight 2009ms Test Files 1 passed (1) - Tests 1 passed | 1 skipped (2) - Start at 10:38:20 - Duration 2.61s (transform 186ms, setup 15ms, collect 322ms, tests 2.12s, environment 0ms, prepare 42ms) + Tests 1 passed | 3 skipped (4) + Start at 23:37:30 + Duration 2.41s (transform 158ms, setup 15ms, collect 260ms, tests 2.01s, environment 0ms, prepare 34ms) exit=0 ``` @@ -218,7 +410,7 @@ With: REVERTED $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartbeat timeout' --maxWorkers=1 --minWorkers=1 - RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 20ms × a heartbeat timeout is lease loss rather than a worker body failure 20ms @@ -240,22 +432,22 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 10:38:23 - Duration 509ms (transform 171ms, setup 14ms, collect 315ms, tests 20ms, environment 0ms, prepare 45ms) + Start at 23:37:33 + Duration 464ms (transform 162ms, setup 16ms, collect 283ms, tests 20ms, environment 0ms, prepare 41ms) exit=1 RESTORED SHA256: 44c13f9f61ac13cb8c0f872f4c22b1c212e020d05880ed080ea7205ba6007213 $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartbeat timeout' --maxWorkers=1 --minWorkers=1 - RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 10:38:24 - Duration 491ms (transform 169ms, setup 15ms, collect 300ms, tests 15ms, environment 0ms, prepare 43ms) + Start at 23:37:33 + Duration 440ms (transform 163ms, setup 15ms, collect 272ms, tests 16ms, environment 0ms, prepare 37ms) exit=0 ``` @@ -264,9 +456,9 @@ exit=0 ```text Source: packages/sdk/src/authored-root.ts -Original SHA256: ef090c86e849b5b4d77700672b60f9a42979e6649e6207a800a1e7561ca6be1e +Original SHA256: 999d3eca1236966ca84d0c4a07694a7449bf0a35dd1e7fac455ab0f510804f1a Replaced: - if ((error instanceof JournalRequestTimeoutError && (READ_ONLY_VERBS.has(error.verb) || error.verb === 'run.watch')) + if (isReadInterruptionError(error) || (error instanceof AuthoredFlowExecutionError && error.code === 'daemon_unresponsive')) { const parked = new AuthoredFlowExecutionError('daemon_unresponsive', `${error.message}. The run remains resumable. Continue with: ${resumeCommand(dispatch.run_id, options.dataDir, options.localAgentStream !== undefined)}.`); @@ -278,18 +470,24 @@ With: REVERTED -$ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the root resumable after a read timeout' --maxWorkers=1 --minWorkers=1 +$ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the root resumable' --maxWorkers=1 --minWorkers=1 - RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (25 tests | 1 failed | 24 skipped) 22ms - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch 21ms + ❯ tests/authored-root.test.ts (27 tests | 3 failed | 24 skipped) 31ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 22ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms + → expected AuthoredFlowExecutionError: daemon_unresp… { …(7) } to match object { code: 'daemon_unresponsive', …(2) } +(6 matching properties omitted from actual) + × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 4ms + → expected JournalReadInterruptedError: journal clie… { …(4) } to match object { code: 'daemon_unresponsive', …(2) } +(5 matching properties omitted from actual) -⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ +⎯⎯⎯⎯⎯⎯⎯ Failed Tests 3 ⎯⎯⎯⎯⎯⎯⎯ - FAIL tests/authored-root.test.ts > durable authored root > leaves the root resumable after a read timeout without waiting for redispatch + FAIL tests/authored-root.test.ts > durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) @@ -303,34 +501,82 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } - } + [JournalRequestTimeoutError: journal client: run.get timed out after 10ms] - ❯ tests/authored-root.test.ts:411:5 - 409| const loaded = await fixture(false, 0, async () => { throw new Jou… - 410| const journal = new RootJournal(); - 411| await expect(executeDurableAuthoredFlow(loaded, journal as unknown… + ❯ tests/authored-root.test.ts:415:5 + 413| }); + 414| const journal = new RootJournal(); + 415| await expect(executeDurableAuthoredFlow(loaded, journal as unknown… | ^ - 412| { dataDir: '/unused', admissionKey: 'read-timeout' })).rejects.t… - 413| code: 'daemon_unresponsive', rootRunId: 'root-run', message: e… + 416| { dataDir: '/unused', admissionKey: 'read-timeout' })).rejects.t… + 417| code: 'daemon_unresponsive', rootRunId: 'root-run', message: e… -⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/3]⎯ + + FAIL tests/authored-root.test.ts > durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) +AssertionError: expected AuthoredFlowExecutionError: daemon_unresp… { …(7) } to match object { code: 'daemon_unresponsive', …(2) } +(6 matching properties omitted from actual) + +- Expected ++ Received + +- Object { ++ AuthoredFlowExecutionError { + "code": "daemon_unresponsive", +- "message": StringContaining "flows resume", + "rootRunId": "root-run", + } + + ❯ tests/authored-root.test.ts:415:5 + 413| }); + 414| const journal = new RootJournal(); + 415| await expect(executeDurableAuthoredFlow(loaded, journal as unknown… + | ^ + 416| { dataDir: '/unused', admissionKey: 'read-timeout' })).rejects.t… + 417| code: 'daemon_unresponsive', rootRunId: 'root-run', message: e… + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[2/3]⎯ + + FAIL tests/authored-root.test.ts > durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy +AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } to match object { code: 'daemon_unresponsive', …(2) } +(5 matching properties omitted from actual) + +- Expected ++ Received + +- Object { +- "code": "daemon_unresponsive", +- "message": StringContaining "flows resume", +- "rootRunId": "root-run", +- } ++ [JournalReadInterruptedError: journal client: run.get read session was interrupted after 2 attempts in 40ms (read budget 300000ms): journal client: connection closed] + + ❯ tests/authored-root.test.ts:428:5 + 426| const loaded = await fixture(false, 0, async () => { throw interru… + 427| const journal = new RootJournal(); + 428| await expect(executeDurableAuthoredFlow(loaded, journal as unknown… + | ^ + 429| { dataDir: '/unused', admissionKey: 'read-disconnect' })).reject… + 430| code: 'daemon_unresponsive', rootRunId: 'root-run', message: e… + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[3/3]⎯ Test Files 1 failed (1) - Tests 1 failed | 24 skipped (25) - Start at 10:38:25 - Duration 1.36s (transform 704ms, setup 18ms, collect 1.16s, tests 22ms, environment 0ms, prepare 46ms) + Tests 3 failed | 24 skipped (27) + Start at 23:37:34 + Duration 1.28s (transform 656ms, setup 16ms, collect 1.08s, tests 31ms, environment 0ms, prepare 43ms) exit=1 -RESTORED SHA256: ef090c86e849b5b4d77700672b60f9a42979e6649e6207a800a1e7561ca6be1e -$ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the root resumable after a read timeout' --maxWorkers=1 --minWorkers=1 +RESTORED SHA256: 999d3eca1236966ca84d0c4a07694a7449bf0a35dd1e7fac455ab0f510804f1a +$ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the root resumable' --maxWorkers=1 --minWorkers=1 - RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (25 tests | 24 skipped) 18ms + ✓ tests/authored-root.test.ts (27 tests | 24 skipped) 25ms Test Files 1 passed (1) - Tests 1 passed | 24 skipped (25) - Start at 10:38:27 - Duration 1.32s (transform 674ms, setup 15ms, collect 1.13s, tests 18ms, environment 0ms, prepare 45ms) + Tests 3 passed | 24 skipped (27) + Start at 23:37:36 + Duration 1.24s (transform 647ms, setup 16ms, collect 1.06s, tests 25ms, environment 0ms, prepare 41ms) exit=0 ``` diff --git a/packages/sdk/src/journal-client.ts b/packages/sdk/src/journal-client.ts index 19affe969..9113459ae 100644 --- a/packages/sdk/src/journal-client.ts +++ b/packages/sdk/src/journal-client.ts @@ -203,7 +203,18 @@ export class JournalClient extends EventEmitter { const reader = this.createPeer(); this.reader = reader; this.readerReady = reader.connect().then(() => reader.hello('flows-reader')).then(() => reader) - .catch(() => { reader.close(); return undefined; }); + .catch(error => { + reader.close(); + // A daemon that refuses the session will keep refusing it: read on the primary. + if (error instanceof JournalProtocolError) return undefined; + // A connect or hello that failed under load is retried with a fresh session, + // never by queueing reads behind the primary's unbounded commands for good. + if (this.reader === reader) { + this.reader = undefined; + this.readerReady = undefined; + } + throw error; + }); } return this.readerReady; } @@ -221,7 +232,10 @@ export class JournalClient extends EventEmitter { if (this.socket && !this.socket.destroyed && this.readBudgetMs !== undefined && timeoutMs !== null && READ_ONLY_VERBS.has(verb)) { return this.reads.read(verb, timeoutMs, this.readBudgetMs, async (bound, attemptSignal) => { const started = performance.now(); - const reader = await this.readSession(); + const reader = await this.readSession().catch(error => { + attemptSignal.throwIfAborted(); + throw new JournalReadInterruptedError(verb, 1, performance.now() - started, undefined, { cause: error }); + }); attemptSignal.throwIfAborted(); const remaining = bound - (performance.now() - started); if (remaining <= 0) throw new JournalRequestTimeoutError(verb, bound); @@ -339,8 +353,10 @@ export class JournalClient extends EventEmitter { } /** - * Open a push stream of every appended entry. Resolves once subscribed; - * entries arrive as `'entry'` events: `client.on('entry', (entry) => …)`. + * Open a push stream of every appended entry. The daemon writes the run's + * existing entries before this resolves (server.rs `watch_with_replay`), so + * the replay is fully delivered first; later entries arrive as they are + * appended. Entries arrive as `'entry'` events: `client.on('entry', (entry) => …)`. */ runWatch(runId: string): Promise { return this.request('run.watch', { run_id: runId }); diff --git a/packages/sdk/tests/journal-client-read-timeout.test.ts b/packages/sdk/tests/journal-client-read-timeout.test.ts index 95f81ea40..47877d7c0 100644 --- a/packages/sdk/tests/journal-client-read-timeout.test.ts +++ b/packages/sdk/tests/journal-client-read-timeout.test.ts @@ -186,3 +186,20 @@ it('a canceled read settles promptly and frees the serial read queue', async () expect(performance.now() - begun).toBeLessThan(40); expect(await client.runGet('run')).toMatchObject({ status: 'completed' }); }); + +it('retries a reader setup that timed out instead of reading on the primary for good', async () => { + let hellos = 0; + let primary: unknown; + const served: unknown[] = []; + const client = await setup({ + hello: ctx => { if (++hellos > 1) sendOk(ctx); }, // the first reader hello times out under load + 'run.get': ctx => { served.push(ctx.socket); sendResult(ctx, { status: 'completed' }); }, + 'stream.append': ctx => { primary = ctx.socket; sendResult(ctx, { seq: 1 }); }, + }, 5_000); + await client.streamAppend('run', 'stream', {}); + expect(await client.runGet('run')).toMatchObject({ status: 'completed' }); + expect(await client.runGet('run')).toMatchObject({ status: 'completed' }); + expect(hellos).toBe(2); + expect(served).toHaveLength(2); + expect(served.every(socket => socket !== primary)).toBe(true); +}); diff --git a/packages/sdk/tests/running-step-watch.test.ts b/packages/sdk/tests/running-step-watch.test.ts index fda2b458a..bba2347ed 100644 --- a/packages/sdk/tests/running-step-watch.test.ts +++ b/packages/sdk/tests/running-step-watch.test.ts @@ -82,3 +82,29 @@ it('cancels promptly while a lease snapshot read is in flight', async () => { expect((settled as Error).message).toContain('was canceled'); } finally { client.close(); await new Promise(resolve => server.close(() => resolve())); } }); + +it('cancelling during the watch handshake leaves no unhandled rejection', async () => { + const path = sockPath(); + let greeted!: () => void; + const hello = new Promise(resolve => { greeted = resolve; }); + const server = startLoopback(path, { hello: () => greeted() }); // never answers the watch hello + await once(server, 'listening'); + const client = new JournalClient(path); + const controller = new AbortController(); + const unhandled: unknown[] = []; + const onUnhandled = (reason: unknown) => { unhandled.push(reason); }; + process.on('unhandledRejection', onUnhandled); + try { + await client.connect(); + const waiting = waitForRunningStep(client, 'run', { id: 'step', type: 'agent', leaseDeadlineMs: Date.now() + 30_000 }, { signal: controller.signal }); + const rejected = expect(waiting).rejects.toThrow('was canceled'); + await hello; + controller.abort(); + await rejected; + await sleep(50); + expect(unhandled).toEqual([]); + } finally { + process.off('unhandledRejection', onUnhandled); + client.close(); await new Promise(resolve => server.close(() => resolve())); + } +}); From 1a6a9f7e1f637eca26c1434987b5ad5ece106fd4 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Wed, 7 Oct 2026 23:54:08 -0700 Subject: [PATCH 04/34] docs(evidence): re-capture focused and live verification at the review-fix source final-focused.log (20 files, 218/218) and resume-live.log (1/1) are re-run at this tree with a relayflowd built from it, and verification.md is rebuilt from them and the regenerated mutation logs. Its header now points at the existing full-suite-final.log and states that transcript predates the review fixes, instead of citing a summary.md that is not in the tree. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: b8262468-ea26-4a6e-bee9-d193832e77dc --- evidence/run-read-timeout/final-focused.log | 91 ++++++++--------- evidence/run-read-timeout/resume-live.log | 12 +-- evidence/run-read-timeout/verification.md | 105 +++++++++----------- 3 files changed, 97 insertions(+), 111 deletions(-) diff --git a/evidence/run-read-timeout/final-focused.log b/evidence/run-read-timeout/final-focused.log index ce9b87422..1513cd872 100644 --- a/evidence/run-read-timeout/final-focused.log +++ b/evidence/run-read-timeout/final-focused.log @@ -1,59 +1,54 @@ -$ cd packages/sdk && RELAYFLOWD_BIN=/home/daytona/.relayflows-toolchain/target/2962130851/debug/relayflowd npx vitest run tests/journal-client-read-timeout.test.ts tests/journal-client.test.ts tests/journal-client-completion.test.ts tests/journal-client-subscriptions.test.ts tests/running-step-watch.test.ts tests/heartbeat-timeout.test.ts tests/run-daemon-unresponsive.test.ts tests/authored-root.test.ts tests/classify-outcome.test.ts tests/cli.test.ts tests/direct-run-worker-lease.test.ts tests/resume-worker-lease.test.ts tests/worker-lease.test.ts tests/worker-lease-lost.test.ts tests/worker-lease-sweep.test.ts tests/run-read-load-live.test.ts tests/worker-lease-lost-live.test.ts tests/flow-executor-chain.test.ts tests/agent-transcript-live.test.ts tests/human-live.test.ts --maxWorkers=1 --minWorkers=1 +$ cd packages/sdk && RELAYFLOWD_BIN= npx vitest run tests/journal-client-read-timeout.test.ts tests/journal-client.test.ts tests/journal-client-completion.test.ts tests/journal-client-subscriptions.test.ts tests/running-step-watch.test.ts tests/heartbeat-timeout.test.ts tests/run-daemon-unresponsive.test.ts tests/authored-root.test.ts tests/classify-outcome.test.ts tests/cli.test.ts tests/direct-run-worker-lease.test.ts tests/resume-worker-lease.test.ts tests/worker-lease.test.ts tests/worker-lease-lost.test.ts tests/worker-lease-lost-live.test.ts tests/worker-lease-sweep.test.ts tests/run-read-load-live.test.ts tests/flow-executor-chain.test.ts tests/agent-transcript-live.test.ts tests/human-live.test.ts --maxWorkers=1 --minWorkers=1 - RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + RUN v2.1.9 /packages/sdk - ✓ tests/cli.test.ts (71 tests) 6026ms - ✓ flows check CLI > binds a checked relative wrapper to the flow directory for worker execution 566ms - ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 488ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2053ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2045ms -(node:44939) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5227ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2036ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2040ms +(node:2803772) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (26 tests) 398ms - ✓ tests/journal-client.test.ts (17 tests) 89ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 10315ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 1022ms - ✓ flow executor LLM and output-binding chain > runs a dollar-budgeted authored Claude agent with the same default used by preflight 700ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1595ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 3382ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 779ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 1068ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 3394ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 875ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 818ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 851ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 848ms - ✓ tests/classify-outcome.test.ts (11 tests) 7422ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2009ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3005ms - ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms - ✓ tests/human-live.test.ts (3 tests) 7854ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 4785ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1916ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 1152ms - ✓ tests/worker-lease.test.ts (7 tests) 20ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 27ms - ✓ tests/journal-client-read-timeout.test.ts (13 tests) 1104ms - ✓ a recovered read timeout does not become an authored callback failure 368ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 943ms - ✓ reports journal success after completion rejects with lease_conflict 327ms - ✓ reports journal success when a renewal rejects after completion landed 327ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2821ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2095ms - ✓ drains read and watch promises before an authored flow completes 725ms + ✓ tests/authored-root.test.ts (27 tests) 401ms + ✓ tests/journal-client.test.ts (17 tests) 83ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 7193ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 486ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 993ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2824ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 391ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 863ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2510ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 617ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 630ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 624ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 638ms + ✓ tests/classify-outcome.test.ts (11 tests) 7426ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2010ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3007ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2002ms + ✓ tests/journal-client-read-timeout.test.ts (17 tests) 1388ms + ✓ tests/human-live.test.ts (3 tests) 4585ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2697ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1115ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 772ms + ✓ tests/worker-lease.test.ts (7 tests) 19ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 19ms + ✓ tests/running-step-watch.test.ts (4 tests) 4184ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms + ✓ cancels promptly while a lease snapshot read is in flight 2008ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 666ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2457ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2062ms + ✓ drains read and watch promises before an authored flow completes 393ms ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms - ✓ tests/journal-client-completion.test.ts (6 tests) 102ms - ✓ tests/resume-worker-lease.test.ts (3 tests) 6ms + ✓ tests/journal-client-completion.test.ts (6 tests) 103ms + ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms ✓ tests/direct-run-worker-lease.test.ts (3 tests) 10ms - ✓ tests/running-step-watch.test.ts (2 tests) 2122ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms - ✓ tests/run-daemon-unresponsive.test.ts (2 tests) 4ms + ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 4ms ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms Test Files 20 passed (20) - Tests 210 passed (210) - Start at 10:41:25 - Duration 54.29s (transform 1.45s, setup 97ms, collect 8.09s, tests 42.69s, environment 3ms, prepare 1.05s) + Tests 218 passed (218) + Start at 23:53:02 + Duration 44.59s (transform 1.06s, setup 120ms, collect 5.68s, tests 36.32s, environment 3ms, prepare 753ms) exit=0 diff --git a/evidence/run-read-timeout/resume-live.log b/evidence/run-read-timeout/resume-live.log index 1fe792d96..a6e215eba 100644 --- a/evidence/run-read-timeout/resume-live.log +++ b/evidence/run-read-timeout/resume-live.log @@ -1,15 +1,13 @@ -$ cd packages/sdk && npx vitest run tests/read-timeout-resume-live.test.ts $ cd packages/sdk && npx vitest run tests/read-timeout-resume-live.test.ts - RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + RUN v2.1.9 /packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 1512ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 1511ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 955ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 954ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 10:49:12 - Duration 3.20s (transform 891ms, setup 54ms, collect 1.39s, tests 1.51s, environment 0ms, prepare 43ms) + Start at 23:53:47 + Duration 2.32s (transform 650ms, setup 14ms, collect 1.10s, tests 955ms, environment 0ms, prepare 60ms) exit=0 -exit=0 diff --git a/evidence/run-read-timeout/verification.md b/evidence/run-read-timeout/verification.md index b40a84f3d..198817853 100644 --- a/evidence/run-read-timeout/verification.md +++ b/evidence/run-read-timeout/verification.md @@ -1,65 +1,60 @@ -Captured verification. Commands and output below are verbatim; the full-suite log is linked from summary.md. +Captured verification at the review-fix source. Commands and output below are verbatim, and both live suites use a relayflowd built from this tree. [full-suite-final.log](full-suite-final.log) is the earlier full-suite transcript from before the review fixes (head a50a8de); it records that run's environment failures and does not cover the review-fix changes. ## final-focused.log ```text -$ cd packages/sdk && RELAYFLOWD_BIN=/home/daytona/.relayflows-toolchain/target/2962130851/debug/relayflowd npx vitest run tests/journal-client-read-timeout.test.ts tests/journal-client.test.ts tests/journal-client-completion.test.ts tests/journal-client-subscriptions.test.ts tests/running-step-watch.test.ts tests/heartbeat-timeout.test.ts tests/run-daemon-unresponsive.test.ts tests/authored-root.test.ts tests/classify-outcome.test.ts tests/cli.test.ts tests/direct-run-worker-lease.test.ts tests/resume-worker-lease.test.ts tests/worker-lease.test.ts tests/worker-lease-lost.test.ts tests/worker-lease-sweep.test.ts tests/run-read-load-live.test.ts tests/worker-lease-lost-live.test.ts tests/flow-executor-chain.test.ts tests/agent-transcript-live.test.ts tests/human-live.test.ts --maxWorkers=1 --minWorkers=1 +$ cd packages/sdk && RELAYFLOWD_BIN= npx vitest run tests/journal-client-read-timeout.test.ts tests/journal-client.test.ts tests/journal-client-completion.test.ts tests/journal-client-subscriptions.test.ts tests/running-step-watch.test.ts tests/heartbeat-timeout.test.ts tests/run-daemon-unresponsive.test.ts tests/authored-root.test.ts tests/classify-outcome.test.ts tests/cli.test.ts tests/direct-run-worker-lease.test.ts tests/resume-worker-lease.test.ts tests/worker-lease.test.ts tests/worker-lease-lost.test.ts tests/worker-lease-lost-live.test.ts tests/worker-lease-sweep.test.ts tests/run-read-load-live.test.ts tests/flow-executor-chain.test.ts tests/agent-transcript-live.test.ts tests/human-live.test.ts --maxWorkers=1 --minWorkers=1 - RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + RUN v2.1.9 /packages/sdk - ✓ tests/cli.test.ts (71 tests) 6026ms - ✓ flows check CLI > binds a checked relative wrapper to the flow directory for worker execution 566ms - ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 488ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2053ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2045ms -(node:44939) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5227ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2036ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2040ms +(node:2803772) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (26 tests) 398ms - ✓ tests/journal-client.test.ts (17 tests) 89ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 10315ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 1022ms - ✓ flow executor LLM and output-binding chain > runs a dollar-budgeted authored Claude agent with the same default used by preflight 700ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1595ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 3382ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 779ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 1068ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 3394ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 875ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 818ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 851ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 848ms - ✓ tests/classify-outcome.test.ts (11 tests) 7422ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2009ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3005ms - ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms - ✓ tests/human-live.test.ts (3 tests) 7854ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 4785ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1916ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 1152ms - ✓ tests/worker-lease.test.ts (7 tests) 20ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 27ms - ✓ tests/journal-client-read-timeout.test.ts (13 tests) 1104ms - ✓ a recovered read timeout does not become an authored callback failure 368ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 943ms - ✓ reports journal success after completion rejects with lease_conflict 327ms - ✓ reports journal success when a renewal rejects after completion landed 327ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2821ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2095ms - ✓ drains read and watch promises before an authored flow completes 725ms + ✓ tests/authored-root.test.ts (27 tests) 401ms + ✓ tests/journal-client.test.ts (17 tests) 83ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 7193ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 486ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 993ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2824ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 391ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 863ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2510ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 617ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 630ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 624ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 638ms + ✓ tests/classify-outcome.test.ts (11 tests) 7426ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2010ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3007ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2002ms + ✓ tests/journal-client-read-timeout.test.ts (17 tests) 1388ms + ✓ tests/human-live.test.ts (3 tests) 4585ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2697ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1115ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 772ms + ✓ tests/worker-lease.test.ts (7 tests) 19ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 19ms + ✓ tests/running-step-watch.test.ts (4 tests) 4184ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms + ✓ cancels promptly while a lease snapshot read is in flight 2008ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 666ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2457ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2062ms + ✓ drains read and watch promises before an authored flow completes 393ms ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms - ✓ tests/journal-client-completion.test.ts (6 tests) 102ms - ✓ tests/resume-worker-lease.test.ts (3 tests) 6ms + ✓ tests/journal-client-completion.test.ts (6 tests) 103ms + ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms ✓ tests/direct-run-worker-lease.test.ts (3 tests) 10ms - ✓ tests/running-step-watch.test.ts (2 tests) 2122ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms - ✓ tests/run-daemon-unresponsive.test.ts (2 tests) 4ms + ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 4ms ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms Test Files 20 passed (20) - Tests 210 passed (210) - Start at 10:41:25 - Duration 54.29s (transform 1.45s, setup 97ms, collect 8.09s, tests 42.69s, environment 3ms, prepare 1.05s) + Tests 218 passed (218) + Start at 23:53:02 + Duration 44.59s (transform 1.06s, setup 120ms, collect 5.68s, tests 36.32s, environment 3ms, prepare 753ms) exit=0 ``` @@ -67,20 +62,18 @@ exit=0 ## resume-live.log ```text -$ cd packages/sdk && npx vitest run tests/read-timeout-resume-live.test.ts $ cd packages/sdk && npx vitest run tests/read-timeout-resume-live.test.ts - RUN v2.1.9 /home/daytona/.relayflow-v2-supervisor/durable/repository/packages/sdk + RUN v2.1.9 /packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 1512ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 1511ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 955ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 954ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 10:49:12 - Duration 3.20s (transform 891ms, setup 54ms, collect 1.39s, tests 1.51s, environment 0ms, prepare 43ms) + Start at 23:53:47 + Duration 2.32s (transform 650ms, setup 14ms, collect 1.10s, tests 955ms, environment 0ms, prepare 60ms) -exit=0 exit=0 ``` From c962fc9ce6c9635187c2d8bfb57e6aaddefc8e3e Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 00:11:54 -0700 Subject: [PATCH 05/34] fix(sdk): keep a classified run outcome when the reuse summary read is interrupted The reuse summary is an extra journal read after classifyOutcome has already decided the run. A read interruption there fell to the outer catch, which recast a completed, failed or parked run as `status: running` with a resume command. attachReuseSummary now keeps the classified execution and adds a reuse_summary_unavailable warning; any other failure still propagates. Evidence: kernel-build.log records the literal cargo command and target, and final-focused.log / resume-live.log are re-captured verbatim with the literal RELAYFLOWD_BIN path (22 files, 278/278; 1/1). verification.md rebuilt. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: b8262468-ea26-4a6e-bee9-d193832e77dc --- evidence/run-read-timeout/final-focused.log | 86 +++++++------- evidence/run-read-timeout/resume-live.log | 12 +- evidence/run-read-timeout/verification.md | 108 ++++++++++-------- packages/sdk/src/cli/reuse.ts | 20 ++++ packages/sdk/src/cli/run.ts | 8 +- packages/sdk/src/failure-kinds.ts | 1 + .../tests/reuse-summary-interruption.test.ts | 32 ++++++ 7 files changed, 167 insertions(+), 100 deletions(-) create mode 100644 packages/sdk/tests/reuse-summary-interruption.test.ts diff --git a/evidence/run-read-timeout/final-focused.log b/evidence/run-read-timeout/final-focused.log index 1513cd872..9ac60b357 100644 --- a/evidence/run-read-timeout/final-focused.log +++ b/evidence/run-read-timeout/final-focused.log @@ -1,54 +1,58 @@ -$ cd packages/sdk && RELAYFLOWD_BIN= npx vitest run tests/journal-client-read-timeout.test.ts tests/journal-client.test.ts tests/journal-client-completion.test.ts tests/journal-client-subscriptions.test.ts tests/running-step-watch.test.ts tests/heartbeat-timeout.test.ts tests/run-daemon-unresponsive.test.ts tests/authored-root.test.ts tests/classify-outcome.test.ts tests/cli.test.ts tests/direct-run-worker-lease.test.ts tests/resume-worker-lease.test.ts tests/worker-lease.test.ts tests/worker-lease-lost.test.ts tests/worker-lease-lost-live.test.ts tests/worker-lease-sweep.test.ts tests/run-read-load-live.test.ts tests/flow-executor-chain.test.ts tests/agent-transcript-live.test.ts tests/human-live.test.ts --maxWorkers=1 --minWorkers=1 +$ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/kernel-target/debug/relayflowd npx vitest run tests/journal-client-read-timeout.test.ts tests/journal-client.test.ts tests/journal-client-completion.test.ts tests/journal-client-subscriptions.test.ts tests/running-step-watch.test.ts tests/heartbeat-timeout.test.ts tests/run-daemon-unresponsive.test.ts tests/authored-root.test.ts tests/classify-outcome.test.ts tests/cli.test.ts tests/direct-run-worker-lease.test.ts tests/resume-worker-lease.test.ts tests/worker-lease.test.ts tests/worker-lease-lost.test.ts tests/worker-lease-lost-live.test.ts tests/worker-lease-sweep.test.ts tests/run-read-load-live.test.ts tests/flow-executor-chain.test.ts tests/agent-transcript-live.test.ts tests/human-live.test.ts tests/reuse-summary-interruption.test.ts tests/memoization.test.ts --maxWorkers=1 --minWorkers=1 - RUN v2.1.9 /packages/sdk + RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5227ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2036ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2040ms -(node:2803772) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5725ms + ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 319ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2049ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2041ms +(node:2935253) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (27 tests) 401ms + ✓ tests/authored-root.test.ts (27 tests) 404ms ✓ tests/journal-client.test.ts (17 tests) 83ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 7193ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 486ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 993ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2824ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 391ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 863ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2510ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 617ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 630ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 624ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 638ms - ✓ tests/classify-outcome.test.ts (11 tests) 7426ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 7385ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 510ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1048ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2909ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 376ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 912ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2804ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 713ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 638ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 750ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 702ms + ✓ tests/classify-outcome.test.ts (11 tests) 7422ms ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2010ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3007ms - ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2002ms - ✓ tests/journal-client-read-timeout.test.ts (17 tests) 1388ms - ✓ tests/human-live.test.ts (3 tests) 4585ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2697ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1115ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 772ms - ✓ tests/worker-lease.test.ts (7 tests) 19ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 19ms - ✓ tests/running-step-watch.test.ts (4 tests) 4184ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms - ✓ cancels promptly while a lease snapshot read is in flight 2008ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 666ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2457ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2062ms - ✓ drains read and watch promises before an authored flow completes 393ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3003ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms + ✓ tests/journal-client-read-timeout.test.ts (17 tests) 1458ms + ✓ a recovered read timeout does not become an authored callback failure 300ms + ✓ tests/human-live.test.ts (3 tests) 5014ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2989ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1204ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 820ms + ✓ tests/worker-lease.test.ts (7 tests) 20ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 31ms + ✓ tests/running-step-watch.test.ts (4 tests) 4178ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms + ✓ cancels promptly while a lease snapshot read is in flight 2003ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 703ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2431ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2058ms + ✓ drains read and watch promises before an authored flow completes 372ms ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms ✓ tests/journal-client-completion.test.ts (6 tests) 103ms - ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms - ✓ tests/direct-run-worker-lease.test.ts (3 tests) 10ms + ✓ tests/resume-worker-lease.test.ts (3 tests) 8ms + ✓ tests/direct-run-worker-lease.test.ts (3 tests) 8ms + ✓ tests/memoization.test.ts (57 tests) 59ms ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 4ms + ✓ tests/reuse-summary-interruption.test.ts (3 tests) 3ms ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms - Test Files 20 passed (20) - Tests 218 passed (218) - Start at 23:53:02 - Duration 44.59s (transform 1.06s, setup 120ms, collect 5.68s, tests 36.32s, environment 3ms, prepare 753ms) + Test Files 22 passed (22) + Tests 278 passed (278) + Start at 00:10:50 + Duration 46.66s (transform 1.04s, setup 129ms, collect 5.94s, tests 37.87s, environment 3ms, prepare 786ms) exit=0 diff --git a/evidence/run-read-timeout/resume-live.log b/evidence/run-read-timeout/resume-live.log index a6e215eba..a6bdfe376 100644 --- a/evidence/run-read-timeout/resume-live.log +++ b/evidence/run-read-timeout/resume-live.log @@ -1,13 +1,13 @@ -$ cd packages/sdk && npx vitest run tests/read-timeout-resume-live.test.ts +$ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/kernel-target/debug/relayflowd npx vitest run tests/read-timeout-resume-live.test.ts - RUN v2.1.9 /packages/sdk + RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 955ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 954ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 920ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 919ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 23:53:47 - Duration 2.32s (transform 650ms, setup 14ms, collect 1.10s, tests 955ms, environment 0ms, prepare 60ms) + Start at 00:11:37 + Duration 2.24s (transform 656ms, setup 20ms, collect 1.08s, tests 920ms, environment 0ms, prepare 82ms) exit=0 diff --git a/evidence/run-read-timeout/verification.md b/evidence/run-read-timeout/verification.md index 198817853..4c17f9766 100644 --- a/evidence/run-read-timeout/verification.md +++ b/evidence/run-read-timeout/verification.md @@ -1,60 +1,72 @@ -Captured verification at the review-fix source. Commands and output below are verbatim, and both live suites use a relayflowd built from this tree. [full-suite-final.log](full-suite-final.log) is the earlier full-suite transcript from before the review fixes (head a50a8de); it records that run's environment failures and does not cover the review-fix changes. +Captured verification at the review-fix source. Commands and output below are verbatim (absolute paths included): relayflowd is built from this tree by the kernel-build command, and both live suites run against that binary. [full-suite-final.log](full-suite-final.log) is the earlier full-suite transcript from before the review fixes (head a50a8de); it records that run's environment failures and does not cover the review-fix changes. + +## kernel-build.log + +```text +$ cd kernel && CARGO_TARGET_DIR=/home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/kernel-target cargo build + Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.06s +exit=0 +``` ## final-focused.log ```text -$ cd packages/sdk && RELAYFLOWD_BIN= npx vitest run tests/journal-client-read-timeout.test.ts tests/journal-client.test.ts tests/journal-client-completion.test.ts tests/journal-client-subscriptions.test.ts tests/running-step-watch.test.ts tests/heartbeat-timeout.test.ts tests/run-daemon-unresponsive.test.ts tests/authored-root.test.ts tests/classify-outcome.test.ts tests/cli.test.ts tests/direct-run-worker-lease.test.ts tests/resume-worker-lease.test.ts tests/worker-lease.test.ts tests/worker-lease-lost.test.ts tests/worker-lease-lost-live.test.ts tests/worker-lease-sweep.test.ts tests/run-read-load-live.test.ts tests/flow-executor-chain.test.ts tests/agent-transcript-live.test.ts tests/human-live.test.ts --maxWorkers=1 --minWorkers=1 +$ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/kernel-target/debug/relayflowd npx vitest run tests/journal-client-read-timeout.test.ts tests/journal-client.test.ts tests/journal-client-completion.test.ts tests/journal-client-subscriptions.test.ts tests/running-step-watch.test.ts tests/heartbeat-timeout.test.ts tests/run-daemon-unresponsive.test.ts tests/authored-root.test.ts tests/classify-outcome.test.ts tests/cli.test.ts tests/direct-run-worker-lease.test.ts tests/resume-worker-lease.test.ts tests/worker-lease.test.ts tests/worker-lease-lost.test.ts tests/worker-lease-lost-live.test.ts tests/worker-lease-sweep.test.ts tests/run-read-load-live.test.ts tests/flow-executor-chain.test.ts tests/agent-transcript-live.test.ts tests/human-live.test.ts tests/reuse-summary-interruption.test.ts tests/memoization.test.ts --maxWorkers=1 --minWorkers=1 - RUN v2.1.9 /packages/sdk + RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5227ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2036ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2040ms -(node:2803772) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5725ms + ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 319ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2049ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2041ms +(node:2935253) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (27 tests) 401ms + ✓ tests/authored-root.test.ts (27 tests) 404ms ✓ tests/journal-client.test.ts (17 tests) 83ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 7193ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 486ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 993ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2824ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 391ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 863ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2510ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 617ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 630ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 624ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 638ms - ✓ tests/classify-outcome.test.ts (11 tests) 7426ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 7385ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 510ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1048ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2909ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 376ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 912ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2804ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 713ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 638ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 750ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 702ms + ✓ tests/classify-outcome.test.ts (11 tests) 7422ms ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2010ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3007ms - ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2002ms - ✓ tests/journal-client-read-timeout.test.ts (17 tests) 1388ms - ✓ tests/human-live.test.ts (3 tests) 4585ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2697ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1115ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 772ms - ✓ tests/worker-lease.test.ts (7 tests) 19ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 19ms - ✓ tests/running-step-watch.test.ts (4 tests) 4184ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms - ✓ cancels promptly while a lease snapshot read is in flight 2008ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 666ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2457ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2062ms - ✓ drains read and watch promises before an authored flow completes 393ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3003ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms + ✓ tests/journal-client-read-timeout.test.ts (17 tests) 1458ms + ✓ a recovered read timeout does not become an authored callback failure 300ms + ✓ tests/human-live.test.ts (3 tests) 5014ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2989ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1204ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 820ms + ✓ tests/worker-lease.test.ts (7 tests) 20ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 31ms + ✓ tests/running-step-watch.test.ts (4 tests) 4178ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms + ✓ cancels promptly while a lease snapshot read is in flight 2003ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 703ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2431ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2058ms + ✓ drains read and watch promises before an authored flow completes 372ms ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms ✓ tests/journal-client-completion.test.ts (6 tests) 103ms - ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms - ✓ tests/direct-run-worker-lease.test.ts (3 tests) 10ms + ✓ tests/resume-worker-lease.test.ts (3 tests) 8ms + ✓ tests/direct-run-worker-lease.test.ts (3 tests) 8ms + ✓ tests/memoization.test.ts (57 tests) 59ms ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 4ms + ✓ tests/reuse-summary-interruption.test.ts (3 tests) 3ms ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms - Test Files 20 passed (20) - Tests 218 passed (218) - Start at 23:53:02 - Duration 44.59s (transform 1.06s, setup 120ms, collect 5.68s, tests 36.32s, environment 3ms, prepare 753ms) + Test Files 22 passed (22) + Tests 278 passed (278) + Start at 00:10:50 + Duration 46.66s (transform 1.04s, setup 129ms, collect 5.94s, tests 37.87s, environment 3ms, prepare 786ms) exit=0 ``` @@ -62,17 +74,17 @@ exit=0 ## resume-live.log ```text -$ cd packages/sdk && npx vitest run tests/read-timeout-resume-live.test.ts +$ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/kernel-target/debug/relayflowd npx vitest run tests/read-timeout-resume-live.test.ts - RUN v2.1.9 /packages/sdk + RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 955ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 954ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 920ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 919ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 23:53:47 - Duration 2.32s (transform 650ms, setup 14ms, collect 1.10s, tests 955ms, environment 0ms, prepare 60ms) + Start at 00:11:37 + Duration 2.24s (transform 656ms, setup 20ms, collect 1.08s, tests 920ms, environment 0ms, prepare 82ms) exit=0 ``` diff --git a/packages/sdk/src/cli/reuse.ts b/packages/sdk/src/cli/reuse.ts index cc9df22ba..6f1c4d04a 100644 --- a/packages/sdk/src/cli/reuse.ts +++ b/packages/sdk/src/cli/reuse.ts @@ -1,4 +1,6 @@ import type { JournalClient } from '../journal-client.js'; +import { isReadInterruptionError } from '../journal-read-policy.js'; +import type { RunExecution } from './run.js'; /** Count durable facts, including failed executions, across paginated reads. */ export async function reuseSummary(client: JournalClient, runId: string, fromRunId: string): Promise<{ @@ -21,3 +23,21 @@ export async function reuseSummary(client: JournalClient, runId: string, fromRun } return { fromRunId, reusedSteps: reused.size, executedSteps: executed.size }; } + +/** + * Add the reuse summary to an already classified run. The summary is an extra + * journal read after the outcome is known: if it cannot be answered, report + * that as a warning and keep the classified outcome rather than recasting a + * finished run as resumable. + */ +export async function attachReuseSummary(execution: RunExecution, client: JournalClient, + runId: string, fromRunId: string): Promise { + try { + execution.report.reuse = await reuseSummary(client, runId, fromRunId); + } catch (error) { + if (!isReadInterruptionError(error)) throw error; + execution.report.diagnostics = [...execution.report.diagnostics, { severity: 'warning', kind: 'reuse_summary_unavailable', + message: `The run's outcome is final, but the summary of steps reused from ${fromRunId} could not be read: ${error.message}` }]; + } + return execution; +} diff --git a/packages/sdk/src/cli/run.ts b/packages/sdk/src/cli/run.ts index abc8d54f5..238d493c5 100644 --- a/packages/sdk/src/cli/run.ts +++ b/packages/sdk/src/cli/run.ts @@ -8,7 +8,7 @@ import { checkCommunicationEnvironment, CommunicationEnvironmentError } from '.. import { parseHumanRecipient } from '../human-to.js'; import { parseDigestReference } from '../bundle-transport.js'; import { prepareDigestRun } from './run-digest.js'; -import { reuseSummary } from './reuse.js'; +import { attachReuseSummary } from './reuse.js'; import { resumeHelperEffect } from '../authored-helper-effect.js'; import { AuthoredFlowExecutionError, AuthoredHumanParked, type AuthoredHumanWait } from '../authored-flow-error.js'; import { answerCommand, resumeCommand } from '../authored-human.js'; @@ -239,10 +239,8 @@ async function executeCheckedFlow( // truthful "not projected" diagnostic. options.onRunReceipt?.({ runId: outcome.run_id, flow: spec.name ?? 'flow' }); const execution = await classifyOutcome(client, 'run', outcome, base, socketPath, { ...options, dataDir }); - if (options.reuseFromRunId !== undefined) { - execution.report.reuse = await reuseSummary(client, outcome.run_id, options.reuseFromRunId); - } - return execution; + return options.reuseFromRunId === undefined ? execution + : await attachReuseSummary(execution, client, outcome.run_id, options.reuseFromRunId); } catch (error) { if (isReadInterruption(error)) return daemonUnresponsiveReport('run', base, socketPath, error, base.runId, options, dataDir); if (error instanceof CommunicationEnvironmentError) return { exitCode: 2, report: { ...base, diff --git a/packages/sdk/src/failure-kinds.ts b/packages/sdk/src/failure-kinds.ts index 09cc78fda..054d47fa8 100644 --- a/packages/sdk/src/failure-kinds.ts +++ b/packages/sdk/src/failure-kinds.ts @@ -161,6 +161,7 @@ export const RUN_FAILURE_KINDS = [ */ export const RUN_WARNING_KINDS = [ 'connection_file_stale', + 'reuse_summary_unavailable', ] as const; /** diff --git a/packages/sdk/tests/reuse-summary-interruption.test.ts b/packages/sdk/tests/reuse-summary-interruption.test.ts new file mode 100644 index 000000000..f75000368 --- /dev/null +++ b/packages/sdk/tests/reuse-summary-interruption.test.ts @@ -0,0 +1,32 @@ +import { expect, it } from 'vitest'; +import { attachReuseSummary } from '../src/cli/reuse.js'; +import { JournalReadInterruptedError, JournalRequestTimeoutError, type JournalClient } from '../src/journal-client.js'; +import type { RunExecution } from '../src/cli/run.js'; + +const completed = (): RunExecution => ({ exitCode: 0, report: { + command: 'run', ok: true, status: 'completed', completionReason: 'success', runId: 'run-1', diagnostics: [], +} as never }); + +it.each([ + ['timeout', new JournalRequestTimeoutError('journal.read', 10, 3, 300_000, 300_000)], + ['interruption', new JournalReadInterruptedError('journal.read', 3, 300_000, 300_000)], +])('keeps the classified outcome when the reuse summary read hits a %s', async (_name, error) => { + const client = { journalRead: async () => { throw error; } } as unknown as JournalClient; + const execution = await attachReuseSummary(completed(), client, 'run-1', 'prior'); + expect(execution.exitCode).toBe(0); + expect(execution.report).toMatchObject({ ok: true, status: 'completed', completionReason: 'success' }); + expect(execution.report.reuse).toBeUndefined(); + expect(execution.report.diagnostics).toEqual([expect.objectContaining({ + severity: 'warning', kind: 'reuse_summary_unavailable', message: expect.stringContaining('prior'), + })]); + expect(JSON.stringify(execution.report)).not.toContain('flows resume'); +}); + +it('still attaches the summary when the read succeeds, and rethrows other failures', async () => { + const ok = { journalRead: async (_run: string, from: number) => ({ entries: from === 1 + ? [{ seq: 1, entry_type: 'step.completed', step_id: 'a', payload: { reused_from: 'prior' } }] : [] }) } as unknown as JournalClient; + expect((await attachReuseSummary(completed(), ok, 'run-1', 'prior')).report.reuse) + .toEqual({ fromRunId: 'prior', reusedSteps: 1, executedSteps: 0 }); + const broken = { journalRead: async () => { throw new Error('invalid journal sequence'); } } as unknown as JournalClient; + await expect(attachReuseSummary(completed(), broken, 'run-1', 'prior')).rejects.toThrow('invalid journal sequence'); +}); From 36ff10ebebdd8e3214eeb3a5b2fb5745a229ec86 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 00:20:49 -0700 Subject: [PATCH 06/34] fix(sdk): a watch connection lost during registration is a read interruption The completion watch mapped transport failures to daemon_unresponsive only up to `hello`; a socket that dropped before `run.watch` answered surfaced as a raw connection error, which an authored child rethrows and the root then records as worker_error. The mapping now covers the whole handshake including registration; protocol refusals still propagate unchanged. Evidence re-captured at this tree: mutations (7/7 fail-then-pass), focused 22 files 279/279, resume-live 1/1, verification.md rebuilt. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: b8262468-ea26-4a6e-bee9-d193832e77dc --- evidence/run-read-timeout/final-focused.log | 90 +++---- .../run-read-timeout/mutation-heartbeat.log | 14 +- evidence/run-read-timeout/mutation-reader.log | 14 +- .../mutation-root-parking.log | 18 +- .../mutation-watch-cadence.log | 34 +-- evidence/run-read-timeout/resume-live.log | 8 +- evidence/run-read-timeout/verification.md | 250 +++++++++--------- packages/sdk/src/cli/running-step.ts | 13 +- packages/sdk/tests/running-step-watch.test.ts | 22 ++ 9 files changed, 244 insertions(+), 219 deletions(-) diff --git a/evidence/run-read-timeout/final-focused.log b/evidence/run-read-timeout/final-focused.log index 9ac60b357..fd567eddc 100644 --- a/evidence/run-read-timeout/final-focused.log +++ b/evidence/run-read-timeout/final-focused.log @@ -2,57 +2,57 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5725ms - ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 319ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2049ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2041ms -(node:2935253) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5624ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2057ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2050ms +(node:3018634) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (27 tests) 404ms - ✓ tests/journal-client.test.ts (17 tests) 83ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 7385ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 510ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1048ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2909ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 376ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 912ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2804ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 713ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 638ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 750ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 702ms - ✓ tests/classify-outcome.test.ts (11 tests) 7422ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2010ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3003ms + ✓ tests/authored-root.test.ts (27 tests) 414ms + ✓ tests/journal-client.test.ts (17 tests) 77ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 8994ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 596ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1638ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 3077ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 398ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 1204ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 3511ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 938ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 856ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 1026ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 691ms + ✓ tests/classify-outcome.test.ts (11 tests) 7431ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2014ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3005ms ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms - ✓ tests/journal-client-read-timeout.test.ts (17 tests) 1458ms - ✓ a recovered read timeout does not become an authored callback failure 300ms - ✓ tests/human-live.test.ts (3 tests) 5014ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2989ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1204ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 820ms - ✓ tests/worker-lease.test.ts (7 tests) 20ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 31ms - ✓ tests/running-step-watch.test.ts (4 tests) 4178ms + ✓ tests/journal-client-read-timeout.test.ts (17 tests) 1474ms + ✓ a recovered read timeout does not become an authored callback failure 328ms + ✓ tests/human-live.test.ts (3 tests) 9715ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 7375ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1477ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 862ms + ✓ tests/worker-lease.test.ts (7 tests) 19ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 27ms + ✓ tests/running-step-watch.test.ts (5 tests) 4194ms ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms - ✓ cancels promptly while a lease snapshot read is in flight 2003ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 703ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2431ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2058ms - ✓ drains read and watch promises before an authored flow completes 372ms - ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms + ✓ cancels promptly while a lease snapshot read is in flight 2009ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 962ms + ✓ reports journal success after completion rejects with lease_conflict 429ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2477ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2062ms + ✓ drains read and watch promises before an authored flow completes 415ms + ✓ tests/worker-lease-sweep.test.ts (4 tests) 6ms ✓ tests/journal-client-completion.test.ts (6 tests) 103ms - ✓ tests/resume-worker-lease.test.ts (3 tests) 8ms - ✓ tests/direct-run-worker-lease.test.ts (3 tests) 8ms - ✓ tests/memoization.test.ts (57 tests) 59ms - ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms - ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 4ms - ✓ tests/reuse-summary-interruption.test.ts (3 tests) 3ms + ✓ tests/resume-worker-lease.test.ts (3 tests) 10ms + ✓ tests/direct-run-worker-lease.test.ts (3 tests) 14ms + ✓ tests/memoization.test.ts (57 tests) 62ms + ✓ tests/journal-client-subscriptions.test.ts (1 test) 6ms + ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 5ms + ✓ tests/reuse-summary-interruption.test.ts (3 tests) 7ms ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms Test Files 22 passed (22) - Tests 278 passed (278) - Start at 00:10:50 - Duration 46.66s (transform 1.04s, setup 129ms, collect 5.94s, tests 37.87s, environment 3ms, prepare 786ms) + Tests 279 passed (279) + Start at 00:19:38 + Duration 56.60s (transform 1.38s, setup 185ms, collect 7.88s, tests 45.15s, environment 4ms, prepare 980ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-heartbeat.log b/evidence/run-read-timeout/mutation-heartbeat.log index 67af0f235..d0cf1e839 100644 --- a/evidence/run-read-timeout/mutation-heartbeat.log +++ b/evidence/run-read-timeout/mutation-heartbeat.log @@ -13,8 +13,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 20ms - × a heartbeat timeout is lease loss rather than a worker body failure 20ms + ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 22ms + × a heartbeat timeout is lease loss rather than a worker body failure 21ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -33,8 +33,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 23:37:33 - Duration 464ms (transform 162ms, setup 16ms, collect 283ms, tests 20ms, environment 0ms, prepare 41ms) + Start at 00:19:25 + Duration 635ms (transform 246ms, setup 17ms, collect 437ms, tests 22ms, environment 0ms, prepare 43ms) exit=1 @@ -43,11 +43,11 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 19ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 23:37:33 - Duration 440ms (transform 163ms, setup 15ms, collect 272ms, tests 16ms, environment 0ms, prepare 37ms) + Start at 00:19:26 + Duration 695ms (transform 233ms, setup 20ms, collect 412ms, tests 19ms, environment 0ms, prepare 60ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader.log b/evidence/run-read-timeout/mutation-reader.log index fecd14783..55f3f3ba4 100644 --- a/evidence/run-read-timeout/mutation-reader.log +++ b/evidence/run-read-timeout/mutation-reader.log @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (17 tests | 1 failed | 16 skipped) 113ms - × serves a bounded read while an unbounded command is in flight 112ms + ❯ tests/journal-client-read-timeout.test.ts (17 tests | 1 failed | 16 skipped) 114ms + × serves a bounded read while an unbounded command is in flight 113ms → expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -37,8 +37,8 @@ AssertionError: expected true to be false // Object.is equality Test Files 1 failed (1) Tests 1 failed | 16 skipped (17) - Start at 23:37:15 - Duration 572ms (transform 170ms, setup 17ms, collect 295ms, tests 113ms, environment 0ms, prepare 42ms) + Start at 00:19:07 + Duration 556ms (transform 161ms, setup 17ms, collect 282ms, tests 114ms, environment 0ms, prepare 40ms) exit=1 @@ -47,11 +47,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 107ms + ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 105ms Test Files 1 passed (1) Tests 1 passed | 16 skipped (17) - Start at 23:37:16 - Duration 556ms (transform 169ms, setup 17ms, collect 293ms, tests 107ms, environment 0ms, prepare 40ms) + Start at 00:19:07 + Duration 543ms (transform 168ms, setup 16ms, collect 289ms, tests 105ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-root-parking.log b/evidence/run-read-timeout/mutation-root-parking.log index 7b3d68745..6f81d5b35 100644 --- a/evidence/run-read-timeout/mutation-root-parking.log +++ b/evidence/run-read-timeout/mutation-root-parking.log @@ -17,14 +17,14 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (27 tests | 3 failed | 24 skipped) 31ms - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 22ms + ❯ tests/authored-root.test.ts (27 tests | 3 failed | 24 skipped) 40ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 28ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 6ms → expected AuthoredFlowExecutionError: daemon_unresp… { …(7) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 4ms + × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 5ms → expected JournalReadInterruptedError: journal clie… { …(4) } to match object { code: 'daemon_unresponsive', …(2) } (5 matching properties omitted from actual) @@ -104,8 +104,8 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } Test Files 1 failed (1) Tests 3 failed | 24 skipped (27) - Start at 23:37:34 - Duration 1.28s (transform 656ms, setup 16ms, collect 1.08s, tests 31ms, environment 0ms, prepare 43ms) + Start at 00:19:27 + Duration 1.50s (transform 788ms, setup 20ms, collect 1.30s, tests 40ms, environment 0ms, prepare 33ms) exit=1 @@ -114,11 +114,11 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (27 tests | 24 skipped) 25ms + ✓ tests/authored-root.test.ts (27 tests | 24 skipped) 55ms Test Files 1 passed (1) Tests 3 passed | 24 skipped (27) - Start at 23:37:36 - Duration 1.24s (transform 647ms, setup 16ms, collect 1.06s, tests 25ms, environment 0ms, prepare 41ms) + Start at 00:19:29 + Duration 1.97s (transform 982ms, setup 20ms, collect 1.70s, tests 55ms, environment 0ms, prepare 54ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-watch-cadence.log b/evidence/run-read-timeout/mutation-watch-cadence.log index 3975b1b05..d4e434706 100644 --- a/evidence/run-read-timeout/mutation-watch-cadence.log +++ b/evidence/run-read-timeout/mutation-watch-cadence.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/cli/running-step.ts -Original SHA256: a57da3dce86c9724d83bd3547a2d9186e05e7ea10409b743f89aaa25bf461bcd +Original SHA256: 5ae78a777c69e5f83a259df2f256305a0e9fd9c60525e661e66e3997d0f3801a Replaced: const LEASE_POLL_MS = 2_000; With: @@ -10,42 +10,42 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (4 tests | 1 failed | 3 skipped) 2110ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2110ms + ❯ tests/running-step-watch.test.ts (5 tests | 1 failed | 4 skipped) 2112ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2111ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/running-step-watch.test.ts > uses pushes for completion with lease-cadence reads and releases its watcher AssertionError: expected 41 to be less than or equal to 1 - ❯ tests/running-step-watch.test.ts:37:19 - 35| await waitForRunningStep(client, 'run', { id: 'step', type: 'agent… - 36| expect(performance.now() - start).toBeGreaterThan(2000); - 37| expect(reads).toBeLessThanOrEqual(1); + ❯ tests/running-step-watch.test.ts:39:19 + 37| await waitForRunningStep(client, 'run', { id: 'step', type: 'agent… + 38| expect(performance.now() - start).toBeGreaterThan(2000); + 39| expect(reads).toBeLessThanOrEqual(1); | ^ - 38| await sleep(10); - 39| expect(watchClosed).toBe(true); + 40| await sleep(10); + 41| expect(watchClosed).toBe(true); ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 3 skipped (4) - Start at 23:37:21 - Duration 2.57s (transform 168ms, setup 15ms, collect 294ms, tests 2.11s, environment 0ms, prepare 40ms) + Tests 1 failed | 4 skipped (5) + Start at 00:19:13 + Duration 2.59s (transform 188ms, setup 17ms, collect 315ms, tests 2.11s, environment 0ms, prepare 42ms) exit=1 -RESTORED SHA256: a57da3dce86c9724d83bd3547a2d9186e05e7ea10409b743f89aaa25bf461bcd +RESTORED SHA256: 5ae78a777c69e5f83a259df2f256305a0e9fd9c60525e661e66e3997d0f3801a $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pushes for completion' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (4 tests | 3 skipped) 2117ms + ✓ tests/running-step-watch.test.ts (5 tests | 4 skipped) 2117ms ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2116ms Test Files 1 passed (1) - Tests 1 passed | 3 skipped (4) - Start at 23:37:24 - Duration 2.57s (transform 165ms, setup 17ms, collect 291ms, tests 2.12s, environment 0ms, prepare 41ms) + Tests 1 passed | 4 skipped (5) + Start at 00:19:16 + Duration 2.60s (transform 184ms, setup 18ms, collect 324ms, tests 2.12s, environment 0ms, prepare 45ms) exit=0 diff --git a/evidence/run-read-timeout/resume-live.log b/evidence/run-read-timeout/resume-live.log index a6bdfe376..013a8a8dc 100644 --- a/evidence/run-read-timeout/resume-live.log +++ b/evidence/run-read-timeout/resume-live.log @@ -2,12 +2,12 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 920ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 919ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 1110ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 1109ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 00:11:37 - Duration 2.24s (transform 656ms, setup 20ms, collect 1.08s, tests 920ms, environment 0ms, prepare 82ms) + Start at 00:20:35 + Duration 2.81s (transform 802ms, setup 28ms, collect 1.35s, tests 1.11s, environment 0ms, prepare 56ms) exit=0 diff --git a/evidence/run-read-timeout/verification.md b/evidence/run-read-timeout/verification.md index 4c17f9766..d2e17c0ce 100644 --- a/evidence/run-read-timeout/verification.md +++ b/evidence/run-read-timeout/verification.md @@ -4,7 +4,7 @@ Captured verification at the review-fix source. Commands and output below are ve ```text $ cd kernel && CARGO_TARGET_DIR=/home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/kernel-target cargo build - Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.06s + Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.09s exit=0 ``` @@ -15,58 +15,58 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5725ms - ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 319ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2049ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2041ms -(node:2935253) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5624ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2057ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2050ms +(node:3018634) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (27 tests) 404ms - ✓ tests/journal-client.test.ts (17 tests) 83ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 7385ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 510ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1048ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2909ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 376ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 912ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2804ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 713ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 638ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 750ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 702ms - ✓ tests/classify-outcome.test.ts (11 tests) 7422ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2010ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3003ms + ✓ tests/authored-root.test.ts (27 tests) 414ms + ✓ tests/journal-client.test.ts (17 tests) 77ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 8994ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 596ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1638ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 3077ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 398ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 1204ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 3511ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 938ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 856ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 1026ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 691ms + ✓ tests/classify-outcome.test.ts (11 tests) 7431ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2014ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3005ms ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms - ✓ tests/journal-client-read-timeout.test.ts (17 tests) 1458ms - ✓ a recovered read timeout does not become an authored callback failure 300ms - ✓ tests/human-live.test.ts (3 tests) 5014ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2989ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1204ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 820ms - ✓ tests/worker-lease.test.ts (7 tests) 20ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 31ms - ✓ tests/running-step-watch.test.ts (4 tests) 4178ms + ✓ tests/journal-client-read-timeout.test.ts (17 tests) 1474ms + ✓ a recovered read timeout does not become an authored callback failure 328ms + ✓ tests/human-live.test.ts (3 tests) 9715ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 7375ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1477ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 862ms + ✓ tests/worker-lease.test.ts (7 tests) 19ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 27ms + ✓ tests/running-step-watch.test.ts (5 tests) 4194ms ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms - ✓ cancels promptly while a lease snapshot read is in flight 2003ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 703ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2431ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2058ms - ✓ drains read and watch promises before an authored flow completes 372ms - ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms + ✓ cancels promptly while a lease snapshot read is in flight 2009ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 962ms + ✓ reports journal success after completion rejects with lease_conflict 429ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2477ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2062ms + ✓ drains read and watch promises before an authored flow completes 415ms + ✓ tests/worker-lease-sweep.test.ts (4 tests) 6ms ✓ tests/journal-client-completion.test.ts (6 tests) 103ms - ✓ tests/resume-worker-lease.test.ts (3 tests) 8ms - ✓ tests/direct-run-worker-lease.test.ts (3 tests) 8ms - ✓ tests/memoization.test.ts (57 tests) 59ms - ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms - ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 4ms - ✓ tests/reuse-summary-interruption.test.ts (3 tests) 3ms + ✓ tests/resume-worker-lease.test.ts (3 tests) 10ms + ✓ tests/direct-run-worker-lease.test.ts (3 tests) 14ms + ✓ tests/memoization.test.ts (57 tests) 62ms + ✓ tests/journal-client-subscriptions.test.ts (1 test) 6ms + ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 5ms + ✓ tests/reuse-summary-interruption.test.ts (3 tests) 7ms ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms Test Files 22 passed (22) - Tests 278 passed (278) - Start at 00:10:50 - Duration 46.66s (transform 1.04s, setup 129ms, collect 5.94s, tests 37.87s, environment 3ms, prepare 786ms) + Tests 279 passed (279) + Start at 00:19:38 + Duration 56.60s (transform 1.38s, setup 185ms, collect 7.88s, tests 45.15s, environment 4ms, prepare 980ms) exit=0 ``` @@ -78,13 +78,13 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 920ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 919ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 1110ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 1109ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 00:11:37 - Duration 2.24s (transform 656ms, setup 20ms, collect 1.08s, tests 920ms, environment 0ms, prepare 82ms) + Start at 00:20:35 + Duration 2.81s (transform 802ms, setup 28ms, collect 1.35s, tests 1.11s, environment 0ms, prepare 56ms) exit=0 ``` @@ -106,8 +106,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (17 tests | 1 failed | 16 skipped) 113ms - × serves a bounded read while an unbounded command is in flight 112ms + ❯ tests/journal-client-read-timeout.test.ts (17 tests | 1 failed | 16 skipped) 114ms + × serves a bounded read while an unbounded command is in flight 113ms → expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -133,8 +133,8 @@ AssertionError: expected true to be false // Object.is equality Test Files 1 failed (1) Tests 1 failed | 16 skipped (17) - Start at 23:37:15 - Duration 572ms (transform 170ms, setup 17ms, collect 295ms, tests 113ms, environment 0ms, prepare 42ms) + Start at 00:19:07 + Duration 556ms (transform 161ms, setup 17ms, collect 282ms, tests 114ms, environment 0ms, prepare 40ms) exit=1 @@ -143,12 +143,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 107ms + ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 105ms Test Files 1 passed (1) Tests 1 passed | 16 skipped (17) - Start at 23:37:16 - Duration 556ms (transform 169ms, setup 17ms, collect 293ms, tests 107ms, environment 0ms, prepare 40ms) + Start at 00:19:07 + Duration 543ms (transform 168ms, setup 16ms, collect 289ms, tests 105ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -169,14 +169,14 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (17 tests | 1 failed | 16 skipped) 1007ms - × reconnects the reader after it disconnects within the read budget 1006ms - → journal client: run.get read session was interrupted after 79 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ tests/journal-client-read-timeout.test.ts (17 tests | 1 failed | 16 skipped) 1008ms + × reconnects the reader after it disconnects within the read budget 1007ms + → journal client: run.get read session was interrupted after 82 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 79 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 82 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:70:9 68| let last: unknown; 69| const exhausted = () => last instanceof JournalReadInterruptedError @@ -184,7 +184,7 @@ JournalReadInterruptedError: journal client: run.get read session was interrupte | ^ 71| : new JournalRequestTimeoutError(verb, timeoutMs, attempts, perf… 72| // A caller's cancellation settles this read and drains its queued… - ❯ Timeout. src/journal-read-policy.ts:81:39 + ❯ src/journal-read-policy.ts:99:32 Caused by: Error: journal client: not connected (run.get): journal client: connection closed ❯ src/journal-client.ts:270:16 @@ -199,8 +199,8 @@ Caused by: Error: journal client: connection closed Test Files 1 failed (1) Tests 1 failed | 16 skipped (17) - Start at 23:37:16 - Duration 1.45s (transform 170ms, setup 18ms, collect 289ms, tests 1.01s, environment 0ms, prepare 42ms) + Start at 00:19:08 + Duration 1.43s (transform 163ms, setup 17ms, collect 284ms, tests 1.01s, environment 0ms, prepare 33ms) exit=1 @@ -209,12 +209,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 76ms + ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 84ms Test Files 1 passed (1) Tests 1 passed | 16 skipped (17) - Start at 23:37:18 - Duration 532ms (transform 166ms, setup 16ms, collect 295ms, tests 76ms, environment 0ms, prepare 41ms) + Start at 00:19:10 + Duration 531ms (transform 170ms, setup 17ms, collect 290ms, tests 84ms, environment 0ms, prepare 42ms) exit=0 ``` @@ -240,8 +240,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (17 tests | 1 failed | 16 skipped) 72ms - × retries a reader setup that timed out instead of reading on the primary for good 71ms + ❯ tests/journal-client-read-timeout.test.ts (17 tests | 1 failed | 16 skipped) 82ms + × retries a reader setup that timed out instead of reading on the primary for good 81ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -267,8 +267,8 @@ AssertionError: expected 1 to be 2 // Object.is equality Test Files 1 failed (1) Tests 1 failed | 16 skipped (17) - Start at 23:37:19 - Duration 495ms (transform 161ms, setup 17ms, collect 270ms, tests 72ms, environment 0ms, prepare 41ms) + Start at 00:19:11 + Duration 533ms (transform 167ms, setup 15ms, collect 289ms, tests 82ms, environment 0ms, prepare 41ms) exit=1 @@ -277,12 +277,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 75ms + ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 72ms Test Files 1 passed (1) Tests 1 passed | 16 skipped (17) - Start at 23:37:20 - Duration 513ms (transform 164ms, setup 17ms, collect 287ms, tests 75ms, environment 0ms, prepare 40ms) + Start at 00:19:12 + Duration 510ms (transform 169ms, setup 22ms, collect 277ms, tests 72ms, environment 0ms, prepare 43ms) exit=0 ``` @@ -291,7 +291,7 @@ exit=0 ```text Source: packages/sdk/src/cli/running-step.ts -Original SHA256: a57da3dce86c9724d83bd3547a2d9186e05e7ea10409b743f89aaa25bf461bcd +Original SHA256: 5ae78a777c69e5f83a259df2f256305a0e9fd9c60525e661e66e3997d0f3801a Replaced: const LEASE_POLL_MS = 2_000; With: @@ -302,43 +302,43 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (4 tests | 1 failed | 3 skipped) 2110ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2110ms + ❯ tests/running-step-watch.test.ts (5 tests | 1 failed | 4 skipped) 2112ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2111ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/running-step-watch.test.ts > uses pushes for completion with lease-cadence reads and releases its watcher AssertionError: expected 41 to be less than or equal to 1 - ❯ tests/running-step-watch.test.ts:37:19 - 35| await waitForRunningStep(client, 'run', { id: 'step', type: 'agent… - 36| expect(performance.now() - start).toBeGreaterThan(2000); - 37| expect(reads).toBeLessThanOrEqual(1); + ❯ tests/running-step-watch.test.ts:39:19 + 37| await waitForRunningStep(client, 'run', { id: 'step', type: 'agent… + 38| expect(performance.now() - start).toBeGreaterThan(2000); + 39| expect(reads).toBeLessThanOrEqual(1); | ^ - 38| await sleep(10); - 39| expect(watchClosed).toBe(true); + 40| await sleep(10); + 41| expect(watchClosed).toBe(true); ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 3 skipped (4) - Start at 23:37:21 - Duration 2.57s (transform 168ms, setup 15ms, collect 294ms, tests 2.11s, environment 0ms, prepare 40ms) + Tests 1 failed | 4 skipped (5) + Start at 00:19:13 + Duration 2.59s (transform 188ms, setup 17ms, collect 315ms, tests 2.11s, environment 0ms, prepare 42ms) exit=1 -RESTORED SHA256: a57da3dce86c9724d83bd3547a2d9186e05e7ea10409b743f89aaa25bf461bcd +RESTORED SHA256: 5ae78a777c69e5f83a259df2f256305a0e9fd9c60525e661e66e3997d0f3801a $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pushes for completion' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (4 tests | 3 skipped) 2117ms + ✓ tests/running-step-watch.test.ts (5 tests | 4 skipped) 2117ms ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2116ms Test Files 1 passed (1) - Tests 1 passed | 3 skipped (4) - Start at 23:37:24 - Duration 2.57s (transform 165ms, setup 17ms, collect 291ms, tests 2.12s, environment 0ms, prepare 41ms) + Tests 1 passed | 4 skipped (5) + Start at 00:19:16 + Duration 2.60s (transform 184ms, setup 18ms, collect 324ms, tests 2.12s, environment 0ms, prepare 45ms) exit=0 ``` @@ -347,7 +347,7 @@ exit=0 ```text Source: packages/sdk/src/cli/running-step.ts -Original SHA256: a57da3dce86c9724d83bd3547a2d9186e05e7ea10409b743f89aaa25bf461bcd +Original SHA256: 5ae78a777c69e5f83a259df2f256305a0e9fd9c60525e661e66e3997d0f3801a Replaced: client.runGet(runId, options.signal === undefined ? {} : { signal: options.signal }) With: @@ -358,43 +358,43 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (4 tests | 1 failed | 3 skipped) 2215ms - × cancels promptly while a lease snapshot read is in flight 2214ms + ❯ tests/running-step-watch.test.ts (5 tests | 1 failed | 4 skipped) 2217ms + × cancels promptly while a lease snapshot read is in flight 2216ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/running-step-watch.test.ts > cancels promptly while a lease snapshot read is in flight AssertionError: expected 'still waiting' to be an instance of Error - ❯ tests/running-step-watch.test.ts:81:21 - 79| controller.abort(); - 80| const settled = await Promise.race([outcome, sleep(200).then(() =>… - 81| expect(settled).toBeInstanceOf(Error); + ❯ tests/running-step-watch.test.ts:83:21 + 81| controller.abort(); + 82| const settled = await Promise.race([outcome, sleep(200).then(() =>… + 83| expect(settled).toBeInstanceOf(Error); | ^ - 82| expect((settled as Error).message).toContain('was canceled'); - 83| } finally { client.close(); await new Promise(resolve => serve… + 84| expect((settled as Error).message).toContain('was canceled'); + 85| } finally { client.close(); await new Promise(resolve => serve… ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 3 skipped (4) - Start at 23:37:27 - Duration 2.65s (transform 164ms, setup 16ms, collect 281ms, tests 2.21s, environment 0ms, prepare 40ms) + Tests 1 failed | 4 skipped (5) + Start at 00:19:19 + Duration 2.74s (transform 201ms, setup 18ms, collect 339ms, tests 2.22s, environment 0ms, prepare 45ms) exit=1 -RESTORED SHA256: a57da3dce86c9724d83bd3547a2d9186e05e7ea10409b743f89aaa25bf461bcd +RESTORED SHA256: 5ae78a777c69e5f83a259df2f256305a0e9fd9c60525e661e66e3997d0f3801a $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels promptly while a lease snapshot read is in flight' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (4 tests | 3 skipped) 2010ms - ✓ cancels promptly while a lease snapshot read is in flight 2009ms + ✓ tests/running-step-watch.test.ts (5 tests | 4 skipped) 2012ms + ✓ cancels promptly while a lease snapshot read is in flight 2011ms Test Files 1 passed (1) - Tests 1 passed | 3 skipped (4) - Start at 23:37:30 - Duration 2.41s (transform 158ms, setup 15ms, collect 260ms, tests 2.01s, environment 0ms, prepare 34ms) + Tests 1 passed | 4 skipped (5) + Start at 00:19:22 + Duration 2.55s (transform 214ms, setup 23ms, collect 356ms, tests 2.01s, environment 0ms, prepare 45ms) exit=0 ``` @@ -417,8 +417,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 20ms - × a heartbeat timeout is lease loss rather than a worker body failure 20ms + ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 22ms + × a heartbeat timeout is lease loss rather than a worker body failure 21ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -437,8 +437,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 23:37:33 - Duration 464ms (transform 162ms, setup 16ms, collect 283ms, tests 20ms, environment 0ms, prepare 41ms) + Start at 00:19:25 + Duration 635ms (transform 246ms, setup 17ms, collect 437ms, tests 22ms, environment 0ms, prepare 43ms) exit=1 @@ -447,12 +447,12 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 19ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 23:37:33 - Duration 440ms (transform 163ms, setup 15ms, collect 272ms, tests 16ms, environment 0ms, prepare 37ms) + Start at 00:19:26 + Duration 695ms (transform 233ms, setup 20ms, collect 412ms, tests 19ms, environment 0ms, prepare 60ms) exit=0 ``` @@ -479,14 +479,14 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (27 tests | 3 failed | 24 skipped) 31ms - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 22ms + ❯ tests/authored-root.test.ts (27 tests | 3 failed | 24 skipped) 40ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 28ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 6ms → expected AuthoredFlowExecutionError: daemon_unresp… { …(7) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 4ms + × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 5ms → expected JournalReadInterruptedError: journal clie… { …(4) } to match object { code: 'daemon_unresponsive', …(2) } (5 matching properties omitted from actual) @@ -566,8 +566,8 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } Test Files 1 failed (1) Tests 3 failed | 24 skipped (27) - Start at 23:37:34 - Duration 1.28s (transform 656ms, setup 16ms, collect 1.08s, tests 31ms, environment 0ms, prepare 43ms) + Start at 00:19:27 + Duration 1.50s (transform 788ms, setup 20ms, collect 1.30s, tests 40ms, environment 0ms, prepare 33ms) exit=1 @@ -576,12 +576,12 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (27 tests | 24 skipped) 25ms + ✓ tests/authored-root.test.ts (27 tests | 24 skipped) 55ms Test Files 1 passed (1) Tests 3 passed | 24 skipped (27) - Start at 23:37:36 - Duration 1.24s (transform 647ms, setup 16ms, collect 1.06s, tests 25ms, environment 0ms, prepare 41ms) + Start at 00:19:29 + Duration 1.97s (transform 982ms, setup 20ms, collect 1.70s, tests 55ms, environment 0ms, prepare 54ms) exit=0 ``` diff --git a/packages/sdk/src/cli/running-step.ts b/packages/sdk/src/cli/running-step.ts index 3621c3ef8..2401d1de5 100644 --- a/packages/sdk/src/cli/running-step.ts +++ b/packages/sdk/src/cli/running-step.ts @@ -47,11 +47,14 @@ export async function waitForRunningStep(client: JournalClient, runId: string, options.signal?.addEventListener('abort', cancel, { once: true }); if (options.signal?.aborted) cancel(); try { - await Promise.race([watch.connect().then(() => watch.hello('flows-step-watch')).catch(error => { - if (error instanceof JournalProtocolError) throw error; - throw new AuthoredFlowExecutionError('daemon_unresponsive', - `could not establish the completion watch: ${error instanceof Error ? error.message : String(error)}`); - }).then(() => watch.runWatch(runId)), completed]); + // Every transport failure up to and including the watch registration is a + // read interruption: the step may still be running and the run resumable. + await Promise.race([watch.connect().then(() => watch.hello('flows-step-watch')) + .then(() => watch.runWatch(runId)).catch(error => { + if (error instanceof JournalProtocolError) throw error; + throw new AuthoredFlowExecutionError('daemon_unresponsive', + `could not establish the completion watch: ${error instanceof Error ? error.message : String(error)}`); + }), completed]); ready = true; if (settled) return; while (true) { diff --git a/packages/sdk/tests/running-step-watch.test.ts b/packages/sdk/tests/running-step-watch.test.ts index bba2347ed..37b982786 100644 --- a/packages/sdk/tests/running-step-watch.test.ts +++ b/packages/sdk/tests/running-step-watch.test.ts @@ -3,6 +3,8 @@ import { setTimeout as sleep } from 'node:timers/promises'; import { expect, it } from 'vitest'; import { JournalClient } from '../src/journal-client.js'; import { waitForRunningStep } from '../src/cli/running-step.js'; +import { isReadInterruption } from '../src/cli/journal-timeout.js'; +import { AuthoredFlowExecutionError } from '../src/authored-flow-error.js'; import { sendOk, sendResult, sockPath, startLoopback } from './journal-client-loopback.js'; it('uses pushes for completion with lease-cadence reads and releases its watcher', async () => { @@ -108,3 +110,23 @@ it('cancelling during the watch handshake leaves no unhandled rejection', async client.close(); await new Promise(resolve => server.close(() => resolve())); } }); + +it('treats a watch connection lost during registration as a read interruption', async () => { + const path = sockPath(); + let hellos = 0; + const server = startLoopback(path, { + hello: ctx => { hellos += 1; sendOk(ctx); }, + 'run.watch': ctx => { ctx.socket.destroy(); }, // registered hello, then the socket drops + }); + await once(server, 'listening'); + const client = new JournalClient(path); + try { + await client.connect(); + const error = await waitForRunningStep(client, 'run', { id: 'step', type: 'agent', leaseDeadlineMs: Date.now() + 30_000 }, {}) + .catch(caught => caught); + expect(error).toBeInstanceOf(AuthoredFlowExecutionError); + expect(error.code).toBe('daemon_unresponsive'); + expect(isReadInterruption(error)).toBe(true); + expect(hellos).toBe(1); + } finally { client.close(); await new Promise(resolve => server.close(() => resolve())); } +}); From 9ed7a6e71185297b480d6c8f5fb1804c176f39f2 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 00:42:53 -0700 Subject: [PATCH 07/34] docs(evidence): commit the mutation and kernel-build transcripts verification.md names *.log is gitignored, so the three new mutation logs and kernel-build.log were written but never tracked. Force-added; every transcript heading in verification.md now resolves to a committed file. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: b8262468-ea26-4a6e-bee9-d193832e77dc --- evidence/run-read-timeout/kernel-build.log | 3 + .../mutation-reader-reconnect.log | 61 ++++++++++++++++++ .../mutation-reader-setup-retry.log | 63 +++++++++++++++++++ .../mutation-snapshot-cancel.log | 51 +++++++++++++++ 4 files changed, 178 insertions(+) create mode 100644 evidence/run-read-timeout/kernel-build.log create mode 100644 evidence/run-read-timeout/mutation-reader-reconnect.log create mode 100644 evidence/run-read-timeout/mutation-reader-setup-retry.log create mode 100644 evidence/run-read-timeout/mutation-snapshot-cancel.log diff --git a/evidence/run-read-timeout/kernel-build.log b/evidence/run-read-timeout/kernel-build.log new file mode 100644 index 000000000..dbf40b19c --- /dev/null +++ b/evidence/run-read-timeout/kernel-build.log @@ -0,0 +1,3 @@ +$ cd kernel && CARGO_TARGET_DIR=/home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/kernel-target cargo build + Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.09s +exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-reconnect.log b/evidence/run-read-timeout/mutation-reader-reconnect.log new file mode 100644 index 000000000..1855b250c --- /dev/null +++ b/evidence/run-read-timeout/mutation-reader-reconnect.log @@ -0,0 +1,61 @@ +Source: packages/sdk/src/journal-client.ts +Original SHA256: e840ea971905def84fdecf0c3826430f058e0a88cf5650627fe0ca4fc4faa299 +Replaced: + this.dropReader(reader); + +With: + + +REVERTED +$ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'reconnects the reader after it disconnects' --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk + + ❯ tests/journal-client-read-timeout.test.ts (17 tests | 1 failed | 16 skipped) 1008ms + × reconnects the reader after it disconnects within the read budget 1007ms + → journal client: run.get read session was interrupted after 82 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + +⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget +JournalReadInterruptedError: journal client: run.get read session was interrupted after 82 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ exhausted src/journal-read-policy.ts:70:9 + 68| let last: unknown; + 69| const exhausted = () => last instanceof JournalReadInterruptedError + 70| ? new JournalReadInterruptedError(verb, attempts, performance.no… + | ^ + 71| : new JournalRequestTimeoutError(verb, timeoutMs, attempts, perf… + 72| // A caller's cancellation settles this read and drains its queued… + ❯ src/journal-read-policy.ts:99:32 + +Caused by: Error: journal client: not connected (run.get): journal client: connection closed + ❯ src/journal-client.ts:270:16 + ❯ JournalClient.requestOnce src/journal-client.ts:263:12 + ❯ src/journal-client.ts:245:31 + ❯ src/journal-read-policy.ts:93:18 + +Caused by: Error: journal client: connection closed + ❯ Socket. src/journal-client.ts:125:26 + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ + + Test Files 1 failed (1) + Tests 1 failed | 16 skipped (17) + Start at 00:19:08 + Duration 1.43s (transform 163ms, setup 17ms, collect 284ms, tests 1.01s, environment 0ms, prepare 33ms) + +exit=1 + +RESTORED SHA256: e840ea971905def84fdecf0c3826430f058e0a88cf5650627fe0ca4fc4faa299 +$ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'reconnects the reader after it disconnects' --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk + + ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 84ms + + Test Files 1 passed (1) + Tests 1 passed | 16 skipped (17) + Start at 00:19:10 + Duration 531ms (transform 170ms, setup 17ms, collect 290ms, tests 84ms, environment 0ms, prepare 42ms) + +exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-setup-retry.log b/evidence/run-read-timeout/mutation-reader-setup-retry.log new file mode 100644 index 000000000..989b1d9c0 --- /dev/null +++ b/evidence/run-read-timeout/mutation-reader-setup-retry.log @@ -0,0 +1,63 @@ +Source: packages/sdk/src/journal-client.ts +Original SHA256: e840ea971905def84fdecf0c3826430f058e0a88cf5650627fe0ca4fc4faa299 +Replaced: + if (this.reader === reader) { + this.reader = undefined; + this.readerReady = undefined; + } + throw error; + +With: + return undefined; + + +REVERTED +$ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'retries a reader setup that timed out' --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk + + ❯ tests/journal-client-read-timeout.test.ts (17 tests | 1 failed | 16 skipped) 82ms + × retries a reader setup that timed out instead of reading on the primary for good 81ms + → expected 1 to be 2 // Object.is equality + +⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/journal-client-read-timeout.test.ts > retries a reader setup that timed out instead of reading on the primary for good +AssertionError: expected 1 to be 2 // Object.is equality + +- Expected ++ Received + +- 2 ++ 1 + + ❯ tests/journal-client-read-timeout.test.ts:202:18 + 200| expect(await client.runGet('run')).toMatchObject({ status: 'complete… + 201| expect(await client.runGet('run')).toMatchObject({ status: 'complete… + 202| expect(hellos).toBe(2); + | ^ + 203| expect(served).toHaveLength(2); + 204| expect(served.every(socket => socket !== primary)).toBe(true); + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ + + Test Files 1 failed (1) + Tests 1 failed | 16 skipped (17) + Start at 00:19:11 + Duration 533ms (transform 167ms, setup 15ms, collect 289ms, tests 82ms, environment 0ms, prepare 41ms) + +exit=1 + +RESTORED SHA256: e840ea971905def84fdecf0c3826430f058e0a88cf5650627fe0ca4fc4faa299 +$ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'retries a reader setup that timed out' --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk + + ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 72ms + + Test Files 1 passed (1) + Tests 1 passed | 16 skipped (17) + Start at 00:19:12 + Duration 510ms (transform 169ms, setup 22ms, collect 277ms, tests 72ms, environment 0ms, prepare 43ms) + +exit=0 diff --git a/evidence/run-read-timeout/mutation-snapshot-cancel.log b/evidence/run-read-timeout/mutation-snapshot-cancel.log new file mode 100644 index 000000000..7b36b4f4f --- /dev/null +++ b/evidence/run-read-timeout/mutation-snapshot-cancel.log @@ -0,0 +1,51 @@ +Source: packages/sdk/src/cli/running-step.ts +Original SHA256: 5ae78a777c69e5f83a259df2f256305a0e9fd9c60525e661e66e3997d0f3801a +Replaced: +client.runGet(runId, options.signal === undefined ? {} : { signal: options.signal }) +With: +client.runGet(runId) + +REVERTED +$ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels promptly while a lease snapshot read is in flight' --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk + + ❯ tests/running-step-watch.test.ts (5 tests | 1 failed | 4 skipped) 2217ms + × cancels promptly while a lease snapshot read is in flight 2216ms + → expected 'still waiting' to be an instance of Error + +⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/running-step-watch.test.ts > cancels promptly while a lease snapshot read is in flight +AssertionError: expected 'still waiting' to be an instance of Error + ❯ tests/running-step-watch.test.ts:83:21 + 81| controller.abort(); + 82| const settled = await Promise.race([outcome, sleep(200).then(() =>… + 83| expect(settled).toBeInstanceOf(Error); + | ^ + 84| expect((settled as Error).message).toContain('was canceled'); + 85| } finally { client.close(); await new Promise(resolve => serve… + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ + + Test Files 1 failed (1) + Tests 1 failed | 4 skipped (5) + Start at 00:19:19 + Duration 2.74s (transform 201ms, setup 18ms, collect 339ms, tests 2.22s, environment 0ms, prepare 45ms) + +exit=1 + +RESTORED SHA256: 5ae78a777c69e5f83a259df2f256305a0e9fd9c60525e661e66e3997d0f3801a +$ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels promptly while a lease snapshot read is in flight' --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk + + ✓ tests/running-step-watch.test.ts (5 tests | 4 skipped) 2012ms + ✓ cancels promptly while a lease snapshot read is in flight 2011ms + + Test Files 1 passed (1) + Tests 1 passed | 4 skipped (5) + Start at 00:19:22 + Duration 2.55s (transform 214ms, setup 23ms, collect 356ms, tests 2.01s, environment 0ms, prepare 45ms) + +exit=0 From a2cff9d432a3a3dca2c27b8a3c57c71a1748504b Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 00:55:06 -0700 Subject: [PATCH 08/34] fix(sdk): a completion push aborts the lease snapshot in flight MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the watch observed step.completed while the periodic run.get was in flight, the wait still awaited that read — up to the 300 s budget under the CPU load this change targets. The snapshot now runs under a per-read abort the completion push triggers, so the wait returns at once and the read drains. Evidence re-captured at this tree (8/8 mutations fail-then-pass incl. the new completion-aborts-snapshot; focused 22 files 280/280; resume-live 1/1). Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: b8262468-ea26-4a6e-bee9-d193832e77dc --- evidence/run-read-timeout/final-focused.log | 95 +++--- evidence/run-read-timeout/kernel-build.log | 2 +- .../mutation-completion-aborts-snapshot.log | 55 ++++ .../run-read-timeout/mutation-heartbeat.log | 14 +- .../mutation-reader-reconnect.log | 16 +- .../mutation-reader-setup-retry.log | 14 +- evidence/run-read-timeout/mutation-reader.log | 14 +- .../mutation-root-parking.log | 18 +- .../mutation-snapshot-cancel.log | 29 +- .../mutation-watch-cadence.log | 24 +- evidence/run-read-timeout/mutations.py | 8 +- evidence/run-read-timeout/resume-live.log | 8 +- evidence/run-read-timeout/verification.md | 294 +++++++++++------- packages/sdk/src/cli/running-step.ts | 15 +- packages/sdk/tests/running-step-watch.test.ts | 23 ++ 15 files changed, 390 insertions(+), 239 deletions(-) create mode 100644 evidence/run-read-timeout/mutation-completion-aborts-snapshot.log diff --git a/evidence/run-read-timeout/final-focused.log b/evidence/run-read-timeout/final-focused.log index fd567eddc..6b5eab720 100644 --- a/evidence/run-read-timeout/final-focused.log +++ b/evidence/run-read-timeout/final-focused.log @@ -2,57 +2,56 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5624ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2057ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2050ms -(node:3018634) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5258ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2039ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2036ms +(node:3253842) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (27 tests) 414ms - ✓ tests/journal-client.test.ts (17 tests) 77ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 8994ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 596ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1638ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 3077ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 398ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 1204ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 3511ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 938ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 856ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 1026ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 691ms - ✓ tests/classify-outcome.test.ts (11 tests) 7431ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2014ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3005ms - ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms - ✓ tests/journal-client-read-timeout.test.ts (17 tests) 1474ms - ✓ a recovered read timeout does not become an authored callback failure 328ms - ✓ tests/human-live.test.ts (3 tests) 9715ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 7375ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1477ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 862ms - ✓ tests/worker-lease.test.ts (7 tests) 19ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 27ms - ✓ tests/running-step-watch.test.ts (5 tests) 4194ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms - ✓ cancels promptly while a lease snapshot read is in flight 2009ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 962ms - ✓ reports journal success after completion rejects with lease_conflict 429ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2477ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2062ms - ✓ drains read and watch promises before an authored flow completes 415ms + ✓ tests/authored-root.test.ts (27 tests) 392ms + ✓ tests/journal-client.test.ts (17 tests) 87ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 6781ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 487ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1068ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2618ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 338ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 738ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2536ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 617ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 635ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 633ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 650ms + ✓ tests/classify-outcome.test.ts (11 tests) 7428ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2013ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3007ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2002ms + ✓ tests/journal-client-read-timeout.test.ts (17 tests) 1407ms + ✓ tests/human-live.test.ts (3 tests) 5520ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 3662ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1130ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 728ms + ✓ tests/running-step-watch.test.ts (6 tests) 6235ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms + ✓ cancels promptly while a lease snapshot read is in flight 2006ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2054ms + ✓ tests/worker-lease.test.ts (7 tests) 21ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 26ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 654ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2496ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2053ms + ✓ drains read and watch promises before an authored flow completes 442ms ✓ tests/worker-lease-sweep.test.ts (4 tests) 6ms - ✓ tests/journal-client-completion.test.ts (6 tests) 103ms - ✓ tests/resume-worker-lease.test.ts (3 tests) 10ms - ✓ tests/direct-run-worker-lease.test.ts (3 tests) 14ms - ✓ tests/memoization.test.ts (57 tests) 62ms - ✓ tests/journal-client-subscriptions.test.ts (1 test) 6ms - ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 5ms - ✓ tests/reuse-summary-interruption.test.ts (3 tests) 7ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms + ✓ tests/journal-client-completion.test.ts (6 tests) 105ms + ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms + ✓ tests/direct-run-worker-lease.test.ts (3 tests) 9ms + ✓ tests/memoization.test.ts (57 tests) 55ms + ✓ tests/journal-client-subscriptions.test.ts (1 test) 7ms + ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 4ms + ✓ tests/reuse-summary-interruption.test.ts (3 tests) 4ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms Test Files 22 passed (22) - Tests 279 passed (279) - Start at 00:19:38 - Duration 56.60s (transform 1.38s, setup 185ms, collect 7.88s, tests 45.15s, environment 4ms, prepare 980ms) + Tests 280 passed (280) + Start at 00:54:04 + Duration 47.77s (transform 1.05s, setup 123ms, collect 5.93s, tests 39.05s, environment 3ms, prepare 786ms) exit=0 diff --git a/evidence/run-read-timeout/kernel-build.log b/evidence/run-read-timeout/kernel-build.log index dbf40b19c..bc4bf8a8f 100644 --- a/evidence/run-read-timeout/kernel-build.log +++ b/evidence/run-read-timeout/kernel-build.log @@ -1,3 +1,3 @@ $ cd kernel && CARGO_TARGET_DIR=/home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/kernel-target cargo build - Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.09s + Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.05s exit=0 diff --git a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log new file mode 100644 index 000000000..f7cc4d4dc --- /dev/null +++ b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log @@ -0,0 +1,55 @@ +Source: packages/sdk/src/cli/running-step.ts +Original SHA256: b84dc4f0d83e3f8c31d55a38f42a0d68c25299970d05ec7f3be772290f8d1a6e +Replaced: +snapshotRead?.abort(new Error(`step "${runningStep.id}" completed during the lease snapshot`)); +With: + + +REVERTED +$ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a completion push ends the wait' --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk + + ❯ tests/running-step-watch.test.ts (6 tests | 1 failed | 5 skipped) 3013ms + × a completion push ends the wait without waiting out an in-flight snapshot 3012ms + → expected 'still waiting' to be 'returned' // Object.is equality + +⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/running-step-watch.test.ts > a completion push ends the wait without waiting out an in-flight snapshot +AssertionError: expected 'still waiting' to be 'returned' // Object.is equality + +Expected: "returned" +Received: "still waiting" + + ❯ tests/running-step-watch.test.ts:152:21 + 150| const waiting = waitForRunningStep(client, 'run', { id: 'step', ty… + 151| const outcome = await Promise.race([waiting.then(() => 'returned')… + 152| expect(outcome).toBe('returned'); + | ^ + 153| expect(await client.runGet('run', { signal: AbortSignal.timeout(50… + 154| } finally { client.close(); await new Promise(resolve => serve… + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ + + Test Files 1 failed (1) + Tests 1 failed | 5 skipped (6) + Start at 00:53:49 + Duration 3.49s (transform 191ms, setup 16ms, collect 320ms, tests 3.01s, environment 0ms, prepare 41ms) + +exit=1 + +RESTORED SHA256: b84dc4f0d83e3f8c31d55a38f42a0d68c25299970d05ec7f3be772290f8d1a6e +$ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a completion push ends the wait' --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk + + ✓ tests/running-step-watch.test.ts (6 tests | 5 skipped) 2062ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2061ms + + Test Files 1 passed (1) + Tests 1 passed | 5 skipped (6) + Start at 00:53:52 + Duration 2.50s (transform 182ms, setup 16ms, collect 305ms, tests 2.06s, environment 0ms, prepare 29ms) + +exit=0 diff --git a/evidence/run-read-timeout/mutation-heartbeat.log b/evidence/run-read-timeout/mutation-heartbeat.log index d0cf1e839..8ffaef622 100644 --- a/evidence/run-read-timeout/mutation-heartbeat.log +++ b/evidence/run-read-timeout/mutation-heartbeat.log @@ -13,8 +13,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 22ms - × a heartbeat timeout is lease loss rather than a worker body failure 21ms + ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 18ms + × a heartbeat timeout is lease loss rather than a worker body failure 18ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -33,8 +33,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 00:19:25 - Duration 635ms (transform 246ms, setup 17ms, collect 437ms, tests 22ms, environment 0ms, prepare 43ms) + Start at 00:53:55 + Duration 453ms (transform 168ms, setup 18ms, collect 273ms, tests 18ms, environment 0ms, prepare 41ms) exit=1 @@ -43,11 +43,11 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/heartbeat-timeout.test.ts (1 test) 19ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 00:19:26 - Duration 695ms (transform 233ms, setup 20ms, collect 412ms, tests 19ms, environment 0ms, prepare 60ms) + Start at 00:53:56 + Duration 453ms (transform 165ms, setup 17ms, collect 278ms, tests 15ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-reconnect.log b/evidence/run-read-timeout/mutation-reader-reconnect.log index 1855b250c..a26cd31f1 100644 --- a/evidence/run-read-timeout/mutation-reader-reconnect.log +++ b/evidence/run-read-timeout/mutation-reader-reconnect.log @@ -13,12 +13,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t ❯ tests/journal-client-read-timeout.test.ts (17 tests | 1 failed | 16 skipped) 1008ms × reconnects the reader after it disconnects within the read budget 1007ms - → journal client: run.get read session was interrupted after 82 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + → journal client: run.get read session was interrupted after 80 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 82 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 80 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:70:9 68| let last: unknown; 69| const exhausted = () => last instanceof JournalReadInterruptedError @@ -26,7 +26,7 @@ JournalReadInterruptedError: journal client: run.get read session was interrupte | ^ 71| : new JournalRequestTimeoutError(verb, timeoutMs, attempts, perf… 72| // A caller's cancellation settles this read and drains its queued… - ❯ src/journal-read-policy.ts:99:32 + ❯ Timeout. src/journal-read-policy.ts:81:39 Caused by: Error: journal client: not connected (run.get): journal client: connection closed ❯ src/journal-client.ts:270:16 @@ -41,8 +41,8 @@ Caused by: Error: journal client: connection closed Test Files 1 failed (1) Tests 1 failed | 16 skipped (17) - Start at 00:19:08 - Duration 1.43s (transform 163ms, setup 17ms, collect 284ms, tests 1.01s, environment 0ms, prepare 33ms) + Start at 00:53:32 + Duration 1.46s (transform 166ms, setup 18ms, collect 286ms, tests 1.01s, environment 0ms, prepare 42ms) exit=1 @@ -51,11 +51,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 84ms + ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 78ms Test Files 1 passed (1) Tests 1 passed | 16 skipped (17) - Start at 00:19:10 - Duration 531ms (transform 170ms, setup 17ms, collect 290ms, tests 84ms, environment 0ms, prepare 42ms) + Start at 00:53:34 + Duration 522ms (transform 156ms, setup 16ms, collect 276ms, tests 78ms, environment 0ms, prepare 43ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-setup-retry.log b/evidence/run-read-timeout/mutation-reader-setup-retry.log index 989b1d9c0..941acdfed 100644 --- a/evidence/run-read-timeout/mutation-reader-setup-retry.log +++ b/evidence/run-read-timeout/mutation-reader-setup-retry.log @@ -16,8 +16,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (17 tests | 1 failed | 16 skipped) 82ms - × retries a reader setup that timed out instead of reading on the primary for good 81ms + ❯ tests/journal-client-read-timeout.test.ts (17 tests | 1 failed | 16 skipped) 73ms + × retries a reader setup that timed out instead of reading on the primary for good 72ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -43,8 +43,8 @@ AssertionError: expected 1 to be 2 // Object.is equality Test Files 1 failed (1) Tests 1 failed | 16 skipped (17) - Start at 00:19:11 - Duration 533ms (transform 167ms, setup 15ms, collect 289ms, tests 82ms, environment 0ms, prepare 41ms) + Start at 00:53:35 + Duration 523ms (transform 166ms, setup 18ms, collect 281ms, tests 73ms, environment 0ms, prepare 46ms) exit=1 @@ -53,11 +53,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 72ms + ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 67ms Test Files 1 passed (1) Tests 1 passed | 16 skipped (17) - Start at 00:19:12 - Duration 510ms (transform 169ms, setup 22ms, collect 277ms, tests 72ms, environment 0ms, prepare 43ms) + Start at 00:53:36 + Duration 520ms (transform 167ms, setup 17ms, collect 290ms, tests 67ms, environment 0ms, prepare 44ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader.log b/evidence/run-read-timeout/mutation-reader.log index 55f3f3ba4..a1036004b 100644 --- a/evidence/run-read-timeout/mutation-reader.log +++ b/evidence/run-read-timeout/mutation-reader.log @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (17 tests | 1 failed | 16 skipped) 114ms - × serves a bounded read while an unbounded command is in flight 113ms + ❯ tests/journal-client-read-timeout.test.ts (17 tests | 1 failed | 16 skipped) 115ms + × serves a bounded read while an unbounded command is in flight 114ms → expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -37,8 +37,8 @@ AssertionError: expected true to be false // Object.is equality Test Files 1 failed (1) Tests 1 failed | 16 skipped (17) - Start at 00:19:07 - Duration 556ms (transform 161ms, setup 17ms, collect 282ms, tests 114ms, environment 0ms, prepare 40ms) + Start at 00:53:30 + Duration 610ms (transform 180ms, setup 19ms, collect 302ms, tests 115ms, environment 0ms, prepare 50ms) exit=1 @@ -47,11 +47,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 105ms + ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 106ms Test Files 1 passed (1) Tests 1 passed | 16 skipped (17) - Start at 00:19:07 - Duration 543ms (transform 168ms, setup 16ms, collect 289ms, tests 105ms, environment 0ms, prepare 41ms) + Start at 00:53:31 + Duration 529ms (transform 161ms, setup 13ms, collect 262ms, tests 106ms, environment 0ms, prepare 43ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-root-parking.log b/evidence/run-read-timeout/mutation-root-parking.log index 6f81d5b35..1e3ed7fac 100644 --- a/evidence/run-read-timeout/mutation-root-parking.log +++ b/evidence/run-read-timeout/mutation-root-parking.log @@ -17,14 +17,14 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (27 tests | 3 failed | 24 skipped) 40ms - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 28ms + ❯ tests/authored-root.test.ts (27 tests | 3 failed | 24 skipped) 30ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 21ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 6ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms → expected AuthoredFlowExecutionError: daemon_unresp… { …(7) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 5ms + × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 4ms → expected JournalReadInterruptedError: journal clie… { …(4) } to match object { code: 'daemon_unresponsive', …(2) } (5 matching properties omitted from actual) @@ -104,8 +104,8 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } Test Files 1 failed (1) Tests 3 failed | 24 skipped (27) - Start at 00:19:27 - Duration 1.50s (transform 788ms, setup 20ms, collect 1.30s, tests 40ms, environment 0ms, prepare 33ms) + Start at 00:53:57 + Duration 1.24s (transform 648ms, setup 16ms, collect 1.06s, tests 30ms, environment 0ms, prepare 28ms) exit=1 @@ -114,11 +114,11 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (27 tests | 24 skipped) 55ms + ✓ tests/authored-root.test.ts (27 tests | 24 skipped) 23ms Test Files 1 passed (1) Tests 3 passed | 24 skipped (27) - Start at 00:19:29 - Duration 1.97s (transform 982ms, setup 20ms, collect 1.70s, tests 55ms, environment 0ms, prepare 54ms) + Start at 00:53:59 + Duration 1.22s (transform 650ms, setup 16ms, collect 1.04s, tests 23ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-snapshot-cancel.log b/evidence/run-read-timeout/mutation-snapshot-cancel.log index 7b36b4f4f..af5b5a0d0 100644 --- a/evidence/run-read-timeout/mutation-snapshot-cancel.log +++ b/evidence/run-read-timeout/mutation-snapshot-cancel.log @@ -1,17 +1,18 @@ Source: packages/sdk/src/cli/running-step.ts -Original SHA256: 5ae78a777c69e5f83a259df2f256305a0e9fd9c60525e661e66e3997d0f3801a +Original SHA256: b84dc4f0d83e3f8c31d55a38f42a0d68c25299970d05ec7f3be772290f8d1a6e Replaced: -client.runGet(runId, options.signal === undefined ? {} : { signal: options.signal }) +const signal = options.signal === undefined ? snapshotRead.signal + : AbortSignal.any([options.signal, snapshotRead.signal]); With: -client.runGet(runId) +const signal = snapshotRead.signal; REVERTED $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels promptly while a lease snapshot read is in flight' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (5 tests | 1 failed | 4 skipped) 2217ms - × cancels promptly while a lease snapshot read is in flight 2216ms + ❯ tests/running-step-watch.test.ts (6 tests | 1 failed | 5 skipped) 2215ms + × cancels promptly while a lease snapshot read is in flight 2215ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -29,23 +30,23 @@ AssertionError: expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 4 skipped (5) - Start at 00:19:19 - Duration 2.74s (transform 201ms, setup 18ms, collect 339ms, tests 2.22s, environment 0ms, prepare 45ms) + Tests 1 failed | 5 skipped (6) + Start at 00:53:43 + Duration 2.68s (transform 182ms, setup 17ms, collect 307ms, tests 2.22s, environment 0ms, prepare 41ms) exit=1 -RESTORED SHA256: 5ae78a777c69e5f83a259df2f256305a0e9fd9c60525e661e66e3997d0f3801a +RESTORED SHA256: b84dc4f0d83e3f8c31d55a38f42a0d68c25299970d05ec7f3be772290f8d1a6e $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels promptly while a lease snapshot read is in flight' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (5 tests | 4 skipped) 2012ms - ✓ cancels promptly while a lease snapshot read is in flight 2011ms + ✓ tests/running-step-watch.test.ts (6 tests | 5 skipped) 2011ms + ✓ cancels promptly while a lease snapshot read is in flight 2010ms Test Files 1 passed (1) - Tests 1 passed | 4 skipped (5) - Start at 00:19:22 - Duration 2.55s (transform 214ms, setup 23ms, collect 356ms, tests 2.01s, environment 0ms, prepare 45ms) + Tests 1 passed | 5 skipped (6) + Start at 00:53:46 + Duration 2.48s (transform 183ms, setup 16ms, collect 311ms, tests 2.01s, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-watch-cadence.log b/evidence/run-read-timeout/mutation-watch-cadence.log index d4e434706..de2092581 100644 --- a/evidence/run-read-timeout/mutation-watch-cadence.log +++ b/evidence/run-read-timeout/mutation-watch-cadence.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/cli/running-step.ts -Original SHA256: 5ae78a777c69e5f83a259df2f256305a0e9fd9c60525e661e66e3997d0f3801a +Original SHA256: b84dc4f0d83e3f8c31d55a38f42a0d68c25299970d05ec7f3be772290f8d1a6e Replaced: const LEASE_POLL_MS = 2_000; With: @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (5 tests | 1 failed | 4 skipped) 2112ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2111ms + ❯ tests/running-step-watch.test.ts (6 tests | 1 failed | 5 skipped) 2111ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2110ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -29,23 +29,23 @@ AssertionError: expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 4 skipped (5) - Start at 00:19:13 - Duration 2.59s (transform 188ms, setup 17ms, collect 315ms, tests 2.11s, environment 0ms, prepare 42ms) + Tests 1 failed | 5 skipped (6) + Start at 00:53:37 + Duration 2.59s (transform 186ms, setup 17ms, collect 316ms, tests 2.11s, environment 0ms, prepare 43ms) exit=1 -RESTORED SHA256: 5ae78a777c69e5f83a259df2f256305a0e9fd9c60525e661e66e3997d0f3801a +RESTORED SHA256: b84dc4f0d83e3f8c31d55a38f42a0d68c25299970d05ec7f3be772290f8d1a6e $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pushes for completion' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (5 tests | 4 skipped) 2117ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2116ms + ✓ tests/running-step-watch.test.ts (6 tests | 5 skipped) 2118ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms Test Files 1 passed (1) - Tests 1 passed | 4 skipped (5) - Start at 00:19:16 - Duration 2.60s (transform 184ms, setup 18ms, collect 324ms, tests 2.12s, environment 0ms, prepare 45ms) + Tests 1 passed | 5 skipped (6) + Start at 00:53:40 + Duration 2.59s (transform 187ms, setup 16ms, collect 316ms, tests 2.12s, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/mutations.py b/evidence/run-read-timeout/mutations.py index 8efcbfdf1..be6ace558 100644 --- a/evidence/run-read-timeout/mutations.py +++ b/evidence/run-read-timeout/mutations.py @@ -26,9 +26,13 @@ 'const LEASE_POLL_MS = 2_000;', 'const LEASE_POLL_MS = 50;', 'tests/running-step-watch.test.ts', 'uses pushes for completion'), ('snapshot-cancel', 'packages/sdk/src/cli/running-step.ts', - 'client.runGet(runId, options.signal === undefined ? {} : { signal: options.signal })', - 'client.runGet(runId)', + 'const signal = options.signal === undefined ? snapshotRead.signal\n : AbortSignal.any([options.signal, snapshotRead.signal]);', + 'const signal = snapshotRead.signal;', 'tests/running-step-watch.test.ts', 'cancels promptly while a lease snapshot read is in flight'), + ('completion-aborts-snapshot', 'packages/sdk/src/cli/running-step.ts', + 'snapshotRead?.abort(new Error(`step "${runningStep.id}" completed during the lease snapshot`));', + '', + 'tests/running-step-watch.test.ts', 'a completion push ends the wait'), ('heartbeat', 'packages/sdk/src/worker-lease.ts', """ if (error instanceof JournalRequestTimeoutError && error.verb === 'step.heartbeat') { throw new WorkerLeaseLostError('renewal_expired', error.message, { cause: error }); diff --git a/evidence/run-read-timeout/resume-live.log b/evidence/run-read-timeout/resume-live.log index 013a8a8dc..df36c5119 100644 --- a/evidence/run-read-timeout/resume-live.log +++ b/evidence/run-read-timeout/resume-live.log @@ -2,12 +2,12 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 1110ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 1109ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 942ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 942ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 00:20:35 - Duration 2.81s (transform 802ms, setup 28ms, collect 1.35s, tests 1.11s, environment 0ms, prepare 56ms) + Start at 00:54:52 + Duration 2.21s (transform 624ms, setup 19ms, collect 1.02s, tests 942ms, environment 0ms, prepare 82ms) exit=0 diff --git a/evidence/run-read-timeout/verification.md b/evidence/run-read-timeout/verification.md index d2e17c0ce..ea3b3c9c0 100644 --- a/evidence/run-read-timeout/verification.md +++ b/evidence/run-read-timeout/verification.md @@ -4,7 +4,7 @@ Captured verification at the review-fix source. Commands and output below are ve ```text $ cd kernel && CARGO_TARGET_DIR=/home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/kernel-target cargo build - Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.09s + Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.05s exit=0 ``` @@ -15,58 +15,57 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5624ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2057ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2050ms -(node:3018634) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5258ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2039ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2036ms +(node:3253842) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (27 tests) 414ms - ✓ tests/journal-client.test.ts (17 tests) 77ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 8994ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 596ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1638ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 3077ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 398ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 1204ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 3511ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 938ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 856ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 1026ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 691ms - ✓ tests/classify-outcome.test.ts (11 tests) 7431ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2014ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3005ms - ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms - ✓ tests/journal-client-read-timeout.test.ts (17 tests) 1474ms - ✓ a recovered read timeout does not become an authored callback failure 328ms - ✓ tests/human-live.test.ts (3 tests) 9715ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 7375ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1477ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 862ms - ✓ tests/worker-lease.test.ts (7 tests) 19ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 27ms - ✓ tests/running-step-watch.test.ts (5 tests) 4194ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms - ✓ cancels promptly while a lease snapshot read is in flight 2009ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 962ms - ✓ reports journal success after completion rejects with lease_conflict 429ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2477ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2062ms - ✓ drains read and watch promises before an authored flow completes 415ms + ✓ tests/authored-root.test.ts (27 tests) 392ms + ✓ tests/journal-client.test.ts (17 tests) 87ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 6781ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 487ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1068ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2618ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 338ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 738ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2536ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 617ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 635ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 633ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 650ms + ✓ tests/classify-outcome.test.ts (11 tests) 7428ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2013ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3007ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2002ms + ✓ tests/journal-client-read-timeout.test.ts (17 tests) 1407ms + ✓ tests/human-live.test.ts (3 tests) 5520ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 3662ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1130ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 728ms + ✓ tests/running-step-watch.test.ts (6 tests) 6235ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms + ✓ cancels promptly while a lease snapshot read is in flight 2006ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2054ms + ✓ tests/worker-lease.test.ts (7 tests) 21ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 26ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 654ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2496ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2053ms + ✓ drains read and watch promises before an authored flow completes 442ms ✓ tests/worker-lease-sweep.test.ts (4 tests) 6ms - ✓ tests/journal-client-completion.test.ts (6 tests) 103ms - ✓ tests/resume-worker-lease.test.ts (3 tests) 10ms - ✓ tests/direct-run-worker-lease.test.ts (3 tests) 14ms - ✓ tests/memoization.test.ts (57 tests) 62ms - ✓ tests/journal-client-subscriptions.test.ts (1 test) 6ms - ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 5ms - ✓ tests/reuse-summary-interruption.test.ts (3 tests) 7ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms + ✓ tests/journal-client-completion.test.ts (6 tests) 105ms + ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms + ✓ tests/direct-run-worker-lease.test.ts (3 tests) 9ms + ✓ tests/memoization.test.ts (57 tests) 55ms + ✓ tests/journal-client-subscriptions.test.ts (1 test) 7ms + ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 4ms + ✓ tests/reuse-summary-interruption.test.ts (3 tests) 4ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms Test Files 22 passed (22) - Tests 279 passed (279) - Start at 00:19:38 - Duration 56.60s (transform 1.38s, setup 185ms, collect 7.88s, tests 45.15s, environment 4ms, prepare 980ms) + Tests 280 passed (280) + Start at 00:54:04 + Duration 47.77s (transform 1.05s, setup 123ms, collect 5.93s, tests 39.05s, environment 3ms, prepare 786ms) exit=0 ``` @@ -78,13 +77,13 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 1110ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 1109ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 942ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 942ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 00:20:35 - Duration 2.81s (transform 802ms, setup 28ms, collect 1.35s, tests 1.11s, environment 0ms, prepare 56ms) + Start at 00:54:52 + Duration 2.21s (transform 624ms, setup 19ms, collect 1.02s, tests 942ms, environment 0ms, prepare 82ms) exit=0 ``` @@ -106,8 +105,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (17 tests | 1 failed | 16 skipped) 114ms - × serves a bounded read while an unbounded command is in flight 113ms + ❯ tests/journal-client-read-timeout.test.ts (17 tests | 1 failed | 16 skipped) 115ms + × serves a bounded read while an unbounded command is in flight 114ms → expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -133,8 +132,8 @@ AssertionError: expected true to be false // Object.is equality Test Files 1 failed (1) Tests 1 failed | 16 skipped (17) - Start at 00:19:07 - Duration 556ms (transform 161ms, setup 17ms, collect 282ms, tests 114ms, environment 0ms, prepare 40ms) + Start at 00:53:30 + Duration 610ms (transform 180ms, setup 19ms, collect 302ms, tests 115ms, environment 0ms, prepare 50ms) exit=1 @@ -143,12 +142,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 105ms + ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 106ms Test Files 1 passed (1) Tests 1 passed | 16 skipped (17) - Start at 00:19:07 - Duration 543ms (transform 168ms, setup 16ms, collect 289ms, tests 105ms, environment 0ms, prepare 41ms) + Start at 00:53:31 + Duration 529ms (transform 161ms, setup 13ms, collect 262ms, tests 106ms, environment 0ms, prepare 43ms) exit=0 ``` @@ -171,12 +170,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t ❯ tests/journal-client-read-timeout.test.ts (17 tests | 1 failed | 16 skipped) 1008ms × reconnects the reader after it disconnects within the read budget 1007ms - → journal client: run.get read session was interrupted after 82 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + → journal client: run.get read session was interrupted after 80 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 82 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 80 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:70:9 68| let last: unknown; 69| const exhausted = () => last instanceof JournalReadInterruptedError @@ -184,7 +183,7 @@ JournalReadInterruptedError: journal client: run.get read session was interrupte | ^ 71| : new JournalRequestTimeoutError(verb, timeoutMs, attempts, perf… 72| // A caller's cancellation settles this read and drains its queued… - ❯ src/journal-read-policy.ts:99:32 + ❯ Timeout. src/journal-read-policy.ts:81:39 Caused by: Error: journal client: not connected (run.get): journal client: connection closed ❯ src/journal-client.ts:270:16 @@ -199,8 +198,8 @@ Caused by: Error: journal client: connection closed Test Files 1 failed (1) Tests 1 failed | 16 skipped (17) - Start at 00:19:08 - Duration 1.43s (transform 163ms, setup 17ms, collect 284ms, tests 1.01s, environment 0ms, prepare 33ms) + Start at 00:53:32 + Duration 1.46s (transform 166ms, setup 18ms, collect 286ms, tests 1.01s, environment 0ms, prepare 42ms) exit=1 @@ -209,12 +208,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 84ms + ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 78ms Test Files 1 passed (1) Tests 1 passed | 16 skipped (17) - Start at 00:19:10 - Duration 531ms (transform 170ms, setup 17ms, collect 290ms, tests 84ms, environment 0ms, prepare 42ms) + Start at 00:53:34 + Duration 522ms (transform 156ms, setup 16ms, collect 276ms, tests 78ms, environment 0ms, prepare 43ms) exit=0 ``` @@ -240,8 +239,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (17 tests | 1 failed | 16 skipped) 82ms - × retries a reader setup that timed out instead of reading on the primary for good 81ms + ❯ tests/journal-client-read-timeout.test.ts (17 tests | 1 failed | 16 skipped) 73ms + × retries a reader setup that timed out instead of reading on the primary for good 72ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -267,8 +266,8 @@ AssertionError: expected 1 to be 2 // Object.is equality Test Files 1 failed (1) Tests 1 failed | 16 skipped (17) - Start at 00:19:11 - Duration 533ms (transform 167ms, setup 15ms, collect 289ms, tests 82ms, environment 0ms, prepare 41ms) + Start at 00:53:35 + Duration 523ms (transform 166ms, setup 18ms, collect 281ms, tests 73ms, environment 0ms, prepare 46ms) exit=1 @@ -277,12 +276,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 72ms + ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 67ms Test Files 1 passed (1) Tests 1 passed | 16 skipped (17) - Start at 00:19:12 - Duration 510ms (transform 169ms, setup 22ms, collect 277ms, tests 72ms, environment 0ms, prepare 43ms) + Start at 00:53:36 + Duration 520ms (transform 167ms, setup 17ms, collect 290ms, tests 67ms, environment 0ms, prepare 44ms) exit=0 ``` @@ -291,7 +290,7 @@ exit=0 ```text Source: packages/sdk/src/cli/running-step.ts -Original SHA256: 5ae78a777c69e5f83a259df2f256305a0e9fd9c60525e661e66e3997d0f3801a +Original SHA256: b84dc4f0d83e3f8c31d55a38f42a0d68c25299970d05ec7f3be772290f8d1a6e Replaced: const LEASE_POLL_MS = 2_000; With: @@ -302,8 +301,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (5 tests | 1 failed | 4 skipped) 2112ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2111ms + ❯ tests/running-step-watch.test.ts (6 tests | 1 failed | 5 skipped) 2111ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2110ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -321,24 +320,24 @@ AssertionError: expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 4 skipped (5) - Start at 00:19:13 - Duration 2.59s (transform 188ms, setup 17ms, collect 315ms, tests 2.11s, environment 0ms, prepare 42ms) + Tests 1 failed | 5 skipped (6) + Start at 00:53:37 + Duration 2.59s (transform 186ms, setup 17ms, collect 316ms, tests 2.11s, environment 0ms, prepare 43ms) exit=1 -RESTORED SHA256: 5ae78a777c69e5f83a259df2f256305a0e9fd9c60525e661e66e3997d0f3801a +RESTORED SHA256: b84dc4f0d83e3f8c31d55a38f42a0d68c25299970d05ec7f3be772290f8d1a6e $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pushes for completion' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (5 tests | 4 skipped) 2117ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2116ms + ✓ tests/running-step-watch.test.ts (6 tests | 5 skipped) 2118ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms Test Files 1 passed (1) - Tests 1 passed | 4 skipped (5) - Start at 00:19:16 - Duration 2.60s (transform 184ms, setup 18ms, collect 324ms, tests 2.12s, environment 0ms, prepare 45ms) + Tests 1 passed | 5 skipped (6) + Start at 00:53:40 + Duration 2.59s (transform 187ms, setup 16ms, collect 316ms, tests 2.12s, environment 0ms, prepare 40ms) exit=0 ``` @@ -347,19 +346,20 @@ exit=0 ```text Source: packages/sdk/src/cli/running-step.ts -Original SHA256: 5ae78a777c69e5f83a259df2f256305a0e9fd9c60525e661e66e3997d0f3801a +Original SHA256: b84dc4f0d83e3f8c31d55a38f42a0d68c25299970d05ec7f3be772290f8d1a6e Replaced: -client.runGet(runId, options.signal === undefined ? {} : { signal: options.signal }) +const signal = options.signal === undefined ? snapshotRead.signal + : AbortSignal.any([options.signal, snapshotRead.signal]); With: -client.runGet(runId) +const signal = snapshotRead.signal; REVERTED $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels promptly while a lease snapshot read is in flight' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (5 tests | 1 failed | 4 skipped) 2217ms - × cancels promptly while a lease snapshot read is in flight 2216ms + ❯ tests/running-step-watch.test.ts (6 tests | 1 failed | 5 skipped) 2215ms + × cancels promptly while a lease snapshot read is in flight 2215ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -377,24 +377,84 @@ AssertionError: expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 4 skipped (5) - Start at 00:19:19 - Duration 2.74s (transform 201ms, setup 18ms, collect 339ms, tests 2.22s, environment 0ms, prepare 45ms) + Tests 1 failed | 5 skipped (6) + Start at 00:53:43 + Duration 2.68s (transform 182ms, setup 17ms, collect 307ms, tests 2.22s, environment 0ms, prepare 41ms) exit=1 -RESTORED SHA256: 5ae78a777c69e5f83a259df2f256305a0e9fd9c60525e661e66e3997d0f3801a +RESTORED SHA256: b84dc4f0d83e3f8c31d55a38f42a0d68c25299970d05ec7f3be772290f8d1a6e $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels promptly while a lease snapshot read is in flight' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (5 tests | 4 skipped) 2012ms - ✓ cancels promptly while a lease snapshot read is in flight 2011ms + ✓ tests/running-step-watch.test.ts (6 tests | 5 skipped) 2011ms + ✓ cancels promptly while a lease snapshot read is in flight 2010ms + + Test Files 1 passed (1) + Tests 1 passed | 5 skipped (6) + Start at 00:53:46 + Duration 2.48s (transform 183ms, setup 16ms, collect 311ms, tests 2.01s, environment 0ms, prepare 41ms) + +exit=0 +``` + +## mutation-completion-aborts-snapshot.log + +```text +Source: packages/sdk/src/cli/running-step.ts +Original SHA256: b84dc4f0d83e3f8c31d55a38f42a0d68c25299970d05ec7f3be772290f8d1a6e +Replaced: +snapshotRead?.abort(new Error(`step "${runningStep.id}" completed during the lease snapshot`)); +With: + + +REVERTED +$ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a completion push ends the wait' --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk + + ❯ tests/running-step-watch.test.ts (6 tests | 1 failed | 5 skipped) 3013ms + × a completion push ends the wait without waiting out an in-flight snapshot 3012ms + → expected 'still waiting' to be 'returned' // Object.is equality + +⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/running-step-watch.test.ts > a completion push ends the wait without waiting out an in-flight snapshot +AssertionError: expected 'still waiting' to be 'returned' // Object.is equality + +Expected: "returned" +Received: "still waiting" + + ❯ tests/running-step-watch.test.ts:152:21 + 150| const waiting = waitForRunningStep(client, 'run', { id: 'step', ty… + 151| const outcome = await Promise.race([waiting.then(() => 'returned')… + 152| expect(outcome).toBe('returned'); + | ^ + 153| expect(await client.runGet('run', { signal: AbortSignal.timeout(50… + 154| } finally { client.close(); await new Promise(resolve => serve… + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ + + Test Files 1 failed (1) + Tests 1 failed | 5 skipped (6) + Start at 00:53:49 + Duration 3.49s (transform 191ms, setup 16ms, collect 320ms, tests 3.01s, environment 0ms, prepare 41ms) + +exit=1 + +RESTORED SHA256: b84dc4f0d83e3f8c31d55a38f42a0d68c25299970d05ec7f3be772290f8d1a6e +$ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a completion push ends the wait' --maxWorkers=1 --minWorkers=1 + + RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk + + ✓ tests/running-step-watch.test.ts (6 tests | 5 skipped) 2062ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2061ms Test Files 1 passed (1) - Tests 1 passed | 4 skipped (5) - Start at 00:19:22 - Duration 2.55s (transform 214ms, setup 23ms, collect 356ms, tests 2.01s, environment 0ms, prepare 45ms) + Tests 1 passed | 5 skipped (6) + Start at 00:53:52 + Duration 2.50s (transform 182ms, setup 16ms, collect 305ms, tests 2.06s, environment 0ms, prepare 29ms) exit=0 ``` @@ -417,8 +477,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 22ms - × a heartbeat timeout is lease loss rather than a worker body failure 21ms + ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 18ms + × a heartbeat timeout is lease loss rather than a worker body failure 18ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -437,8 +497,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 00:19:25 - Duration 635ms (transform 246ms, setup 17ms, collect 437ms, tests 22ms, environment 0ms, prepare 43ms) + Start at 00:53:55 + Duration 453ms (transform 168ms, setup 18ms, collect 273ms, tests 18ms, environment 0ms, prepare 41ms) exit=1 @@ -447,12 +507,12 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/heartbeat-timeout.test.ts (1 test) 19ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 00:19:26 - Duration 695ms (transform 233ms, setup 20ms, collect 412ms, tests 19ms, environment 0ms, prepare 60ms) + Start at 00:53:56 + Duration 453ms (transform 165ms, setup 17ms, collect 278ms, tests 15ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -479,14 +539,14 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (27 tests | 3 failed | 24 skipped) 40ms - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 28ms + ❯ tests/authored-root.test.ts (27 tests | 3 failed | 24 skipped) 30ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 21ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 6ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms → expected AuthoredFlowExecutionError: daemon_unresp… { …(7) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 5ms + × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 4ms → expected JournalReadInterruptedError: journal clie… { …(4) } to match object { code: 'daemon_unresponsive', …(2) } (5 matching properties omitted from actual) @@ -566,8 +626,8 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } Test Files 1 failed (1) Tests 3 failed | 24 skipped (27) - Start at 00:19:27 - Duration 1.50s (transform 788ms, setup 20ms, collect 1.30s, tests 40ms, environment 0ms, prepare 33ms) + Start at 00:53:57 + Duration 1.24s (transform 648ms, setup 16ms, collect 1.06s, tests 30ms, environment 0ms, prepare 28ms) exit=1 @@ -576,12 +636,12 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (27 tests | 24 skipped) 55ms + ✓ tests/authored-root.test.ts (27 tests | 24 skipped) 23ms Test Files 1 passed (1) Tests 3 passed | 24 skipped (27) - Start at 00:19:29 - Duration 1.97s (transform 982ms, setup 20ms, collect 1.70s, tests 55ms, environment 0ms, prepare 54ms) + Start at 00:53:59 + Duration 1.22s (transform 650ms, setup 16ms, collect 1.04s, tests 23ms, environment 0ms, prepare 41ms) exit=0 ``` diff --git a/packages/sdk/src/cli/running-step.ts b/packages/sdk/src/cli/running-step.ts index 2401d1de5..5d43b67c3 100644 --- a/packages/sdk/src/cli/running-step.ts +++ b/packages/sdk/src/cli/running-step.ts @@ -33,6 +33,8 @@ export async function waitForRunningStep(client: JournalClient, runId: string, }); // The subscription handshake itself may be pending when cancellation arrives. void completed.catch(() => {}); + // A completion push aborts the lease snapshot in flight: the answer is known. + let snapshotRead: AbortController | undefined; const onEntry = (entry: JournalEvent) => { if (entry.run_id !== runId) return; if (entry.entry_type === 'run.completed') settled = true; @@ -41,7 +43,10 @@ export async function waitForRunningStep(client: JournalClient, runId: string, if (entry.entry_type === 'step.completed' || entry.entry_type === 'wait.human') settled = true; } // Fold the entire replay before deciding: an older attempt may have failed. - if (ready && settled) finish(); + if (ready && settled) { + finish(); + snapshotRead?.abort(new Error(`step "${runningStep.id}" completed during the lease snapshot`)); + } }; watch.on('entry', onEntry); options.signal?.addEventListener('abort', cancel, { once: true }); @@ -70,12 +75,16 @@ export async function waitForRunningStep(client: JournalClient, runId: string, // Drain the read before leaving the authored promise scope. A raced but // unresolved read would look like unawaited derived work to that scope, // so cancellation aborts the read itself and it settles at once. - const snapshot = await client.runGet(runId, options.signal === undefined ? {} : { signal: options.signal }) + snapshotRead = new AbortController(); + const signal = options.signal === undefined ? snapshotRead.signal + : AbortSignal.any([options.signal, snapshotRead.signal]); + const snapshot = await client.runGet(runId, { signal }) .catch(error => { if (options.signal?.aborted) throw canceled(); if (settled) return undefined; throw error; - }); + }) + .finally(() => { snapshotRead = undefined; }); if (options.signal?.aborted) throw canceled(); if (settled || snapshot === undefined || snapshot.steps[runningStep.id]?.state !== 'running') return; const deadline = snapshot.steps[runningStep.id]?.lease_deadline_ms; diff --git a/packages/sdk/tests/running-step-watch.test.ts b/packages/sdk/tests/running-step-watch.test.ts index 37b982786..6bd4cad10 100644 --- a/packages/sdk/tests/running-step-watch.test.ts +++ b/packages/sdk/tests/running-step-watch.test.ts @@ -130,3 +130,26 @@ it('treats a watch connection lost during registration as a read interruption', expect(hellos).toBe(1); } finally { client.close(); await new Promise(resolve => server.close(() => resolve())); } }); + +it('a completion push ends the wait without waiting out an in-flight snapshot', async () => { + const path = sockPath(); + let push!: () => void; + const server = startLoopback(path, { + hello: sendOk, + 'run.watch': (ctx, params) => { + sendResult(ctx, {}); + push = () => ctx.send({ event: 'entry', data: { run_id: String(params['run_id']), step_id: 'step', entry_type: 'step.completed' } }); + }, + // The snapshot never answers; the completion arrives while it is in flight. + 'run.get': () => push(), + }); + await once(server, 'listening'); + const client = new JournalClient(path, { readBudgetMs: 300_000 }); + try { + await client.connect(); + const waiting = waitForRunningStep(client, 'run', { id: 'step', type: 'agent', leaseDeadlineMs: Date.now() + 30_000 }, {}); + const outcome = await Promise.race([waiting.then(() => 'returned'), sleep(3_000).then(() => 'still waiting')]); + expect(outcome).toBe('returned'); + expect(await client.runGet('run', { signal: AbortSignal.timeout(50) }).catch(error => error.name)).toBe('TimeoutError'); + } finally { client.close(); await new Promise(resolve => server.close(() => resolve())); } +}); From fb4524d7823aa15584072bc25ac18a57107e5a66 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 01:06:24 -0700 Subject: [PATCH 09/34] fix(sdk): a snapshot aborted by completion ends the wait; verifier reads get the flow budget - running-step: when the completion push aborts the in-flight snapshot, the wait returns even if step.attempt.started arrives in the same turn; it no longer rethrows its own abort as an error (finish() is one-shot and the caller re-inspects the run). - authored-node-runner: verifyAuthoredNodeResult reads the journal with FLOW_READ_BUDGET_MS like the body it verifies, so a CPU-delayed read under Bun retries within the 300 s budget instead of failing after one 30 s try. Evidence re-captured at this tree (8/8 mutations; focused 23 files 282/282; resume-live 1/1). Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: b8262468-ea26-4a6e-bee9-d193832e77dc --- evidence/run-read-timeout/final-focused.log | 88 +++---- evidence/run-read-timeout/kernel-build.log | 2 +- .../mutation-completion-aborts-snapshot.log | 24 +- .../run-read-timeout/mutation-heartbeat.log | 8 +- .../mutation-reader-reconnect.log | 14 +- .../mutation-reader-setup-retry.log | 14 +- evidence/run-read-timeout/mutation-reader.log | 14 +- .../mutation-root-parking.log | 10 +- .../mutation-snapshot-cancel.log | 28 +-- .../mutation-watch-cadence.log | 20 +- evidence/run-read-timeout/mutations.py | 4 +- evidence/run-read-timeout/resume-live.log | 8 +- evidence/run-read-timeout/verification.md | 230 +++++++++--------- packages/sdk/src/authored-node-runner.ts | 4 +- packages/sdk/src/cli/running-step.ts | 10 +- .../authored-verifier-read-budget.test.ts | 23 ++ packages/sdk/tests/running-step-watch.test.ts | 25 ++ 17 files changed, 291 insertions(+), 235 deletions(-) create mode 100644 packages/sdk/tests/authored-verifier-read-budget.test.ts diff --git a/evidence/run-read-timeout/final-focused.log b/evidence/run-read-timeout/final-focused.log index 6b5eab720..9c7a4fb73 100644 --- a/evidence/run-read-timeout/final-focused.log +++ b/evidence/run-read-timeout/final-focused.log @@ -1,57 +1,59 @@ -$ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/kernel-target/debug/relayflowd npx vitest run tests/journal-client-read-timeout.test.ts tests/journal-client.test.ts tests/journal-client-completion.test.ts tests/journal-client-subscriptions.test.ts tests/running-step-watch.test.ts tests/heartbeat-timeout.test.ts tests/run-daemon-unresponsive.test.ts tests/authored-root.test.ts tests/classify-outcome.test.ts tests/cli.test.ts tests/direct-run-worker-lease.test.ts tests/resume-worker-lease.test.ts tests/worker-lease.test.ts tests/worker-lease-lost.test.ts tests/worker-lease-lost-live.test.ts tests/worker-lease-sweep.test.ts tests/run-read-load-live.test.ts tests/flow-executor-chain.test.ts tests/agent-transcript-live.test.ts tests/human-live.test.ts tests/reuse-summary-interruption.test.ts tests/memoization.test.ts --maxWorkers=1 --minWorkers=1 +$ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/kernel-target/debug/relayflowd npx vitest run tests/journal-client-read-timeout.test.ts tests/journal-client.test.ts tests/journal-client-completion.test.ts tests/journal-client-subscriptions.test.ts tests/running-step-watch.test.ts tests/heartbeat-timeout.test.ts tests/run-daemon-unresponsive.test.ts tests/authored-root.test.ts tests/classify-outcome.test.ts tests/cli.test.ts tests/direct-run-worker-lease.test.ts tests/resume-worker-lease.test.ts tests/worker-lease.test.ts tests/worker-lease-lost.test.ts tests/worker-lease-lost-live.test.ts tests/worker-lease-sweep.test.ts tests/run-read-load-live.test.ts tests/flow-executor-chain.test.ts tests/agent-transcript-live.test.ts tests/human-live.test.ts tests/reuse-summary-interruption.test.ts tests/memoization.test.ts tests/authored-verifier-read-budget.test.ts --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5258ms + ✓ tests/cli.test.ts (71 tests) 5277ms ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2039ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2036ms -(node:3253842) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2035ms +(node:3305472) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (27 tests) 392ms - ✓ tests/journal-client.test.ts (17 tests) 87ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 6781ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 487ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1068ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2618ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 338ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 738ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2536ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 617ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 635ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 633ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 650ms - ✓ tests/classify-outcome.test.ts (11 tests) 7428ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2013ms + ✓ tests/authored-root.test.ts (27 tests) 378ms + ✓ tests/journal-client.test.ts (17 tests) 79ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 6804ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 484ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1051ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2650ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 350ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 737ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2547ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 669ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 562ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 659ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 656ms + ✓ tests/classify-outcome.test.ts (11 tests) 7432ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2015ms ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3007ms ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2002ms - ✓ tests/journal-client-read-timeout.test.ts (17 tests) 1407ms - ✓ tests/human-live.test.ts (3 tests) 5520ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 3662ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1130ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 728ms - ✓ tests/running-step-watch.test.ts (6 tests) 6235ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms - ✓ cancels promptly while a lease snapshot read is in flight 2006ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2054ms - ✓ tests/worker-lease.test.ts (7 tests) 21ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 26ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 654ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2496ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2053ms - ✓ drains read and watch promises before an authored flow completes 442ms - ✓ tests/worker-lease-sweep.test.ts (4 tests) 6ms - ✓ tests/journal-client-completion.test.ts (6 tests) 105ms + ✓ tests/journal-client-read-timeout.test.ts (17 tests) 1413ms + ✓ tests/human-live.test.ts (3 tests) 4712ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2781ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1177ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 753ms + ✓ tests/running-step-watch.test.ts (7 tests) 8233ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2116ms + ✓ cancels promptly while a lease snapshot read is in flight 2004ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2053ms + ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2003ms + ✓ tests/worker-lease.test.ts (7 tests) 16ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 24ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 682ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2503ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2059ms + ✓ drains read and watch promises before an authored flow completes 443ms + ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms + ✓ tests/journal-client-completion.test.ts (6 tests) 106ms ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms ✓ tests/direct-run-worker-lease.test.ts (3 tests) 9ms - ✓ tests/memoization.test.ts (57 tests) 55ms - ✓ tests/journal-client-subscriptions.test.ts (1 test) 7ms + ✓ tests/memoization.test.ts (57 tests) 49ms + ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 4ms ✓ tests/reuse-summary-interruption.test.ts (3 tests) 4ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms + ✓ tests/authored-verifier-read-budget.test.ts (1 test) 30ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms - Test Files 22 passed (22) - Tests 280 passed (280) - Start at 00:54:04 - Duration 47.77s (transform 1.05s, setup 123ms, collect 5.93s, tests 39.05s, environment 3ms, prepare 786ms) + Test Files 23 passed (23) + Tests 282 passed (282) + Start at 01:04:32 + Duration 48.66s (transform 1.05s, setup 125ms, collect 5.48s, tests 40.34s, environment 3ms, prepare 814ms) exit=0 diff --git a/evidence/run-read-timeout/kernel-build.log b/evidence/run-read-timeout/kernel-build.log index bc4bf8a8f..6b859fb94 100644 --- a/evidence/run-read-timeout/kernel-build.log +++ b/evidence/run-read-timeout/kernel-build.log @@ -1,3 +1,3 @@ $ cd kernel && CARGO_TARGET_DIR=/home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/kernel-target cargo build - Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.05s + Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.04s exit=0 diff --git a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log index f7cc4d4dc..6d3dde365 100644 --- a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log +++ b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/cli/running-step.ts -Original SHA256: b84dc4f0d83e3f8c31d55a38f42a0d68c25299970d05ec7f3be772290f8d1a6e +Original SHA256: 5f7ca7c0a29e5dcd0be4650369e994a21aa715f7dbdc45363924abf5df30d96b Replaced: snapshotRead?.abort(new Error(`step "${runningStep.id}" completed during the lease snapshot`)); With: @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (6 tests | 1 failed | 5 skipped) 3013ms - × a completion push ends the wait without waiting out an in-flight snapshot 3012ms + ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 3012ms + × a completion push ends the wait without waiting out an in-flight snapshot 3011ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -33,23 +33,23 @@ Received: "still waiting" ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 5 skipped (6) - Start at 00:53:49 - Duration 3.49s (transform 191ms, setup 16ms, collect 320ms, tests 3.01s, environment 0ms, prepare 41ms) + Tests 1 failed | 6 skipped (7) + Start at 01:06:00 + Duration 3.48s (transform 181ms, setup 16ms, collect 305ms, tests 3.01s, environment 0ms, prepare 40ms) exit=1 -RESTORED SHA256: b84dc4f0d83e3f8c31d55a38f42a0d68c25299970d05ec7f3be772290f8d1a6e +RESTORED SHA256: 5f7ca7c0a29e5dcd0be4650369e994a21aa715f7dbdc45363924abf5df30d96b $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a completion push ends the wait' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (6 tests | 5 skipped) 2062ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2061ms + ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2060ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2059ms Test Files 1 passed (1) - Tests 1 passed | 5 skipped (6) - Start at 00:53:52 - Duration 2.50s (transform 182ms, setup 16ms, collect 305ms, tests 2.06s, environment 0ms, prepare 29ms) + Tests 1 passed | 6 skipped (7) + Start at 01:06:04 + Duration 2.50s (transform 182ms, setup 15ms, collect 304ms, tests 2.06s, environment 0ms, prepare 35ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-heartbeat.log b/evidence/run-read-timeout/mutation-heartbeat.log index 8ffaef622..10b5d7411 100644 --- a/evidence/run-read-timeout/mutation-heartbeat.log +++ b/evidence/run-read-timeout/mutation-heartbeat.log @@ -33,8 +33,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 00:53:55 - Duration 453ms (transform 168ms, setup 18ms, collect 273ms, tests 18ms, environment 0ms, prepare 41ms) + Start at 01:06:07 + Duration 459ms (transform 167ms, setup 15ms, collect 281ms, tests 18ms, environment 0ms, prepare 43ms) exit=1 @@ -47,7 +47,7 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb Test Files 1 passed (1) Tests 1 passed (1) - Start at 00:53:56 - Duration 453ms (transform 165ms, setup 17ms, collect 278ms, tests 15ms, environment 0ms, prepare 41ms) + Start at 01:06:08 + Duration 448ms (transform 162ms, setup 17ms, collect 279ms, tests 15ms, environment 0ms, prepare 42ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-reconnect.log b/evidence/run-read-timeout/mutation-reader-reconnect.log index a26cd31f1..59cf83261 100644 --- a/evidence/run-read-timeout/mutation-reader-reconnect.log +++ b/evidence/run-read-timeout/mutation-reader-reconnect.log @@ -13,12 +13,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t ❯ tests/journal-client-read-timeout.test.ts (17 tests | 1 failed | 16 skipped) 1008ms × reconnects the reader after it disconnects within the read budget 1007ms - → journal client: run.get read session was interrupted after 80 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + → journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 80 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:70:9 68| let last: unknown; 69| const exhausted = () => last instanceof JournalReadInterruptedError @@ -41,8 +41,8 @@ Caused by: Error: journal client: connection closed Test Files 1 failed (1) Tests 1 failed | 16 skipped (17) - Start at 00:53:32 - Duration 1.46s (transform 166ms, setup 18ms, collect 286ms, tests 1.01s, environment 0ms, prepare 42ms) + Start at 01:05:44 + Duration 1.44s (transform 158ms, setup 16ms, collect 276ms, tests 1.01s, environment 0ms, prepare 42ms) exit=1 @@ -51,11 +51,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 78ms + ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 79ms Test Files 1 passed (1) Tests 1 passed | 16 skipped (17) - Start at 00:53:34 - Duration 522ms (transform 156ms, setup 16ms, collect 276ms, tests 78ms, environment 0ms, prepare 43ms) + Start at 01:05:46 + Duration 486ms (transform 158ms, setup 18ms, collect 267ms, tests 79ms, environment 0ms, prepare 30ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-setup-retry.log b/evidence/run-read-timeout/mutation-reader-setup-retry.log index 941acdfed..0ad615a03 100644 --- a/evidence/run-read-timeout/mutation-reader-setup-retry.log +++ b/evidence/run-read-timeout/mutation-reader-setup-retry.log @@ -16,8 +16,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (17 tests | 1 failed | 16 skipped) 73ms - × retries a reader setup that timed out instead of reading on the primary for good 72ms + ❯ tests/journal-client-read-timeout.test.ts (17 tests | 1 failed | 16 skipped) 79ms + × retries a reader setup that timed out instead of reading on the primary for good 78ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -43,8 +43,8 @@ AssertionError: expected 1 to be 2 // Object.is equality Test Files 1 failed (1) Tests 1 failed | 16 skipped (17) - Start at 00:53:35 - Duration 523ms (transform 166ms, setup 18ms, collect 281ms, tests 73ms, environment 0ms, prepare 46ms) + Start at 01:05:47 + Duration 467ms (transform 151ms, setup 16ms, collect 251ms, tests 79ms, environment 0ms, prepare 30ms) exit=1 @@ -53,11 +53,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 67ms + ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 76ms Test Files 1 passed (1) Tests 1 passed | 16 skipped (17) - Start at 00:53:36 - Duration 520ms (transform 167ms, setup 17ms, collect 290ms, tests 67ms, environment 0ms, prepare 44ms) + Start at 01:05:48 + Duration 507ms (transform 160ms, setup 15ms, collect 284ms, tests 76ms, environment 0ms, prepare 30ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader.log b/evidence/run-read-timeout/mutation-reader.log index a1036004b..2b644d8cf 100644 --- a/evidence/run-read-timeout/mutation-reader.log +++ b/evidence/run-read-timeout/mutation-reader.log @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (17 tests | 1 failed | 16 skipped) 115ms - × serves a bounded read while an unbounded command is in flight 114ms + ❯ tests/journal-client-read-timeout.test.ts (17 tests | 1 failed | 16 skipped) 112ms + × serves a bounded read while an unbounded command is in flight 112ms → expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -37,8 +37,8 @@ AssertionError: expected true to be false // Object.is equality Test Files 1 failed (1) Tests 1 failed | 16 skipped (17) - Start at 00:53:30 - Duration 610ms (transform 180ms, setup 19ms, collect 302ms, tests 115ms, environment 0ms, prepare 50ms) + Start at 01:05:42 + Duration 533ms (transform 160ms, setup 16ms, collect 271ms, tests 112ms, environment 0ms, prepare 41ms) exit=1 @@ -47,11 +47,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 106ms + ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 107ms Test Files 1 passed (1) Tests 1 passed | 16 skipped (17) - Start at 00:53:31 - Duration 529ms (transform 161ms, setup 13ms, collect 262ms, tests 106ms, environment 0ms, prepare 43ms) + Start at 01:05:43 + Duration 563ms (transform 170ms, setup 17ms, collect 291ms, tests 107ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-root-parking.log b/evidence/run-read-timeout/mutation-root-parking.log index 1e3ed7fac..76b801d9a 100644 --- a/evidence/run-read-timeout/mutation-root-parking.log +++ b/evidence/run-read-timeout/mutation-root-parking.log @@ -18,7 +18,7 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk ❯ tests/authored-root.test.ts (27 tests | 3 failed | 24 skipped) 30ms - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 21ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 22ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms @@ -104,8 +104,8 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } Test Files 1 failed (1) Tests 3 failed | 24 skipped (27) - Start at 00:53:57 - Duration 1.24s (transform 648ms, setup 16ms, collect 1.06s, tests 30ms, environment 0ms, prepare 28ms) + Start at 01:06:09 + Duration 1.23s (transform 654ms, setup 17ms, collect 1.04s, tests 30ms, environment 0ms, prepare 41ms) exit=1 @@ -118,7 +118,7 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r Test Files 1 passed (1) Tests 3 passed | 24 skipped (27) - Start at 00:53:59 - Duration 1.22s (transform 650ms, setup 16ms, collect 1.04s, tests 23ms, environment 0ms, prepare 41ms) + Start at 01:06:10 + Duration 1.27s (transform 658ms, setup 17ms, collect 1.08s, tests 23ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-snapshot-cancel.log b/evidence/run-read-timeout/mutation-snapshot-cancel.log index af5b5a0d0..aaa76c02c 100644 --- a/evidence/run-read-timeout/mutation-snapshot-cancel.log +++ b/evidence/run-read-timeout/mutation-snapshot-cancel.log @@ -1,18 +1,18 @@ Source: packages/sdk/src/cli/running-step.ts -Original SHA256: b84dc4f0d83e3f8c31d55a38f42a0d68c25299970d05ec7f3be772290f8d1a6e +Original SHA256: 5f7ca7c0a29e5dcd0be4650369e994a21aa715f7dbdc45363924abf5df30d96b Replaced: -const signal = options.signal === undefined ? snapshotRead.signal - : AbortSignal.any([options.signal, snapshotRead.signal]); +const signal = options.signal === undefined ? read.signal + : AbortSignal.any([options.signal, read.signal]); With: -const signal = snapshotRead.signal; +const signal = read.signal; REVERTED $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels promptly while a lease snapshot read is in flight' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (6 tests | 1 failed | 5 skipped) 2215ms - × cancels promptly while a lease snapshot read is in flight 2215ms + ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 2217ms + × cancels promptly while a lease snapshot read is in flight 2216ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -30,23 +30,23 @@ AssertionError: expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 5 skipped (6) - Start at 00:53:43 - Duration 2.68s (transform 182ms, setup 17ms, collect 307ms, tests 2.22s, environment 0ms, prepare 41ms) + Tests 1 failed | 6 skipped (7) + Start at 01:05:54 + Duration 2.69s (transform 190ms, setup 16ms, collect 323ms, tests 2.22s, environment 0ms, prepare 42ms) exit=1 -RESTORED SHA256: b84dc4f0d83e3f8c31d55a38f42a0d68c25299970d05ec7f3be772290f8d1a6e +RESTORED SHA256: 5f7ca7c0a29e5dcd0be4650369e994a21aa715f7dbdc45363924abf5df30d96b $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels promptly while a lease snapshot read is in flight' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (6 tests | 5 skipped) 2011ms + ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2010ms ✓ cancels promptly while a lease snapshot read is in flight 2010ms Test Files 1 passed (1) - Tests 1 passed | 5 skipped (6) - Start at 00:53:46 - Duration 2.48s (transform 183ms, setup 16ms, collect 311ms, tests 2.01s, environment 0ms, prepare 41ms) + Tests 1 passed | 6 skipped (7) + Start at 01:05:57 + Duration 2.49s (transform 189ms, setup 16ms, collect 318ms, tests 2.01s, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-watch-cadence.log b/evidence/run-read-timeout/mutation-watch-cadence.log index de2092581..054711ed8 100644 --- a/evidence/run-read-timeout/mutation-watch-cadence.log +++ b/evidence/run-read-timeout/mutation-watch-cadence.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/cli/running-step.ts -Original SHA256: b84dc4f0d83e3f8c31d55a38f42a0d68c25299970d05ec7f3be772290f8d1a6e +Original SHA256: 5f7ca7c0a29e5dcd0be4650369e994a21aa715f7dbdc45363924abf5df30d96b Replaced: const LEASE_POLL_MS = 2_000; With: @@ -10,7 +10,7 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (6 tests | 1 failed | 5 skipped) 2111ms + ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 2110ms × uses pushes for completion with lease-cadence reads and releases its watcher 2110ms → expected 41 to be less than or equal to 1 @@ -29,23 +29,23 @@ AssertionError: expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 5 skipped (6) - Start at 00:53:37 - Duration 2.59s (transform 186ms, setup 17ms, collect 316ms, tests 2.11s, environment 0ms, prepare 43ms) + Tests 1 failed | 6 skipped (7) + Start at 01:05:48 + Duration 2.58s (transform 185ms, setup 16ms, collect 311ms, tests 2.11s, environment 0ms, prepare 41ms) exit=1 -RESTORED SHA256: b84dc4f0d83e3f8c31d55a38f42a0d68c25299970d05ec7f3be772290f8d1a6e +RESTORED SHA256: 5f7ca7c0a29e5dcd0be4650369e994a21aa715f7dbdc45363924abf5df30d96b $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pushes for completion' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (6 tests | 5 skipped) 2118ms + ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2118ms ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms Test Files 1 passed (1) - Tests 1 passed | 5 skipped (6) - Start at 00:53:40 - Duration 2.59s (transform 187ms, setup 16ms, collect 316ms, tests 2.12s, environment 0ms, prepare 40ms) + Tests 1 passed | 6 skipped (7) + Start at 01:05:51 + Duration 2.58s (transform 177ms, setup 17ms, collect 295ms, tests 2.12s, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutations.py b/evidence/run-read-timeout/mutations.py index be6ace558..360733e19 100644 --- a/evidence/run-read-timeout/mutations.py +++ b/evidence/run-read-timeout/mutations.py @@ -26,8 +26,8 @@ 'const LEASE_POLL_MS = 2_000;', 'const LEASE_POLL_MS = 50;', 'tests/running-step-watch.test.ts', 'uses pushes for completion'), ('snapshot-cancel', 'packages/sdk/src/cli/running-step.ts', - 'const signal = options.signal === undefined ? snapshotRead.signal\n : AbortSignal.any([options.signal, snapshotRead.signal]);', - 'const signal = snapshotRead.signal;', + 'const signal = options.signal === undefined ? read.signal\n : AbortSignal.any([options.signal, read.signal]);', + 'const signal = read.signal;', 'tests/running-step-watch.test.ts', 'cancels promptly while a lease snapshot read is in flight'), ('completion-aborts-snapshot', 'packages/sdk/src/cli/running-step.ts', 'snapshotRead?.abort(new Error(`step "${runningStep.id}" completed during the lease snapshot`));', diff --git a/evidence/run-read-timeout/resume-live.log b/evidence/run-read-timeout/resume-live.log index df36c5119..7e8a13ca0 100644 --- a/evidence/run-read-timeout/resume-live.log +++ b/evidence/run-read-timeout/resume-live.log @@ -2,12 +2,12 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 942ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 942ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 919ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 919ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 00:54:52 - Duration 2.21s (transform 624ms, setup 19ms, collect 1.02s, tests 942ms, environment 0ms, prepare 82ms) + Start at 01:05:21 + Duration 2.18s (transform 615ms, setup 17ms, collect 1.01s, tests 919ms, environment 0ms, prepare 73ms) exit=0 diff --git a/evidence/run-read-timeout/verification.md b/evidence/run-read-timeout/verification.md index ea3b3c9c0..c4bbf46d9 100644 --- a/evidence/run-read-timeout/verification.md +++ b/evidence/run-read-timeout/verification.md @@ -4,68 +4,70 @@ Captured verification at the review-fix source. Commands and output below are ve ```text $ cd kernel && CARGO_TARGET_DIR=/home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/kernel-target cargo build - Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.05s + Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.04s exit=0 ``` ## final-focused.log ```text -$ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/kernel-target/debug/relayflowd npx vitest run tests/journal-client-read-timeout.test.ts tests/journal-client.test.ts tests/journal-client-completion.test.ts tests/journal-client-subscriptions.test.ts tests/running-step-watch.test.ts tests/heartbeat-timeout.test.ts tests/run-daemon-unresponsive.test.ts tests/authored-root.test.ts tests/classify-outcome.test.ts tests/cli.test.ts tests/direct-run-worker-lease.test.ts tests/resume-worker-lease.test.ts tests/worker-lease.test.ts tests/worker-lease-lost.test.ts tests/worker-lease-lost-live.test.ts tests/worker-lease-sweep.test.ts tests/run-read-load-live.test.ts tests/flow-executor-chain.test.ts tests/agent-transcript-live.test.ts tests/human-live.test.ts tests/reuse-summary-interruption.test.ts tests/memoization.test.ts --maxWorkers=1 --minWorkers=1 +$ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/kernel-target/debug/relayflowd npx vitest run tests/journal-client-read-timeout.test.ts tests/journal-client.test.ts tests/journal-client-completion.test.ts tests/journal-client-subscriptions.test.ts tests/running-step-watch.test.ts tests/heartbeat-timeout.test.ts tests/run-daemon-unresponsive.test.ts tests/authored-root.test.ts tests/classify-outcome.test.ts tests/cli.test.ts tests/direct-run-worker-lease.test.ts tests/resume-worker-lease.test.ts tests/worker-lease.test.ts tests/worker-lease-lost.test.ts tests/worker-lease-lost-live.test.ts tests/worker-lease-sweep.test.ts tests/run-read-load-live.test.ts tests/flow-executor-chain.test.ts tests/agent-transcript-live.test.ts tests/human-live.test.ts tests/reuse-summary-interruption.test.ts tests/memoization.test.ts tests/authored-verifier-read-budget.test.ts --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5258ms + ✓ tests/cli.test.ts (71 tests) 5277ms ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2039ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2036ms -(node:3253842) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2035ms +(node:3305472) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (27 tests) 392ms - ✓ tests/journal-client.test.ts (17 tests) 87ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 6781ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 487ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1068ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2618ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 338ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 738ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2536ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 617ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 635ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 633ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 650ms - ✓ tests/classify-outcome.test.ts (11 tests) 7428ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2013ms + ✓ tests/authored-root.test.ts (27 tests) 378ms + ✓ tests/journal-client.test.ts (17 tests) 79ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 6804ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 484ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1051ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2650ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 350ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 737ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2547ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 669ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 562ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 659ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 656ms + ✓ tests/classify-outcome.test.ts (11 tests) 7432ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2015ms ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3007ms ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2002ms - ✓ tests/journal-client-read-timeout.test.ts (17 tests) 1407ms - ✓ tests/human-live.test.ts (3 tests) 5520ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 3662ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1130ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 728ms - ✓ tests/running-step-watch.test.ts (6 tests) 6235ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms - ✓ cancels promptly while a lease snapshot read is in flight 2006ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2054ms - ✓ tests/worker-lease.test.ts (7 tests) 21ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 26ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 654ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2496ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2053ms - ✓ drains read and watch promises before an authored flow completes 442ms - ✓ tests/worker-lease-sweep.test.ts (4 tests) 6ms - ✓ tests/journal-client-completion.test.ts (6 tests) 105ms + ✓ tests/journal-client-read-timeout.test.ts (17 tests) 1413ms + ✓ tests/human-live.test.ts (3 tests) 4712ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2781ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1177ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 753ms + ✓ tests/running-step-watch.test.ts (7 tests) 8233ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2116ms + ✓ cancels promptly while a lease snapshot read is in flight 2004ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2053ms + ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2003ms + ✓ tests/worker-lease.test.ts (7 tests) 16ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 24ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 682ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2503ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2059ms + ✓ drains read and watch promises before an authored flow completes 443ms + ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms + ✓ tests/journal-client-completion.test.ts (6 tests) 106ms ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms ✓ tests/direct-run-worker-lease.test.ts (3 tests) 9ms - ✓ tests/memoization.test.ts (57 tests) 55ms - ✓ tests/journal-client-subscriptions.test.ts (1 test) 7ms + ✓ tests/memoization.test.ts (57 tests) 49ms + ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 4ms ✓ tests/reuse-summary-interruption.test.ts (3 tests) 4ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms + ✓ tests/authored-verifier-read-budget.test.ts (1 test) 30ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms - Test Files 22 passed (22) - Tests 280 passed (280) - Start at 00:54:04 - Duration 47.77s (transform 1.05s, setup 123ms, collect 5.93s, tests 39.05s, environment 3ms, prepare 786ms) + Test Files 23 passed (23) + Tests 282 passed (282) + Start at 01:04:32 + Duration 48.66s (transform 1.05s, setup 125ms, collect 5.48s, tests 40.34s, environment 3ms, prepare 814ms) exit=0 ``` @@ -77,13 +79,13 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 942ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 942ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 919ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 919ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 00:54:52 - Duration 2.21s (transform 624ms, setup 19ms, collect 1.02s, tests 942ms, environment 0ms, prepare 82ms) + Start at 01:05:21 + Duration 2.18s (transform 615ms, setup 17ms, collect 1.01s, tests 919ms, environment 0ms, prepare 73ms) exit=0 ``` @@ -105,8 +107,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (17 tests | 1 failed | 16 skipped) 115ms - × serves a bounded read while an unbounded command is in flight 114ms + ❯ tests/journal-client-read-timeout.test.ts (17 tests | 1 failed | 16 skipped) 112ms + × serves a bounded read while an unbounded command is in flight 112ms → expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -132,8 +134,8 @@ AssertionError: expected true to be false // Object.is equality Test Files 1 failed (1) Tests 1 failed | 16 skipped (17) - Start at 00:53:30 - Duration 610ms (transform 180ms, setup 19ms, collect 302ms, tests 115ms, environment 0ms, prepare 50ms) + Start at 01:05:42 + Duration 533ms (transform 160ms, setup 16ms, collect 271ms, tests 112ms, environment 0ms, prepare 41ms) exit=1 @@ -142,12 +144,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 106ms + ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 107ms Test Files 1 passed (1) Tests 1 passed | 16 skipped (17) - Start at 00:53:31 - Duration 529ms (transform 161ms, setup 13ms, collect 262ms, tests 106ms, environment 0ms, prepare 43ms) + Start at 01:05:43 + Duration 563ms (transform 170ms, setup 17ms, collect 291ms, tests 107ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -170,12 +172,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t ❯ tests/journal-client-read-timeout.test.ts (17 tests | 1 failed | 16 skipped) 1008ms × reconnects the reader after it disconnects within the read budget 1007ms - → journal client: run.get read session was interrupted after 80 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + → journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 80 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:70:9 68| let last: unknown; 69| const exhausted = () => last instanceof JournalReadInterruptedError @@ -198,8 +200,8 @@ Caused by: Error: journal client: connection closed Test Files 1 failed (1) Tests 1 failed | 16 skipped (17) - Start at 00:53:32 - Duration 1.46s (transform 166ms, setup 18ms, collect 286ms, tests 1.01s, environment 0ms, prepare 42ms) + Start at 01:05:44 + Duration 1.44s (transform 158ms, setup 16ms, collect 276ms, tests 1.01s, environment 0ms, prepare 42ms) exit=1 @@ -208,12 +210,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 78ms + ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 79ms Test Files 1 passed (1) Tests 1 passed | 16 skipped (17) - Start at 00:53:34 - Duration 522ms (transform 156ms, setup 16ms, collect 276ms, tests 78ms, environment 0ms, prepare 43ms) + Start at 01:05:46 + Duration 486ms (transform 158ms, setup 18ms, collect 267ms, tests 79ms, environment 0ms, prepare 30ms) exit=0 ``` @@ -239,8 +241,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (17 tests | 1 failed | 16 skipped) 73ms - × retries a reader setup that timed out instead of reading on the primary for good 72ms + ❯ tests/journal-client-read-timeout.test.ts (17 tests | 1 failed | 16 skipped) 79ms + × retries a reader setup that timed out instead of reading on the primary for good 78ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -266,8 +268,8 @@ AssertionError: expected 1 to be 2 // Object.is equality Test Files 1 failed (1) Tests 1 failed | 16 skipped (17) - Start at 00:53:35 - Duration 523ms (transform 166ms, setup 18ms, collect 281ms, tests 73ms, environment 0ms, prepare 46ms) + Start at 01:05:47 + Duration 467ms (transform 151ms, setup 16ms, collect 251ms, tests 79ms, environment 0ms, prepare 30ms) exit=1 @@ -276,12 +278,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 67ms + ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 76ms Test Files 1 passed (1) Tests 1 passed | 16 skipped (17) - Start at 00:53:36 - Duration 520ms (transform 167ms, setup 17ms, collect 290ms, tests 67ms, environment 0ms, prepare 44ms) + Start at 01:05:48 + Duration 507ms (transform 160ms, setup 15ms, collect 284ms, tests 76ms, environment 0ms, prepare 30ms) exit=0 ``` @@ -290,7 +292,7 @@ exit=0 ```text Source: packages/sdk/src/cli/running-step.ts -Original SHA256: b84dc4f0d83e3f8c31d55a38f42a0d68c25299970d05ec7f3be772290f8d1a6e +Original SHA256: 5f7ca7c0a29e5dcd0be4650369e994a21aa715f7dbdc45363924abf5df30d96b Replaced: const LEASE_POLL_MS = 2_000; With: @@ -301,7 +303,7 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (6 tests | 1 failed | 5 skipped) 2111ms + ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 2110ms × uses pushes for completion with lease-cadence reads and releases its watcher 2110ms → expected 41 to be less than or equal to 1 @@ -320,24 +322,24 @@ AssertionError: expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 5 skipped (6) - Start at 00:53:37 - Duration 2.59s (transform 186ms, setup 17ms, collect 316ms, tests 2.11s, environment 0ms, prepare 43ms) + Tests 1 failed | 6 skipped (7) + Start at 01:05:48 + Duration 2.58s (transform 185ms, setup 16ms, collect 311ms, tests 2.11s, environment 0ms, prepare 41ms) exit=1 -RESTORED SHA256: b84dc4f0d83e3f8c31d55a38f42a0d68c25299970d05ec7f3be772290f8d1a6e +RESTORED SHA256: 5f7ca7c0a29e5dcd0be4650369e994a21aa715f7dbdc45363924abf5df30d96b $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pushes for completion' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (6 tests | 5 skipped) 2118ms + ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2118ms ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms Test Files 1 passed (1) - Tests 1 passed | 5 skipped (6) - Start at 00:53:40 - Duration 2.59s (transform 187ms, setup 16ms, collect 316ms, tests 2.12s, environment 0ms, prepare 40ms) + Tests 1 passed | 6 skipped (7) + Start at 01:05:51 + Duration 2.58s (transform 177ms, setup 17ms, collect 295ms, tests 2.12s, environment 0ms, prepare 41ms) exit=0 ``` @@ -346,20 +348,20 @@ exit=0 ```text Source: packages/sdk/src/cli/running-step.ts -Original SHA256: b84dc4f0d83e3f8c31d55a38f42a0d68c25299970d05ec7f3be772290f8d1a6e +Original SHA256: 5f7ca7c0a29e5dcd0be4650369e994a21aa715f7dbdc45363924abf5df30d96b Replaced: -const signal = options.signal === undefined ? snapshotRead.signal - : AbortSignal.any([options.signal, snapshotRead.signal]); +const signal = options.signal === undefined ? read.signal + : AbortSignal.any([options.signal, read.signal]); With: -const signal = snapshotRead.signal; +const signal = read.signal; REVERTED $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels promptly while a lease snapshot read is in flight' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (6 tests | 1 failed | 5 skipped) 2215ms - × cancels promptly while a lease snapshot read is in flight 2215ms + ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 2217ms + × cancels promptly while a lease snapshot read is in flight 2216ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -377,24 +379,24 @@ AssertionError: expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 5 skipped (6) - Start at 00:53:43 - Duration 2.68s (transform 182ms, setup 17ms, collect 307ms, tests 2.22s, environment 0ms, prepare 41ms) + Tests 1 failed | 6 skipped (7) + Start at 01:05:54 + Duration 2.69s (transform 190ms, setup 16ms, collect 323ms, tests 2.22s, environment 0ms, prepare 42ms) exit=1 -RESTORED SHA256: b84dc4f0d83e3f8c31d55a38f42a0d68c25299970d05ec7f3be772290f8d1a6e +RESTORED SHA256: 5f7ca7c0a29e5dcd0be4650369e994a21aa715f7dbdc45363924abf5df30d96b $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels promptly while a lease snapshot read is in flight' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (6 tests | 5 skipped) 2011ms + ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2010ms ✓ cancels promptly while a lease snapshot read is in flight 2010ms Test Files 1 passed (1) - Tests 1 passed | 5 skipped (6) - Start at 00:53:46 - Duration 2.48s (transform 183ms, setup 16ms, collect 311ms, tests 2.01s, environment 0ms, prepare 41ms) + Tests 1 passed | 6 skipped (7) + Start at 01:05:57 + Duration 2.49s (transform 189ms, setup 16ms, collect 318ms, tests 2.01s, environment 0ms, prepare 41ms) exit=0 ``` @@ -403,7 +405,7 @@ exit=0 ```text Source: packages/sdk/src/cli/running-step.ts -Original SHA256: b84dc4f0d83e3f8c31d55a38f42a0d68c25299970d05ec7f3be772290f8d1a6e +Original SHA256: 5f7ca7c0a29e5dcd0be4650369e994a21aa715f7dbdc45363924abf5df30d96b Replaced: snapshotRead?.abort(new Error(`step "${runningStep.id}" completed during the lease snapshot`)); With: @@ -414,8 +416,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (6 tests | 1 failed | 5 skipped) 3013ms - × a completion push ends the wait without waiting out an in-flight snapshot 3012ms + ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 3012ms + × a completion push ends the wait without waiting out an in-flight snapshot 3011ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -437,24 +439,24 @@ Received: "still waiting" ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 5 skipped (6) - Start at 00:53:49 - Duration 3.49s (transform 191ms, setup 16ms, collect 320ms, tests 3.01s, environment 0ms, prepare 41ms) + Tests 1 failed | 6 skipped (7) + Start at 01:06:00 + Duration 3.48s (transform 181ms, setup 16ms, collect 305ms, tests 3.01s, environment 0ms, prepare 40ms) exit=1 -RESTORED SHA256: b84dc4f0d83e3f8c31d55a38f42a0d68c25299970d05ec7f3be772290f8d1a6e +RESTORED SHA256: 5f7ca7c0a29e5dcd0be4650369e994a21aa715f7dbdc45363924abf5df30d96b $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a completion push ends the wait' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (6 tests | 5 skipped) 2062ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2061ms + ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2060ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2059ms Test Files 1 passed (1) - Tests 1 passed | 5 skipped (6) - Start at 00:53:52 - Duration 2.50s (transform 182ms, setup 16ms, collect 305ms, tests 2.06s, environment 0ms, prepare 29ms) + Tests 1 passed | 6 skipped (7) + Start at 01:06:04 + Duration 2.50s (transform 182ms, setup 15ms, collect 304ms, tests 2.06s, environment 0ms, prepare 35ms) exit=0 ``` @@ -497,8 +499,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 00:53:55 - Duration 453ms (transform 168ms, setup 18ms, collect 273ms, tests 18ms, environment 0ms, prepare 41ms) + Start at 01:06:07 + Duration 459ms (transform 167ms, setup 15ms, collect 281ms, tests 18ms, environment 0ms, prepare 43ms) exit=1 @@ -511,8 +513,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb Test Files 1 passed (1) Tests 1 passed (1) - Start at 00:53:56 - Duration 453ms (transform 165ms, setup 17ms, collect 278ms, tests 15ms, environment 0ms, prepare 41ms) + Start at 01:06:08 + Duration 448ms (transform 162ms, setup 17ms, collect 279ms, tests 15ms, environment 0ms, prepare 42ms) exit=0 ``` @@ -540,7 +542,7 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk ❯ tests/authored-root.test.ts (27 tests | 3 failed | 24 skipped) 30ms - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 21ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 22ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms @@ -626,8 +628,8 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } Test Files 1 failed (1) Tests 3 failed | 24 skipped (27) - Start at 00:53:57 - Duration 1.24s (transform 648ms, setup 16ms, collect 1.06s, tests 30ms, environment 0ms, prepare 28ms) + Start at 01:06:09 + Duration 1.23s (transform 654ms, setup 17ms, collect 1.04s, tests 30ms, environment 0ms, prepare 41ms) exit=1 @@ -640,8 +642,8 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r Test Files 1 passed (1) Tests 3 passed | 24 skipped (27) - Start at 00:53:59 - Duration 1.22s (transform 650ms, setup 16ms, collect 1.04s, tests 23ms, environment 0ms, prepare 41ms) + Start at 01:06:10 + Duration 1.27s (transform 658ms, setup 17ms, collect 1.08s, tests 23ms, environment 0ms, prepare 41ms) exit=0 ``` diff --git a/packages/sdk/src/authored-node-runner.ts b/packages/sdk/src/authored-node-runner.ts index cf2c8f075..fdd0c967a 100644 --- a/packages/sdk/src/authored-node-runner.ts +++ b/packages/sdk/src/authored-node-runner.ts @@ -1,4 +1,5 @@ import { JournalClient } from './journal-client.js'; +import { FLOW_READ_BUDGET_MS } from './journal-read-policy.js'; import { spawn, spawnSync } from 'node:child_process'; import { createHash, createHmac, randomBytes, timingSafeEqual } from 'node:crypto'; import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; @@ -354,7 +355,8 @@ export async function verifyAuthoredNodeResult( if (!authoredIds.has(gate.id.slice(0, -'.gate'.length))) invalid(`orphan ${gate.id}`); } const runs = new Set(); - const journal = new JournalClient(socketPath); + // The verifier reads the same journal under the same CPU load as the body. + const journal = new JournalClient(socketPath, { readBudgetMs: FLOW_READ_BUDGET_MS }); await journal.connect(); try { await journal.hello('flows-authored-result-verifier'); diff --git a/packages/sdk/src/cli/running-step.ts b/packages/sdk/src/cli/running-step.ts index 5d43b67c3..98f35d10d 100644 --- a/packages/sdk/src/cli/running-step.ts +++ b/packages/sdk/src/cli/running-step.ts @@ -75,13 +75,15 @@ export async function waitForRunningStep(client: JournalClient, runId: string, // Drain the read before leaving the authored promise scope. A raced but // unresolved read would look like unawaited derived work to that scope, // so cancellation aborts the read itself and it settles at once. - snapshotRead = new AbortController(); - const signal = options.signal === undefined ? snapshotRead.signal - : AbortSignal.any([options.signal, snapshotRead.signal]); + const read = snapshotRead = new AbortController(); + const signal = options.signal === undefined ? read.signal + : AbortSignal.any([options.signal, read.signal]); const snapshot = await client.runGet(runId, { signal }) .catch(error => { if (options.signal?.aborted) throw canceled(); - if (settled) return undefined; + // A completion push aborted this read: completion was observed, and + // finish() is one-shot, even if a retry attempt started right after. + if (settled || read.signal.aborted) return undefined; throw error; }) .finally(() => { snapshotRead = undefined; }); diff --git a/packages/sdk/tests/authored-verifier-read-budget.test.ts b/packages/sdk/tests/authored-verifier-read-budget.test.ts new file mode 100644 index 000000000..e2cb95069 --- /dev/null +++ b/packages/sdk/tests/authored-verifier-read-budget.test.ts @@ -0,0 +1,23 @@ +import { expect, it, vi } from 'vitest'; +import { FLOW_READ_BUDGET_MS } from '../src/journal-read-policy.js'; + +const constructed = vi.hoisted(() => [] as unknown[]); +vi.mock('../src/journal-client.js', async original => { + const actual = await original(); + class RecordingClient extends actual.JournalClient { + constructor(socketPath: string, options?: ConstructorParameters[1]) { + super(socketPath, options); + constructed.push(options); + } + override async connect(): Promise { throw new Error('stop after construction'); } + } + return { ...actual, JournalClient: RecordingClient }; +}); + +it('verifies a Bun-run authored result with the flow read budget, not single-shot reads', async () => { + const { verifyAuthoredNodeResult } = await import('../src/authored-node-runner.js'); + const result = { rootRunId: 'root', name: 'flow', completionReason: 'success', + journalSteps: [{ id: 'complete-1' }] }; + await verifyAuthoredNodeResult(result as never, { flowName: 'flow' } as never, 'root', '/unused.sock').catch(() => undefined); + expect(constructed).toContainEqual(expect.objectContaining({ readBudgetMs: FLOW_READ_BUDGET_MS })); +}); diff --git a/packages/sdk/tests/running-step-watch.test.ts b/packages/sdk/tests/running-step-watch.test.ts index 6bd4cad10..a97cab67a 100644 --- a/packages/sdk/tests/running-step-watch.test.ts +++ b/packages/sdk/tests/running-step-watch.test.ts @@ -153,3 +153,28 @@ it('a completion push ends the wait without waiting out an in-flight snapshot', expect(await client.runGet('run', { signal: AbortSignal.timeout(50) }).catch(error => error.name)).toBe('TimeoutError'); } finally { client.close(); await new Promise(resolve => server.close(() => resolve())); } }); + +it('a completion that aborts the snapshot is not an error even if a retry starts at once', async () => { + const path = sockPath(); + let push!: () => void; + const server = startLoopback(path, { + hello: sendOk, + 'run.watch': (ctx, params) => { + sendResult(ctx, {}); + const entry = (type: string) => ctx.send({ event: 'entry', data: { run_id: String(params['run_id']), step_id: 'step', entry_type: type } }); + push = () => { entry('step.completed'); entry('step.attempt.started'); }; + }, + 'run.get': () => push(), + }); + await once(server, 'listening'); + const client = new JournalClient(path, { readBudgetMs: 300_000 }); + try { + await client.connect(); + const outcome = await Promise.race([ + waitForRunningStep(client, 'run', { id: 'step', type: 'agent', leaseDeadlineMs: Date.now() + 30_000 }, {}) + .then(() => 'returned', error => `threw: ${error.message}`), + sleep(3_000).then(() => 'still waiting'), + ]); + expect(outcome).toBe('returned'); + } finally { client.close(); await new Promise(resolve => server.close(() => resolve())); } +}); From 0cfcc0d299188e923a4cfdbaceb1edd9298c0613 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 01:14:18 -0700 Subject: [PATCH 10/34] fix(sdk): no reuse-summary read after classification reported an unanswered read A --reuse-from run whose classification already exhausted the read budget returns a resumable daemon_unresponsive/daemon_unreachable report; attachReuseSummary then started another journal.read with a fresh 300 s budget. It now returns that execution unchanged. Evidence re-captured at this tree (focused 23 files, resume-live 1/1). Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: b8262468-ea26-4a6e-bee9-d193832e77dc --- evidence/run-read-timeout/final-focused.log | 90 ++++++++--------- evidence/run-read-timeout/resume-live.log | 8 +- evidence/run-read-timeout/verification.md | 98 +++++++++---------- packages/sdk/src/cli/reuse.ts | 4 + .../tests/reuse-summary-interruption.test.ts | 11 +++ 5 files changed, 113 insertions(+), 98 deletions(-) diff --git a/evidence/run-read-timeout/final-focused.log b/evidence/run-read-timeout/final-focused.log index 9c7a4fb73..bc05590e9 100644 --- a/evidence/run-read-timeout/final-focused.log +++ b/evidence/run-read-timeout/final-focused.log @@ -2,58 +2,58 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5277ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2039ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2035ms -(node:3305472) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5237ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2041ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2038ms +(node:3394253) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (27 tests) 378ms - ✓ tests/journal-client.test.ts (17 tests) 79ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 6804ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 484ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1051ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2650ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 350ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 737ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2547ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 669ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 562ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 659ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 656ms - ✓ tests/classify-outcome.test.ts (11 tests) 7432ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2015ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3007ms - ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2002ms - ✓ tests/journal-client-read-timeout.test.ts (17 tests) 1413ms - ✓ tests/human-live.test.ts (3 tests) 4712ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2781ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1177ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 753ms - ✓ tests/running-step-watch.test.ts (7 tests) 8233ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2116ms - ✓ cancels promptly while a lease snapshot read is in flight 2004ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2053ms + ✓ tests/authored-root.test.ts (27 tests) 438ms + ✓ tests/journal-client.test.ts (17 tests) 106ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 6940ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 504ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1021ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2693ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 364ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 820ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2469ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 633ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 591ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 684ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 560ms + ✓ tests/classify-outcome.test.ts (11 tests) 7426ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2011ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3006ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms + ✓ tests/journal-client-read-timeout.test.ts (17 tests) 1445ms + ✓ tests/human-live.test.ts (3 tests) 4493ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2750ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1090ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 652ms + ✓ tests/running-step-watch.test.ts (7 tests) 8238ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms + ✓ cancels promptly while a lease snapshot read is in flight 2007ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2054ms ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2003ms - ✓ tests/worker-lease.test.ts (7 tests) 16ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 24ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 682ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2503ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2059ms - ✓ drains read and watch promises before an authored flow completes 443ms + ✓ tests/worker-lease.test.ts (7 tests) 13ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 29ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 688ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2434ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2065ms + ✓ drains read and watch promises before an authored flow completes 368ms ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms - ✓ tests/journal-client-completion.test.ts (6 tests) 106ms + ✓ tests/journal-client-completion.test.ts (6 tests) 104ms ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms ✓ tests/direct-run-worker-lease.test.ts (3 tests) 9ms - ✓ tests/memoization.test.ts (57 tests) 49ms - ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms - ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 4ms - ✓ tests/reuse-summary-interruption.test.ts (3 tests) 4ms - ✓ tests/authored-verifier-read-budget.test.ts (1 test) 30ms + ✓ tests/reuse-summary-interruption.test.ts (5 tests) 4ms + ✓ tests/memoization.test.ts (57 tests) 53ms + ✓ tests/journal-client-subscriptions.test.ts (1 test) 5ms + ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 3ms + ✓ tests/authored-verifier-read-budget.test.ts (1 test) 44ms ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms Test Files 23 passed (23) - Tests 282 passed (282) - Start at 01:04:32 - Duration 48.66s (transform 1.05s, setup 125ms, collect 5.48s, tests 40.34s, environment 3ms, prepare 814ms) + Tests 284 passed (284) + Start at 01:13:26 + Duration 49.32s (transform 1.07s, setup 137ms, collect 6.12s, tests 40.21s, environment 3ms, prepare 880ms) exit=0 diff --git a/evidence/run-read-timeout/resume-live.log b/evidence/run-read-timeout/resume-live.log index 7e8a13ca0..e1b82fae2 100644 --- a/evidence/run-read-timeout/resume-live.log +++ b/evidence/run-read-timeout/resume-live.log @@ -2,12 +2,12 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 919ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 919ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 910ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 910ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 01:05:21 - Duration 2.18s (transform 615ms, setup 17ms, collect 1.01s, tests 919ms, environment 0ms, prepare 73ms) + Start at 01:14:16 + Duration 2.12s (transform 593ms, setup 16ms, collect 968ms, tests 910ms, environment 0ms, prepare 59ms) exit=0 diff --git a/evidence/run-read-timeout/verification.md b/evidence/run-read-timeout/verification.md index c4bbf46d9..de25216ab 100644 --- a/evidence/run-read-timeout/verification.md +++ b/evidence/run-read-timeout/verification.md @@ -15,59 +15,59 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5277ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2039ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2035ms -(node:3305472) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5237ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2041ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2038ms +(node:3394253) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (27 tests) 378ms - ✓ tests/journal-client.test.ts (17 tests) 79ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 6804ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 484ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1051ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2650ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 350ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 737ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2547ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 669ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 562ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 659ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 656ms - ✓ tests/classify-outcome.test.ts (11 tests) 7432ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2015ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3007ms - ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2002ms - ✓ tests/journal-client-read-timeout.test.ts (17 tests) 1413ms - ✓ tests/human-live.test.ts (3 tests) 4712ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2781ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1177ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 753ms - ✓ tests/running-step-watch.test.ts (7 tests) 8233ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2116ms - ✓ cancels promptly while a lease snapshot read is in flight 2004ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2053ms + ✓ tests/authored-root.test.ts (27 tests) 438ms + ✓ tests/journal-client.test.ts (17 tests) 106ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 6940ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 504ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1021ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2693ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 364ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 820ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2469ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 633ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 591ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 684ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 560ms + ✓ tests/classify-outcome.test.ts (11 tests) 7426ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2011ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3006ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms + ✓ tests/journal-client-read-timeout.test.ts (17 tests) 1445ms + ✓ tests/human-live.test.ts (3 tests) 4493ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2750ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1090ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 652ms + ✓ tests/running-step-watch.test.ts (7 tests) 8238ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms + ✓ cancels promptly while a lease snapshot read is in flight 2007ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2054ms ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2003ms - ✓ tests/worker-lease.test.ts (7 tests) 16ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 24ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 682ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2503ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2059ms - ✓ drains read and watch promises before an authored flow completes 443ms + ✓ tests/worker-lease.test.ts (7 tests) 13ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 29ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 688ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2434ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2065ms + ✓ drains read and watch promises before an authored flow completes 368ms ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms - ✓ tests/journal-client-completion.test.ts (6 tests) 106ms + ✓ tests/journal-client-completion.test.ts (6 tests) 104ms ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms ✓ tests/direct-run-worker-lease.test.ts (3 tests) 9ms - ✓ tests/memoization.test.ts (57 tests) 49ms - ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms - ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 4ms - ✓ tests/reuse-summary-interruption.test.ts (3 tests) 4ms - ✓ tests/authored-verifier-read-budget.test.ts (1 test) 30ms + ✓ tests/reuse-summary-interruption.test.ts (5 tests) 4ms + ✓ tests/memoization.test.ts (57 tests) 53ms + ✓ tests/journal-client-subscriptions.test.ts (1 test) 5ms + ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 3ms + ✓ tests/authored-verifier-read-budget.test.ts (1 test) 44ms ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms Test Files 23 passed (23) - Tests 282 passed (282) - Start at 01:04:32 - Duration 48.66s (transform 1.05s, setup 125ms, collect 5.48s, tests 40.34s, environment 3ms, prepare 814ms) + Tests 284 passed (284) + Start at 01:13:26 + Duration 49.32s (transform 1.07s, setup 137ms, collect 6.12s, tests 40.21s, environment 3ms, prepare 880ms) exit=0 ``` @@ -79,13 +79,13 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 919ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 919ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 910ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 910ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 01:05:21 - Duration 2.18s (transform 615ms, setup 17ms, collect 1.01s, tests 919ms, environment 0ms, prepare 73ms) + Start at 01:14:16 + Duration 2.12s (transform 593ms, setup 16ms, collect 968ms, tests 910ms, environment 0ms, prepare 59ms) exit=0 ``` diff --git a/packages/sdk/src/cli/reuse.ts b/packages/sdk/src/cli/reuse.ts index 6f1c4d04a..3e15c276d 100644 --- a/packages/sdk/src/cli/reuse.ts +++ b/packages/sdk/src/cli/reuse.ts @@ -32,6 +32,10 @@ export async function reuseSummary(client: JournalClient, runId: string, fromRun */ export async function attachReuseSummary(execution: RunExecution, client: JournalClient, runId: string, fromRunId: string): Promise { + // Classification already ended on an unanswered read: the run is resumable and + // another read would only spend a fresh budget against the same delayed daemon. + if (execution.report.diagnostics.some(diagnostic => + diagnostic.kind === 'daemon_unresponsive' || diagnostic.kind === 'daemon_unreachable')) return execution; try { execution.report.reuse = await reuseSummary(client, runId, fromRunId); } catch (error) { diff --git a/packages/sdk/tests/reuse-summary-interruption.test.ts b/packages/sdk/tests/reuse-summary-interruption.test.ts index f75000368..106444d46 100644 --- a/packages/sdk/tests/reuse-summary-interruption.test.ts +++ b/packages/sdk/tests/reuse-summary-interruption.test.ts @@ -30,3 +30,14 @@ it('still attaches the summary when the read succeeds, and rethrows other failur const broken = { journalRead: async () => { throw new Error('invalid journal sequence'); } } as unknown as JournalClient; await expect(attachReuseSummary(completed(), broken, 'run-1', 'prior')).rejects.toThrow('invalid journal sequence'); }); + +it.each(['daemon_unresponsive', 'daemon_unreachable'])('does not start another read after classification reported %s', async kind => { + let reads = 0; + const client = { journalRead: async () => { reads += 1; return { entries: [] }; } } as unknown as JournalClient; + const interrupted: RunExecution = { exitCode: 1, report: { command: 'run', ok: false, status: 'running', runId: 'run-1', + diagnostics: [{ severity: 'failure', kind, message: 'read timed out' }] } as never }; + const execution = await attachReuseSummary(interrupted, client, 'run-1', 'prior'); + expect(reads).toBe(0); + expect(execution).toBe(interrupted); + expect(execution.report.reuse).toBeUndefined(); +}); From 35c9e76777c1668f873230ebdf5f5d448290e844 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 01:25:21 -0700 Subject: [PATCH 11/34] fix(sdk): fail closed on malformed reader frames; run cancellation reaches every budgeted read - A malformed frame is now a JournalFrameError; the reader no longer treats it as a transport loss to reconnect and retry, matching the primary session. - JournalClient takes a readSignal that every budgeted read honours; flows run/resume (declarative and authored) pass the lifecycle signal, so an abort during a delayed classification read returns promptly instead of waiting out the 300 s budget. Evidence re-captured at this tree (8/8 mutations; focused 23 files 287/287; resume-live 1/1). Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: b8262468-ea26-4a6e-bee9-d193832e77dc --- evidence/run-read-timeout/final-focused.log | 92 +++--- .../mutation-completion-aborts-snapshot.log | 16 +- .../run-read-timeout/mutation-heartbeat.log | 12 +- .../mutation-reader-reconnect.log | 34 +-- .../mutation-reader-setup-retry.log | 32 +-- evidence/run-read-timeout/mutation-reader.log | 34 +-- .../mutation-root-parking.log | 12 +- .../mutation-snapshot-cancel.log | 16 +- .../mutation-watch-cadence.log | 16 +- evidence/run-read-timeout/resume-live.log | 8 +- evidence/run-read-timeout/verification.md | 272 +++++++++--------- packages/sdk/src/cli/direct-run.ts | 3 +- packages/sdk/src/cli/run.ts | 6 +- packages/sdk/src/journal-client.ts | 19 +- .../tests/journal-client-read-timeout.test.ts | 29 ++ packages/sdk/tests/memoization.test.ts | 26 ++ 16 files changed, 349 insertions(+), 278 deletions(-) diff --git a/evidence/run-read-timeout/final-focused.log b/evidence/run-read-timeout/final-focused.log index bc05590e9..97c57b2a0 100644 --- a/evidence/run-read-timeout/final-focused.log +++ b/evidence/run-read-timeout/final-focused.log @@ -2,58 +2,58 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5237ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2041ms + ✓ tests/cli.test.ts (71 tests) 5251ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2039ms ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2038ms -(node:3394253) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. +(node:3460729) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (27 tests) 438ms - ✓ tests/journal-client.test.ts (17 tests) 106ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 6940ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 504ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1021ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2693ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 364ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 820ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2469ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 633ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 591ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 684ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 560ms - ✓ tests/classify-outcome.test.ts (11 tests) 7426ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2011ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3006ms - ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms - ✓ tests/journal-client-read-timeout.test.ts (17 tests) 1445ms - ✓ tests/human-live.test.ts (3 tests) 4493ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2750ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1090ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 652ms - ✓ tests/running-step-watch.test.ts (7 tests) 8238ms + ✓ tests/authored-root.test.ts (27 tests) 398ms + ✓ tests/journal-client.test.ts (17 tests) 82ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 7251ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 575ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1120ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2705ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 358ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 761ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2529ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 629ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 559ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 678ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 663ms + ✓ tests/classify-outcome.test.ts (11 tests) 7433ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2014ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3009ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2003ms + ✓ tests/journal-client-read-timeout.test.ts (19 tests) 1422ms + ✓ tests/human-live.test.ts (3 tests) 4521ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2654ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1134ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 732ms + ✓ tests/running-step-watch.test.ts (7 tests) 8237ms ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms - ✓ cancels promptly while a lease snapshot read is in flight 2007ms + ✓ cancels promptly while a lease snapshot read is in flight 2005ms ✓ a completion push ends the wait without waiting out an in-flight snapshot 2054ms - ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2003ms - ✓ tests/worker-lease.test.ts (7 tests) 13ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 29ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 688ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2434ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2065ms - ✓ drains read and watch promises before an authored flow completes 368ms - ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms - ✓ tests/journal-client-completion.test.ts (6 tests) 104ms - ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms - ✓ tests/direct-run-worker-lease.test.ts (3 tests) 9ms - ✓ tests/reuse-summary-interruption.test.ts (5 tests) 4ms - ✓ tests/memoization.test.ts (57 tests) 53ms - ✓ tests/journal-client-subscriptions.test.ts (1 test) 5ms + ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms + ✓ tests/worker-lease.test.ts (7 tests) 21ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 27ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 781ms + ✓ tests/memoization.test.ts (58 tests) 70ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2461ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2057ms + ✓ drains read and watch promises before an authored flow completes 404ms + ✓ tests/worker-lease-sweep.test.ts (4 tests) 10ms + ✓ tests/journal-client-completion.test.ts (6 tests) 106ms + ✓ tests/resume-worker-lease.test.ts (3 tests) 8ms + ✓ tests/direct-run-worker-lease.test.ts (3 tests) 15ms + ✓ tests/reuse-summary-interruption.test.ts (5 tests) 5ms + ✓ tests/journal-client-subscriptions.test.ts (1 test) 9ms ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 3ms - ✓ tests/authored-verifier-read-budget.test.ts (1 test) 44ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms + ✓ tests/authored-verifier-read-budget.test.ts (1 test) 52ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 14ms Test Files 23 passed (23) - Tests 284 passed (284) - Start at 01:13:26 - Duration 49.32s (transform 1.07s, setup 137ms, collect 6.12s, tests 40.21s, environment 3ms, prepare 880ms) + Tests 287 passed (287) + Start at 01:24:17 + Duration 49.98s (transform 1.07s, setup 139ms, collect 6.17s, tests 40.71s, environment 3ms, prepare 879ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log index 6d3dde365..bd4189f2f 100644 --- a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log +++ b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 3012ms - × a completion push ends the wait without waiting out an in-flight snapshot 3011ms + ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 3010ms + × a completion push ends the wait without waiting out an in-flight snapshot 3010ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -34,8 +34,8 @@ Received: "still waiting" Test Files 1 failed (1) Tests 1 failed | 6 skipped (7) - Start at 01:06:00 - Duration 3.48s (transform 181ms, setup 16ms, collect 305ms, tests 3.01s, environment 0ms, prepare 40ms) + Start at 01:24:02 + Duration 3.48s (transform 186ms, setup 17ms, collect 316ms, tests 3.01s, environment 0ms, prepare 41ms) exit=1 @@ -44,12 +44,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2060ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2059ms + ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2058ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2058ms Test Files 1 passed (1) Tests 1 passed | 6 skipped (7) - Start at 01:06:04 - Duration 2.50s (transform 182ms, setup 15ms, collect 304ms, tests 2.06s, environment 0ms, prepare 35ms) + Start at 01:24:05 + Duration 2.53s (transform 190ms, setup 17ms, collect 316ms, tests 2.06s, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-heartbeat.log b/evidence/run-read-timeout/mutation-heartbeat.log index 10b5d7411..f954fc142 100644 --- a/evidence/run-read-timeout/mutation-heartbeat.log +++ b/evidence/run-read-timeout/mutation-heartbeat.log @@ -13,8 +13,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 18ms - × a heartbeat timeout is lease loss rather than a worker body failure 18ms + ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 19ms + × a heartbeat timeout is lease loss rather than a worker body failure 19ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -33,8 +33,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 01:06:07 - Duration 459ms (transform 167ms, setup 15ms, collect 281ms, tests 18ms, environment 0ms, prepare 43ms) + Start at 01:24:08 + Duration 469ms (transform 168ms, setup 16ms, collect 284ms, tests 19ms, environment 0ms, prepare 44ms) exit=1 @@ -47,7 +47,7 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb Test Files 1 passed (1) Tests 1 passed (1) - Start at 01:06:08 - Duration 448ms (transform 162ms, setup 17ms, collect 279ms, tests 15ms, environment 0ms, prepare 42ms) + Start at 01:24:09 + Duration 439ms (transform 165ms, setup 17ms, collect 274ms, tests 15ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-reconnect.log b/evidence/run-read-timeout/mutation-reader-reconnect.log index 59cf83261..5a8d6a4f2 100644 --- a/evidence/run-read-timeout/mutation-reader-reconnect.log +++ b/evidence/run-read-timeout/mutation-reader-reconnect.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/journal-client.ts -Original SHA256: e840ea971905def84fdecf0c3826430f058e0a88cf5650627fe0ca4fc4faa299 +Original SHA256: 7398cebb5ccc218341966f90586a40e862bf1fe85bf31b77684198906f10a329 Replaced: this.dropReader(reader); @@ -11,14 +11,14 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (17 tests | 1 failed | 16 skipped) 1008ms - × reconnects the reader after it disconnects within the read budget 1007ms - → journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ tests/journal-client-read-timeout.test.ts (19 tests | 1 failed | 18 skipped) 1007ms + × reconnects the reader after it disconnects within the read budget 1006ms + → journal client: run.get read session was interrupted after 78 attempts in 999ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 78 attempts in 999ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:70:9 68| let last: unknown; 69| const exhausted = () => last instanceof JournalReadInterruptedError @@ -29,33 +29,33 @@ JournalReadInterruptedError: journal client: run.get read session was interrupte ❯ Timeout. src/journal-read-policy.ts:81:39 Caused by: Error: journal client: not connected (run.get): journal client: connection closed - ❯ src/journal-client.ts:270:16 - ❯ JournalClient.requestOnce src/journal-client.ts:263:12 - ❯ src/journal-client.ts:245:31 + ❯ src/journal-client.ts:283:16 + ❯ JournalClient.requestOnce src/journal-client.ts:276:12 + ❯ src/journal-client.ts:257:31 ❯ src/journal-read-policy.ts:93:18 Caused by: Error: journal client: connection closed - ❯ Socket. src/journal-client.ts:125:26 + ❯ Socket. src/journal-client.ts:137:26 ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 16 skipped (17) - Start at 01:05:44 - Duration 1.44s (transform 158ms, setup 16ms, collect 276ms, tests 1.01s, environment 0ms, prepare 42ms) + Tests 1 failed | 18 skipped (19) + Start at 01:23:45 + Duration 1.45s (transform 165ms, setup 17ms, collect 282ms, tests 1.01s, environment 0ms, prepare 42ms) exit=1 -RESTORED SHA256: e840ea971905def84fdecf0c3826430f058e0a88cf5650627fe0ca4fc4faa299 +RESTORED SHA256: 7398cebb5ccc218341966f90586a40e862bf1fe85bf31b77684198906f10a329 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'reconnects the reader after it disconnects' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 79ms + ✓ tests/journal-client-read-timeout.test.ts (19 tests | 18 skipped) 83ms Test Files 1 passed (1) - Tests 1 passed | 16 skipped (17) - Start at 01:05:46 - Duration 486ms (transform 158ms, setup 18ms, collect 267ms, tests 79ms, environment 0ms, prepare 30ms) + Tests 1 passed | 18 skipped (19) + Start at 01:23:47 + Duration 536ms (transform 167ms, setup 18ms, collect 291ms, tests 83ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-setup-retry.log b/evidence/run-read-timeout/mutation-reader-setup-retry.log index 0ad615a03..4019dcf96 100644 --- a/evidence/run-read-timeout/mutation-reader-setup-retry.log +++ b/evidence/run-read-timeout/mutation-reader-setup-retry.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/journal-client.ts -Original SHA256: e840ea971905def84fdecf0c3826430f058e0a88cf5650627fe0ca4fc4faa299 +Original SHA256: 7398cebb5ccc218341966f90586a40e862bf1fe85bf31b77684198906f10a329 Replaced: if (this.reader === reader) { this.reader = undefined; @@ -16,8 +16,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (17 tests | 1 failed | 16 skipped) 79ms - × retries a reader setup that timed out instead of reading on the primary for good 78ms + ❯ tests/journal-client-read-timeout.test.ts (19 tests | 1 failed | 18 skipped) 74ms + × retries a reader setup that timed out instead of reading on the primary for good 73ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -31,33 +31,33 @@ AssertionError: expected 1 to be 2 // Object.is equality - 2 + 1 - ❯ tests/journal-client-read-timeout.test.ts:202:18 - 200| expect(await client.runGet('run')).toMatchObject({ status: 'complete… + ❯ tests/journal-client-read-timeout.test.ts:203:18 201| expect(await client.runGet('run')).toMatchObject({ status: 'complete… - 202| expect(hellos).toBe(2); + 202| expect(await client.runGet('run')).toMatchObject({ status: 'complete… + 203| expect(hellos).toBe(2); | ^ - 203| expect(served).toHaveLength(2); - 204| expect(served.every(socket => socket !== primary)).toBe(true); + 204| expect(served).toHaveLength(2); + 205| expect(served.every(socket => socket !== primary)).toBe(true); ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 16 skipped (17) - Start at 01:05:47 - Duration 467ms (transform 151ms, setup 16ms, collect 251ms, tests 79ms, environment 0ms, prepare 30ms) + Tests 1 failed | 18 skipped (19) + Start at 01:23:48 + Duration 504ms (transform 163ms, setup 17ms, collect 279ms, tests 74ms, environment 0ms, prepare 40ms) exit=1 -RESTORED SHA256: e840ea971905def84fdecf0c3826430f058e0a88cf5650627fe0ca4fc4faa299 +RESTORED SHA256: 7398cebb5ccc218341966f90586a40e862bf1fe85bf31b77684198906f10a329 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'retries a reader setup that timed out' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 76ms + ✓ tests/journal-client-read-timeout.test.ts (19 tests | 18 skipped) 67ms Test Files 1 passed (1) - Tests 1 passed | 16 skipped (17) - Start at 01:05:48 - Duration 507ms (transform 160ms, setup 15ms, collect 284ms, tests 76ms, environment 0ms, prepare 30ms) + Tests 1 passed | 18 skipped (19) + Start at 01:23:49 + Duration 515ms (transform 165ms, setup 17ms, collect 286ms, tests 67ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader.log b/evidence/run-read-timeout/mutation-reader.log index 2b644d8cf..4aa9dac17 100644 --- a/evidence/run-read-timeout/mutation-reader.log +++ b/evidence/run-read-timeout/mutation-reader.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/journal-client.ts -Original SHA256: e840ea971905def84fdecf0c3826430f058e0a88cf5650627fe0ca4fc4faa299 +Original SHA256: 7398cebb5ccc218341966f90586a40e862bf1fe85bf31b77684198906f10a329 Replaced: return await reader.requestOnce(verb, params, remaining, attemptSignal); With: @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (17 tests | 1 failed | 16 skipped) 112ms - × serves a bounded read while an unbounded command is in flight 112ms + ❯ tests/journal-client-read-timeout.test.ts (19 tests | 1 failed | 18 skipped) 114ms + × serves a bounded read while an unbounded command is in flight 113ms → expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -25,33 +25,33 @@ AssertionError: expected true to be false // Object.is equality - false + true - ❯ tests/journal-client-read-timeout.test.ts:37:23 - 35| await inFlight; - 36| expect(await client.runGet('run')).toMatchObject({ status: 'running'… - 37| expect(commandDone).toBe(false); + ❯ tests/journal-client-read-timeout.test.ts:38:23 + 36| await inFlight; + 37| expect(await client.runGet('run')).toMatchObject({ status: 'running'… + 38| expect(commandDone).toBe(false); | ^ - 38| await command; - 39| }); + 39| await command; + 40| }); ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 16 skipped (17) - Start at 01:05:42 - Duration 533ms (transform 160ms, setup 16ms, collect 271ms, tests 112ms, environment 0ms, prepare 41ms) + Tests 1 failed | 18 skipped (19) + Start at 01:23:44 + Duration 560ms (transform 162ms, setup 17ms, collect 282ms, tests 114ms, environment 0ms, prepare 42ms) exit=1 -RESTORED SHA256: e840ea971905def84fdecf0c3826430f058e0a88cf5650627fe0ca4fc4faa299 +RESTORED SHA256: 7398cebb5ccc218341966f90586a40e862bf1fe85bf31b77684198906f10a329 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'serves a bounded read' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 107ms + ✓ tests/journal-client-read-timeout.test.ts (19 tests | 18 skipped) 106ms Test Files 1 passed (1) - Tests 1 passed | 16 skipped (17) - Start at 01:05:43 - Duration 563ms (transform 170ms, setup 17ms, collect 291ms, tests 107ms, environment 0ms, prepare 41ms) + Tests 1 passed | 18 skipped (19) + Start at 01:23:45 + Duration 558ms (transform 163ms, setup 16ms, collect 289ms, tests 106ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-root-parking.log b/evidence/run-read-timeout/mutation-root-parking.log index 76b801d9a..137d3acf1 100644 --- a/evidence/run-read-timeout/mutation-root-parking.log +++ b/evidence/run-read-timeout/mutation-root-parking.log @@ -17,11 +17,11 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (27 tests | 3 failed | 24 skipped) 30ms + ❯ tests/authored-root.test.ts (27 tests | 3 failed | 24 skipped) 31ms × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 22ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 5ms → expected AuthoredFlowExecutionError: daemon_unresp… { …(7) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 4ms @@ -104,8 +104,8 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } Test Files 1 failed (1) Tests 3 failed | 24 skipped (27) - Start at 01:06:09 - Duration 1.23s (transform 654ms, setup 17ms, collect 1.04s, tests 30ms, environment 0ms, prepare 41ms) + Start at 01:24:10 + Duration 1.25s (transform 640ms, setup 16ms, collect 1.05s, tests 31ms, environment 0ms, prepare 41ms) exit=1 @@ -118,7 +118,7 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r Test Files 1 passed (1) Tests 3 passed | 24 skipped (27) - Start at 01:06:10 - Duration 1.27s (transform 658ms, setup 17ms, collect 1.08s, tests 23ms, environment 0ms, prepare 41ms) + Start at 01:24:12 + Duration 1.30s (transform 672ms, setup 17ms, collect 1.10s, tests 23ms, environment 0ms, prepare 47ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-snapshot-cancel.log b/evidence/run-read-timeout/mutation-snapshot-cancel.log index aaa76c02c..fe46e3ec9 100644 --- a/evidence/run-read-timeout/mutation-snapshot-cancel.log +++ b/evidence/run-read-timeout/mutation-snapshot-cancel.log @@ -11,8 +11,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 2217ms - × cancels promptly while a lease snapshot read is in flight 2216ms + ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 2216ms + × cancels promptly while a lease snapshot read is in flight 2215ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -31,8 +31,8 @@ AssertionError: expected 'still waiting' to be an instance of Error Test Files 1 failed (1) Tests 1 failed | 6 skipped (7) - Start at 01:05:54 - Duration 2.69s (transform 190ms, setup 16ms, collect 323ms, tests 2.22s, environment 0ms, prepare 42ms) + Start at 01:23:56 + Duration 2.68s (transform 187ms, setup 16ms, collect 316ms, tests 2.22s, environment 0ms, prepare 40ms) exit=1 @@ -41,12 +41,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2010ms - ✓ cancels promptly while a lease snapshot read is in flight 2010ms + ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2011ms + ✓ cancels promptly while a lease snapshot read is in flight 2011ms Test Files 1 passed (1) Tests 1 passed | 6 skipped (7) - Start at 01:05:57 - Duration 2.49s (transform 189ms, setup 16ms, collect 318ms, tests 2.01s, environment 0ms, prepare 41ms) + Start at 01:23:59 + Duration 2.48s (transform 188ms, setup 17ms, collect 309ms, tests 2.01s, environment 0ms, prepare 42ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-watch-cadence.log b/evidence/run-read-timeout/mutation-watch-cadence.log index 054711ed8..d6700441d 100644 --- a/evidence/run-read-timeout/mutation-watch-cadence.log +++ b/evidence/run-read-timeout/mutation-watch-cadence.log @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 2110ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2110ms + ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 2109ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2108ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -30,8 +30,8 @@ AssertionError: expected 41 to be less than or equal to 1 Test Files 1 failed (1) Tests 1 failed | 6 skipped (7) - Start at 01:05:48 - Duration 2.58s (transform 185ms, setup 16ms, collect 311ms, tests 2.11s, environment 0ms, prepare 41ms) + Start at 01:23:50 + Duration 2.59s (transform 194ms, setup 16ms, collect 330ms, tests 2.11s, environment 0ms, prepare 43ms) exit=1 @@ -40,12 +40,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2118ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms + ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2119ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms Test Files 1 passed (1) Tests 1 passed | 6 skipped (7) - Start at 01:05:51 - Duration 2.58s (transform 177ms, setup 17ms, collect 295ms, tests 2.12s, environment 0ms, prepare 41ms) + Start at 01:23:53 + Duration 2.60s (transform 189ms, setup 17ms, collect 312ms, tests 2.12s, environment 0ms, prepare 45ms) exit=0 diff --git a/evidence/run-read-timeout/resume-live.log b/evidence/run-read-timeout/resume-live.log index e1b82fae2..9824a7253 100644 --- a/evidence/run-read-timeout/resume-live.log +++ b/evidence/run-read-timeout/resume-live.log @@ -2,12 +2,12 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 910ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 910ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 906ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 905ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 01:14:16 - Duration 2.12s (transform 593ms, setup 16ms, collect 968ms, tests 910ms, environment 0ms, prepare 59ms) + Start at 01:25:07 + Duration 2.25s (transform 657ms, setup 19ms, collect 1.09s, tests 906ms, environment 0ms, prepare 81ms) exit=0 diff --git a/evidence/run-read-timeout/verification.md b/evidence/run-read-timeout/verification.md index de25216ab..f6091bfd6 100644 --- a/evidence/run-read-timeout/verification.md +++ b/evidence/run-read-timeout/verification.md @@ -15,59 +15,59 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5237ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2041ms + ✓ tests/cli.test.ts (71 tests) 5251ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2039ms ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2038ms -(node:3394253) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. +(node:3460729) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (27 tests) 438ms - ✓ tests/journal-client.test.ts (17 tests) 106ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 6940ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 504ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1021ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2693ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 364ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 820ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2469ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 633ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 591ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 684ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 560ms - ✓ tests/classify-outcome.test.ts (11 tests) 7426ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2011ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3006ms - ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms - ✓ tests/journal-client-read-timeout.test.ts (17 tests) 1445ms - ✓ tests/human-live.test.ts (3 tests) 4493ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2750ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1090ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 652ms - ✓ tests/running-step-watch.test.ts (7 tests) 8238ms + ✓ tests/authored-root.test.ts (27 tests) 398ms + ✓ tests/journal-client.test.ts (17 tests) 82ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 7251ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 575ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1120ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2705ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 358ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 761ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2529ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 629ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 559ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 678ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 663ms + ✓ tests/classify-outcome.test.ts (11 tests) 7433ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2014ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3009ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2003ms + ✓ tests/journal-client-read-timeout.test.ts (19 tests) 1422ms + ✓ tests/human-live.test.ts (3 tests) 4521ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2654ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1134ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 732ms + ✓ tests/running-step-watch.test.ts (7 tests) 8237ms ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms - ✓ cancels promptly while a lease snapshot read is in flight 2007ms + ✓ cancels promptly while a lease snapshot read is in flight 2005ms ✓ a completion push ends the wait without waiting out an in-flight snapshot 2054ms - ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2003ms - ✓ tests/worker-lease.test.ts (7 tests) 13ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 29ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 688ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2434ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2065ms - ✓ drains read and watch promises before an authored flow completes 368ms - ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms - ✓ tests/journal-client-completion.test.ts (6 tests) 104ms - ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms - ✓ tests/direct-run-worker-lease.test.ts (3 tests) 9ms - ✓ tests/reuse-summary-interruption.test.ts (5 tests) 4ms - ✓ tests/memoization.test.ts (57 tests) 53ms - ✓ tests/journal-client-subscriptions.test.ts (1 test) 5ms + ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms + ✓ tests/worker-lease.test.ts (7 tests) 21ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 27ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 781ms + ✓ tests/memoization.test.ts (58 tests) 70ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2461ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2057ms + ✓ drains read and watch promises before an authored flow completes 404ms + ✓ tests/worker-lease-sweep.test.ts (4 tests) 10ms + ✓ tests/journal-client-completion.test.ts (6 tests) 106ms + ✓ tests/resume-worker-lease.test.ts (3 tests) 8ms + ✓ tests/direct-run-worker-lease.test.ts (3 tests) 15ms + ✓ tests/reuse-summary-interruption.test.ts (5 tests) 5ms + ✓ tests/journal-client-subscriptions.test.ts (1 test) 9ms ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 3ms - ✓ tests/authored-verifier-read-budget.test.ts (1 test) 44ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms + ✓ tests/authored-verifier-read-budget.test.ts (1 test) 52ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 14ms Test Files 23 passed (23) - Tests 284 passed (284) - Start at 01:13:26 - Duration 49.32s (transform 1.07s, setup 137ms, collect 6.12s, tests 40.21s, environment 3ms, prepare 880ms) + Tests 287 passed (287) + Start at 01:24:17 + Duration 49.98s (transform 1.07s, setup 139ms, collect 6.17s, tests 40.71s, environment 3ms, prepare 879ms) exit=0 ``` @@ -79,13 +79,13 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 910ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 910ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 906ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 905ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 01:14:16 - Duration 2.12s (transform 593ms, setup 16ms, collect 968ms, tests 910ms, environment 0ms, prepare 59ms) + Start at 01:25:07 + Duration 2.25s (transform 657ms, setup 19ms, collect 1.09s, tests 906ms, environment 0ms, prepare 81ms) exit=0 ``` @@ -96,7 +96,7 @@ The mutation sections below were regenerated by `python3 evidence/run-read-timeo ```text Source: packages/sdk/src/journal-client.ts -Original SHA256: e840ea971905def84fdecf0c3826430f058e0a88cf5650627fe0ca4fc4faa299 +Original SHA256: 7398cebb5ccc218341966f90586a40e862bf1fe85bf31b77684198906f10a329 Replaced: return await reader.requestOnce(verb, params, remaining, attemptSignal); With: @@ -107,8 +107,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (17 tests | 1 failed | 16 skipped) 112ms - × serves a bounded read while an unbounded command is in flight 112ms + ❯ tests/journal-client-read-timeout.test.ts (19 tests | 1 failed | 18 skipped) 114ms + × serves a bounded read while an unbounded command is in flight 113ms → expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -122,34 +122,34 @@ AssertionError: expected true to be false // Object.is equality - false + true - ❯ tests/journal-client-read-timeout.test.ts:37:23 - 35| await inFlight; - 36| expect(await client.runGet('run')).toMatchObject({ status: 'running'… - 37| expect(commandDone).toBe(false); + ❯ tests/journal-client-read-timeout.test.ts:38:23 + 36| await inFlight; + 37| expect(await client.runGet('run')).toMatchObject({ status: 'running'… + 38| expect(commandDone).toBe(false); | ^ - 38| await command; - 39| }); + 39| await command; + 40| }); ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 16 skipped (17) - Start at 01:05:42 - Duration 533ms (transform 160ms, setup 16ms, collect 271ms, tests 112ms, environment 0ms, prepare 41ms) + Tests 1 failed | 18 skipped (19) + Start at 01:23:44 + Duration 560ms (transform 162ms, setup 17ms, collect 282ms, tests 114ms, environment 0ms, prepare 42ms) exit=1 -RESTORED SHA256: e840ea971905def84fdecf0c3826430f058e0a88cf5650627fe0ca4fc4faa299 +RESTORED SHA256: 7398cebb5ccc218341966f90586a40e862bf1fe85bf31b77684198906f10a329 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'serves a bounded read' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 107ms + ✓ tests/journal-client-read-timeout.test.ts (19 tests | 18 skipped) 106ms Test Files 1 passed (1) - Tests 1 passed | 16 skipped (17) - Start at 01:05:43 - Duration 563ms (transform 170ms, setup 17ms, collect 291ms, tests 107ms, environment 0ms, prepare 41ms) + Tests 1 passed | 18 skipped (19) + Start at 01:23:45 + Duration 558ms (transform 163ms, setup 16ms, collect 289ms, tests 106ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -158,7 +158,7 @@ exit=0 ```text Source: packages/sdk/src/journal-client.ts -Original SHA256: e840ea971905def84fdecf0c3826430f058e0a88cf5650627fe0ca4fc4faa299 +Original SHA256: 7398cebb5ccc218341966f90586a40e862bf1fe85bf31b77684198906f10a329 Replaced: this.dropReader(reader); @@ -170,14 +170,14 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (17 tests | 1 failed | 16 skipped) 1008ms - × reconnects the reader after it disconnects within the read budget 1007ms - → journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ tests/journal-client-read-timeout.test.ts (19 tests | 1 failed | 18 skipped) 1007ms + × reconnects the reader after it disconnects within the read budget 1006ms + → journal client: run.get read session was interrupted after 78 attempts in 999ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 78 attempts in 999ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:70:9 68| let last: unknown; 69| const exhausted = () => last instanceof JournalReadInterruptedError @@ -188,34 +188,34 @@ JournalReadInterruptedError: journal client: run.get read session was interrupte ❯ Timeout. src/journal-read-policy.ts:81:39 Caused by: Error: journal client: not connected (run.get): journal client: connection closed - ❯ src/journal-client.ts:270:16 - ❯ JournalClient.requestOnce src/journal-client.ts:263:12 - ❯ src/journal-client.ts:245:31 + ❯ src/journal-client.ts:283:16 + ❯ JournalClient.requestOnce src/journal-client.ts:276:12 + ❯ src/journal-client.ts:257:31 ❯ src/journal-read-policy.ts:93:18 Caused by: Error: journal client: connection closed - ❯ Socket. src/journal-client.ts:125:26 + ❯ Socket. src/journal-client.ts:137:26 ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 16 skipped (17) - Start at 01:05:44 - Duration 1.44s (transform 158ms, setup 16ms, collect 276ms, tests 1.01s, environment 0ms, prepare 42ms) + Tests 1 failed | 18 skipped (19) + Start at 01:23:45 + Duration 1.45s (transform 165ms, setup 17ms, collect 282ms, tests 1.01s, environment 0ms, prepare 42ms) exit=1 -RESTORED SHA256: e840ea971905def84fdecf0c3826430f058e0a88cf5650627fe0ca4fc4faa299 +RESTORED SHA256: 7398cebb5ccc218341966f90586a40e862bf1fe85bf31b77684198906f10a329 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'reconnects the reader after it disconnects' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 79ms + ✓ tests/journal-client-read-timeout.test.ts (19 tests | 18 skipped) 83ms Test Files 1 passed (1) - Tests 1 passed | 16 skipped (17) - Start at 01:05:46 - Duration 486ms (transform 158ms, setup 18ms, collect 267ms, tests 79ms, environment 0ms, prepare 30ms) + Tests 1 passed | 18 skipped (19) + Start at 01:23:47 + Duration 536ms (transform 167ms, setup 18ms, collect 291ms, tests 83ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -224,7 +224,7 @@ exit=0 ```text Source: packages/sdk/src/journal-client.ts -Original SHA256: e840ea971905def84fdecf0c3826430f058e0a88cf5650627fe0ca4fc4faa299 +Original SHA256: 7398cebb5ccc218341966f90586a40e862bf1fe85bf31b77684198906f10a329 Replaced: if (this.reader === reader) { this.reader = undefined; @@ -241,8 +241,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (17 tests | 1 failed | 16 skipped) 79ms - × retries a reader setup that timed out instead of reading on the primary for good 78ms + ❯ tests/journal-client-read-timeout.test.ts (19 tests | 1 failed | 18 skipped) 74ms + × retries a reader setup that timed out instead of reading on the primary for good 73ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -256,34 +256,34 @@ AssertionError: expected 1 to be 2 // Object.is equality - 2 + 1 - ❯ tests/journal-client-read-timeout.test.ts:202:18 - 200| expect(await client.runGet('run')).toMatchObject({ status: 'complete… + ❯ tests/journal-client-read-timeout.test.ts:203:18 201| expect(await client.runGet('run')).toMatchObject({ status: 'complete… - 202| expect(hellos).toBe(2); + 202| expect(await client.runGet('run')).toMatchObject({ status: 'complete… + 203| expect(hellos).toBe(2); | ^ - 203| expect(served).toHaveLength(2); - 204| expect(served.every(socket => socket !== primary)).toBe(true); + 204| expect(served).toHaveLength(2); + 205| expect(served.every(socket => socket !== primary)).toBe(true); ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 16 skipped (17) - Start at 01:05:47 - Duration 467ms (transform 151ms, setup 16ms, collect 251ms, tests 79ms, environment 0ms, prepare 30ms) + Tests 1 failed | 18 skipped (19) + Start at 01:23:48 + Duration 504ms (transform 163ms, setup 17ms, collect 279ms, tests 74ms, environment 0ms, prepare 40ms) exit=1 -RESTORED SHA256: e840ea971905def84fdecf0c3826430f058e0a88cf5650627fe0ca4fc4faa299 +RESTORED SHA256: 7398cebb5ccc218341966f90586a40e862bf1fe85bf31b77684198906f10a329 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'retries a reader setup that timed out' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (17 tests | 16 skipped) 76ms + ✓ tests/journal-client-read-timeout.test.ts (19 tests | 18 skipped) 67ms Test Files 1 passed (1) - Tests 1 passed | 16 skipped (17) - Start at 01:05:48 - Duration 507ms (transform 160ms, setup 15ms, collect 284ms, tests 76ms, environment 0ms, prepare 30ms) + Tests 1 passed | 18 skipped (19) + Start at 01:23:49 + Duration 515ms (transform 165ms, setup 17ms, collect 286ms, tests 67ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -303,8 +303,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 2110ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2110ms + ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 2109ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2108ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -323,8 +323,8 @@ AssertionError: expected 41 to be less than or equal to 1 Test Files 1 failed (1) Tests 1 failed | 6 skipped (7) - Start at 01:05:48 - Duration 2.58s (transform 185ms, setup 16ms, collect 311ms, tests 2.11s, environment 0ms, prepare 41ms) + Start at 01:23:50 + Duration 2.59s (transform 194ms, setup 16ms, collect 330ms, tests 2.11s, environment 0ms, prepare 43ms) exit=1 @@ -333,13 +333,13 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2118ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms + ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2119ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms Test Files 1 passed (1) Tests 1 passed | 6 skipped (7) - Start at 01:05:51 - Duration 2.58s (transform 177ms, setup 17ms, collect 295ms, tests 2.12s, environment 0ms, prepare 41ms) + Start at 01:23:53 + Duration 2.60s (transform 189ms, setup 17ms, collect 312ms, tests 2.12s, environment 0ms, prepare 45ms) exit=0 ``` @@ -360,8 +360,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 2217ms - × cancels promptly while a lease snapshot read is in flight 2216ms + ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 2216ms + × cancels promptly while a lease snapshot read is in flight 2215ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -380,8 +380,8 @@ AssertionError: expected 'still waiting' to be an instance of Error Test Files 1 failed (1) Tests 1 failed | 6 skipped (7) - Start at 01:05:54 - Duration 2.69s (transform 190ms, setup 16ms, collect 323ms, tests 2.22s, environment 0ms, prepare 42ms) + Start at 01:23:56 + Duration 2.68s (transform 187ms, setup 16ms, collect 316ms, tests 2.22s, environment 0ms, prepare 40ms) exit=1 @@ -390,13 +390,13 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2010ms - ✓ cancels promptly while a lease snapshot read is in flight 2010ms + ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2011ms + ✓ cancels promptly while a lease snapshot read is in flight 2011ms Test Files 1 passed (1) Tests 1 passed | 6 skipped (7) - Start at 01:05:57 - Duration 2.49s (transform 189ms, setup 16ms, collect 318ms, tests 2.01s, environment 0ms, prepare 41ms) + Start at 01:23:59 + Duration 2.48s (transform 188ms, setup 17ms, collect 309ms, tests 2.01s, environment 0ms, prepare 42ms) exit=0 ``` @@ -416,8 +416,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 3012ms - × a completion push ends the wait without waiting out an in-flight snapshot 3011ms + ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 3010ms + × a completion push ends the wait without waiting out an in-flight snapshot 3010ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -440,8 +440,8 @@ Received: "still waiting" Test Files 1 failed (1) Tests 1 failed | 6 skipped (7) - Start at 01:06:00 - Duration 3.48s (transform 181ms, setup 16ms, collect 305ms, tests 3.01s, environment 0ms, prepare 40ms) + Start at 01:24:02 + Duration 3.48s (transform 186ms, setup 17ms, collect 316ms, tests 3.01s, environment 0ms, prepare 41ms) exit=1 @@ -450,13 +450,13 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2060ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2059ms + ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2058ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2058ms Test Files 1 passed (1) Tests 1 passed | 6 skipped (7) - Start at 01:06:04 - Duration 2.50s (transform 182ms, setup 15ms, collect 304ms, tests 2.06s, environment 0ms, prepare 35ms) + Start at 01:24:05 + Duration 2.53s (transform 190ms, setup 17ms, collect 316ms, tests 2.06s, environment 0ms, prepare 40ms) exit=0 ``` @@ -479,8 +479,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 18ms - × a heartbeat timeout is lease loss rather than a worker body failure 18ms + ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 19ms + × a heartbeat timeout is lease loss rather than a worker body failure 19ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -499,8 +499,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 01:06:07 - Duration 459ms (transform 167ms, setup 15ms, collect 281ms, tests 18ms, environment 0ms, prepare 43ms) + Start at 01:24:08 + Duration 469ms (transform 168ms, setup 16ms, collect 284ms, tests 19ms, environment 0ms, prepare 44ms) exit=1 @@ -513,8 +513,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb Test Files 1 passed (1) Tests 1 passed (1) - Start at 01:06:08 - Duration 448ms (transform 162ms, setup 17ms, collect 279ms, tests 15ms, environment 0ms, prepare 42ms) + Start at 01:24:09 + Duration 439ms (transform 165ms, setup 17ms, collect 274ms, tests 15ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -541,11 +541,11 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (27 tests | 3 failed | 24 skipped) 30ms + ❯ tests/authored-root.test.ts (27 tests | 3 failed | 24 skipped) 31ms × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 22ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 5ms → expected AuthoredFlowExecutionError: daemon_unresp… { …(7) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 4ms @@ -628,8 +628,8 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } Test Files 1 failed (1) Tests 3 failed | 24 skipped (27) - Start at 01:06:09 - Duration 1.23s (transform 654ms, setup 17ms, collect 1.04s, tests 30ms, environment 0ms, prepare 41ms) + Start at 01:24:10 + Duration 1.25s (transform 640ms, setup 16ms, collect 1.05s, tests 31ms, environment 0ms, prepare 41ms) exit=1 @@ -642,8 +642,8 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r Test Files 1 passed (1) Tests 3 passed | 24 skipped (27) - Start at 01:06:10 - Duration 1.27s (transform 658ms, setup 17ms, collect 1.08s, tests 23ms, environment 0ms, prepare 41ms) + Start at 01:24:12 + Duration 1.30s (transform 672ms, setup 17ms, collect 1.10s, tests 23ms, environment 0ms, prepare 47ms) exit=0 ``` diff --git a/packages/sdk/src/cli/direct-run.ts b/packages/sdk/src/cli/direct-run.ts index 7109ff599..248a54581 100644 --- a/packages/sdk/src/cli/direct-run.ts +++ b/packages/sdk/src/cli/direct-run.ts @@ -69,7 +69,8 @@ export async function runDirectFlow( } const socketPath = socketFor(dataDir); const base: RunReport = { ...emptyReport('run'), path }; - const client = new JournalClient(socketPath, { readBudgetMs: FLOW_READ_BUDGET_MS }); + const client = new JournalClient(socketPath, { readBudgetMs: FLOW_READ_BUDGET_MS, + ...(options.signal === undefined ? {} : { readSignal: options.signal }) }); const connected = await connect(client, 'run', dataDir, base, options); if (connected !== undefined) return connected; diff --git a/packages/sdk/src/cli/run.ts b/packages/sdk/src/cli/run.ts index 238d493c5..2746e4d1f 100644 --- a/packages/sdk/src/cli/run.ts +++ b/packages/sdk/src/cli/run.ts @@ -207,7 +207,8 @@ async function executeCheckedFlow( catch (error) { return { exitCode: 2, report: { ...base, diagnostics: [...base.diagnostics, { severity: 'refusal', kind: 'probe_failed', message: error instanceof Error ? error.message : 'Communication environment could not be checked.' }] } }; } } - const client = new JournalClient(socketPath, { readBudgetMs: FLOW_READ_BUDGET_MS }); + const client = new JournalClient(socketPath, { readBudgetMs: FLOW_READ_BUDGET_MS, + ...(options.signal === undefined ? {} : { readSignal: options.signal }) }); const connected = await connect(client, 'run', dataDir, base, options); if (connected !== undefined) return connected; @@ -293,7 +294,8 @@ export async function resumeFlow( const agentEnvironment = localAgentEnvironment(); const socketPath = socketFor(dataDir); const base = emptyReport('resume'); - const client = new JournalClient(socketPath, { readBudgetMs: FLOW_READ_BUDGET_MS }); + const client = new JournalClient(socketPath, { readBudgetMs: FLOW_READ_BUDGET_MS, + ...(options.signal === undefined ? {} : { readSignal: options.signal }) }); const connected = await connect(client, 'resume', dataDir, base, options); if (connected !== undefined) return connected; diff --git a/packages/sdk/src/journal-client.ts b/packages/sdk/src/journal-client.ts index 9113459ae..63a5d7e39 100644 --- a/packages/sdk/src/journal-client.ts +++ b/packages/sdk/src/journal-client.ts @@ -31,6 +31,8 @@ import type { KernelRunSpec, StepType } from './spec.js'; export interface JournalClientOptions { /** Opt-in total budget for read-only requests; interactive clients stay single-shot. */ readBudgetMs?: number; + /** Cancels every budgeted read, so a run's lifecycle abort never waits out the budget. */ + readSignal?: AbortSignal; /** Override the timeout for bounded protocol requests (ms). Default 30000. */ requestTimeoutMs?: number; /** @@ -64,11 +66,20 @@ export class JournalProtocolError extends Error { } } +/** The daemon sent a frame this client cannot parse: a protocol violation, never retried. */ +export class JournalFrameError extends Error { + constructor(message = 'journal client: malformed frame from server') { + super(message); + this.name = 'JournalFrameError'; + } +} + export class JournalClient extends EventEmitter { private readonly reads = new JournalReadPolicy(); private reader: JournalClient | undefined; private readerReady: Promise | undefined; private readonly readBudgetMs?: number; + private readonly readSignal?: AbortSignal; /** Additive request capabilities the daemon advertised at `hello`; none until then. */ private features: ReadonlySet = new Set(); private socket: Socket | null = null; @@ -86,6 +97,7 @@ export class JournalClient extends EventEmitter { ) { super(); this.readBudgetMs = options.readBudgetMs; + this.readSignal = options.readSignal; this.requestTimeoutMs = options.requestTimeoutMs ?? 30_000; this.connectTimeoutMs = options.connectTimeoutMs ?? 2_000; } @@ -171,7 +183,7 @@ export class JournalClient extends EventEmitter { msg = JSON.parse(line) as Response | ServerEvent; } catch { // A malformed frame is a protocol violation; fail closed. - this.failAll(new Error('journal client: malformed frame from server')); + this.failAll(new JournalFrameError()); return; } @@ -245,12 +257,13 @@ export class JournalClient extends EventEmitter { return await reader.requestOnce(verb, params, remaining, attemptSignal); } catch (error) { if (attemptSignal.aborted || error instanceof JournalRequestTimeoutError - || error instanceof JournalProtocolError) throw error; + || error instanceof JournalProtocolError || error instanceof JournalFrameError) throw error; // Only the dedicated reader's transport failed; reconnect within the budget. this.dropReader(reader); throw new JournalReadInterruptedError(verb, 1, performance.now() - started, undefined, { cause: error }); } - }, signal); + }, signal === undefined ? this.readSignal + : this.readSignal === undefined ? signal : AbortSignal.any([signal, this.readSignal])); } return this.requestOnce(verb, params, timeoutMs, signal); } diff --git a/packages/sdk/tests/journal-client-read-timeout.test.ts b/packages/sdk/tests/journal-client-read-timeout.test.ts index 47877d7c0..d4775ad61 100644 --- a/packages/sdk/tests/journal-client-read-timeout.test.ts +++ b/packages/sdk/tests/journal-client-read-timeout.test.ts @@ -4,6 +4,7 @@ import type { Server } from 'node:net'; import { afterEach, expect, it } from 'vitest'; import { JournalClient, JournalRequestTimeoutError } from '../src/journal-client.js'; import { JournalReadInterruptedError, isReadInterruptionError } from '../src/journal-read-policy.js'; +import { JournalFrameError } from '../src/journal-client.js'; import { HELLO_SPEC, sendOk, sendResult, sockPath, startLoopback, type LoopbackHandlers } from './journal-client-loopback.js'; const clients: JournalClient[] = []; @@ -203,3 +204,31 @@ it('retries a reader setup that timed out instead of reading on the primary for expect(served).toHaveLength(2); expect(served.every(socket => socket !== primary)).toBe(true); }); + +it('fails closed on a malformed reader frame instead of retrying it as a transport loss', async () => { + let reads = 0; + const client = await setup({ 'run.get': ctx => { reads += 1; ctx.socket.write('{not json\n'); } }, 5_000); + const error = await client.runGet('run').catch(caught => caught); + expect(error).toBeInstanceOf(JournalFrameError); + expect(isReadInterruptionError(error)).toBe(false); + expect(reads).toBe(1); +}); + +it('a client read signal cancels every budgeted read promptly', async () => { + let entered!: () => void; + const started = new Promise(resolve => { entered = resolve; }); + const path = sockPath(); + const server = startLoopback(path, { hello: sendOk, 'journal.read': () => entered() }, { serialize: true }); + servers.push(server); + await once(server, 'listening'); + const controller = new AbortController(); + const client = new JournalClient(path, { requestTimeoutMs: 50, readBudgetMs: 60_000, readSignal: controller.signal }); + clients.push(client); + await client.connect(); + const pending = client.journalRead('run', 1); + await started; + const begun = performance.now(); + controller.abort(new Error('run canceled')); + await expect(pending).rejects.toThrow('run canceled'); + expect(performance.now() - begun).toBeLessThan(40); +}); diff --git a/packages/sdk/tests/memoization.test.ts b/packages/sdk/tests/memoization.test.ts index 2622e1eae..11a85e3db 100644 --- a/packages/sdk/tests/memoization.test.ts +++ b/packages/sdk/tests/memoization.test.ts @@ -56,3 +56,29 @@ it.each([ rmSync(dir, { recursive: true, force: true }); } }); + +it('cancels runFlow promptly while a classification read is delayed', async () => { + const dir = mkdtempSync(join(tmpdir(), 'run-cancel-read-')); + let reading!: () => void; + const inRead = new Promise(resolve => { reading = resolve; }); + const server = startLoopback(socketPathFor(dir), { + hello: ctx => sendOk(ctx), + 'run.start': ctx => ctx.send({ id: ctx.id, ok: true, result: { run_id: 'r1', status: 'running', completion_reason: null, completed_steps: 0 } }), + 'run.get': () => reading(), // the daemon never answers under load + }); + await once(server, 'listening'); + const controller = new AbortController(); + try { + const running = runFlow(fileURLToPath(new URL('../../../testdata/hello-deterministic.flow.yaml', import.meta.url)), + dir, { daemon: { spawn: false }, signal: controller.signal }); + await inRead; + const begun = performance.now(); + controller.abort(); + const settled = await Promise.race([running.then(() => 'settled', () => 'settled'), new Promise(resolve => setTimeout(() => resolve('still reading'), 2_000))]); + expect(settled).toBe('settled'); + expect(performance.now() - begun).toBeLessThan(2_000); + } finally { + await new Promise(resolve => server.close(() => resolve())); + rmSync(dir, { recursive: true, force: true }); + } +}); From bfcc879bd4a0b195eea364b2f72a39a030893396 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 01:35:28 -0700 Subject: [PATCH 12/34] fix(sdk): malformed handshake frames fail closed; the verifier honours run cancellation - A JournalFrameError during reader setup is rethrown instead of being retried as a read interruption. - verifyAuthoredNodeResult takes the lifecycle signal and hands it to its budgeted client, so cancelling or losing the root lease while a Bun-run body is verified stops the reads instead of waiting out the budget. Evidence re-captured at this tree. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: b8262468-ea26-4a6e-bee9-d193832e77dc --- evidence/run-read-timeout/final-focused.log | 97 +++---- .../mutation-completion-aborts-snapshot.log | 16 +- .../run-read-timeout/mutation-heartbeat.log | 12 +- .../mutation-reader-reconnect.log | 32 +-- .../mutation-reader-setup-retry.log | 22 +- evidence/run-read-timeout/mutation-reader.log | 20 +- .../mutation-root-parking.log | 8 +- .../mutation-snapshot-cancel.log | 14 +- .../mutation-watch-cadence.log | 14 +- evidence/run-read-timeout/resume-live.log | 8 +- evidence/run-read-timeout/verification.md | 243 +++++++++--------- packages/sdk/src/authored-node-runner.ts | 7 +- packages/sdk/src/journal-client.ts | 2 + .../authored-verifier-read-budget.test.ts | 10 + .../tests/journal-client-read-timeout.test.ts | 13 + 15 files changed, 273 insertions(+), 245 deletions(-) diff --git a/evidence/run-read-timeout/final-focused.log b/evidence/run-read-timeout/final-focused.log index 97c57b2a0..af316f867 100644 --- a/evidence/run-read-timeout/final-focused.log +++ b/evidence/run-read-timeout/final-focused.log @@ -2,58 +2,59 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5251ms + ✓ tests/cli.test.ts (71 tests) 5255ms + ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 322ms ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2039ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2038ms -(node:3460729) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2033ms +(node:3512367) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (27 tests) 398ms - ✓ tests/journal-client.test.ts (17 tests) 82ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 7251ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 575ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1120ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2705ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 358ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 761ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2529ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 629ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 559ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 678ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 663ms - ✓ tests/classify-outcome.test.ts (11 tests) 7433ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2014ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3009ms - ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2003ms - ✓ tests/journal-client-read-timeout.test.ts (19 tests) 1422ms - ✓ tests/human-live.test.ts (3 tests) 4521ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2654ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1134ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 732ms - ✓ tests/running-step-watch.test.ts (7 tests) 8237ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms - ✓ cancels promptly while a lease snapshot read is in flight 2005ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2054ms - ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms - ✓ tests/worker-lease.test.ts (7 tests) 21ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 27ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 781ms - ✓ tests/memoization.test.ts (58 tests) 70ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2461ms + ✓ tests/authored-root.test.ts (27 tests) 390ms + ✓ tests/journal-client.test.ts (17 tests) 80ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 6940ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 468ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1067ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2741ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 349ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 827ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2544ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 651ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 595ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 624ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 673ms + ✓ tests/journal-client-read-timeout.test.ts (20 tests) 1427ms + ✓ tests/classify-outcome.test.ts (11 tests) 7425ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2011ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3006ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms + ✓ tests/human-live.test.ts (3 tests) 4580ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2734ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1166ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 679ms + ✓ tests/running-step-watch.test.ts (7 tests) 8236ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2116ms + ✓ cancels promptly while a lease snapshot read is in flight 2006ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2055ms + ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2003ms + ✓ tests/worker-lease.test.ts (7 tests) 20ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 26ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 697ms + ✓ tests/memoization.test.ts (58 tests) 64ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2471ms ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2057ms - ✓ drains read and watch promises before an authored flow completes 404ms - ✓ tests/worker-lease-sweep.test.ts (4 tests) 10ms - ✓ tests/journal-client-completion.test.ts (6 tests) 106ms - ✓ tests/resume-worker-lease.test.ts (3 tests) 8ms - ✓ tests/direct-run-worker-lease.test.ts (3 tests) 15ms - ✓ tests/reuse-summary-interruption.test.ts (5 tests) 5ms - ✓ tests/journal-client-subscriptions.test.ts (1 test) 9ms - ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 3ms - ✓ tests/authored-verifier-read-budget.test.ts (1 test) 52ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 14ms + ✓ drains read and watch promises before an authored flow completes 414ms + ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms + ✓ tests/journal-client-completion.test.ts (6 tests) 102ms + ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms + ✓ tests/direct-run-worker-lease.test.ts (3 tests) 9ms + ✓ tests/reuse-summary-interruption.test.ts (5 tests) 4ms + ✓ tests/journal-client-subscriptions.test.ts (1 test) 7ms + ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 4ms + ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 49ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms Test Files 23 passed (23) - Tests 287 passed (287) - Start at 01:24:17 - Duration 49.98s (transform 1.07s, setup 139ms, collect 6.17s, tests 40.71s, environment 3ms, prepare 879ms) + Tests 289 passed (289) + Start at 01:34:37 + Duration 48.85s (transform 1.05s, setup 122ms, collect 5.61s, tests 40.36s, environment 3ms, prepare 812ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log index bd4189f2f..e6a2c1749 100644 --- a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log +++ b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 3010ms - × a completion push ends the wait without waiting out an in-flight snapshot 3010ms + ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 3012ms + × a completion push ends the wait without waiting out an in-flight snapshot 3012ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -34,8 +34,8 @@ Received: "still waiting" Test Files 1 failed (1) Tests 1 failed | 6 skipped (7) - Start at 01:24:02 - Duration 3.48s (transform 186ms, setup 17ms, collect 316ms, tests 3.01s, environment 0ms, prepare 41ms) + Start at 01:34:22 + Duration 3.48s (transform 186ms, setup 17ms, collect 313ms, tests 3.01s, environment 0ms, prepare 41ms) exit=1 @@ -44,12 +44,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2058ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2058ms + ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2061ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2061ms Test Files 1 passed (1) Tests 1 passed | 6 skipped (7) - Start at 01:24:05 - Duration 2.53s (transform 190ms, setup 17ms, collect 316ms, tests 2.06s, environment 0ms, prepare 40ms) + Start at 01:34:26 + Duration 2.52s (transform 183ms, setup 17ms, collect 312ms, tests 2.06s, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-heartbeat.log b/evidence/run-read-timeout/mutation-heartbeat.log index f954fc142..3ac149d36 100644 --- a/evidence/run-read-timeout/mutation-heartbeat.log +++ b/evidence/run-read-timeout/mutation-heartbeat.log @@ -14,7 +14,7 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 19ms - × a heartbeat timeout is lease loss rather than a worker body failure 19ms + × a heartbeat timeout is lease loss rather than a worker body failure 18ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -33,8 +33,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 01:24:08 - Duration 469ms (transform 168ms, setup 16ms, collect 284ms, tests 19ms, environment 0ms, prepare 44ms) + Start at 01:34:28 + Duration 465ms (transform 168ms, setup 15ms, collect 287ms, tests 19ms, environment 0ms, prepare 40ms) exit=1 @@ -43,11 +43,11 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 13ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 01:24:09 - Duration 439ms (transform 165ms, setup 17ms, collect 274ms, tests 15ms, environment 0ms, prepare 41ms) + Start at 01:34:29 + Duration 431ms (transform 160ms, setup 17ms, collect 261ms, tests 13ms, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-reconnect.log b/evidence/run-read-timeout/mutation-reader-reconnect.log index 5a8d6a4f2..3cdedf783 100644 --- a/evidence/run-read-timeout/mutation-reader-reconnect.log +++ b/evidence/run-read-timeout/mutation-reader-reconnect.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/journal-client.ts -Original SHA256: 7398cebb5ccc218341966f90586a40e862bf1fe85bf31b77684198906f10a329 +Original SHA256: 9c77a946018fecb31ee86b19e343b78cb65f077e87bd03395b5dcb30f2e3a0a3 Replaced: this.dropReader(reader); @@ -11,14 +11,14 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (19 tests | 1 failed | 18 skipped) 1007ms - × reconnects the reader after it disconnects within the read budget 1006ms - → journal client: run.get read session was interrupted after 78 attempts in 999ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ tests/journal-client-read-timeout.test.ts (20 tests | 1 failed | 19 skipped) 1009ms + × reconnects the reader after it disconnects within the read budget 1009ms + → journal client: run.get read session was interrupted after 79 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 78 attempts in 999ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 79 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:70:9 68| let last: unknown; 69| const exhausted = () => last instanceof JournalReadInterruptedError @@ -29,9 +29,9 @@ JournalReadInterruptedError: journal client: run.get read session was interrupte ❯ Timeout. src/journal-read-policy.ts:81:39 Caused by: Error: journal client: not connected (run.get): journal client: connection closed - ❯ src/journal-client.ts:283:16 - ❯ JournalClient.requestOnce src/journal-client.ts:276:12 - ❯ src/journal-client.ts:257:31 + ❯ src/journal-client.ts:285:16 + ❯ JournalClient.requestOnce src/journal-client.ts:278:12 + ❯ src/journal-client.ts:259:31 ❯ src/journal-read-policy.ts:93:18 Caused by: Error: journal client: connection closed @@ -40,22 +40,22 @@ Caused by: Error: journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 18 skipped (19) - Start at 01:23:45 - Duration 1.45s (transform 165ms, setup 17ms, collect 282ms, tests 1.01s, environment 0ms, prepare 42ms) + Tests 1 failed | 19 skipped (20) + Start at 01:34:06 + Duration 1.45s (transform 166ms, setup 18ms, collect 283ms, tests 1.01s, environment 0ms, prepare 41ms) exit=1 -RESTORED SHA256: 7398cebb5ccc218341966f90586a40e862bf1fe85bf31b77684198906f10a329 +RESTORED SHA256: 9c77a946018fecb31ee86b19e343b78cb65f077e87bd03395b5dcb30f2e3a0a3 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'reconnects the reader after it disconnects' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (19 tests | 18 skipped) 83ms + ✓ tests/journal-client-read-timeout.test.ts (20 tests | 19 skipped) 93ms Test Files 1 passed (1) - Tests 1 passed | 18 skipped (19) - Start at 01:23:47 - Duration 536ms (transform 167ms, setup 18ms, collect 291ms, tests 83ms, environment 0ms, prepare 41ms) + Tests 1 passed | 19 skipped (20) + Start at 01:34:07 + Duration 535ms (transform 165ms, setup 16ms, collect 282ms, tests 93ms, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-setup-retry.log b/evidence/run-read-timeout/mutation-reader-setup-retry.log index 4019dcf96..080cc251d 100644 --- a/evidence/run-read-timeout/mutation-reader-setup-retry.log +++ b/evidence/run-read-timeout/mutation-reader-setup-retry.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/journal-client.ts -Original SHA256: 7398cebb5ccc218341966f90586a40e862bf1fe85bf31b77684198906f10a329 +Original SHA256: 9c77a946018fecb31ee86b19e343b78cb65f077e87bd03395b5dcb30f2e3a0a3 Replaced: if (this.reader === reader) { this.reader = undefined; @@ -16,8 +16,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (19 tests | 1 failed | 18 skipped) 74ms - × retries a reader setup that timed out instead of reading on the primary for good 73ms + ❯ tests/journal-client-read-timeout.test.ts (20 tests | 1 failed | 19 skipped) 77ms + × retries a reader setup that timed out instead of reading on the primary for good 76ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -42,22 +42,22 @@ AssertionError: expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 18 skipped (19) - Start at 01:23:48 - Duration 504ms (transform 163ms, setup 17ms, collect 279ms, tests 74ms, environment 0ms, prepare 40ms) + Tests 1 failed | 19 skipped (20) + Start at 01:34:08 + Duration 520ms (transform 162ms, setup 17ms, collect 280ms, tests 77ms, environment 0ms, prepare 40ms) exit=1 -RESTORED SHA256: 7398cebb5ccc218341966f90586a40e862bf1fe85bf31b77684198906f10a329 +RESTORED SHA256: 9c77a946018fecb31ee86b19e343b78cb65f077e87bd03395b5dcb30f2e3a0a3 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'retries a reader setup that timed out' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (19 tests | 18 skipped) 67ms + ✓ tests/journal-client-read-timeout.test.ts (20 tests | 19 skipped) 74ms Test Files 1 passed (1) - Tests 1 passed | 18 skipped (19) - Start at 01:23:49 - Duration 515ms (transform 165ms, setup 17ms, collect 286ms, tests 67ms, environment 0ms, prepare 41ms) + Tests 1 passed | 19 skipped (20) + Start at 01:34:09 + Duration 523ms (transform 170ms, setup 17ms, collect 297ms, tests 74ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader.log b/evidence/run-read-timeout/mutation-reader.log index 4aa9dac17..9cabcdd41 100644 --- a/evidence/run-read-timeout/mutation-reader.log +++ b/evidence/run-read-timeout/mutation-reader.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/journal-client.ts -Original SHA256: 7398cebb5ccc218341966f90586a40e862bf1fe85bf31b77684198906f10a329 +Original SHA256: 9c77a946018fecb31ee86b19e343b78cb65f077e87bd03395b5dcb30f2e3a0a3 Replaced: return await reader.requestOnce(verb, params, remaining, attemptSignal); With: @@ -10,7 +10,7 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (19 tests | 1 failed | 18 skipped) 114ms + ❯ tests/journal-client-read-timeout.test.ts (20 tests | 1 failed | 19 skipped) 114ms × serves a bounded read while an unbounded command is in flight 113ms → expected true to be false // Object.is equality @@ -36,22 +36,22 @@ AssertionError: expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 18 skipped (19) - Start at 01:23:44 - Duration 560ms (transform 162ms, setup 17ms, collect 282ms, tests 114ms, environment 0ms, prepare 42ms) + Tests 1 failed | 19 skipped (20) + Start at 01:34:04 + Duration 542ms (transform 158ms, setup 17ms, collect 274ms, tests 114ms, environment 0ms, prepare 41ms) exit=1 -RESTORED SHA256: 7398cebb5ccc218341966f90586a40e862bf1fe85bf31b77684198906f10a329 +RESTORED SHA256: 9c77a946018fecb31ee86b19e343b78cb65f077e87bd03395b5dcb30f2e3a0a3 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'serves a bounded read' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (19 tests | 18 skipped) 106ms + ✓ tests/journal-client-read-timeout.test.ts (20 tests | 19 skipped) 106ms Test Files 1 passed (1) - Tests 1 passed | 18 skipped (19) - Start at 01:23:45 - Duration 558ms (transform 163ms, setup 16ms, collect 289ms, tests 106ms, environment 0ms, prepare 41ms) + Tests 1 passed | 19 skipped (20) + Start at 01:34:05 + Duration 544ms (transform 163ms, setup 16ms, collect 281ms, tests 106ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-root-parking.log b/evidence/run-read-timeout/mutation-root-parking.log index 137d3acf1..412230697 100644 --- a/evidence/run-read-timeout/mutation-root-parking.log +++ b/evidence/run-read-timeout/mutation-root-parking.log @@ -104,8 +104,8 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } Test Files 1 failed (1) Tests 3 failed | 24 skipped (27) - Start at 01:24:10 - Duration 1.25s (transform 640ms, setup 16ms, collect 1.05s, tests 31ms, environment 0ms, prepare 41ms) + Start at 01:34:30 + Duration 1.29s (transform 670ms, setup 16ms, collect 1.10s, tests 31ms, environment 0ms, prepare 41ms) exit=1 @@ -118,7 +118,7 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r Test Files 1 passed (1) Tests 3 passed | 24 skipped (27) - Start at 01:24:12 - Duration 1.30s (transform 672ms, setup 17ms, collect 1.10s, tests 23ms, environment 0ms, prepare 47ms) + Start at 01:34:32 + Duration 1.28s (transform 664ms, setup 16ms, collect 1.09s, tests 23ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-snapshot-cancel.log b/evidence/run-read-timeout/mutation-snapshot-cancel.log index fe46e3ec9..345c766e6 100644 --- a/evidence/run-read-timeout/mutation-snapshot-cancel.log +++ b/evidence/run-read-timeout/mutation-snapshot-cancel.log @@ -11,8 +11,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 2216ms - × cancels promptly while a lease snapshot read is in flight 2215ms + ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 2217ms + × cancels promptly while a lease snapshot read is in flight 2216ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -31,8 +31,8 @@ AssertionError: expected 'still waiting' to be an instance of Error Test Files 1 failed (1) Tests 1 failed | 6 skipped (7) - Start at 01:23:56 - Duration 2.68s (transform 187ms, setup 16ms, collect 316ms, tests 2.22s, environment 0ms, prepare 40ms) + Start at 01:34:16 + Duration 2.70s (transform 191ms, setup 16ms, collect 321ms, tests 2.22s, environment 0ms, prepare 41ms) exit=1 @@ -42,11 +42,11 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2011ms - ✓ cancels promptly while a lease snapshot read is in flight 2011ms + ✓ cancels promptly while a lease snapshot read is in flight 2010ms Test Files 1 passed (1) Tests 1 passed | 6 skipped (7) - Start at 01:23:59 - Duration 2.48s (transform 188ms, setup 17ms, collect 309ms, tests 2.01s, environment 0ms, prepare 42ms) + Start at 01:34:19 + Duration 2.46s (transform 178ms, setup 17ms, collect 300ms, tests 2.01s, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-watch-cadence.log b/evidence/run-read-timeout/mutation-watch-cadence.log index d6700441d..b9f6fbe06 100644 --- a/evidence/run-read-timeout/mutation-watch-cadence.log +++ b/evidence/run-read-timeout/mutation-watch-cadence.log @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 2109ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2108ms + ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 2111ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2111ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -30,8 +30,8 @@ AssertionError: expected 41 to be less than or equal to 1 Test Files 1 failed (1) Tests 1 failed | 6 skipped (7) - Start at 01:23:50 - Duration 2.59s (transform 194ms, setup 16ms, collect 330ms, tests 2.11s, environment 0ms, prepare 43ms) + Start at 01:34:10 + Duration 2.59s (transform 182ms, setup 17ms, collect 311ms, tests 2.11s, environment 0ms, prepare 42ms) exit=1 @@ -40,12 +40,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2119ms + ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2118ms ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms Test Files 1 passed (1) Tests 1 passed | 6 skipped (7) - Start at 01:23:53 - Duration 2.60s (transform 189ms, setup 17ms, collect 312ms, tests 2.12s, environment 0ms, prepare 45ms) + Start at 01:34:13 + Duration 2.58s (transform 184ms, setup 16ms, collect 313ms, tests 2.12s, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/resume-live.log b/evidence/run-read-timeout/resume-live.log index 9824a7253..409399fcd 100644 --- a/evidence/run-read-timeout/resume-live.log +++ b/evidence/run-read-timeout/resume-live.log @@ -2,12 +2,12 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 906ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 905ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 931ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 930ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 01:25:07 - Duration 2.25s (transform 657ms, setup 19ms, collect 1.09s, tests 906ms, environment 0ms, prepare 81ms) + Start at 01:35:26 + Duration 2.18s (transform 616ms, setup 20ms, collect 1.01s, tests 931ms, environment 0ms, prepare 80ms) exit=0 diff --git a/evidence/run-read-timeout/verification.md b/evidence/run-read-timeout/verification.md index f6091bfd6..8485de4dd 100644 --- a/evidence/run-read-timeout/verification.md +++ b/evidence/run-read-timeout/verification.md @@ -15,59 +15,60 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5251ms + ✓ tests/cli.test.ts (71 tests) 5255ms + ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 322ms ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2039ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2038ms -(node:3460729) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2033ms +(node:3512367) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (27 tests) 398ms - ✓ tests/journal-client.test.ts (17 tests) 82ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 7251ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 575ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1120ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2705ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 358ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 761ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2529ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 629ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 559ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 678ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 663ms - ✓ tests/classify-outcome.test.ts (11 tests) 7433ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2014ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3009ms - ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2003ms - ✓ tests/journal-client-read-timeout.test.ts (19 tests) 1422ms - ✓ tests/human-live.test.ts (3 tests) 4521ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2654ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1134ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 732ms - ✓ tests/running-step-watch.test.ts (7 tests) 8237ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms - ✓ cancels promptly while a lease snapshot read is in flight 2005ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2054ms - ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms - ✓ tests/worker-lease.test.ts (7 tests) 21ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 27ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 781ms - ✓ tests/memoization.test.ts (58 tests) 70ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2461ms + ✓ tests/authored-root.test.ts (27 tests) 390ms + ✓ tests/journal-client.test.ts (17 tests) 80ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 6940ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 468ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1067ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2741ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 349ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 827ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2544ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 651ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 595ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 624ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 673ms + ✓ tests/journal-client-read-timeout.test.ts (20 tests) 1427ms + ✓ tests/classify-outcome.test.ts (11 tests) 7425ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2011ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3006ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms + ✓ tests/human-live.test.ts (3 tests) 4580ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2734ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1166ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 679ms + ✓ tests/running-step-watch.test.ts (7 tests) 8236ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2116ms + ✓ cancels promptly while a lease snapshot read is in flight 2006ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2055ms + ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2003ms + ✓ tests/worker-lease.test.ts (7 tests) 20ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 26ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 697ms + ✓ tests/memoization.test.ts (58 tests) 64ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2471ms ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2057ms - ✓ drains read and watch promises before an authored flow completes 404ms - ✓ tests/worker-lease-sweep.test.ts (4 tests) 10ms - ✓ tests/journal-client-completion.test.ts (6 tests) 106ms - ✓ tests/resume-worker-lease.test.ts (3 tests) 8ms - ✓ tests/direct-run-worker-lease.test.ts (3 tests) 15ms - ✓ tests/reuse-summary-interruption.test.ts (5 tests) 5ms - ✓ tests/journal-client-subscriptions.test.ts (1 test) 9ms - ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 3ms - ✓ tests/authored-verifier-read-budget.test.ts (1 test) 52ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 14ms + ✓ drains read and watch promises before an authored flow completes 414ms + ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms + ✓ tests/journal-client-completion.test.ts (6 tests) 102ms + ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms + ✓ tests/direct-run-worker-lease.test.ts (3 tests) 9ms + ✓ tests/reuse-summary-interruption.test.ts (5 tests) 4ms + ✓ tests/journal-client-subscriptions.test.ts (1 test) 7ms + ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 4ms + ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 49ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms Test Files 23 passed (23) - Tests 287 passed (287) - Start at 01:24:17 - Duration 49.98s (transform 1.07s, setup 139ms, collect 6.17s, tests 40.71s, environment 3ms, prepare 879ms) + Tests 289 passed (289) + Start at 01:34:37 + Duration 48.85s (transform 1.05s, setup 122ms, collect 5.61s, tests 40.36s, environment 3ms, prepare 812ms) exit=0 ``` @@ -79,13 +80,13 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 906ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 905ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 931ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 930ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 01:25:07 - Duration 2.25s (transform 657ms, setup 19ms, collect 1.09s, tests 906ms, environment 0ms, prepare 81ms) + Start at 01:35:26 + Duration 2.18s (transform 616ms, setup 20ms, collect 1.01s, tests 931ms, environment 0ms, prepare 80ms) exit=0 ``` @@ -96,7 +97,7 @@ The mutation sections below were regenerated by `python3 evidence/run-read-timeo ```text Source: packages/sdk/src/journal-client.ts -Original SHA256: 7398cebb5ccc218341966f90586a40e862bf1fe85bf31b77684198906f10a329 +Original SHA256: 9c77a946018fecb31ee86b19e343b78cb65f077e87bd03395b5dcb30f2e3a0a3 Replaced: return await reader.requestOnce(verb, params, remaining, attemptSignal); With: @@ -107,7 +108,7 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (19 tests | 1 failed | 18 skipped) 114ms + ❯ tests/journal-client-read-timeout.test.ts (20 tests | 1 failed | 19 skipped) 114ms × serves a bounded read while an unbounded command is in flight 113ms → expected true to be false // Object.is equality @@ -133,23 +134,23 @@ AssertionError: expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 18 skipped (19) - Start at 01:23:44 - Duration 560ms (transform 162ms, setup 17ms, collect 282ms, tests 114ms, environment 0ms, prepare 42ms) + Tests 1 failed | 19 skipped (20) + Start at 01:34:04 + Duration 542ms (transform 158ms, setup 17ms, collect 274ms, tests 114ms, environment 0ms, prepare 41ms) exit=1 -RESTORED SHA256: 7398cebb5ccc218341966f90586a40e862bf1fe85bf31b77684198906f10a329 +RESTORED SHA256: 9c77a946018fecb31ee86b19e343b78cb65f077e87bd03395b5dcb30f2e3a0a3 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'serves a bounded read' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (19 tests | 18 skipped) 106ms + ✓ tests/journal-client-read-timeout.test.ts (20 tests | 19 skipped) 106ms Test Files 1 passed (1) - Tests 1 passed | 18 skipped (19) - Start at 01:23:45 - Duration 558ms (transform 163ms, setup 16ms, collect 289ms, tests 106ms, environment 0ms, prepare 41ms) + Tests 1 passed | 19 skipped (20) + Start at 01:34:05 + Duration 544ms (transform 163ms, setup 16ms, collect 281ms, tests 106ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -158,7 +159,7 @@ exit=0 ```text Source: packages/sdk/src/journal-client.ts -Original SHA256: 7398cebb5ccc218341966f90586a40e862bf1fe85bf31b77684198906f10a329 +Original SHA256: 9c77a946018fecb31ee86b19e343b78cb65f077e87bd03395b5dcb30f2e3a0a3 Replaced: this.dropReader(reader); @@ -170,14 +171,14 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (19 tests | 1 failed | 18 skipped) 1007ms - × reconnects the reader after it disconnects within the read budget 1006ms - → journal client: run.get read session was interrupted after 78 attempts in 999ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ tests/journal-client-read-timeout.test.ts (20 tests | 1 failed | 19 skipped) 1009ms + × reconnects the reader after it disconnects within the read budget 1009ms + → journal client: run.get read session was interrupted after 79 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 78 attempts in 999ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 79 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:70:9 68| let last: unknown; 69| const exhausted = () => last instanceof JournalReadInterruptedError @@ -188,9 +189,9 @@ JournalReadInterruptedError: journal client: run.get read session was interrupte ❯ Timeout. src/journal-read-policy.ts:81:39 Caused by: Error: journal client: not connected (run.get): journal client: connection closed - ❯ src/journal-client.ts:283:16 - ❯ JournalClient.requestOnce src/journal-client.ts:276:12 - ❯ src/journal-client.ts:257:31 + ❯ src/journal-client.ts:285:16 + ❯ JournalClient.requestOnce src/journal-client.ts:278:12 + ❯ src/journal-client.ts:259:31 ❯ src/journal-read-policy.ts:93:18 Caused by: Error: journal client: connection closed @@ -199,23 +200,23 @@ Caused by: Error: journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 18 skipped (19) - Start at 01:23:45 - Duration 1.45s (transform 165ms, setup 17ms, collect 282ms, tests 1.01s, environment 0ms, prepare 42ms) + Tests 1 failed | 19 skipped (20) + Start at 01:34:06 + Duration 1.45s (transform 166ms, setup 18ms, collect 283ms, tests 1.01s, environment 0ms, prepare 41ms) exit=1 -RESTORED SHA256: 7398cebb5ccc218341966f90586a40e862bf1fe85bf31b77684198906f10a329 +RESTORED SHA256: 9c77a946018fecb31ee86b19e343b78cb65f077e87bd03395b5dcb30f2e3a0a3 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'reconnects the reader after it disconnects' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (19 tests | 18 skipped) 83ms + ✓ tests/journal-client-read-timeout.test.ts (20 tests | 19 skipped) 93ms Test Files 1 passed (1) - Tests 1 passed | 18 skipped (19) - Start at 01:23:47 - Duration 536ms (transform 167ms, setup 18ms, collect 291ms, tests 83ms, environment 0ms, prepare 41ms) + Tests 1 passed | 19 skipped (20) + Start at 01:34:07 + Duration 535ms (transform 165ms, setup 16ms, collect 282ms, tests 93ms, environment 0ms, prepare 40ms) exit=0 ``` @@ -224,7 +225,7 @@ exit=0 ```text Source: packages/sdk/src/journal-client.ts -Original SHA256: 7398cebb5ccc218341966f90586a40e862bf1fe85bf31b77684198906f10a329 +Original SHA256: 9c77a946018fecb31ee86b19e343b78cb65f077e87bd03395b5dcb30f2e3a0a3 Replaced: if (this.reader === reader) { this.reader = undefined; @@ -241,8 +242,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (19 tests | 1 failed | 18 skipped) 74ms - × retries a reader setup that timed out instead of reading on the primary for good 73ms + ❯ tests/journal-client-read-timeout.test.ts (20 tests | 1 failed | 19 skipped) 77ms + × retries a reader setup that timed out instead of reading on the primary for good 76ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -267,23 +268,23 @@ AssertionError: expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 18 skipped (19) - Start at 01:23:48 - Duration 504ms (transform 163ms, setup 17ms, collect 279ms, tests 74ms, environment 0ms, prepare 40ms) + Tests 1 failed | 19 skipped (20) + Start at 01:34:08 + Duration 520ms (transform 162ms, setup 17ms, collect 280ms, tests 77ms, environment 0ms, prepare 40ms) exit=1 -RESTORED SHA256: 7398cebb5ccc218341966f90586a40e862bf1fe85bf31b77684198906f10a329 +RESTORED SHA256: 9c77a946018fecb31ee86b19e343b78cb65f077e87bd03395b5dcb30f2e3a0a3 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'retries a reader setup that timed out' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (19 tests | 18 skipped) 67ms + ✓ tests/journal-client-read-timeout.test.ts (20 tests | 19 skipped) 74ms Test Files 1 passed (1) - Tests 1 passed | 18 skipped (19) - Start at 01:23:49 - Duration 515ms (transform 165ms, setup 17ms, collect 286ms, tests 67ms, environment 0ms, prepare 41ms) + Tests 1 passed | 19 skipped (20) + Start at 01:34:09 + Duration 523ms (transform 170ms, setup 17ms, collect 297ms, tests 74ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -303,8 +304,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 2109ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2108ms + ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 2111ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2111ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -323,8 +324,8 @@ AssertionError: expected 41 to be less than or equal to 1 Test Files 1 failed (1) Tests 1 failed | 6 skipped (7) - Start at 01:23:50 - Duration 2.59s (transform 194ms, setup 16ms, collect 330ms, tests 2.11s, environment 0ms, prepare 43ms) + Start at 01:34:10 + Duration 2.59s (transform 182ms, setup 17ms, collect 311ms, tests 2.11s, environment 0ms, prepare 42ms) exit=1 @@ -333,13 +334,13 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2119ms + ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2118ms ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms Test Files 1 passed (1) Tests 1 passed | 6 skipped (7) - Start at 01:23:53 - Duration 2.60s (transform 189ms, setup 17ms, collect 312ms, tests 2.12s, environment 0ms, prepare 45ms) + Start at 01:34:13 + Duration 2.58s (transform 184ms, setup 16ms, collect 313ms, tests 2.12s, environment 0ms, prepare 40ms) exit=0 ``` @@ -360,8 +361,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 2216ms - × cancels promptly while a lease snapshot read is in flight 2215ms + ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 2217ms + × cancels promptly while a lease snapshot read is in flight 2216ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -380,8 +381,8 @@ AssertionError: expected 'still waiting' to be an instance of Error Test Files 1 failed (1) Tests 1 failed | 6 skipped (7) - Start at 01:23:56 - Duration 2.68s (transform 187ms, setup 16ms, collect 316ms, tests 2.22s, environment 0ms, prepare 40ms) + Start at 01:34:16 + Duration 2.70s (transform 191ms, setup 16ms, collect 321ms, tests 2.22s, environment 0ms, prepare 41ms) exit=1 @@ -391,12 +392,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2011ms - ✓ cancels promptly while a lease snapshot read is in flight 2011ms + ✓ cancels promptly while a lease snapshot read is in flight 2010ms Test Files 1 passed (1) Tests 1 passed | 6 skipped (7) - Start at 01:23:59 - Duration 2.48s (transform 188ms, setup 17ms, collect 309ms, tests 2.01s, environment 0ms, prepare 42ms) + Start at 01:34:19 + Duration 2.46s (transform 178ms, setup 17ms, collect 300ms, tests 2.01s, environment 0ms, prepare 41ms) exit=0 ``` @@ -416,8 +417,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 3010ms - × a completion push ends the wait without waiting out an in-flight snapshot 3010ms + ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 3012ms + × a completion push ends the wait without waiting out an in-flight snapshot 3012ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -440,8 +441,8 @@ Received: "still waiting" Test Files 1 failed (1) Tests 1 failed | 6 skipped (7) - Start at 01:24:02 - Duration 3.48s (transform 186ms, setup 17ms, collect 316ms, tests 3.01s, environment 0ms, prepare 41ms) + Start at 01:34:22 + Duration 3.48s (transform 186ms, setup 17ms, collect 313ms, tests 3.01s, environment 0ms, prepare 41ms) exit=1 @@ -450,13 +451,13 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2058ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2058ms + ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2061ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2061ms Test Files 1 passed (1) Tests 1 passed | 6 skipped (7) - Start at 01:24:05 - Duration 2.53s (transform 190ms, setup 17ms, collect 316ms, tests 2.06s, environment 0ms, prepare 40ms) + Start at 01:34:26 + Duration 2.52s (transform 183ms, setup 17ms, collect 312ms, tests 2.06s, environment 0ms, prepare 40ms) exit=0 ``` @@ -480,7 +481,7 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 19ms - × a heartbeat timeout is lease loss rather than a worker body failure 19ms + × a heartbeat timeout is lease loss rather than a worker body failure 18ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -499,8 +500,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 01:24:08 - Duration 469ms (transform 168ms, setup 16ms, collect 284ms, tests 19ms, environment 0ms, prepare 44ms) + Start at 01:34:28 + Duration 465ms (transform 168ms, setup 15ms, collect 287ms, tests 19ms, environment 0ms, prepare 40ms) exit=1 @@ -509,12 +510,12 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 13ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 01:24:09 - Duration 439ms (transform 165ms, setup 17ms, collect 274ms, tests 15ms, environment 0ms, prepare 41ms) + Start at 01:34:29 + Duration 431ms (transform 160ms, setup 17ms, collect 261ms, tests 13ms, environment 0ms, prepare 40ms) exit=0 ``` @@ -628,8 +629,8 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } Test Files 1 failed (1) Tests 3 failed | 24 skipped (27) - Start at 01:24:10 - Duration 1.25s (transform 640ms, setup 16ms, collect 1.05s, tests 31ms, environment 0ms, prepare 41ms) + Start at 01:34:30 + Duration 1.29s (transform 670ms, setup 16ms, collect 1.10s, tests 31ms, environment 0ms, prepare 41ms) exit=1 @@ -642,8 +643,8 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r Test Files 1 passed (1) Tests 3 passed | 24 skipped (27) - Start at 01:24:12 - Duration 1.30s (transform 672ms, setup 17ms, collect 1.10s, tests 23ms, environment 0ms, prepare 47ms) + Start at 01:34:32 + Duration 1.28s (transform 664ms, setup 16ms, collect 1.09s, tests 23ms, environment 0ms, prepare 41ms) exit=0 ``` diff --git a/packages/sdk/src/authored-node-runner.ts b/packages/sdk/src/authored-node-runner.ts index fdd0c967a..8c3c893c5 100644 --- a/packages/sdk/src/authored-node-runner.ts +++ b/packages/sdk/src/authored-node-runner.ts @@ -172,7 +172,7 @@ export async function runAuthoredInNode( else resolve(result); }); }); - await verifyAuthoredNodeResult(result, metadata, rootRunId, socketPath); + await verifyAuthoredNodeResult(result, metadata, rootRunId, socketPath, options.signal); return result; } finally { await rm(directory, { recursive: true, force: true }); } } @@ -324,7 +324,7 @@ function frameCount(frame: Record, key: string): number | undef /** The IPC frame is a claim, not a durable terminal fact or a sandbox boundary. */ export async function verifyAuthoredNodeResult( result: AuthoredFlowExecutionResult, metadata: AuthoredRootMetadata, - rootRunId: string, socketPath: string, + rootRunId: string, socketPath: string, signal?: AbortSignal, ): Promise { const invalid = (why = ''): never => { throw new Error(`authored runtime result has no matching durable completion${process.env['FLOWS_VERIFIER_DEBUG'] && why ? ` (${why})` : ''}`); }; if (result.rootRunId !== rootRunId || result.name !== metadata.flowName @@ -356,7 +356,8 @@ export async function verifyAuthoredNodeResult( } const runs = new Set(); // The verifier reads the same journal under the same CPU load as the body. - const journal = new JournalClient(socketPath, { readBudgetMs: FLOW_READ_BUDGET_MS }); + const journal = new JournalClient(socketPath, { readBudgetMs: FLOW_READ_BUDGET_MS, + ...(signal === undefined ? {} : { readSignal: signal }) }); await journal.connect(); try { await journal.hello('flows-authored-result-verifier'); diff --git a/packages/sdk/src/journal-client.ts b/packages/sdk/src/journal-client.ts index 63a5d7e39..c4e5f096e 100644 --- a/packages/sdk/src/journal-client.ts +++ b/packages/sdk/src/journal-client.ts @@ -246,6 +246,8 @@ export class JournalClient extends EventEmitter { const started = performance.now(); const reader = await this.readSession().catch(error => { attemptSignal.throwIfAborted(); + // A malformed handshake frame is a protocol violation: fail closed. + if (error instanceof JournalFrameError) throw error; throw new JournalReadInterruptedError(verb, 1, performance.now() - started, undefined, { cause: error }); }); attemptSignal.throwIfAborted(); diff --git a/packages/sdk/tests/authored-verifier-read-budget.test.ts b/packages/sdk/tests/authored-verifier-read-budget.test.ts index e2cb95069..de6028b00 100644 --- a/packages/sdk/tests/authored-verifier-read-budget.test.ts +++ b/packages/sdk/tests/authored-verifier-read-budget.test.ts @@ -21,3 +21,13 @@ it('verifies a Bun-run authored result with the flow read budget, not single-sho await verifyAuthoredNodeResult(result as never, { flowName: 'flow' } as never, 'root', '/unused.sock').catch(() => undefined); expect(constructed).toContainEqual(expect.objectContaining({ readBudgetMs: FLOW_READ_BUDGET_MS })); }); + +it('hands the lifecycle signal to the verifier so cancellation stops its reads', async () => { + const { verifyAuthoredNodeResult } = await import('../src/authored-node-runner.js'); + constructed.length = 0; + const controller = new AbortController(); + const result = { rootRunId: 'root', name: 'flow', completionReason: 'success', journalSteps: [{ id: 'complete-1' }] }; + await verifyAuthoredNodeResult(result as never, { flowName: 'flow' } as never, 'root', '/unused.sock', controller.signal) + .catch(() => undefined); + expect(constructed).toContainEqual(expect.objectContaining({ readSignal: controller.signal })); +}); diff --git a/packages/sdk/tests/journal-client-read-timeout.test.ts b/packages/sdk/tests/journal-client-read-timeout.test.ts index d4775ad61..81e059e90 100644 --- a/packages/sdk/tests/journal-client-read-timeout.test.ts +++ b/packages/sdk/tests/journal-client-read-timeout.test.ts @@ -232,3 +232,16 @@ it('a client read signal cancels every budgeted read promptly', async () => { await expect(pending).rejects.toThrow('run canceled'); expect(performance.now() - begun).toBeLessThan(40); }); + +it('fails closed on a malformed frame during reader setup', async () => { + let hellos = 0; + let reads = 0; + const client = await setup({ + hello: ctx => { hellos += 1; ctx.socket.write('{not json\n'); }, + 'run.get': ctx => { reads += 1; sendResult(ctx, { status: 'completed' }); }, + }, 5_000); + const error = await client.runGet('run').catch(caught => caught); + expect(error).toBeInstanceOf(JournalFrameError); + expect(hellos).toBe(1); + expect(reads).toBe(0); +}); From 90355c93a175a9a97d476eeb6822d5cd2ce6f8f2 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 01:45:44 -0700 Subject: [PATCH 13/34] fix(sdk): cancellation never discards a classified outcome or terminalizes the root Threading the lifecycle signal into budgeted reads made two optional or post-success reads able to destroy a known result: - attachReuseSummary: any failure of the optional summary read (abort, bad frame, interruption, bad sequence) is now a reuse_summary_unavailable warning; the classified outcome stands. - driveRoot: an error that is the caller's cancellation (the lifecycle signal's reason or an AbortError while it is aborted) is rethrown with the root id and never recorded as terminal worker_error, so a Bun verifier aborted after the child succeeded leaves the root resumable. Evidence re-captured at this tree. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: b8262468-ea26-4a6e-bee9-d193832e77dc --- evidence/run-read-timeout/final-focused.log | 84 +++---- .../mutation-completion-aborts-snapshot.log | 14 +- .../run-read-timeout/mutation-heartbeat.log | 14 +- .../mutation-reader-reconnect.log | 18 +- .../mutation-reader-setup-retry.log | 14 +- evidence/run-read-timeout/mutation-reader.log | 8 +- .../mutation-root-parking.log | 24 +- .../mutation-snapshot-cancel.log | 10 +- .../mutation-watch-cadence.log | 14 +- evidence/run-read-timeout/resume-live.log | 8 +- evidence/run-read-timeout/verification.md | 208 +++++++++--------- packages/sdk/src/authored-root.ts | 7 + packages/sdk/src/cli/reuse.ts | 9 +- packages/sdk/tests/authored-root.test.ts | 13 ++ .../tests/reuse-summary-interruption.test.ts | 13 +- 15 files changed, 242 insertions(+), 216 deletions(-) diff --git a/evidence/run-read-timeout/final-focused.log b/evidence/run-read-timeout/final-focused.log index af316f867..595090827 100644 --- a/evidence/run-read-timeout/final-focused.log +++ b/evidence/run-read-timeout/final-focused.log @@ -2,59 +2,59 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5255ms - ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 322ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2039ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2033ms -(node:3512367) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5289ms + ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 303ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2036ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2036ms +(node:3553150) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (27 tests) 390ms + ✓ tests/authored-root.test.ts (28 tests) 391ms ✓ tests/journal-client.test.ts (17 tests) 80ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 6940ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 468ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1067ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2741ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 349ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 827ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2544ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 651ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 595ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 624ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 673ms - ✓ tests/journal-client-read-timeout.test.ts (20 tests) 1427ms - ✓ tests/classify-outcome.test.ts (11 tests) 7425ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2011ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3006ms - ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms - ✓ tests/human-live.test.ts (3 tests) 4580ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2734ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1166ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 679ms - ✓ tests/running-step-watch.test.ts (7 tests) 8236ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2116ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 6834ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 483ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 969ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2765ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 331ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 783ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2656ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 661ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 638ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 692ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 664ms + ✓ tests/journal-client-read-timeout.test.ts (20 tests) 1428ms + ✓ tests/classify-outcome.test.ts (11 tests) 7436ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2019ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3008ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2002ms + ✓ tests/human-live.test.ts (3 tests) 4374ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2609ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1072ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 692ms + ✓ tests/running-step-watch.test.ts (7 tests) 8239ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms ✓ cancels promptly while a lease snapshot read is in flight 2006ms ✓ a completion push ends the wait without waiting out an in-flight snapshot 2055ms - ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2003ms - ✓ tests/worker-lease.test.ts (7 tests) 20ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 26ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 697ms - ✓ tests/memoization.test.ts (58 tests) 64ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2471ms + ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms + ✓ tests/worker-lease.test.ts (7 tests) 12ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 25ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 700ms + ✓ tests/memoization.test.ts (58 tests) 63ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2418ms ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2057ms - ✓ drains read and watch promises before an authored flow completes 414ms + ✓ drains read and watch promises before an authored flow completes 360ms ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms - ✓ tests/journal-client-completion.test.ts (6 tests) 102ms + ✓ tests/journal-client-completion.test.ts (6 tests) 103ms + ✓ tests/reuse-summary-interruption.test.ts (5 tests) 4ms ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms ✓ tests/direct-run-worker-lease.test.ts (3 tests) 9ms - ✓ tests/reuse-summary-interruption.test.ts (5 tests) 4ms ✓ tests/journal-client-subscriptions.test.ts (1 test) 7ms ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 4ms - ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 49ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms + ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 48ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms Test Files 23 passed (23) - Tests 289 passed (289) - Start at 01:34:37 - Duration 48.85s (transform 1.05s, setup 122ms, collect 5.61s, tests 40.36s, environment 3ms, prepare 812ms) + Tests 290 passed (290) + Start at 01:44:53 + Duration 48.73s (transform 1.06s, setup 131ms, collect 5.73s, tests 40.15s, environment 3ms, prepare 826ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log index e6a2c1749..bfab4fc7d 100644 --- a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log +++ b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 3012ms - × a completion push ends the wait without waiting out an in-flight snapshot 3012ms + ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 3010ms + × a completion push ends the wait without waiting out an in-flight snapshot 3009ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -34,8 +34,8 @@ Received: "still waiting" Test Files 1 failed (1) Tests 1 failed | 6 skipped (7) - Start at 01:34:22 - Duration 3.48s (transform 186ms, setup 17ms, collect 313ms, tests 3.01s, environment 0ms, prepare 41ms) + Start at 01:44:37 + Duration 3.49s (transform 190ms, setup 16ms, collect 318ms, tests 3.01s, environment 0ms, prepare 41ms) exit=1 @@ -44,12 +44,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2061ms + ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2062ms ✓ a completion push ends the wait without waiting out an in-flight snapshot 2061ms Test Files 1 passed (1) Tests 1 passed | 6 skipped (7) - Start at 01:34:26 - Duration 2.52s (transform 183ms, setup 17ms, collect 312ms, tests 2.06s, environment 0ms, prepare 40ms) + Start at 01:44:41 + Duration 2.51s (transform 180ms, setup 18ms, collect 292ms, tests 2.06s, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-heartbeat.log b/evidence/run-read-timeout/mutation-heartbeat.log index 3ac149d36..129c18f8d 100644 --- a/evidence/run-read-timeout/mutation-heartbeat.log +++ b/evidence/run-read-timeout/mutation-heartbeat.log @@ -13,8 +13,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 19ms - × a heartbeat timeout is lease loss rather than a worker body failure 18ms + ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 18ms + × a heartbeat timeout is lease loss rather than a worker body failure 17ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -33,8 +33,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 01:34:28 - Duration 465ms (transform 168ms, setup 15ms, collect 287ms, tests 19ms, environment 0ms, prepare 40ms) + Start at 01:44:44 + Duration 464ms (transform 165ms, setup 16ms, collect 291ms, tests 18ms, environment 0ms, prepare 40ms) exit=1 @@ -43,11 +43,11 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/heartbeat-timeout.test.ts (1 test) 13ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 01:34:29 - Duration 431ms (transform 160ms, setup 17ms, collect 261ms, tests 13ms, environment 0ms, prepare 40ms) + Start at 01:44:45 + Duration 447ms (transform 167ms, setup 17ms, collect 281ms, tests 15ms, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-reconnect.log b/evidence/run-read-timeout/mutation-reader-reconnect.log index 3cdedf783..a2f704438 100644 --- a/evidence/run-read-timeout/mutation-reader-reconnect.log +++ b/evidence/run-read-timeout/mutation-reader-reconnect.log @@ -11,14 +11,14 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (20 tests | 1 failed | 19 skipped) 1009ms - × reconnects the reader after it disconnects within the read budget 1009ms - → journal client: run.get read session was interrupted after 79 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ tests/journal-client-read-timeout.test.ts (20 tests | 1 failed | 19 skipped) 1006ms + × reconnects the reader after it disconnects within the read budget 1006ms + → journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 79 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:70:9 68| let last: unknown; 69| const exhausted = () => last instanceof JournalReadInterruptedError @@ -41,8 +41,8 @@ Caused by: Error: journal client: connection closed Test Files 1 failed (1) Tests 1 failed | 19 skipped (20) - Start at 01:34:06 - Duration 1.45s (transform 166ms, setup 18ms, collect 283ms, tests 1.01s, environment 0ms, prepare 41ms) + Start at 01:44:21 + Duration 1.42s (transform 158ms, setup 17ms, collect 260ms, tests 1.01s, environment 0ms, prepare 41ms) exit=1 @@ -51,11 +51,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (20 tests | 19 skipped) 93ms + ✓ tests/journal-client-read-timeout.test.ts (20 tests | 19 skipped) 74ms Test Files 1 passed (1) Tests 1 passed | 19 skipped (20) - Start at 01:34:07 - Duration 535ms (transform 165ms, setup 16ms, collect 282ms, tests 93ms, environment 0ms, prepare 40ms) + Start at 01:44:23 + Duration 499ms (transform 170ms, setup 17ms, collect 282ms, tests 74ms, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-setup-retry.log b/evidence/run-read-timeout/mutation-reader-setup-retry.log index 080cc251d..c59fee983 100644 --- a/evidence/run-read-timeout/mutation-reader-setup-retry.log +++ b/evidence/run-read-timeout/mutation-reader-setup-retry.log @@ -16,8 +16,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (20 tests | 1 failed | 19 skipped) 77ms - × retries a reader setup that timed out instead of reading on the primary for good 76ms + ❯ tests/journal-client-read-timeout.test.ts (20 tests | 1 failed | 19 skipped) 80ms + × retries a reader setup that timed out instead of reading on the primary for good 79ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -43,8 +43,8 @@ AssertionError: expected 1 to be 2 // Object.is equality Test Files 1 failed (1) Tests 1 failed | 19 skipped (20) - Start at 01:34:08 - Duration 520ms (transform 162ms, setup 17ms, collect 280ms, tests 77ms, environment 0ms, prepare 40ms) + Start at 01:44:24 + Duration 507ms (transform 163ms, setup 17ms, collect 277ms, tests 80ms, environment 0ms, prepare 40ms) exit=1 @@ -53,11 +53,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (20 tests | 19 skipped) 74ms + ✓ tests/journal-client-read-timeout.test.ts (20 tests | 19 skipped) 70ms Test Files 1 passed (1) Tests 1 passed | 19 skipped (20) - Start at 01:34:09 - Duration 523ms (transform 170ms, setup 17ms, collect 297ms, tests 74ms, environment 0ms, prepare 41ms) + Start at 01:44:25 + Duration 509ms (transform 166ms, setup 17ms, collect 288ms, tests 70ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader.log b/evidence/run-read-timeout/mutation-reader.log index 9cabcdd41..a79fad9a0 100644 --- a/evidence/run-read-timeout/mutation-reader.log +++ b/evidence/run-read-timeout/mutation-reader.log @@ -37,8 +37,8 @@ AssertionError: expected true to be false // Object.is equality Test Files 1 failed (1) Tests 1 failed | 19 skipped (20) - Start at 01:34:04 - Duration 542ms (transform 158ms, setup 17ms, collect 274ms, tests 114ms, environment 0ms, prepare 41ms) + Start at 01:44:20 + Duration 546ms (transform 170ms, setup 17ms, collect 294ms, tests 114ms, environment 0ms, prepare 30ms) exit=1 @@ -51,7 +51,7 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t Test Files 1 passed (1) Tests 1 passed | 19 skipped (20) - Start at 01:34:05 - Duration 544ms (transform 163ms, setup 16ms, collect 281ms, tests 106ms, environment 0ms, prepare 41ms) + Start at 01:44:21 + Duration 552ms (transform 163ms, setup 17ms, collect 285ms, tests 106ms, environment 0ms, prepare 38ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-root-parking.log b/evidence/run-read-timeout/mutation-root-parking.log index 412230697..36d712e15 100644 --- a/evidence/run-read-timeout/mutation-root-parking.log +++ b/evidence/run-read-timeout/mutation-root-parking.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/authored-root.ts -Original SHA256: 999d3eca1236966ca84d0c4a07694a7449bf0a35dd1e7fac455ab0f510804f1a +Original SHA256: 79b644e3c53dc5ec73fa859720e6da87008efb4ce9c4dd1447aa362729624dc8 Replaced: if (isReadInterruptionError(error) || (error instanceof AuthoredFlowExecutionError && error.code === 'daemon_unresponsive')) { @@ -17,11 +17,11 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (27 tests | 3 failed | 24 skipped) 31ms - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 22ms + ❯ tests/authored-root.test.ts (28 tests | 3 failed | 25 skipped) 31ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 21ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 5ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms → expected AuthoredFlowExecutionError: daemon_unresp… { …(7) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 4ms @@ -103,22 +103,22 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[3/3]⎯ Test Files 1 failed (1) - Tests 3 failed | 24 skipped (27) - Start at 01:34:30 - Duration 1.29s (transform 670ms, setup 16ms, collect 1.10s, tests 31ms, environment 0ms, prepare 41ms) + Tests 3 failed | 25 skipped (28) + Start at 01:44:46 + Duration 1.29s (transform 662ms, setup 17ms, collect 1.09s, tests 31ms, environment 0ms, prepare 41ms) exit=1 -RESTORED SHA256: 999d3eca1236966ca84d0c4a07694a7449bf0a35dd1e7fac455ab0f510804f1a +RESTORED SHA256: 79b644e3c53dc5ec73fa859720e6da87008efb4ce9c4dd1447aa362729624dc8 $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the root resumable' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (27 tests | 24 skipped) 23ms + ✓ tests/authored-root.test.ts (28 tests | 25 skipped) 23ms Test Files 1 passed (1) - Tests 3 passed | 24 skipped (27) - Start at 01:34:32 - Duration 1.28s (transform 664ms, setup 16ms, collect 1.09s, tests 23ms, environment 0ms, prepare 41ms) + Tests 3 passed | 25 skipped (28) + Start at 01:44:47 + Duration 1.26s (transform 655ms, setup 16ms, collect 1.07s, tests 23ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-snapshot-cancel.log b/evidence/run-read-timeout/mutation-snapshot-cancel.log index 345c766e6..bc7572ca5 100644 --- a/evidence/run-read-timeout/mutation-snapshot-cancel.log +++ b/evidence/run-read-timeout/mutation-snapshot-cancel.log @@ -12,7 +12,7 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 2217ms - × cancels promptly while a lease snapshot read is in flight 2216ms + × cancels promptly while a lease snapshot read is in flight 2217ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -31,8 +31,8 @@ AssertionError: expected 'still waiting' to be an instance of Error Test Files 1 failed (1) Tests 1 failed | 6 skipped (7) - Start at 01:34:16 - Duration 2.70s (transform 191ms, setup 16ms, collect 321ms, tests 2.22s, environment 0ms, prepare 41ms) + Start at 01:44:32 + Duration 2.67s (transform 192ms, setup 16ms, collect 311ms, tests 2.22s, environment 0ms, prepare 35ms) exit=1 @@ -46,7 +46,7 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels Test Files 1 passed (1) Tests 1 passed | 6 skipped (7) - Start at 01:34:19 - Duration 2.46s (transform 178ms, setup 17ms, collect 300ms, tests 2.01s, environment 0ms, prepare 41ms) + Start at 01:44:35 + Duration 2.48s (transform 184ms, setup 17ms, collect 308ms, tests 2.01s, environment 0ms, prepare 42ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-watch-cadence.log b/evidence/run-read-timeout/mutation-watch-cadence.log index b9f6fbe06..ad6886d12 100644 --- a/evidence/run-read-timeout/mutation-watch-cadence.log +++ b/evidence/run-read-timeout/mutation-watch-cadence.log @@ -11,7 +11,7 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 2111ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2111ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2110ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -30,8 +30,8 @@ AssertionError: expected 41 to be less than or equal to 1 Test Files 1 failed (1) Tests 1 failed | 6 skipped (7) - Start at 01:34:10 - Duration 2.59s (transform 182ms, setup 17ms, collect 311ms, tests 2.11s, environment 0ms, prepare 42ms) + Start at 01:44:26 + Duration 2.59s (transform 186ms, setup 17ms, collect 313ms, tests 2.11s, environment 0ms, prepare 41ms) exit=1 @@ -40,12 +40,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2118ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms + ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2116ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2116ms Test Files 1 passed (1) Tests 1 passed | 6 skipped (7) - Start at 01:34:13 - Duration 2.58s (transform 184ms, setup 16ms, collect 313ms, tests 2.12s, environment 0ms, prepare 40ms) + Start at 01:44:29 + Duration 2.56s (transform 187ms, setup 16ms, collect 307ms, tests 2.12s, environment 0ms, prepare 35ms) exit=0 diff --git a/evidence/run-read-timeout/resume-live.log b/evidence/run-read-timeout/resume-live.log index 409399fcd..df47425e3 100644 --- a/evidence/run-read-timeout/resume-live.log +++ b/evidence/run-read-timeout/resume-live.log @@ -2,12 +2,12 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 931ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 930ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 929ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 929ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 01:35:26 - Duration 2.18s (transform 616ms, setup 20ms, collect 1.01s, tests 931ms, environment 0ms, prepare 80ms) + Start at 01:45:42 + Duration 2.16s (transform 602ms, setup 16ms, collect 993ms, tests 929ms, environment 0ms, prepare 61ms) exit=0 diff --git a/evidence/run-read-timeout/verification.md b/evidence/run-read-timeout/verification.md index 8485de4dd..76470c8f1 100644 --- a/evidence/run-read-timeout/verification.md +++ b/evidence/run-read-timeout/verification.md @@ -15,60 +15,60 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5255ms - ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 322ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2039ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2033ms -(node:3512367) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5289ms + ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 303ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2036ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2036ms +(node:3553150) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (27 tests) 390ms + ✓ tests/authored-root.test.ts (28 tests) 391ms ✓ tests/journal-client.test.ts (17 tests) 80ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 6940ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 468ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1067ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2741ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 349ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 827ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2544ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 651ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 595ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 624ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 673ms - ✓ tests/journal-client-read-timeout.test.ts (20 tests) 1427ms - ✓ tests/classify-outcome.test.ts (11 tests) 7425ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2011ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3006ms - ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms - ✓ tests/human-live.test.ts (3 tests) 4580ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2734ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1166ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 679ms - ✓ tests/running-step-watch.test.ts (7 tests) 8236ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2116ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 6834ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 483ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 969ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2765ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 331ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 783ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2656ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 661ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 638ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 692ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 664ms + ✓ tests/journal-client-read-timeout.test.ts (20 tests) 1428ms + ✓ tests/classify-outcome.test.ts (11 tests) 7436ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2019ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3008ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2002ms + ✓ tests/human-live.test.ts (3 tests) 4374ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2609ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1072ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 692ms + ✓ tests/running-step-watch.test.ts (7 tests) 8239ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms ✓ cancels promptly while a lease snapshot read is in flight 2006ms ✓ a completion push ends the wait without waiting out an in-flight snapshot 2055ms - ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2003ms - ✓ tests/worker-lease.test.ts (7 tests) 20ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 26ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 697ms - ✓ tests/memoization.test.ts (58 tests) 64ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2471ms + ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms + ✓ tests/worker-lease.test.ts (7 tests) 12ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 25ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 700ms + ✓ tests/memoization.test.ts (58 tests) 63ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2418ms ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2057ms - ✓ drains read and watch promises before an authored flow completes 414ms + ✓ drains read and watch promises before an authored flow completes 360ms ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms - ✓ tests/journal-client-completion.test.ts (6 tests) 102ms + ✓ tests/journal-client-completion.test.ts (6 tests) 103ms + ✓ tests/reuse-summary-interruption.test.ts (5 tests) 4ms ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms ✓ tests/direct-run-worker-lease.test.ts (3 tests) 9ms - ✓ tests/reuse-summary-interruption.test.ts (5 tests) 4ms ✓ tests/journal-client-subscriptions.test.ts (1 test) 7ms ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 4ms - ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 49ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms + ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 48ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms Test Files 23 passed (23) - Tests 289 passed (289) - Start at 01:34:37 - Duration 48.85s (transform 1.05s, setup 122ms, collect 5.61s, tests 40.36s, environment 3ms, prepare 812ms) + Tests 290 passed (290) + Start at 01:44:53 + Duration 48.73s (transform 1.06s, setup 131ms, collect 5.73s, tests 40.15s, environment 3ms, prepare 826ms) exit=0 ``` @@ -80,13 +80,13 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 931ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 930ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 929ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 929ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 01:35:26 - Duration 2.18s (transform 616ms, setup 20ms, collect 1.01s, tests 931ms, environment 0ms, prepare 80ms) + Start at 01:45:42 + Duration 2.16s (transform 602ms, setup 16ms, collect 993ms, tests 929ms, environment 0ms, prepare 61ms) exit=0 ``` @@ -135,8 +135,8 @@ AssertionError: expected true to be false // Object.is equality Test Files 1 failed (1) Tests 1 failed | 19 skipped (20) - Start at 01:34:04 - Duration 542ms (transform 158ms, setup 17ms, collect 274ms, tests 114ms, environment 0ms, prepare 41ms) + Start at 01:44:20 + Duration 546ms (transform 170ms, setup 17ms, collect 294ms, tests 114ms, environment 0ms, prepare 30ms) exit=1 @@ -149,8 +149,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t Test Files 1 passed (1) Tests 1 passed | 19 skipped (20) - Start at 01:34:05 - Duration 544ms (transform 163ms, setup 16ms, collect 281ms, tests 106ms, environment 0ms, prepare 41ms) + Start at 01:44:21 + Duration 552ms (transform 163ms, setup 17ms, collect 285ms, tests 106ms, environment 0ms, prepare 38ms) exit=0 ``` @@ -171,14 +171,14 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (20 tests | 1 failed | 19 skipped) 1009ms - × reconnects the reader after it disconnects within the read budget 1009ms - → journal client: run.get read session was interrupted after 79 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ tests/journal-client-read-timeout.test.ts (20 tests | 1 failed | 19 skipped) 1006ms + × reconnects the reader after it disconnects within the read budget 1006ms + → journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 79 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:70:9 68| let last: unknown; 69| const exhausted = () => last instanceof JournalReadInterruptedError @@ -201,8 +201,8 @@ Caused by: Error: journal client: connection closed Test Files 1 failed (1) Tests 1 failed | 19 skipped (20) - Start at 01:34:06 - Duration 1.45s (transform 166ms, setup 18ms, collect 283ms, tests 1.01s, environment 0ms, prepare 41ms) + Start at 01:44:21 + Duration 1.42s (transform 158ms, setup 17ms, collect 260ms, tests 1.01s, environment 0ms, prepare 41ms) exit=1 @@ -211,12 +211,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (20 tests | 19 skipped) 93ms + ✓ tests/journal-client-read-timeout.test.ts (20 tests | 19 skipped) 74ms Test Files 1 passed (1) Tests 1 passed | 19 skipped (20) - Start at 01:34:07 - Duration 535ms (transform 165ms, setup 16ms, collect 282ms, tests 93ms, environment 0ms, prepare 40ms) + Start at 01:44:23 + Duration 499ms (transform 170ms, setup 17ms, collect 282ms, tests 74ms, environment 0ms, prepare 40ms) exit=0 ``` @@ -242,8 +242,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (20 tests | 1 failed | 19 skipped) 77ms - × retries a reader setup that timed out instead of reading on the primary for good 76ms + ❯ tests/journal-client-read-timeout.test.ts (20 tests | 1 failed | 19 skipped) 80ms + × retries a reader setup that timed out instead of reading on the primary for good 79ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -269,8 +269,8 @@ AssertionError: expected 1 to be 2 // Object.is equality Test Files 1 failed (1) Tests 1 failed | 19 skipped (20) - Start at 01:34:08 - Duration 520ms (transform 162ms, setup 17ms, collect 280ms, tests 77ms, environment 0ms, prepare 40ms) + Start at 01:44:24 + Duration 507ms (transform 163ms, setup 17ms, collect 277ms, tests 80ms, environment 0ms, prepare 40ms) exit=1 @@ -279,12 +279,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (20 tests | 19 skipped) 74ms + ✓ tests/journal-client-read-timeout.test.ts (20 tests | 19 skipped) 70ms Test Files 1 passed (1) Tests 1 passed | 19 skipped (20) - Start at 01:34:09 - Duration 523ms (transform 170ms, setup 17ms, collect 297ms, tests 74ms, environment 0ms, prepare 41ms) + Start at 01:44:25 + Duration 509ms (transform 166ms, setup 17ms, collect 288ms, tests 70ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -305,7 +305,7 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 2111ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2111ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2110ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -324,8 +324,8 @@ AssertionError: expected 41 to be less than or equal to 1 Test Files 1 failed (1) Tests 1 failed | 6 skipped (7) - Start at 01:34:10 - Duration 2.59s (transform 182ms, setup 17ms, collect 311ms, tests 2.11s, environment 0ms, prepare 42ms) + Start at 01:44:26 + Duration 2.59s (transform 186ms, setup 17ms, collect 313ms, tests 2.11s, environment 0ms, prepare 41ms) exit=1 @@ -334,13 +334,13 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2118ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms + ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2116ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2116ms Test Files 1 passed (1) Tests 1 passed | 6 skipped (7) - Start at 01:34:13 - Duration 2.58s (transform 184ms, setup 16ms, collect 313ms, tests 2.12s, environment 0ms, prepare 40ms) + Start at 01:44:29 + Duration 2.56s (transform 187ms, setup 16ms, collect 307ms, tests 2.12s, environment 0ms, prepare 35ms) exit=0 ``` @@ -362,7 +362,7 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 2217ms - × cancels promptly while a lease snapshot read is in flight 2216ms + × cancels promptly while a lease snapshot read is in flight 2217ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -381,8 +381,8 @@ AssertionError: expected 'still waiting' to be an instance of Error Test Files 1 failed (1) Tests 1 failed | 6 skipped (7) - Start at 01:34:16 - Duration 2.70s (transform 191ms, setup 16ms, collect 321ms, tests 2.22s, environment 0ms, prepare 41ms) + Start at 01:44:32 + Duration 2.67s (transform 192ms, setup 16ms, collect 311ms, tests 2.22s, environment 0ms, prepare 35ms) exit=1 @@ -396,8 +396,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels Test Files 1 passed (1) Tests 1 passed | 6 skipped (7) - Start at 01:34:19 - Duration 2.46s (transform 178ms, setup 17ms, collect 300ms, tests 2.01s, environment 0ms, prepare 41ms) + Start at 01:44:35 + Duration 2.48s (transform 184ms, setup 17ms, collect 308ms, tests 2.01s, environment 0ms, prepare 42ms) exit=0 ``` @@ -417,8 +417,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 3012ms - × a completion push ends the wait without waiting out an in-flight snapshot 3012ms + ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 3010ms + × a completion push ends the wait without waiting out an in-flight snapshot 3009ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -441,8 +441,8 @@ Received: "still waiting" Test Files 1 failed (1) Tests 1 failed | 6 skipped (7) - Start at 01:34:22 - Duration 3.48s (transform 186ms, setup 17ms, collect 313ms, tests 3.01s, environment 0ms, prepare 41ms) + Start at 01:44:37 + Duration 3.49s (transform 190ms, setup 16ms, collect 318ms, tests 3.01s, environment 0ms, prepare 41ms) exit=1 @@ -451,13 +451,13 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2061ms + ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2062ms ✓ a completion push ends the wait without waiting out an in-flight snapshot 2061ms Test Files 1 passed (1) Tests 1 passed | 6 skipped (7) - Start at 01:34:26 - Duration 2.52s (transform 183ms, setup 17ms, collect 312ms, tests 2.06s, environment 0ms, prepare 40ms) + Start at 01:44:41 + Duration 2.51s (transform 180ms, setup 18ms, collect 292ms, tests 2.06s, environment 0ms, prepare 41ms) exit=0 ``` @@ -480,8 +480,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 19ms - × a heartbeat timeout is lease loss rather than a worker body failure 18ms + ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 18ms + × a heartbeat timeout is lease loss rather than a worker body failure 17ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -500,8 +500,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 01:34:28 - Duration 465ms (transform 168ms, setup 15ms, collect 287ms, tests 19ms, environment 0ms, prepare 40ms) + Start at 01:44:44 + Duration 464ms (transform 165ms, setup 16ms, collect 291ms, tests 18ms, environment 0ms, prepare 40ms) exit=1 @@ -510,12 +510,12 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/heartbeat-timeout.test.ts (1 test) 13ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 01:34:29 - Duration 431ms (transform 160ms, setup 17ms, collect 261ms, tests 13ms, environment 0ms, prepare 40ms) + Start at 01:44:45 + Duration 447ms (transform 167ms, setup 17ms, collect 281ms, tests 15ms, environment 0ms, prepare 40ms) exit=0 ``` @@ -524,7 +524,7 @@ exit=0 ```text Source: packages/sdk/src/authored-root.ts -Original SHA256: 999d3eca1236966ca84d0c4a07694a7449bf0a35dd1e7fac455ab0f510804f1a +Original SHA256: 79b644e3c53dc5ec73fa859720e6da87008efb4ce9c4dd1447aa362729624dc8 Replaced: if (isReadInterruptionError(error) || (error instanceof AuthoredFlowExecutionError && error.code === 'daemon_unresponsive')) { @@ -542,11 +542,11 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (27 tests | 3 failed | 24 skipped) 31ms - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 22ms + ❯ tests/authored-root.test.ts (28 tests | 3 failed | 25 skipped) 31ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 21ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 5ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms → expected AuthoredFlowExecutionError: daemon_unresp… { …(7) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 4ms @@ -628,23 +628,23 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[3/3]⎯ Test Files 1 failed (1) - Tests 3 failed | 24 skipped (27) - Start at 01:34:30 - Duration 1.29s (transform 670ms, setup 16ms, collect 1.10s, tests 31ms, environment 0ms, prepare 41ms) + Tests 3 failed | 25 skipped (28) + Start at 01:44:46 + Duration 1.29s (transform 662ms, setup 17ms, collect 1.09s, tests 31ms, environment 0ms, prepare 41ms) exit=1 -RESTORED SHA256: 999d3eca1236966ca84d0c4a07694a7449bf0a35dd1e7fac455ab0f510804f1a +RESTORED SHA256: 79b644e3c53dc5ec73fa859720e6da87008efb4ce9c4dd1447aa362729624dc8 $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the root resumable' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (27 tests | 24 skipped) 23ms + ✓ tests/authored-root.test.ts (28 tests | 25 skipped) 23ms Test Files 1 passed (1) - Tests 3 passed | 24 skipped (27) - Start at 01:34:32 - Duration 1.28s (transform 664ms, setup 16ms, collect 1.09s, tests 23ms, environment 0ms, prepare 41ms) + Tests 3 passed | 25 skipped (28) + Start at 01:44:47 + Duration 1.26s (transform 655ms, setup 16ms, collect 1.07s, tests 23ms, environment 0ms, prepare 41ms) exit=0 ``` diff --git a/packages/sdk/src/authored-root.ts b/packages/sdk/src/authored-root.ts index d3e3bc8d5..715b798a2 100644 --- a/packages/sdk/src/authored-root.ts +++ b/packages/sdk/src/authored-root.ts @@ -369,6 +369,13 @@ async function driveRoot( // A lost lease is owned by the kernel's retry path, not by this attempt's // terminalization or suspension handling. if (isLeaseLost(error)) throw error; + // A caller cancel is the caller's, not a body failure: reads aborted by it + // (including the Bun verifier's) leave the root resumable, never terminal. + const signal = options.lifecycle?.signal; + if (aborted(signal) && (error === signal!.reason || (error instanceof Error && error.name === 'AbortError'))) { + if (error instanceof Error) (error as Error & { rootRunId?: string }).rootRunId = dispatch.run_id; + throw error; + } if (isReadInterruptionError(error) || (error instanceof AuthoredFlowExecutionError && error.code === 'daemon_unresponsive')) { const parked = new AuthoredFlowExecutionError('daemon_unresponsive', diff --git a/packages/sdk/src/cli/reuse.ts b/packages/sdk/src/cli/reuse.ts index 3e15c276d..26f012188 100644 --- a/packages/sdk/src/cli/reuse.ts +++ b/packages/sdk/src/cli/reuse.ts @@ -1,5 +1,4 @@ import type { JournalClient } from '../journal-client.js'; -import { isReadInterruptionError } from '../journal-read-policy.js'; import type { RunExecution } from './run.js'; /** Count durable facts, including failed executions, across paginated reads. */ @@ -28,7 +27,7 @@ export async function reuseSummary(client: JournalClient, runId: string, fromRun * Add the reuse summary to an already classified run. The summary is an extra * journal read after the outcome is known: if it cannot be answered, report * that as a warning and keep the classified outcome rather than recasting a - * finished run as resumable. + * finished run as resumable or failed. */ export async function attachReuseSummary(execution: RunExecution, client: JournalClient, runId: string, fromRunId: string): Promise { @@ -39,9 +38,11 @@ export async function attachReuseSummary(execution: RunExecution, client: Journa try { execution.report.reuse = await reuseSummary(client, runId, fromRunId); } catch (error) { - if (!isReadInterruptionError(error)) throw error; + // Optional detail after the outcome is decided: no failure here — an + // interruption, a cancellation, a bad frame — may replace that outcome. + const why = error instanceof Error ? error.message : String(error); execution.report.diagnostics = [...execution.report.diagnostics, { severity: 'warning', kind: 'reuse_summary_unavailable', - message: `The run's outcome is final, but the summary of steps reused from ${fromRunId} could not be read: ${error.message}` }]; + message: `The run's outcome is final, but the summary of steps reused from ${fromRunId} could not be read: ${why}` }]; } return execution; } diff --git a/packages/sdk/tests/authored-root.test.ts b/packages/sdk/tests/authored-root.test.ts index 7acad6a38..6bb0c741f 100644 --- a/packages/sdk/tests/authored-root.test.ts +++ b/packages/sdk/tests/authored-root.test.ts @@ -432,6 +432,19 @@ describe('durable authored root', () => { expect(journal.peer.completions).toEqual([]); }); + it('never terminalizes the root for a caller cancellation', async () => { + const controller = new AbortController(); + const loaded = await fixture(false, 0, async () => { + controller.abort(new Error('caller canceled the run')); + throw controller.signal.reason; + }); + const journal = new RootJournal(); + await expect(executeDurableAuthoredFlow(loaded, journal as unknown as JournalClient, undefined, + { dataDir: '/unused', admissionKey: 'caller-cancel', lifecycle: { signal: controller.signal } })) + .rejects.toMatchObject({ message: 'caller canceled the run', rootRunId: 'root-run' }); + expect(journal.peer.completions).toEqual([]); + }); + it('terminalizes a returned body failure without replaying semantic side effects', async () => { const loaded = await fixture(true); const journal = new RootJournal(); diff --git a/packages/sdk/tests/reuse-summary-interruption.test.ts b/packages/sdk/tests/reuse-summary-interruption.test.ts index 106444d46..5c9d2926d 100644 --- a/packages/sdk/tests/reuse-summary-interruption.test.ts +++ b/packages/sdk/tests/reuse-summary-interruption.test.ts @@ -1,6 +1,6 @@ import { expect, it } from 'vitest'; import { attachReuseSummary } from '../src/cli/reuse.js'; -import { JournalReadInterruptedError, JournalRequestTimeoutError, type JournalClient } from '../src/journal-client.js'; +import { JournalFrameError, JournalReadInterruptedError, JournalRequestTimeoutError, type JournalClient } from '../src/journal-client.js'; import type { RunExecution } from '../src/cli/run.js'; const completed = (): RunExecution => ({ exitCode: 0, report: { @@ -22,13 +22,18 @@ it.each([ expect(JSON.stringify(execution.report)).not.toContain('flows resume'); }); -it('still attaches the summary when the read succeeds, and rethrows other failures', async () => { +it('still attaches the summary when the read succeeds, and never lets the optional read discard the outcome', async () => { const ok = { journalRead: async (_run: string, from: number) => ({ entries: from === 1 ? [{ seq: 1, entry_type: 'step.completed', step_id: 'a', payload: { reused_from: 'prior' } }] : [] }) } as unknown as JournalClient; expect((await attachReuseSummary(completed(), ok, 'run-1', 'prior')).report.reuse) .toEqual({ fromRunId: 'prior', reusedSteps: 1, executedSteps: 0 }); - const broken = { journalRead: async () => { throw new Error('invalid journal sequence'); } } as unknown as JournalClient; - await expect(attachReuseSummary(completed(), broken, 'run-1', 'prior')).rejects.toThrow('invalid journal sequence'); + for (const failure of [new Error('invalid journal sequence'), new JournalFrameError(), + Object.assign(new Error('run canceled'), { name: 'AbortError' })]) { + const broken = { journalRead: async () => { throw failure; } } as unknown as JournalClient; + const execution = await attachReuseSummary(completed(), broken, 'run-1', 'prior'); + expect(execution.report).toMatchObject({ ok: true, status: 'completed' }); + expect(execution.report.diagnostics.at(-1)).toMatchObject({ kind: 'reuse_summary_unavailable', message: expect.stringContaining(failure.message) }); + } }); it.each(['daemon_unresponsive', 'daemon_unreachable'])('does not start another read after classification reported %s', async kind => { From 2cdf35240007a63174b45dc6aefbb93883f8b100 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 02:22:41 -0700 Subject: [PATCH 14/34] docs(evidence): record the mutation harness command as actually run verification.md said to run evidence/run-read-timeout/mutations.py from packages/sdk, which resolves to a nonexistent path; the command used is `cd packages/sdk && python3 ../../evidence/run-read-timeout/mutations.py`. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: b8262468-ea26-4a6e-bee9-d193832e77dc --- evidence/run-read-timeout/verification.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evidence/run-read-timeout/verification.md b/evidence/run-read-timeout/verification.md index 76470c8f1..ba11f383d 100644 --- a/evidence/run-read-timeout/verification.md +++ b/evidence/run-read-timeout/verification.md @@ -91,7 +91,7 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo exit=0 ``` -The mutation sections below were regenerated by `python3 evidence/run-read-timeout/mutations.py` (run from `packages/sdk`) against the review-fix source; each mutation fails its test and the restored source passes. +The mutation sections below were regenerated by `cd packages/sdk && python3 ../../evidence/run-read-timeout/mutations.py` (the command used) against the review-fix source; each mutation fails its test and the restored source passes. ## mutation-reader.log From db0326fbd7052b2fd9b630202a388c847458a853 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 02:37:08 -0700 Subject: [PATCH 15/34] refactor(sdk): move budgeted reader sessions out of journal-client; watch setup fails closed on bad frames - journal-budgeted-reads.ts (86 lines) now owns the dedicated reader session, its reconnect/fallback rules and the budgeted retry wrapper; journal-client keeps framing, connection lifecycle and the typed verbs, and delegates allowlisted reads when a read budget is configured. journal-client.ts goes from 647 to 591 lines (main: 537); the remaining growth is per-request cancellation in requestOnce and JournalFrameError, both client-level. - waitForRunningStep rethrows JournalFrameError during watch setup instead of recasting it as a resumable daemon_unresponsive. Mutation harness retargeted to the new module; evidence re-captured (8/8 mutations, focused 23 files 291/291, resume-live 1/1). Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: b8262468-ea26-4a6e-bee9-d193832e77dc --- evidence/run-read-timeout/final-focused.log | 89 +++--- .../mutation-completion-aborts-snapshot.log | 24 +- .../run-read-timeout/mutation-heartbeat.log | 12 +- .../mutation-reader-reconnect.log | 34 +-- .../mutation-reader-setup-retry.log | 20 +- evidence/run-read-timeout/mutation-reader.log | 20 +- .../mutation-root-parking.log | 14 +- .../mutation-snapshot-cancel.log | 24 +- .../mutation-watch-cadence.log | 24 +- evidence/run-read-timeout/mutations.py | 10 +- evidence/run-read-timeout/resume-live.log | 8 +- evidence/run-read-timeout/verification.md | 269 +++++++++--------- packages/sdk/src/cli/running-step.ts | 5 +- packages/sdk/src/journal-budgeted-reads.ts | 86 ++++++ packages/sdk/src/journal-client.ts | 78 +---- packages/sdk/tests/running-step-watch.test.ts | 16 +- 16 files changed, 388 insertions(+), 345 deletions(-) create mode 100644 packages/sdk/src/journal-budgeted-reads.ts diff --git a/evidence/run-read-timeout/final-focused.log b/evidence/run-read-timeout/final-focused.log index 595090827..abef16983 100644 --- a/evidence/run-read-timeout/final-focused.log +++ b/evidence/run-read-timeout/final-focused.log @@ -2,59 +2,58 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5289ms - ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 303ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2036ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2036ms -(node:3553150) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5254ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2037ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2038ms +(node:3771421) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (28 tests) 391ms - ✓ tests/journal-client.test.ts (17 tests) 80ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 6834ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 483ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 969ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2765ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 331ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 783ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2656ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 661ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 638ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 692ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 664ms - ✓ tests/journal-client-read-timeout.test.ts (20 tests) 1428ms - ✓ tests/classify-outcome.test.ts (11 tests) 7436ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2019ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3008ms - ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2002ms - ✓ tests/human-live.test.ts (3 tests) 4374ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2609ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1072ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 692ms - ✓ tests/running-step-watch.test.ts (7 tests) 8239ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms - ✓ cancels promptly while a lease snapshot read is in flight 2006ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2055ms + ✓ tests/authored-root.test.ts (28 tests) 371ms + ✓ tests/journal-client.test.ts (17 tests) 78ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 6760ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 434ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1015ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2633ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 356ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 760ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2576ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 663ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 632ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 651ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 630ms + ✓ tests/journal-client-read-timeout.test.ts (20 tests) 1427ms + ✓ tests/classify-outcome.test.ts (11 tests) 7427ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2014ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3003ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms + ✓ tests/human-live.test.ts (3 tests) 4605ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2726ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1114ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 764ms + ✓ tests/running-step-watch.test.ts (8 tests) 8239ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms + ✓ cancels promptly while a lease snapshot read is in flight 2007ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2054ms ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms - ✓ tests/worker-lease.test.ts (7 tests) 12ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 25ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 700ms - ✓ tests/memoization.test.ts (58 tests) 63ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2418ms + ✓ tests/worker-lease.test.ts (7 tests) 15ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 26ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 668ms + ✓ tests/memoization.test.ts (58 tests) 65ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2492ms ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2057ms - ✓ drains read and watch promises before an authored flow completes 360ms + ✓ drains read and watch promises before an authored flow completes 435ms ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms - ✓ tests/journal-client-completion.test.ts (6 tests) 103ms - ✓ tests/reuse-summary-interruption.test.ts (5 tests) 4ms + ✓ tests/journal-client-completion.test.ts (6 tests) 101ms + ✓ tests/reuse-summary-interruption.test.ts (5 tests) 3ms ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms ✓ tests/direct-run-worker-lease.test.ts (3 tests) 9ms - ✓ tests/journal-client-subscriptions.test.ts (1 test) 7ms + ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 4ms - ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 48ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms + ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 47ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 17ms Test Files 23 passed (23) - Tests 290 passed (290) - Start at 01:44:53 - Duration 48.73s (transform 1.06s, setup 131ms, collect 5.73s, tests 40.15s, environment 3ms, prepare 826ms) + Tests 291 passed (291) + Start at 02:36:06 + Duration 48.85s (transform 1.05s, setup 132ms, collect 5.77s, tests 40.21s, environment 3ms, prepare 804ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log index bfab4fc7d..802161169 100644 --- a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log +++ b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/cli/running-step.ts -Original SHA256: 5f7ca7c0a29e5dcd0be4650369e994a21aa715f7dbdc45363924abf5df30d96b +Original SHA256: e535b533e40304d47aeba175c8ac6f6e51808de9fc99d1d0f64648f2b8a39a70 Replaced: snapshotRead?.abort(new Error(`step "${runningStep.id}" completed during the lease snapshot`)); With: @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 3010ms - × a completion push ends the wait without waiting out an in-flight snapshot 3009ms + ❯ tests/running-step-watch.test.ts (8 tests | 1 failed | 7 skipped) 3014ms + × a completion push ends the wait without waiting out an in-flight snapshot 3013ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -33,23 +33,23 @@ Received: "still waiting" ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 6 skipped (7) - Start at 01:44:37 - Duration 3.49s (transform 190ms, setup 16ms, collect 318ms, tests 3.01s, environment 0ms, prepare 41ms) + Tests 1 failed | 7 skipped (8) + Start at 02:35:42 + Duration 3.49s (transform 186ms, setup 19ms, collect 314ms, tests 3.01s, environment 0ms, prepare 41ms) exit=1 -RESTORED SHA256: 5f7ca7c0a29e5dcd0be4650369e994a21aa715f7dbdc45363924abf5df30d96b +RESTORED SHA256: e535b533e40304d47aeba175c8ac6f6e51808de9fc99d1d0f64648f2b8a39a70 $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a completion push ends the wait' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2062ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2061ms + ✓ tests/running-step-watch.test.ts (8 tests | 7 skipped) 2058ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2058ms Test Files 1 passed (1) - Tests 1 passed | 6 skipped (7) - Start at 01:44:41 - Duration 2.51s (transform 180ms, setup 18ms, collect 292ms, tests 2.06s, environment 0ms, prepare 41ms) + Tests 1 passed | 7 skipped (8) + Start at 02:35:46 + Duration 2.53s (transform 188ms, setup 16ms, collect 308ms, tests 2.06s, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-heartbeat.log b/evidence/run-read-timeout/mutation-heartbeat.log index 129c18f8d..2417c64ca 100644 --- a/evidence/run-read-timeout/mutation-heartbeat.log +++ b/evidence/run-read-timeout/mutation-heartbeat.log @@ -13,8 +13,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 18ms - × a heartbeat timeout is lease loss rather than a worker body failure 17ms + ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 19ms + × a heartbeat timeout is lease loss rather than a worker body failure 18ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -33,8 +33,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 01:44:44 - Duration 464ms (transform 165ms, setup 16ms, collect 291ms, tests 18ms, environment 0ms, prepare 40ms) + Start at 02:35:49 + Duration 466ms (transform 169ms, setup 17ms, collect 286ms, tests 19ms, environment 0ms, prepare 41ms) exit=1 @@ -47,7 +47,7 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb Test Files 1 passed (1) Tests 1 passed (1) - Start at 01:44:45 - Duration 447ms (transform 167ms, setup 17ms, collect 281ms, tests 15ms, environment 0ms, prepare 40ms) + Start at 02:35:50 + Duration 451ms (transform 164ms, setup 17ms, collect 284ms, tests 15ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-reconnect.log b/evidence/run-read-timeout/mutation-reader-reconnect.log index a2f704438..3d5a54045 100644 --- a/evidence/run-read-timeout/mutation-reader-reconnect.log +++ b/evidence/run-read-timeout/mutation-reader-reconnect.log @@ -1,7 +1,7 @@ -Source: packages/sdk/src/journal-client.ts -Original SHA256: 9c77a946018fecb31ee86b19e343b78cb65f077e87bd03395b5dcb30f2e3a0a3 +Source: packages/sdk/src/journal-budgeted-reads.ts +Original SHA256: 247f3340fff1711bea34f2ad3478f080daeb35a0751050ef8624d5c22140f285 Replaced: - this.dropReader(reader); + this.drop(reader); With: @@ -11,14 +11,14 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (20 tests | 1 failed | 19 skipped) 1006ms - × reconnects the reader after it disconnects within the read budget 1006ms - → journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ tests/journal-client-read-timeout.test.ts (20 tests | 1 failed | 19 skipped) 1007ms + × reconnects the reader after it disconnects within the read budget 1007ms + → journal client: run.get read session was interrupted after 81 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 81 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:70:9 68| let last: unknown; 69| const exhausted = () => last instanceof JournalReadInterruptedError @@ -29,33 +29,33 @@ JournalReadInterruptedError: journal client: run.get read session was interrupte ❯ Timeout. src/journal-read-policy.ts:81:39 Caused by: Error: journal client: not connected (run.get): journal client: connection closed - ❯ src/journal-client.ts:285:16 - ❯ JournalClient.requestOnce src/journal-client.ts:278:12 - ❯ src/journal-client.ts:259:31 + ❯ src/journal-client.ts:230:16 + ❯ JournalClient.requestOnce src/journal-client.ts:223:12 + ❯ src/journal-budgeted-reads.ts:47:29 ❯ src/journal-read-policy.ts:93:18 Caused by: Error: journal client: connection closed - ❯ Socket. src/journal-client.ts:137:26 + ❯ Socket. src/journal-client.ts:134:26 ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) Tests 1 failed | 19 skipped (20) - Start at 01:44:21 - Duration 1.42s (transform 158ms, setup 17ms, collect 260ms, tests 1.01s, environment 0ms, prepare 41ms) + Start at 02:35:26 + Duration 1.45s (transform 162ms, setup 17ms, collect 287ms, tests 1.01s, environment 0ms, prepare 43ms) exit=1 -RESTORED SHA256: 9c77a946018fecb31ee86b19e343b78cb65f077e87bd03395b5dcb30f2e3a0a3 +RESTORED SHA256: 247f3340fff1711bea34f2ad3478f080daeb35a0751050ef8624d5c22140f285 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'reconnects the reader after it disconnects' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (20 tests | 19 skipped) 74ms + ✓ tests/journal-client-read-timeout.test.ts (20 tests | 19 skipped) 91ms Test Files 1 passed (1) Tests 1 passed | 19 skipped (20) - Start at 01:44:23 - Duration 499ms (transform 170ms, setup 17ms, collect 282ms, tests 74ms, environment 0ms, prepare 40ms) + Start at 02:35:28 + Duration 547ms (transform 167ms, setup 16ms, collect 291ms, tests 91ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-setup-retry.log b/evidence/run-read-timeout/mutation-reader-setup-retry.log index c59fee983..61f7681dc 100644 --- a/evidence/run-read-timeout/mutation-reader-setup-retry.log +++ b/evidence/run-read-timeout/mutation-reader-setup-retry.log @@ -1,5 +1,5 @@ -Source: packages/sdk/src/journal-client.ts -Original SHA256: 9c77a946018fecb31ee86b19e343b78cb65f077e87bd03395b5dcb30f2e3a0a3 +Source: packages/sdk/src/journal-budgeted-reads.ts +Original SHA256: 247f3340fff1711bea34f2ad3478f080daeb35a0751050ef8624d5c22140f285 Replaced: if (this.reader === reader) { this.reader = undefined; @@ -16,8 +16,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (20 tests | 1 failed | 19 skipped) 80ms - × retries a reader setup that timed out instead of reading on the primary for good 79ms + ❯ tests/journal-client-read-timeout.test.ts (20 tests | 1 failed | 19 skipped) 81ms + × retries a reader setup that timed out instead of reading on the primary for good 80ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -43,21 +43,21 @@ AssertionError: expected 1 to be 2 // Object.is equality Test Files 1 failed (1) Tests 1 failed | 19 skipped (20) - Start at 01:44:24 - Duration 507ms (transform 163ms, setup 17ms, collect 277ms, tests 80ms, environment 0ms, prepare 40ms) + Start at 02:35:29 + Duration 533ms (transform 167ms, setup 16ms, collect 290ms, tests 81ms, environment 0ms, prepare 41ms) exit=1 -RESTORED SHA256: 9c77a946018fecb31ee86b19e343b78cb65f077e87bd03395b5dcb30f2e3a0a3 +RESTORED SHA256: 247f3340fff1711bea34f2ad3478f080daeb35a0751050ef8624d5c22140f285 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'retries a reader setup that timed out' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (20 tests | 19 skipped) 70ms + ✓ tests/journal-client-read-timeout.test.ts (20 tests | 19 skipped) 75ms Test Files 1 passed (1) Tests 1 passed | 19 skipped (20) - Start at 01:44:25 - Duration 509ms (transform 166ms, setup 17ms, collect 288ms, tests 70ms, environment 0ms, prepare 41ms) + Start at 02:35:29 + Duration 515ms (transform 165ms, setup 16ms, collect 287ms, tests 75ms, environment 0ms, prepare 43ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader.log b/evidence/run-read-timeout/mutation-reader.log index a79fad9a0..89e76475d 100644 --- a/evidence/run-read-timeout/mutation-reader.log +++ b/evidence/run-read-timeout/mutation-reader.log @@ -1,17 +1,17 @@ -Source: packages/sdk/src/journal-client.ts -Original SHA256: 9c77a946018fecb31ee86b19e343b78cb65f077e87bd03395b5dcb30f2e3a0a3 +Source: packages/sdk/src/journal-budgeted-reads.ts +Original SHA256: 247f3340fff1711bea34f2ad3478f080daeb35a0751050ef8624d5c22140f285 Replaced: return await reader.requestOnce(verb, params, remaining, attemptSignal); With: -return await this.requestOnce(verb, params, remaining, attemptSignal); +return await this.primary.requestOnce(verb, params, remaining, attemptSignal); REVERTED $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'serves a bounded read' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (20 tests | 1 failed | 19 skipped) 114ms - × serves a bounded read while an unbounded command is in flight 113ms + ❯ tests/journal-client-read-timeout.test.ts (20 tests | 1 failed | 19 skipped) 113ms + × serves a bounded read while an unbounded command is in flight 112ms → expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -37,12 +37,12 @@ AssertionError: expected true to be false // Object.is equality Test Files 1 failed (1) Tests 1 failed | 19 skipped (20) - Start at 01:44:20 - Duration 546ms (transform 170ms, setup 17ms, collect 294ms, tests 114ms, environment 0ms, prepare 30ms) + Start at 02:35:24 + Duration 563ms (transform 162ms, setup 16ms, collect 288ms, tests 113ms, environment 0ms, prepare 40ms) exit=1 -RESTORED SHA256: 9c77a946018fecb31ee86b19e343b78cb65f077e87bd03395b5dcb30f2e3a0a3 +RESTORED SHA256: 247f3340fff1711bea34f2ad3478f080daeb35a0751050ef8624d5c22140f285 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'serves a bounded read' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk @@ -51,7 +51,7 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t Test Files 1 passed (1) Tests 1 passed | 19 skipped (20) - Start at 01:44:21 - Duration 552ms (transform 163ms, setup 17ms, collect 285ms, tests 106ms, environment 0ms, prepare 38ms) + Start at 02:35:25 + Duration 553ms (transform 169ms, setup 18ms, collect 292ms, tests 106ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-root-parking.log b/evidence/run-read-timeout/mutation-root-parking.log index 36d712e15..883daa652 100644 --- a/evidence/run-read-timeout/mutation-root-parking.log +++ b/evidence/run-read-timeout/mutation-root-parking.log @@ -17,8 +17,8 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (28 tests | 3 failed | 25 skipped) 31ms - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 21ms + ❯ tests/authored-root.test.ts (28 tests | 3 failed | 25 skipped) 25ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 16ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms @@ -104,8 +104,8 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } Test Files 1 failed (1) Tests 3 failed | 25 skipped (28) - Start at 01:44:46 - Duration 1.29s (transform 662ms, setup 17ms, collect 1.09s, tests 31ms, environment 0ms, prepare 41ms) + Start at 02:35:50 + Duration 1.28s (transform 669ms, setup 17ms, collect 1.09s, tests 25ms, environment 0ms, prepare 41ms) exit=1 @@ -114,11 +114,11 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (28 tests | 25 skipped) 23ms + ✓ tests/authored-root.test.ts (28 tests | 25 skipped) 24ms Test Files 1 passed (1) Tests 3 passed | 25 skipped (28) - Start at 01:44:47 - Duration 1.26s (transform 655ms, setup 16ms, collect 1.07s, tests 23ms, environment 0ms, prepare 41ms) + Start at 02:35:52 + Duration 1.23s (transform 641ms, setup 16ms, collect 1.06s, tests 24ms, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-snapshot-cancel.log b/evidence/run-read-timeout/mutation-snapshot-cancel.log index bc7572ca5..1bac28272 100644 --- a/evidence/run-read-timeout/mutation-snapshot-cancel.log +++ b/evidence/run-read-timeout/mutation-snapshot-cancel.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/cli/running-step.ts -Original SHA256: 5f7ca7c0a29e5dcd0be4650369e994a21aa715f7dbdc45363924abf5df30d96b +Original SHA256: e535b533e40304d47aeba175c8ac6f6e51808de9fc99d1d0f64648f2b8a39a70 Replaced: const signal = options.signal === undefined ? read.signal : AbortSignal.any([options.signal, read.signal]); @@ -11,8 +11,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 2217ms - × cancels promptly while a lease snapshot read is in flight 2217ms + ❯ tests/running-step-watch.test.ts (8 tests | 1 failed | 7 skipped) 2216ms + × cancels promptly while a lease snapshot read is in flight 2215ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -30,23 +30,23 @@ AssertionError: expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 6 skipped (7) - Start at 01:44:32 - Duration 2.67s (transform 192ms, setup 16ms, collect 311ms, tests 2.22s, environment 0ms, prepare 35ms) + Tests 1 failed | 7 skipped (8) + Start at 02:35:36 + Duration 2.69s (transform 192ms, setup 18ms, collect 313ms, tests 2.22s, environment 0ms, prepare 41ms) exit=1 -RESTORED SHA256: 5f7ca7c0a29e5dcd0be4650369e994a21aa715f7dbdc45363924abf5df30d96b +RESTORED SHA256: e535b533e40304d47aeba175c8ac6f6e51808de9fc99d1d0f64648f2b8a39a70 $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels promptly while a lease snapshot read is in flight' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2011ms - ✓ cancels promptly while a lease snapshot read is in flight 2010ms + ✓ tests/running-step-watch.test.ts (8 tests | 7 skipped) 2012ms + ✓ cancels promptly while a lease snapshot read is in flight 2011ms Test Files 1 passed (1) - Tests 1 passed | 6 skipped (7) - Start at 01:44:35 - Duration 2.48s (transform 184ms, setup 17ms, collect 308ms, tests 2.01s, environment 0ms, prepare 42ms) + Tests 1 passed | 7 skipped (8) + Start at 02:35:39 + Duration 2.47s (transform 178ms, setup 17ms, collect 296ms, tests 2.01s, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-watch-cadence.log b/evidence/run-read-timeout/mutation-watch-cadence.log index ad6886d12..5184fe3eb 100644 --- a/evidence/run-read-timeout/mutation-watch-cadence.log +++ b/evidence/run-read-timeout/mutation-watch-cadence.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/cli/running-step.ts -Original SHA256: 5f7ca7c0a29e5dcd0be4650369e994a21aa715f7dbdc45363924abf5df30d96b +Original SHA256: e535b533e40304d47aeba175c8ac6f6e51808de9fc99d1d0f64648f2b8a39a70 Replaced: const LEASE_POLL_MS = 2_000; With: @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 2111ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2110ms + ❯ tests/running-step-watch.test.ts (8 tests | 1 failed | 7 skipped) 2110ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2109ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -29,23 +29,23 @@ AssertionError: expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 6 skipped (7) - Start at 01:44:26 - Duration 2.59s (transform 186ms, setup 17ms, collect 313ms, tests 2.11s, environment 0ms, prepare 41ms) + Tests 1 failed | 7 skipped (8) + Start at 02:35:30 + Duration 2.59s (transform 189ms, setup 17ms, collect 314ms, tests 2.11s, environment 0ms, prepare 41ms) exit=1 -RESTORED SHA256: 5f7ca7c0a29e5dcd0be4650369e994a21aa715f7dbdc45363924abf5df30d96b +RESTORED SHA256: e535b533e40304d47aeba175c8ac6f6e51808de9fc99d1d0f64648f2b8a39a70 $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pushes for completion' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2116ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2116ms + ✓ tests/running-step-watch.test.ts (8 tests | 7 skipped) 2118ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms Test Files 1 passed (1) - Tests 1 passed | 6 skipped (7) - Start at 01:44:29 - Duration 2.56s (transform 187ms, setup 16ms, collect 307ms, tests 2.12s, environment 0ms, prepare 35ms) + Tests 1 passed | 7 skipped (8) + Start at 02:35:33 + Duration 2.60s (transform 191ms, setup 16ms, collect 317ms, tests 2.12s, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutations.py b/evidence/run-read-timeout/mutations.py index 360733e19..f1ac34ac3 100644 --- a/evidence/run-read-timeout/mutations.py +++ b/evidence/run-read-timeout/mutations.py @@ -7,14 +7,14 @@ ROOT = Path(__file__).resolve().parents[2] SDK = ROOT / 'packages/sdk' CASES = [ - ('reader', 'packages/sdk/src/journal-client.ts', + ('reader', 'packages/sdk/src/journal-budgeted-reads.ts', 'return await reader.requestOnce(verb, params, remaining, attemptSignal);', - 'return await this.requestOnce(verb, params, remaining, attemptSignal);', + 'return await this.primary.requestOnce(verb, params, remaining, attemptSignal);', 'tests/journal-client-read-timeout.test.ts', 'serves a bounded read'), - ('reader-reconnect', 'packages/sdk/src/journal-client.ts', - """ this.dropReader(reader); + ('reader-reconnect', 'packages/sdk/src/journal-budgeted-reads.ts', + """ this.drop(reader); """, '', 'tests/journal-client-read-timeout.test.ts', 'reconnects the reader after it disconnects'), - ('reader-setup-retry', 'packages/sdk/src/journal-client.ts', + ('reader-setup-retry', 'packages/sdk/src/journal-budgeted-reads.ts', """ if (this.reader === reader) { this.reader = undefined; this.readerReady = undefined; diff --git a/evidence/run-read-timeout/resume-live.log b/evidence/run-read-timeout/resume-live.log index df47425e3..646e1b4a0 100644 --- a/evidence/run-read-timeout/resume-live.log +++ b/evidence/run-read-timeout/resume-live.log @@ -2,12 +2,12 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 929ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 929ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 903ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 903ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 01:45:42 - Duration 2.16s (transform 602ms, setup 16ms, collect 993ms, tests 929ms, environment 0ms, prepare 61ms) + Start at 02:36:55 + Duration 2.21s (transform 641ms, setup 19ms, collect 1.05s, tests 903ms, environment 0ms, prepare 74ms) exit=0 diff --git a/evidence/run-read-timeout/verification.md b/evidence/run-read-timeout/verification.md index ba11f383d..849f2e725 100644 --- a/evidence/run-read-timeout/verification.md +++ b/evidence/run-read-timeout/verification.md @@ -15,60 +15,59 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5289ms - ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 303ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2036ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2036ms -(node:3553150) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5254ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2037ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2038ms +(node:3771421) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (28 tests) 391ms - ✓ tests/journal-client.test.ts (17 tests) 80ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 6834ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 483ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 969ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2765ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 331ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 783ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2656ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 661ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 638ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 692ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 664ms - ✓ tests/journal-client-read-timeout.test.ts (20 tests) 1428ms - ✓ tests/classify-outcome.test.ts (11 tests) 7436ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2019ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3008ms - ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2002ms - ✓ tests/human-live.test.ts (3 tests) 4374ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2609ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1072ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 692ms - ✓ tests/running-step-watch.test.ts (7 tests) 8239ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms - ✓ cancels promptly while a lease snapshot read is in flight 2006ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2055ms + ✓ tests/authored-root.test.ts (28 tests) 371ms + ✓ tests/journal-client.test.ts (17 tests) 78ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 6760ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 434ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1015ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2633ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 356ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 760ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2576ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 663ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 632ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 651ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 630ms + ✓ tests/journal-client-read-timeout.test.ts (20 tests) 1427ms + ✓ tests/classify-outcome.test.ts (11 tests) 7427ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2014ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3003ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms + ✓ tests/human-live.test.ts (3 tests) 4605ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2726ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1114ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 764ms + ✓ tests/running-step-watch.test.ts (8 tests) 8239ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms + ✓ cancels promptly while a lease snapshot read is in flight 2007ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2054ms ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms - ✓ tests/worker-lease.test.ts (7 tests) 12ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 25ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 700ms - ✓ tests/memoization.test.ts (58 tests) 63ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2418ms + ✓ tests/worker-lease.test.ts (7 tests) 15ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 26ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 668ms + ✓ tests/memoization.test.ts (58 tests) 65ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2492ms ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2057ms - ✓ drains read and watch promises before an authored flow completes 360ms + ✓ drains read and watch promises before an authored flow completes 435ms ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms - ✓ tests/journal-client-completion.test.ts (6 tests) 103ms - ✓ tests/reuse-summary-interruption.test.ts (5 tests) 4ms + ✓ tests/journal-client-completion.test.ts (6 tests) 101ms + ✓ tests/reuse-summary-interruption.test.ts (5 tests) 3ms ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms ✓ tests/direct-run-worker-lease.test.ts (3 tests) 9ms - ✓ tests/journal-client-subscriptions.test.ts (1 test) 7ms + ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 4ms - ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 48ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms + ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 47ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 17ms Test Files 23 passed (23) - Tests 290 passed (290) - Start at 01:44:53 - Duration 48.73s (transform 1.06s, setup 131ms, collect 5.73s, tests 40.15s, environment 3ms, prepare 826ms) + Tests 291 passed (291) + Start at 02:36:06 + Duration 48.85s (transform 1.05s, setup 132ms, collect 5.77s, tests 40.21s, environment 3ms, prepare 804ms) exit=0 ``` @@ -80,13 +79,13 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 929ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 929ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 903ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 903ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 01:45:42 - Duration 2.16s (transform 602ms, setup 16ms, collect 993ms, tests 929ms, environment 0ms, prepare 61ms) + Start at 02:36:55 + Duration 2.21s (transform 641ms, setup 19ms, collect 1.05s, tests 903ms, environment 0ms, prepare 74ms) exit=0 ``` @@ -96,20 +95,20 @@ The mutation sections below were regenerated by `cd packages/sdk && python3 ../. ## mutation-reader.log ```text -Source: packages/sdk/src/journal-client.ts -Original SHA256: 9c77a946018fecb31ee86b19e343b78cb65f077e87bd03395b5dcb30f2e3a0a3 +Source: packages/sdk/src/journal-budgeted-reads.ts +Original SHA256: 247f3340fff1711bea34f2ad3478f080daeb35a0751050ef8624d5c22140f285 Replaced: return await reader.requestOnce(verb, params, remaining, attemptSignal); With: -return await this.requestOnce(verb, params, remaining, attemptSignal); +return await this.primary.requestOnce(verb, params, remaining, attemptSignal); REVERTED $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'serves a bounded read' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (20 tests | 1 failed | 19 skipped) 114ms - × serves a bounded read while an unbounded command is in flight 113ms + ❯ tests/journal-client-read-timeout.test.ts (20 tests | 1 failed | 19 skipped) 113ms + × serves a bounded read while an unbounded command is in flight 112ms → expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -135,12 +134,12 @@ AssertionError: expected true to be false // Object.is equality Test Files 1 failed (1) Tests 1 failed | 19 skipped (20) - Start at 01:44:20 - Duration 546ms (transform 170ms, setup 17ms, collect 294ms, tests 114ms, environment 0ms, prepare 30ms) + Start at 02:35:24 + Duration 563ms (transform 162ms, setup 16ms, collect 288ms, tests 113ms, environment 0ms, prepare 40ms) exit=1 -RESTORED SHA256: 9c77a946018fecb31ee86b19e343b78cb65f077e87bd03395b5dcb30f2e3a0a3 +RESTORED SHA256: 247f3340fff1711bea34f2ad3478f080daeb35a0751050ef8624d5c22140f285 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'serves a bounded read' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk @@ -149,8 +148,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t Test Files 1 passed (1) Tests 1 passed | 19 skipped (20) - Start at 01:44:21 - Duration 552ms (transform 163ms, setup 17ms, collect 285ms, tests 106ms, environment 0ms, prepare 38ms) + Start at 02:35:25 + Duration 553ms (transform 169ms, setup 18ms, collect 292ms, tests 106ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -158,10 +157,10 @@ exit=0 ## mutation-reader-reconnect.log ```text -Source: packages/sdk/src/journal-client.ts -Original SHA256: 9c77a946018fecb31ee86b19e343b78cb65f077e87bd03395b5dcb30f2e3a0a3 +Source: packages/sdk/src/journal-budgeted-reads.ts +Original SHA256: 247f3340fff1711bea34f2ad3478f080daeb35a0751050ef8624d5c22140f285 Replaced: - this.dropReader(reader); + this.drop(reader); With: @@ -171,14 +170,14 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (20 tests | 1 failed | 19 skipped) 1006ms - × reconnects the reader after it disconnects within the read budget 1006ms - → journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ tests/journal-client-read-timeout.test.ts (20 tests | 1 failed | 19 skipped) 1007ms + × reconnects the reader after it disconnects within the read budget 1007ms + → journal client: run.get read session was interrupted after 81 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 81 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:70:9 68| let last: unknown; 69| const exhausted = () => last instanceof JournalReadInterruptedError @@ -189,34 +188,34 @@ JournalReadInterruptedError: journal client: run.get read session was interrupte ❯ Timeout. src/journal-read-policy.ts:81:39 Caused by: Error: journal client: not connected (run.get): journal client: connection closed - ❯ src/journal-client.ts:285:16 - ❯ JournalClient.requestOnce src/journal-client.ts:278:12 - ❯ src/journal-client.ts:259:31 + ❯ src/journal-client.ts:230:16 + ❯ JournalClient.requestOnce src/journal-client.ts:223:12 + ❯ src/journal-budgeted-reads.ts:47:29 ❯ src/journal-read-policy.ts:93:18 Caused by: Error: journal client: connection closed - ❯ Socket. src/journal-client.ts:137:26 + ❯ Socket. src/journal-client.ts:134:26 ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) Tests 1 failed | 19 skipped (20) - Start at 01:44:21 - Duration 1.42s (transform 158ms, setup 17ms, collect 260ms, tests 1.01s, environment 0ms, prepare 41ms) + Start at 02:35:26 + Duration 1.45s (transform 162ms, setup 17ms, collect 287ms, tests 1.01s, environment 0ms, prepare 43ms) exit=1 -RESTORED SHA256: 9c77a946018fecb31ee86b19e343b78cb65f077e87bd03395b5dcb30f2e3a0a3 +RESTORED SHA256: 247f3340fff1711bea34f2ad3478f080daeb35a0751050ef8624d5c22140f285 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'reconnects the reader after it disconnects' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (20 tests | 19 skipped) 74ms + ✓ tests/journal-client-read-timeout.test.ts (20 tests | 19 skipped) 91ms Test Files 1 passed (1) Tests 1 passed | 19 skipped (20) - Start at 01:44:23 - Duration 499ms (transform 170ms, setup 17ms, collect 282ms, tests 74ms, environment 0ms, prepare 40ms) + Start at 02:35:28 + Duration 547ms (transform 167ms, setup 16ms, collect 291ms, tests 91ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -224,8 +223,8 @@ exit=0 ## mutation-reader-setup-retry.log ```text -Source: packages/sdk/src/journal-client.ts -Original SHA256: 9c77a946018fecb31ee86b19e343b78cb65f077e87bd03395b5dcb30f2e3a0a3 +Source: packages/sdk/src/journal-budgeted-reads.ts +Original SHA256: 247f3340fff1711bea34f2ad3478f080daeb35a0751050ef8624d5c22140f285 Replaced: if (this.reader === reader) { this.reader = undefined; @@ -242,8 +241,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (20 tests | 1 failed | 19 skipped) 80ms - × retries a reader setup that timed out instead of reading on the primary for good 79ms + ❯ tests/journal-client-read-timeout.test.ts (20 tests | 1 failed | 19 skipped) 81ms + × retries a reader setup that timed out instead of reading on the primary for good 80ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -269,22 +268,22 @@ AssertionError: expected 1 to be 2 // Object.is equality Test Files 1 failed (1) Tests 1 failed | 19 skipped (20) - Start at 01:44:24 - Duration 507ms (transform 163ms, setup 17ms, collect 277ms, tests 80ms, environment 0ms, prepare 40ms) + Start at 02:35:29 + Duration 533ms (transform 167ms, setup 16ms, collect 290ms, tests 81ms, environment 0ms, prepare 41ms) exit=1 -RESTORED SHA256: 9c77a946018fecb31ee86b19e343b78cb65f077e87bd03395b5dcb30f2e3a0a3 +RESTORED SHA256: 247f3340fff1711bea34f2ad3478f080daeb35a0751050ef8624d5c22140f285 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'retries a reader setup that timed out' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (20 tests | 19 skipped) 70ms + ✓ tests/journal-client-read-timeout.test.ts (20 tests | 19 skipped) 75ms Test Files 1 passed (1) Tests 1 passed | 19 skipped (20) - Start at 01:44:25 - Duration 509ms (transform 166ms, setup 17ms, collect 288ms, tests 70ms, environment 0ms, prepare 41ms) + Start at 02:35:29 + Duration 515ms (transform 165ms, setup 16ms, collect 287ms, tests 75ms, environment 0ms, prepare 43ms) exit=0 ``` @@ -293,7 +292,7 @@ exit=0 ```text Source: packages/sdk/src/cli/running-step.ts -Original SHA256: 5f7ca7c0a29e5dcd0be4650369e994a21aa715f7dbdc45363924abf5df30d96b +Original SHA256: e535b533e40304d47aeba175c8ac6f6e51808de9fc99d1d0f64648f2b8a39a70 Replaced: const LEASE_POLL_MS = 2_000; With: @@ -304,8 +303,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 2111ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2110ms + ❯ tests/running-step-watch.test.ts (8 tests | 1 failed | 7 skipped) 2110ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2109ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -323,24 +322,24 @@ AssertionError: expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 6 skipped (7) - Start at 01:44:26 - Duration 2.59s (transform 186ms, setup 17ms, collect 313ms, tests 2.11s, environment 0ms, prepare 41ms) + Tests 1 failed | 7 skipped (8) + Start at 02:35:30 + Duration 2.59s (transform 189ms, setup 17ms, collect 314ms, tests 2.11s, environment 0ms, prepare 41ms) exit=1 -RESTORED SHA256: 5f7ca7c0a29e5dcd0be4650369e994a21aa715f7dbdc45363924abf5df30d96b +RESTORED SHA256: e535b533e40304d47aeba175c8ac6f6e51808de9fc99d1d0f64648f2b8a39a70 $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pushes for completion' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2116ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2116ms + ✓ tests/running-step-watch.test.ts (8 tests | 7 skipped) 2118ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms Test Files 1 passed (1) - Tests 1 passed | 6 skipped (7) - Start at 01:44:29 - Duration 2.56s (transform 187ms, setup 16ms, collect 307ms, tests 2.12s, environment 0ms, prepare 35ms) + Tests 1 passed | 7 skipped (8) + Start at 02:35:33 + Duration 2.60s (transform 191ms, setup 16ms, collect 317ms, tests 2.12s, environment 0ms, prepare 41ms) exit=0 ``` @@ -349,7 +348,7 @@ exit=0 ```text Source: packages/sdk/src/cli/running-step.ts -Original SHA256: 5f7ca7c0a29e5dcd0be4650369e994a21aa715f7dbdc45363924abf5df30d96b +Original SHA256: e535b533e40304d47aeba175c8ac6f6e51808de9fc99d1d0f64648f2b8a39a70 Replaced: const signal = options.signal === undefined ? read.signal : AbortSignal.any([options.signal, read.signal]); @@ -361,8 +360,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 2217ms - × cancels promptly while a lease snapshot read is in flight 2217ms + ❯ tests/running-step-watch.test.ts (8 tests | 1 failed | 7 skipped) 2216ms + × cancels promptly while a lease snapshot read is in flight 2215ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -380,24 +379,24 @@ AssertionError: expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 6 skipped (7) - Start at 01:44:32 - Duration 2.67s (transform 192ms, setup 16ms, collect 311ms, tests 2.22s, environment 0ms, prepare 35ms) + Tests 1 failed | 7 skipped (8) + Start at 02:35:36 + Duration 2.69s (transform 192ms, setup 18ms, collect 313ms, tests 2.22s, environment 0ms, prepare 41ms) exit=1 -RESTORED SHA256: 5f7ca7c0a29e5dcd0be4650369e994a21aa715f7dbdc45363924abf5df30d96b +RESTORED SHA256: e535b533e40304d47aeba175c8ac6f6e51808de9fc99d1d0f64648f2b8a39a70 $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels promptly while a lease snapshot read is in flight' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2011ms - ✓ cancels promptly while a lease snapshot read is in flight 2010ms + ✓ tests/running-step-watch.test.ts (8 tests | 7 skipped) 2012ms + ✓ cancels promptly while a lease snapshot read is in flight 2011ms Test Files 1 passed (1) - Tests 1 passed | 6 skipped (7) - Start at 01:44:35 - Duration 2.48s (transform 184ms, setup 17ms, collect 308ms, tests 2.01s, environment 0ms, prepare 42ms) + Tests 1 passed | 7 skipped (8) + Start at 02:35:39 + Duration 2.47s (transform 178ms, setup 17ms, collect 296ms, tests 2.01s, environment 0ms, prepare 40ms) exit=0 ``` @@ -406,7 +405,7 @@ exit=0 ```text Source: packages/sdk/src/cli/running-step.ts -Original SHA256: 5f7ca7c0a29e5dcd0be4650369e994a21aa715f7dbdc45363924abf5df30d96b +Original SHA256: e535b533e40304d47aeba175c8ac6f6e51808de9fc99d1d0f64648f2b8a39a70 Replaced: snapshotRead?.abort(new Error(`step "${runningStep.id}" completed during the lease snapshot`)); With: @@ -417,8 +416,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (7 tests | 1 failed | 6 skipped) 3010ms - × a completion push ends the wait without waiting out an in-flight snapshot 3009ms + ❯ tests/running-step-watch.test.ts (8 tests | 1 failed | 7 skipped) 3014ms + × a completion push ends the wait without waiting out an in-flight snapshot 3013ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -440,24 +439,24 @@ Received: "still waiting" ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 6 skipped (7) - Start at 01:44:37 - Duration 3.49s (transform 190ms, setup 16ms, collect 318ms, tests 3.01s, environment 0ms, prepare 41ms) + Tests 1 failed | 7 skipped (8) + Start at 02:35:42 + Duration 3.49s (transform 186ms, setup 19ms, collect 314ms, tests 3.01s, environment 0ms, prepare 41ms) exit=1 -RESTORED SHA256: 5f7ca7c0a29e5dcd0be4650369e994a21aa715f7dbdc45363924abf5df30d96b +RESTORED SHA256: e535b533e40304d47aeba175c8ac6f6e51808de9fc99d1d0f64648f2b8a39a70 $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a completion push ends the wait' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (7 tests | 6 skipped) 2062ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2061ms + ✓ tests/running-step-watch.test.ts (8 tests | 7 skipped) 2058ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2058ms Test Files 1 passed (1) - Tests 1 passed | 6 skipped (7) - Start at 01:44:41 - Duration 2.51s (transform 180ms, setup 18ms, collect 292ms, tests 2.06s, environment 0ms, prepare 41ms) + Tests 1 passed | 7 skipped (8) + Start at 02:35:46 + Duration 2.53s (transform 188ms, setup 16ms, collect 308ms, tests 2.06s, environment 0ms, prepare 41ms) exit=0 ``` @@ -480,8 +479,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 18ms - × a heartbeat timeout is lease loss rather than a worker body failure 17ms + ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 19ms + × a heartbeat timeout is lease loss rather than a worker body failure 18ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -500,8 +499,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 01:44:44 - Duration 464ms (transform 165ms, setup 16ms, collect 291ms, tests 18ms, environment 0ms, prepare 40ms) + Start at 02:35:49 + Duration 466ms (transform 169ms, setup 17ms, collect 286ms, tests 19ms, environment 0ms, prepare 41ms) exit=1 @@ -514,8 +513,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb Test Files 1 passed (1) Tests 1 passed (1) - Start at 01:44:45 - Duration 447ms (transform 167ms, setup 17ms, collect 281ms, tests 15ms, environment 0ms, prepare 40ms) + Start at 02:35:50 + Duration 451ms (transform 164ms, setup 17ms, collect 284ms, tests 15ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -542,8 +541,8 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (28 tests | 3 failed | 25 skipped) 31ms - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 21ms + ❯ tests/authored-root.test.ts (28 tests | 3 failed | 25 skipped) 25ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 16ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms @@ -629,8 +628,8 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } Test Files 1 failed (1) Tests 3 failed | 25 skipped (28) - Start at 01:44:46 - Duration 1.29s (transform 662ms, setup 17ms, collect 1.09s, tests 31ms, environment 0ms, prepare 41ms) + Start at 02:35:50 + Duration 1.28s (transform 669ms, setup 17ms, collect 1.09s, tests 25ms, environment 0ms, prepare 41ms) exit=1 @@ -639,12 +638,12 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (28 tests | 25 skipped) 23ms + ✓ tests/authored-root.test.ts (28 tests | 25 skipped) 24ms Test Files 1 passed (1) Tests 3 passed | 25 skipped (28) - Start at 01:44:47 - Duration 1.26s (transform 655ms, setup 16ms, collect 1.07s, tests 23ms, environment 0ms, prepare 41ms) + Start at 02:35:52 + Duration 1.23s (transform 641ms, setup 16ms, collect 1.06s, tests 24ms, environment 0ms, prepare 40ms) exit=0 ``` diff --git a/packages/sdk/src/cli/running-step.ts b/packages/sdk/src/cli/running-step.ts index 98f35d10d..bd5736a89 100644 --- a/packages/sdk/src/cli/running-step.ts +++ b/packages/sdk/src/cli/running-step.ts @@ -1,4 +1,4 @@ -import { JournalProtocolError, type JournalClient } from '../journal-client.js'; +import { JournalFrameError, JournalProtocolError, type JournalClient } from '../journal-client.js'; import { AuthoredFlowExecutionError } from '../authored-flow-error.js'; import type { JournalEvent } from '../journal-reader.js'; import type { RunLifecycleOptions, ParkedStep } from './run.js'; @@ -56,7 +56,8 @@ export async function waitForRunningStep(client: JournalClient, runId: string, // read interruption: the step may still be running and the run resumable. await Promise.race([watch.connect().then(() => watch.hello('flows-step-watch')) .then(() => watch.runWatch(runId)).catch(error => { - if (error instanceof JournalProtocolError) throw error; + // Refusals and malformed frames are protocol facts, not lost transport: fail closed. + if (error instanceof JournalProtocolError || error instanceof JournalFrameError) throw error; throw new AuthoredFlowExecutionError('daemon_unresponsive', `could not establish the completion watch: ${error instanceof Error ? error.message : String(error)}`); }), completed]); diff --git a/packages/sdk/src/journal-budgeted-reads.ts b/packages/sdk/src/journal-budgeted-reads.ts new file mode 100644 index 000000000..b10e06deb --- /dev/null +++ b/packages/sdk/src/journal-budgeted-reads.ts @@ -0,0 +1,86 @@ +import { JournalFrameError, JournalProtocolError, type JournalClient } from './journal-client.js'; +import { JournalReadInterruptedError, JournalReadPolicy, JournalRequestTimeoutError } from './journal-read-policy.js'; +import type { VerbContract } from './protocol.js'; + +/** + * Budgeted, retried reads for one flow client, on a dedicated reader session. + * + * Reads go to a second connection so they never queue behind the primary's + * unbounded commands; the policy serializes them and retries timeouts and lost + * reader transport within the budget. Protocol refusals and malformed frames + * fail closed. A refused reader handshake falls back to the primary session, + * whose own failures keep their meaning because it also carries writes. + */ +export class BudgetedReads { + private readonly policy = new JournalReadPolicy(); + private reader: JournalClient | undefined; + private readerReady: Promise | undefined; + + constructor( + private readonly primary: JournalClient, + private readonly budgetMs: number, + private readonly lifecycle?: AbortSignal, + ) {} + + close(cause?: unknown): void { + this.policy.close(); + this.reader?.close(cause); + } + + read(verb: V, params: VerbContract[V]['params'], timeoutMs: number, + signal?: AbortSignal): Promise { + const combined = signal === undefined ? this.lifecycle + : this.lifecycle === undefined ? signal : AbortSignal.any([signal, this.lifecycle]); + return this.policy.read(verb, timeoutMs, this.budgetMs, async (bound, attemptSignal) => { + const started = performance.now(); + const reader = await this.session().catch(error => { + attemptSignal.throwIfAborted(); + // A malformed handshake frame is a protocol violation: fail closed. + if (error instanceof JournalFrameError) throw error; + throw new JournalReadInterruptedError(verb, 1, performance.now() - started, undefined, { cause: error }); + }); + attemptSignal.throwIfAborted(); + const remaining = bound - (performance.now() - started); + if (remaining <= 0) throw new JournalRequestTimeoutError(verb, bound); + if (reader === undefined) return this.primary.requestOnce(verb, params, remaining, attemptSignal); + try { + return await reader.requestOnce(verb, params, remaining, attemptSignal); + } catch (error) { + if (attemptSignal.aborted || error instanceof JournalRequestTimeoutError + || error instanceof JournalProtocolError || error instanceof JournalFrameError) throw error; + // Only the dedicated reader's transport failed; reconnect within the budget. + this.drop(reader); + throw new JournalReadInterruptedError(verb, 1, performance.now() - started, undefined, { cause: error }); + } + }, combined); + } + + private session(): Promise { + if (this.readerReady === undefined) { + const reader = this.primary.createPeer(); + this.reader = reader; + this.readerReady = reader.connect().then(() => reader.hello('flows-reader')).then(() => reader) + .catch(error => { + reader.close(); + // A daemon that refuses the session will keep refusing it: read on the primary. + if (error instanceof JournalProtocolError) return undefined; + // A connect or hello that failed under load is retried with a fresh session, + // never by queueing reads behind the primary's unbounded commands for good. + if (this.reader === reader) { + this.reader = undefined; + this.readerReady = undefined; + } + throw error; + }); + } + return this.readerReady; + } + + /** Forget a reader whose transport failed so the next attempt reconnects one. */ + private drop(reader: JournalClient): void { + if (this.reader !== reader) return; + this.reader = undefined; + this.readerReady = undefined; + reader.close(); + } +} diff --git a/packages/sdk/src/journal-client.ts b/packages/sdk/src/journal-client.ts index c4e5f096e..a1a6fd617 100644 --- a/packages/sdk/src/journal-client.ts +++ b/packages/sdk/src/journal-client.ts @@ -9,7 +9,8 @@ // stream plumbing. The client's framing and failure behavior is covered by a // loopback double in tests. -import { JournalReadInterruptedError, JournalReadPolicy, JournalRequestTimeoutError, READ_ONLY_VERBS } from './journal-read-policy.js'; +import { JournalRequestTimeoutError, READ_ONLY_VERBS } from './journal-read-policy.js'; +import { BudgetedReads } from './journal-budgeted-reads.js'; export { JournalReadInterruptedError, JournalRequestTimeoutError } from './journal-read-policy.js'; import { EventEmitter } from 'node:events'; export { walkJournal, JournalReadError, type JournalEvent, type JournalReadFailure } from './journal-reader.js'; @@ -75,11 +76,7 @@ export class JournalFrameError extends Error { } export class JournalClient extends EventEmitter { - private readonly reads = new JournalReadPolicy(); - private reader: JournalClient | undefined; - private readerReady: Promise | undefined; - private readonly readBudgetMs?: number; - private readonly readSignal?: AbortSignal; + private readonly budgeted: BudgetedReads | undefined; /** Additive request capabilities the daemon advertised at `hello`; none until then. */ private features: ReadonlySet = new Set(); private socket: Socket | null = null; @@ -96,8 +93,8 @@ export class JournalClient extends EventEmitter { options: JournalClientOptions = {}, ) { super(); - this.readBudgetMs = options.readBudgetMs; - this.readSignal = options.readSignal; + this.budgeted = options.readBudgetMs === undefined ? undefined + : new BudgetedReads(this, options.readBudgetMs, options.readSignal); this.requestTimeoutMs = options.requestTimeoutMs ?? 30_000; this.connectTimeoutMs = options.connectTimeoutMs ?? 2_000; } @@ -135,8 +132,7 @@ export class JournalClient extends EventEmitter { socket.on('data', (chunk) => this.onData(chunk)); socket.on('close', () => { const closed = new Error('journal client: connection closed'); - this.reads.close(); - this.reader?.close(closed); + this.budgeted?.close(closed); this.disconnectCause ??= closed; this.failAll(closed); }); @@ -157,8 +153,7 @@ export class JournalClient extends EventEmitter { const closed = cause === undefined ? new Error('journal client: closed by caller') : new Error(`journal client: closed after ${cause instanceof Error ? cause.message : String(cause)}`, { cause }); - this.reads.close(); - this.reader?.close(cause); + this.budgeted?.close(cause); this.connectingSocket?.destroy(closed); this.disconnectCause ??= closed; this.failAll(closed); @@ -210,67 +205,16 @@ export class JournalClient extends EventEmitter { this.pending.clear(); } - private async readSession(): Promise { - if (this.readerReady === undefined) { - const reader = this.createPeer(); - this.reader = reader; - this.readerReady = reader.connect().then(() => reader.hello('flows-reader')).then(() => reader) - .catch(error => { - reader.close(); - // A daemon that refuses the session will keep refusing it: read on the primary. - if (error instanceof JournalProtocolError) return undefined; - // A connect or hello that failed under load is retried with a fresh session, - // never by queueing reads behind the primary's unbounded commands for good. - if (this.reader === reader) { - this.reader = undefined; - this.readerReady = undefined; - } - throw error; - }); - } - return this.readerReady; - } - - /** Forget a reader whose transport failed so the next attempt reconnects one. */ - private dropReader(reader: JournalClient): void { - if (this.reader !== reader) return; - this.reader = undefined; - this.readerReady = undefined; - reader.close(); - } - private request(verb: V, params: VerbContract[V]['params'], timeoutMs: number | null = this.requestTimeoutMs, signal?: AbortSignal): Promise { - if (this.socket && !this.socket.destroyed && this.readBudgetMs !== undefined && timeoutMs !== null && READ_ONLY_VERBS.has(verb)) { - return this.reads.read(verb, timeoutMs, this.readBudgetMs, async (bound, attemptSignal) => { - const started = performance.now(); - const reader = await this.readSession().catch(error => { - attemptSignal.throwIfAborted(); - // A malformed handshake frame is a protocol violation: fail closed. - if (error instanceof JournalFrameError) throw error; - throw new JournalReadInterruptedError(verb, 1, performance.now() - started, undefined, { cause: error }); - }); - attemptSignal.throwIfAborted(); - const remaining = bound - (performance.now() - started); - if (remaining <= 0) throw new JournalRequestTimeoutError(verb, bound); - // The primary session's own failures keep their meaning: it also carries writes. - if (reader === undefined) return this.requestOnce(verb, params, remaining, attemptSignal); - try { - return await reader.requestOnce(verb, params, remaining, attemptSignal); - } catch (error) { - if (attemptSignal.aborted || error instanceof JournalRequestTimeoutError - || error instanceof JournalProtocolError || error instanceof JournalFrameError) throw error; - // Only the dedicated reader's transport failed; reconnect within the budget. - this.dropReader(reader); - throw new JournalReadInterruptedError(verb, 1, performance.now() - started, undefined, { cause: error }); - } - }, signal === undefined ? this.readSignal - : this.readSignal === undefined ? signal : AbortSignal.any([signal, this.readSignal])); + if (this.budgeted !== undefined && this.socket && !this.socket.destroyed && timeoutMs !== null && READ_ONLY_VERBS.has(verb)) { + return this.budgeted.read(verb, params, timeoutMs, signal); } return this.requestOnce(verb, params, timeoutMs, signal); } - private requestOnce( + /** @internal One unretried request on this session; flow reads go through `request`. */ + requestOnce( verb: V, params: VerbContract[V]['params'], timeoutMs: number | null = this.requestTimeoutMs, diff --git a/packages/sdk/tests/running-step-watch.test.ts b/packages/sdk/tests/running-step-watch.test.ts index a97cab67a..9c8ac3b7a 100644 --- a/packages/sdk/tests/running-step-watch.test.ts +++ b/packages/sdk/tests/running-step-watch.test.ts @@ -1,7 +1,7 @@ import { once } from 'node:events'; import { setTimeout as sleep } from 'node:timers/promises'; import { expect, it } from 'vitest'; -import { JournalClient } from '../src/journal-client.js'; +import { JournalClient, JournalFrameError } from '../src/journal-client.js'; import { waitForRunningStep } from '../src/cli/running-step.js'; import { isReadInterruption } from '../src/cli/journal-timeout.js'; import { AuthoredFlowExecutionError } from '../src/authored-flow-error.js'; @@ -178,3 +178,17 @@ it('a completion that aborts the snapshot is not an error even if a retry starts expect(outcome).toBe('returned'); } finally { client.close(); await new Promise(resolve => server.close(() => resolve())); } }); + +it('fails closed on a malformed frame during watch setup', async () => { + const path = sockPath(); + const server = startLoopback(path, { hello: sendOk, 'run.watch': ctx => { ctx.socket.write('{not json\n'); } }); + await once(server, 'listening'); + const client = new JournalClient(path); + try { + await client.connect(); + const error = await waitForRunningStep(client, 'run', { id: 'step', type: 'agent', leaseDeadlineMs: Date.now() + 30_000 }, {}) + .catch(caught => caught); + expect(error).toBeInstanceOf(JournalFrameError); + expect(isReadInterruption(error)).toBe(false); + } finally { client.close(); await new Promise(resolve => server.close(() => resolve())); } +}); From e3a63eac57737f9a49b8843a6f5b6eda618f7136 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 02:48:02 -0700 Subject: [PATCH 16/34] fix(sdk): custom workers hand a lost lease back to the kernel withWorkerLease now turns a heartbeat timeout into WorkerLeaseLostError, but the helper-effect driver wrapped it as step_failed/worker_error and the communication worker completed the attempt as worker_error, terminalizing a root the kernel should sweep and retry. Both now rethrow isLeaseLost errors. Evidence re-captured at this tree (8/8 mutations; focused 27 files incl. the helper and communication suites; resume-live 1/1). Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: b8262468-ea26-4a6e-bee9-d193832e77dc --- evidence/run-read-timeout/final-focused.log | 93 ++++---- .../mutation-completion-aborts-snapshot.log | 16 +- .../run-read-timeout/mutation-heartbeat.log | 14 +- .../mutation-reader-reconnect.log | 18 +- .../mutation-reader-setup-retry.log | 14 +- evidence/run-read-timeout/mutation-reader.log | 10 +- .../mutation-root-parking.log | 14 +- .../mutation-snapshot-cancel.log | 12 +- .../mutation-watch-cadence.log | 12 +- evidence/run-read-timeout/resume-live.log | 8 +- evidence/run-read-timeout/verification.md | 211 +++++++++--------- packages/sdk/src/authored-helper-effect.ts | 5 +- packages/sdk/src/communication/worker.ts | 4 +- .../sdk/tests/communication-worker.test.ts | 16 ++ .../tests/helper-effect-lease-loss.test.ts | 54 +++++ 15 files changed, 296 insertions(+), 205 deletions(-) create mode 100644 packages/sdk/tests/helper-effect-lease-loss.test.ts diff --git a/evidence/run-read-timeout/final-focused.log b/evidence/run-read-timeout/final-focused.log index abef16983..99bbb4caa 100644 --- a/evidence/run-read-timeout/final-focused.log +++ b/evidence/run-read-timeout/final-focused.log @@ -1,59 +1,68 @@ -$ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/kernel-target/debug/relayflowd npx vitest run tests/journal-client-read-timeout.test.ts tests/journal-client.test.ts tests/journal-client-completion.test.ts tests/journal-client-subscriptions.test.ts tests/running-step-watch.test.ts tests/heartbeat-timeout.test.ts tests/run-daemon-unresponsive.test.ts tests/authored-root.test.ts tests/classify-outcome.test.ts tests/cli.test.ts tests/direct-run-worker-lease.test.ts tests/resume-worker-lease.test.ts tests/worker-lease.test.ts tests/worker-lease-lost.test.ts tests/worker-lease-lost-live.test.ts tests/worker-lease-sweep.test.ts tests/run-read-load-live.test.ts tests/flow-executor-chain.test.ts tests/agent-transcript-live.test.ts tests/human-live.test.ts tests/reuse-summary-interruption.test.ts tests/memoization.test.ts tests/authored-verifier-read-budget.test.ts --maxWorkers=1 --minWorkers=1 +$ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/kernel-target/debug/relayflowd npx vitest run tests/journal-client-read-timeout.test.ts tests/journal-client.test.ts tests/journal-client-completion.test.ts tests/journal-client-subscriptions.test.ts tests/running-step-watch.test.ts tests/heartbeat-timeout.test.ts tests/run-daemon-unresponsive.test.ts tests/authored-root.test.ts tests/classify-outcome.test.ts tests/cli.test.ts tests/direct-run-worker-lease.test.ts tests/resume-worker-lease.test.ts tests/worker-lease.test.ts tests/worker-lease-lost.test.ts tests/worker-lease-lost-live.test.ts tests/worker-lease-sweep.test.ts tests/run-read-load-live.test.ts tests/flow-executor-chain.test.ts tests/agent-transcript-live.test.ts tests/human-live.test.ts tests/reuse-summary-interruption.test.ts tests/memoization.test.ts tests/authored-verifier-read-budget.test.ts tests/communication-worker.test.ts tests/helper-effect-lease-loss.test.ts tests/authored-helpers.test.ts tests/authored-flow-slack.test.ts --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5254ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2037ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2038ms -(node:3771421) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5300ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2038ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2034ms +(node:3820686) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (28 tests) 371ms - ✓ tests/journal-client.test.ts (17 tests) 78ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 6760ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 434ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1015ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2633ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 356ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 760ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2576ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 663ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 632ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 651ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 630ms - ✓ tests/journal-client-read-timeout.test.ts (20 tests) 1427ms - ✓ tests/classify-outcome.test.ts (11 tests) 7427ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2014ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3003ms + ✓ tests/authored-root.test.ts (28 tests) 395ms + ✓ tests/journal-client.test.ts (17 tests) 79ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 6970ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 480ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 973ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2720ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 370ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 868ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 1089ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 340ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 410ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2573ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 647ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 615ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 643ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 667ms + ✓ tests/authored-helpers.test.ts (6 tests) 1798ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 867ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 347ms + ✓ replays after SIGKILL before complete with the same token and one successful completion 406ms + ✓ tests/journal-client-read-timeout.test.ts (20 tests) 1411ms + ✓ tests/classify-outcome.test.ts (11 tests) 7426ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2009ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3008ms ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms - ✓ tests/human-live.test.ts (3 tests) 4605ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2726ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1114ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 764ms - ✓ tests/running-step-watch.test.ts (8 tests) 8239ms + ✓ tests/human-live.test.ts (3 tests) 5563ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 3518ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1228ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 816ms + ✓ tests/running-step-watch.test.ts (8 tests) 8240ms ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms - ✓ cancels promptly while a lease snapshot read is in flight 2007ms + ✓ cancels promptly while a lease snapshot read is in flight 2006ms ✓ a completion push ends the wait without waiting out an in-flight snapshot 2054ms ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms - ✓ tests/worker-lease.test.ts (7 tests) 15ms + ✓ tests/worker-lease.test.ts (7 tests) 18ms + ✓ tests/communication-worker.test.ts (18 tests) 1720ms ✓ tests/worker-lease-lost.test.ts (17 tests) 26ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 668ms - ✓ tests/memoization.test.ts (58 tests) 65ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2492ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2057ms - ✓ drains read and watch promises before an authored flow completes 435ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 730ms + ✓ tests/memoization.test.ts (58 tests) 66ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2401ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2061ms + ✓ drains read and watch promises before an authored flow completes 340ms ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms - ✓ tests/journal-client-completion.test.ts (6 tests) 101ms - ✓ tests/reuse-summary-interruption.test.ts (5 tests) 3ms + ✓ tests/journal-client-completion.test.ts (6 tests) 99ms + ✓ tests/reuse-summary-interruption.test.ts (5 tests) 4ms ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms ✓ tests/direct-run-worker-lease.test.ts (3 tests) 9ms - ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms + ✓ tests/helper-effect-lease-loss.test.ts (1 test) 237ms + ✓ tests/journal-client-subscriptions.test.ts (1 test) 9ms ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 4ms ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 47ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 17ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms - Test Files 23 passed (23) - Tests 291 passed (291) - Start at 02:36:06 - Duration 48.85s (transform 1.05s, setup 132ms, collect 5.77s, tests 40.21s, environment 3ms, prepare 804ms) + Test Files 27 passed (27) + Tests 323 passed (323) + Start at 02:47:03 + Duration 56.48s (transform 1.13s, setup 155ms, collect 6.83s, tests 46.24s, environment 4ms, prepare 986ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log index 802161169..fb0b86829 100644 --- a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log +++ b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (8 tests | 1 failed | 7 skipped) 3014ms - × a completion push ends the wait without waiting out an in-flight snapshot 3013ms + ❯ tests/running-step-watch.test.ts (8 tests | 1 failed | 7 skipped) 3009ms + × a completion push ends the wait without waiting out an in-flight snapshot 3009ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -34,8 +34,8 @@ Received: "still waiting" Test Files 1 failed (1) Tests 1 failed | 7 skipped (8) - Start at 02:35:42 - Duration 3.49s (transform 186ms, setup 19ms, collect 314ms, tests 3.01s, environment 0ms, prepare 41ms) + Start at 02:46:51 + Duration 3.45s (transform 183ms, setup 17ms, collect 304ms, tests 3.01s, environment 0ms, prepare 29ms) exit=1 @@ -44,12 +44,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (8 tests | 7 skipped) 2058ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2058ms + ✓ tests/running-step-watch.test.ts (8 tests | 7 skipped) 2062ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2061ms Test Files 1 passed (1) Tests 1 passed | 7 skipped (8) - Start at 02:35:46 - Duration 2.53s (transform 188ms, setup 16ms, collect 308ms, tests 2.06s, environment 0ms, prepare 41ms) + Start at 02:46:55 + Duration 2.54s (transform 188ms, setup 16ms, collect 318ms, tests 2.06s, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-heartbeat.log b/evidence/run-read-timeout/mutation-heartbeat.log index 2417c64ca..ad428d77d 100644 --- a/evidence/run-read-timeout/mutation-heartbeat.log +++ b/evidence/run-read-timeout/mutation-heartbeat.log @@ -13,8 +13,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 19ms - × a heartbeat timeout is lease loss rather than a worker body failure 18ms + ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 21ms + × a heartbeat timeout is lease loss rather than a worker body failure 20ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -33,8 +33,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 02:35:49 - Duration 466ms (transform 169ms, setup 17ms, collect 286ms, tests 19ms, environment 0ms, prepare 41ms) + Start at 02:46:58 + Duration 442ms (transform 157ms, setup 13ms, collect 271ms, tests 21ms, environment 0ms, prepare 36ms) exit=1 @@ -43,11 +43,11 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 02:35:50 - Duration 451ms (transform 164ms, setup 17ms, collect 284ms, tests 15ms, environment 0ms, prepare 41ms) + Start at 02:46:59 + Duration 460ms (transform 167ms, setup 17ms, collect 289ms, tests 16ms, environment 0ms, prepare 42ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-reconnect.log b/evidence/run-read-timeout/mutation-reader-reconnect.log index 3d5a54045..0f87af952 100644 --- a/evidence/run-read-timeout/mutation-reader-reconnect.log +++ b/evidence/run-read-timeout/mutation-reader-reconnect.log @@ -11,14 +11,14 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (20 tests | 1 failed | 19 skipped) 1007ms - × reconnects the reader after it disconnects within the read budget 1007ms - → journal client: run.get read session was interrupted after 81 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ tests/journal-client-read-timeout.test.ts (20 tests | 1 failed | 19 skipped) 1006ms + × reconnects the reader after it disconnects within the read budget 1006ms + → journal client: run.get read session was interrupted after 82 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 81 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 82 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:70:9 68| let last: unknown; 69| const exhausted = () => last instanceof JournalReadInterruptedError @@ -41,8 +41,8 @@ Caused by: Error: journal client: connection closed Test Files 1 failed (1) Tests 1 failed | 19 skipped (20) - Start at 02:35:26 - Duration 1.45s (transform 162ms, setup 17ms, collect 287ms, tests 1.01s, environment 0ms, prepare 43ms) + Start at 02:46:35 + Duration 1.44s (transform 164ms, setup 16ms, collect 286ms, tests 1.01s, environment 0ms, prepare 40ms) exit=1 @@ -51,11 +51,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (20 tests | 19 skipped) 91ms + ✓ tests/journal-client-read-timeout.test.ts (20 tests | 19 skipped) 78ms Test Files 1 passed (1) Tests 1 passed | 19 skipped (20) - Start at 02:35:28 - Duration 547ms (transform 167ms, setup 16ms, collect 291ms, tests 91ms, environment 0ms, prepare 41ms) + Start at 02:46:37 + Duration 522ms (transform 168ms, setup 17ms, collect 282ms, tests 78ms, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-setup-retry.log b/evidence/run-read-timeout/mutation-reader-setup-retry.log index 61f7681dc..5ea22f5bf 100644 --- a/evidence/run-read-timeout/mutation-reader-setup-retry.log +++ b/evidence/run-read-timeout/mutation-reader-setup-retry.log @@ -16,8 +16,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (20 tests | 1 failed | 19 skipped) 81ms - × retries a reader setup that timed out instead of reading on the primary for good 80ms + ❯ tests/journal-client-read-timeout.test.ts (20 tests | 1 failed | 19 skipped) 83ms + × retries a reader setup that timed out instead of reading on the primary for good 83ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -43,8 +43,8 @@ AssertionError: expected 1 to be 2 // Object.is equality Test Files 1 failed (1) Tests 1 failed | 19 skipped (20) - Start at 02:35:29 - Duration 533ms (transform 167ms, setup 16ms, collect 290ms, tests 81ms, environment 0ms, prepare 41ms) + Start at 02:46:38 + Duration 534ms (transform 165ms, setup 16ms, collect 295ms, tests 83ms, environment 0ms, prepare 39ms) exit=1 @@ -53,11 +53,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (20 tests | 19 skipped) 75ms + ✓ tests/journal-client-read-timeout.test.ts (20 tests | 19 skipped) 74ms Test Files 1 passed (1) Tests 1 passed | 19 skipped (20) - Start at 02:35:29 - Duration 515ms (transform 165ms, setup 16ms, collect 287ms, tests 75ms, environment 0ms, prepare 43ms) + Start at 02:46:38 + Duration 521ms (transform 165ms, setup 17ms, collect 286ms, tests 74ms, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader.log b/evidence/run-read-timeout/mutation-reader.log index 89e76475d..f3b4026a3 100644 --- a/evidence/run-read-timeout/mutation-reader.log +++ b/evidence/run-read-timeout/mutation-reader.log @@ -37,8 +37,8 @@ AssertionError: expected true to be false // Object.is equality Test Files 1 failed (1) Tests 1 failed | 19 skipped (20) - Start at 02:35:24 - Duration 563ms (transform 162ms, setup 16ms, collect 288ms, tests 113ms, environment 0ms, prepare 40ms) + Start at 02:46:33 + Duration 564ms (transform 166ms, setup 17ms, collect 285ms, tests 113ms, environment 0ms, prepare 42ms) exit=1 @@ -47,11 +47,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (20 tests | 19 skipped) 106ms + ✓ tests/journal-client-read-timeout.test.ts (20 tests | 19 skipped) 108ms Test Files 1 passed (1) Tests 1 passed | 19 skipped (20) - Start at 02:35:25 - Duration 553ms (transform 169ms, setup 18ms, collect 292ms, tests 106ms, environment 0ms, prepare 41ms) + Start at 02:46:34 + Duration 566ms (transform 168ms, setup 16ms, collect 290ms, tests 108ms, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-root-parking.log b/evidence/run-read-timeout/mutation-root-parking.log index 883daa652..40a6d0bc2 100644 --- a/evidence/run-read-timeout/mutation-root-parking.log +++ b/evidence/run-read-timeout/mutation-root-parking.log @@ -17,8 +17,8 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (28 tests | 3 failed | 25 skipped) 25ms - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 16ms + ❯ tests/authored-root.test.ts (28 tests | 3 failed | 25 skipped) 31ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 22ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms @@ -104,8 +104,8 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } Test Files 1 failed (1) Tests 3 failed | 25 skipped (28) - Start at 02:35:50 - Duration 1.28s (transform 669ms, setup 17ms, collect 1.09s, tests 25ms, environment 0ms, prepare 41ms) + Start at 02:47:00 + Duration 1.25s (transform 654ms, setup 16ms, collect 1.06s, tests 31ms, environment 0ms, prepare 30ms) exit=1 @@ -114,11 +114,11 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (28 tests | 25 skipped) 24ms + ✓ tests/authored-root.test.ts (28 tests | 25 skipped) 21ms Test Files 1 passed (1) Tests 3 passed | 25 skipped (28) - Start at 02:35:52 - Duration 1.23s (transform 641ms, setup 16ms, collect 1.06s, tests 24ms, environment 0ms, prepare 40ms) + Start at 02:47:01 + Duration 1.25s (transform 669ms, setup 15ms, collect 1.09s, tests 21ms, environment 0ms, prepare 29ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-snapshot-cancel.log b/evidence/run-read-timeout/mutation-snapshot-cancel.log index 1bac28272..a7bdfa079 100644 --- a/evidence/run-read-timeout/mutation-snapshot-cancel.log +++ b/evidence/run-read-timeout/mutation-snapshot-cancel.log @@ -11,8 +11,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (8 tests | 1 failed | 7 skipped) 2216ms - × cancels promptly while a lease snapshot read is in flight 2215ms + ❯ tests/running-step-watch.test.ts (8 tests | 1 failed | 7 skipped) 2217ms + × cancels promptly while a lease snapshot read is in flight 2216ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -31,8 +31,8 @@ AssertionError: expected 'still waiting' to be an instance of Error Test Files 1 failed (1) Tests 1 failed | 7 skipped (8) - Start at 02:35:36 - Duration 2.69s (transform 192ms, setup 18ms, collect 313ms, tests 2.22s, environment 0ms, prepare 41ms) + Start at 02:46:45 + Duration 2.71s (transform 191ms, setup 16ms, collect 333ms, tests 2.22s, environment 0ms, prepare 42ms) exit=1 @@ -46,7 +46,7 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels Test Files 1 passed (1) Tests 1 passed | 7 skipped (8) - Start at 02:35:39 - Duration 2.47s (transform 178ms, setup 17ms, collect 296ms, tests 2.01s, environment 0ms, prepare 40ms) + Start at 02:46:48 + Duration 2.59s (transform 223ms, setup 19ms, collect 389ms, tests 2.01s, environment 0ms, prepare 45ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-watch-cadence.log b/evidence/run-read-timeout/mutation-watch-cadence.log index 5184fe3eb..b376a09a2 100644 --- a/evidence/run-read-timeout/mutation-watch-cadence.log +++ b/evidence/run-read-timeout/mutation-watch-cadence.log @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (8 tests | 1 failed | 7 skipped) 2110ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2109ms + ❯ tests/running-step-watch.test.ts (8 tests | 1 failed | 7 skipped) 2111ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2110ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -30,8 +30,8 @@ AssertionError: expected 41 to be less than or equal to 1 Test Files 1 failed (1) Tests 1 failed | 7 skipped (8) - Start at 02:35:30 - Duration 2.59s (transform 189ms, setup 17ms, collect 314ms, tests 2.11s, environment 0ms, prepare 41ms) + Start at 02:46:39 + Duration 2.57s (transform 179ms, setup 12ms, collect 316ms, tests 2.11s, environment 0ms, prepare 36ms) exit=1 @@ -45,7 +45,7 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu Test Files 1 passed (1) Tests 1 passed | 7 skipped (8) - Start at 02:35:33 - Duration 2.60s (transform 191ms, setup 16ms, collect 317ms, tests 2.12s, environment 0ms, prepare 41ms) + Start at 02:46:42 + Duration 2.63s (transform 205ms, setup 17ms, collect 345ms, tests 2.12s, environment 0ms, prepare 42ms) exit=0 diff --git a/evidence/run-read-timeout/resume-live.log b/evidence/run-read-timeout/resume-live.log index 646e1b4a0..60d7c3eb2 100644 --- a/evidence/run-read-timeout/resume-live.log +++ b/evidence/run-read-timeout/resume-live.log @@ -2,12 +2,12 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 903ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 903ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 886ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 885ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 02:36:55 - Duration 2.21s (transform 641ms, setup 19ms, collect 1.05s, tests 903ms, environment 0ms, prepare 74ms) + Start at 02:48:00 + Duration 2.16s (transform 630ms, setup 17ms, collect 1.03s, tests 886ms, environment 0ms, prepare 62ms) exit=0 diff --git a/evidence/run-read-timeout/verification.md b/evidence/run-read-timeout/verification.md index 849f2e725..068d43b75 100644 --- a/evidence/run-read-timeout/verification.md +++ b/evidence/run-read-timeout/verification.md @@ -11,63 +11,72 @@ exit=0 ## final-focused.log ```text -$ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/kernel-target/debug/relayflowd npx vitest run tests/journal-client-read-timeout.test.ts tests/journal-client.test.ts tests/journal-client-completion.test.ts tests/journal-client-subscriptions.test.ts tests/running-step-watch.test.ts tests/heartbeat-timeout.test.ts tests/run-daemon-unresponsive.test.ts tests/authored-root.test.ts tests/classify-outcome.test.ts tests/cli.test.ts tests/direct-run-worker-lease.test.ts tests/resume-worker-lease.test.ts tests/worker-lease.test.ts tests/worker-lease-lost.test.ts tests/worker-lease-lost-live.test.ts tests/worker-lease-sweep.test.ts tests/run-read-load-live.test.ts tests/flow-executor-chain.test.ts tests/agent-transcript-live.test.ts tests/human-live.test.ts tests/reuse-summary-interruption.test.ts tests/memoization.test.ts tests/authored-verifier-read-budget.test.ts --maxWorkers=1 --minWorkers=1 +$ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/kernel-target/debug/relayflowd npx vitest run tests/journal-client-read-timeout.test.ts tests/journal-client.test.ts tests/journal-client-completion.test.ts tests/journal-client-subscriptions.test.ts tests/running-step-watch.test.ts tests/heartbeat-timeout.test.ts tests/run-daemon-unresponsive.test.ts tests/authored-root.test.ts tests/classify-outcome.test.ts tests/cli.test.ts tests/direct-run-worker-lease.test.ts tests/resume-worker-lease.test.ts tests/worker-lease.test.ts tests/worker-lease-lost.test.ts tests/worker-lease-lost-live.test.ts tests/worker-lease-sweep.test.ts tests/run-read-load-live.test.ts tests/flow-executor-chain.test.ts tests/agent-transcript-live.test.ts tests/human-live.test.ts tests/reuse-summary-interruption.test.ts tests/memoization.test.ts tests/authored-verifier-read-budget.test.ts tests/communication-worker.test.ts tests/helper-effect-lease-loss.test.ts tests/authored-helpers.test.ts tests/authored-flow-slack.test.ts --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5254ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2037ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2038ms -(node:3771421) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5300ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2038ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2034ms +(node:3820686) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (28 tests) 371ms - ✓ tests/journal-client.test.ts (17 tests) 78ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 6760ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 434ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1015ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2633ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 356ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 760ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2576ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 663ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 632ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 651ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 630ms - ✓ tests/journal-client-read-timeout.test.ts (20 tests) 1427ms - ✓ tests/classify-outcome.test.ts (11 tests) 7427ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2014ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3003ms + ✓ tests/authored-root.test.ts (28 tests) 395ms + ✓ tests/journal-client.test.ts (17 tests) 79ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 6970ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 480ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 973ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2720ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 370ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 868ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 1089ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 340ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 410ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2573ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 647ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 615ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 643ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 667ms + ✓ tests/authored-helpers.test.ts (6 tests) 1798ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 867ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 347ms + ✓ replays after SIGKILL before complete with the same token and one successful completion 406ms + ✓ tests/journal-client-read-timeout.test.ts (20 tests) 1411ms + ✓ tests/classify-outcome.test.ts (11 tests) 7426ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2009ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3008ms ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms - ✓ tests/human-live.test.ts (3 tests) 4605ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2726ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1114ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 764ms - ✓ tests/running-step-watch.test.ts (8 tests) 8239ms + ✓ tests/human-live.test.ts (3 tests) 5563ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 3518ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1228ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 816ms + ✓ tests/running-step-watch.test.ts (8 tests) 8240ms ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms - ✓ cancels promptly while a lease snapshot read is in flight 2007ms + ✓ cancels promptly while a lease snapshot read is in flight 2006ms ✓ a completion push ends the wait without waiting out an in-flight snapshot 2054ms ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms - ✓ tests/worker-lease.test.ts (7 tests) 15ms + ✓ tests/worker-lease.test.ts (7 tests) 18ms + ✓ tests/communication-worker.test.ts (18 tests) 1720ms ✓ tests/worker-lease-lost.test.ts (17 tests) 26ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 668ms - ✓ tests/memoization.test.ts (58 tests) 65ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2492ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2057ms - ✓ drains read and watch promises before an authored flow completes 435ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 730ms + ✓ tests/memoization.test.ts (58 tests) 66ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2401ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2061ms + ✓ drains read and watch promises before an authored flow completes 340ms ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms - ✓ tests/journal-client-completion.test.ts (6 tests) 101ms - ✓ tests/reuse-summary-interruption.test.ts (5 tests) 3ms + ✓ tests/journal-client-completion.test.ts (6 tests) 99ms + ✓ tests/reuse-summary-interruption.test.ts (5 tests) 4ms ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms ✓ tests/direct-run-worker-lease.test.ts (3 tests) 9ms - ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms + ✓ tests/helper-effect-lease-loss.test.ts (1 test) 237ms + ✓ tests/journal-client-subscriptions.test.ts (1 test) 9ms ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 4ms ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 47ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 17ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms - Test Files 23 passed (23) - Tests 291 passed (291) - Start at 02:36:06 - Duration 48.85s (transform 1.05s, setup 132ms, collect 5.77s, tests 40.21s, environment 3ms, prepare 804ms) + Test Files 27 passed (27) + Tests 323 passed (323) + Start at 02:47:03 + Duration 56.48s (transform 1.13s, setup 155ms, collect 6.83s, tests 46.24s, environment 4ms, prepare 986ms) exit=0 ``` @@ -79,13 +88,13 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 903ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 903ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 886ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 885ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 02:36:55 - Duration 2.21s (transform 641ms, setup 19ms, collect 1.05s, tests 903ms, environment 0ms, prepare 74ms) + Start at 02:48:00 + Duration 2.16s (transform 630ms, setup 17ms, collect 1.03s, tests 886ms, environment 0ms, prepare 62ms) exit=0 ``` @@ -134,8 +143,8 @@ AssertionError: expected true to be false // Object.is equality Test Files 1 failed (1) Tests 1 failed | 19 skipped (20) - Start at 02:35:24 - Duration 563ms (transform 162ms, setup 16ms, collect 288ms, tests 113ms, environment 0ms, prepare 40ms) + Start at 02:46:33 + Duration 564ms (transform 166ms, setup 17ms, collect 285ms, tests 113ms, environment 0ms, prepare 42ms) exit=1 @@ -144,12 +153,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (20 tests | 19 skipped) 106ms + ✓ tests/journal-client-read-timeout.test.ts (20 tests | 19 skipped) 108ms Test Files 1 passed (1) Tests 1 passed | 19 skipped (20) - Start at 02:35:25 - Duration 553ms (transform 169ms, setup 18ms, collect 292ms, tests 106ms, environment 0ms, prepare 41ms) + Start at 02:46:34 + Duration 566ms (transform 168ms, setup 16ms, collect 290ms, tests 108ms, environment 0ms, prepare 40ms) exit=0 ``` @@ -170,14 +179,14 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (20 tests | 1 failed | 19 skipped) 1007ms - × reconnects the reader after it disconnects within the read budget 1007ms - → journal client: run.get read session was interrupted after 81 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ tests/journal-client-read-timeout.test.ts (20 tests | 1 failed | 19 skipped) 1006ms + × reconnects the reader after it disconnects within the read budget 1006ms + → journal client: run.get read session was interrupted after 82 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 81 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 82 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:70:9 68| let last: unknown; 69| const exhausted = () => last instanceof JournalReadInterruptedError @@ -200,8 +209,8 @@ Caused by: Error: journal client: connection closed Test Files 1 failed (1) Tests 1 failed | 19 skipped (20) - Start at 02:35:26 - Duration 1.45s (transform 162ms, setup 17ms, collect 287ms, tests 1.01s, environment 0ms, prepare 43ms) + Start at 02:46:35 + Duration 1.44s (transform 164ms, setup 16ms, collect 286ms, tests 1.01s, environment 0ms, prepare 40ms) exit=1 @@ -210,12 +219,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (20 tests | 19 skipped) 91ms + ✓ tests/journal-client-read-timeout.test.ts (20 tests | 19 skipped) 78ms Test Files 1 passed (1) Tests 1 passed | 19 skipped (20) - Start at 02:35:28 - Duration 547ms (transform 167ms, setup 16ms, collect 291ms, tests 91ms, environment 0ms, prepare 41ms) + Start at 02:46:37 + Duration 522ms (transform 168ms, setup 17ms, collect 282ms, tests 78ms, environment 0ms, prepare 40ms) exit=0 ``` @@ -241,8 +250,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (20 tests | 1 failed | 19 skipped) 81ms - × retries a reader setup that timed out instead of reading on the primary for good 80ms + ❯ tests/journal-client-read-timeout.test.ts (20 tests | 1 failed | 19 skipped) 83ms + × retries a reader setup that timed out instead of reading on the primary for good 83ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -268,8 +277,8 @@ AssertionError: expected 1 to be 2 // Object.is equality Test Files 1 failed (1) Tests 1 failed | 19 skipped (20) - Start at 02:35:29 - Duration 533ms (transform 167ms, setup 16ms, collect 290ms, tests 81ms, environment 0ms, prepare 41ms) + Start at 02:46:38 + Duration 534ms (transform 165ms, setup 16ms, collect 295ms, tests 83ms, environment 0ms, prepare 39ms) exit=1 @@ -278,12 +287,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (20 tests | 19 skipped) 75ms + ✓ tests/journal-client-read-timeout.test.ts (20 tests | 19 skipped) 74ms Test Files 1 passed (1) Tests 1 passed | 19 skipped (20) - Start at 02:35:29 - Duration 515ms (transform 165ms, setup 16ms, collect 287ms, tests 75ms, environment 0ms, prepare 43ms) + Start at 02:46:38 + Duration 521ms (transform 165ms, setup 17ms, collect 286ms, tests 74ms, environment 0ms, prepare 40ms) exit=0 ``` @@ -303,8 +312,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (8 tests | 1 failed | 7 skipped) 2110ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2109ms + ❯ tests/running-step-watch.test.ts (8 tests | 1 failed | 7 skipped) 2111ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2110ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -323,8 +332,8 @@ AssertionError: expected 41 to be less than or equal to 1 Test Files 1 failed (1) Tests 1 failed | 7 skipped (8) - Start at 02:35:30 - Duration 2.59s (transform 189ms, setup 17ms, collect 314ms, tests 2.11s, environment 0ms, prepare 41ms) + Start at 02:46:39 + Duration 2.57s (transform 179ms, setup 12ms, collect 316ms, tests 2.11s, environment 0ms, prepare 36ms) exit=1 @@ -338,8 +347,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu Test Files 1 passed (1) Tests 1 passed | 7 skipped (8) - Start at 02:35:33 - Duration 2.60s (transform 191ms, setup 16ms, collect 317ms, tests 2.12s, environment 0ms, prepare 41ms) + Start at 02:46:42 + Duration 2.63s (transform 205ms, setup 17ms, collect 345ms, tests 2.12s, environment 0ms, prepare 42ms) exit=0 ``` @@ -360,8 +369,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (8 tests | 1 failed | 7 skipped) 2216ms - × cancels promptly while a lease snapshot read is in flight 2215ms + ❯ tests/running-step-watch.test.ts (8 tests | 1 failed | 7 skipped) 2217ms + × cancels promptly while a lease snapshot read is in flight 2216ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -380,8 +389,8 @@ AssertionError: expected 'still waiting' to be an instance of Error Test Files 1 failed (1) Tests 1 failed | 7 skipped (8) - Start at 02:35:36 - Duration 2.69s (transform 192ms, setup 18ms, collect 313ms, tests 2.22s, environment 0ms, prepare 41ms) + Start at 02:46:45 + Duration 2.71s (transform 191ms, setup 16ms, collect 333ms, tests 2.22s, environment 0ms, prepare 42ms) exit=1 @@ -395,8 +404,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels Test Files 1 passed (1) Tests 1 passed | 7 skipped (8) - Start at 02:35:39 - Duration 2.47s (transform 178ms, setup 17ms, collect 296ms, tests 2.01s, environment 0ms, prepare 40ms) + Start at 02:46:48 + Duration 2.59s (transform 223ms, setup 19ms, collect 389ms, tests 2.01s, environment 0ms, prepare 45ms) exit=0 ``` @@ -416,8 +425,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (8 tests | 1 failed | 7 skipped) 3014ms - × a completion push ends the wait without waiting out an in-flight snapshot 3013ms + ❯ tests/running-step-watch.test.ts (8 tests | 1 failed | 7 skipped) 3009ms + × a completion push ends the wait without waiting out an in-flight snapshot 3009ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -440,8 +449,8 @@ Received: "still waiting" Test Files 1 failed (1) Tests 1 failed | 7 skipped (8) - Start at 02:35:42 - Duration 3.49s (transform 186ms, setup 19ms, collect 314ms, tests 3.01s, environment 0ms, prepare 41ms) + Start at 02:46:51 + Duration 3.45s (transform 183ms, setup 17ms, collect 304ms, tests 3.01s, environment 0ms, prepare 29ms) exit=1 @@ -450,13 +459,13 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (8 tests | 7 skipped) 2058ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2058ms + ✓ tests/running-step-watch.test.ts (8 tests | 7 skipped) 2062ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2061ms Test Files 1 passed (1) Tests 1 passed | 7 skipped (8) - Start at 02:35:46 - Duration 2.53s (transform 188ms, setup 16ms, collect 308ms, tests 2.06s, environment 0ms, prepare 41ms) + Start at 02:46:55 + Duration 2.54s (transform 188ms, setup 16ms, collect 318ms, tests 2.06s, environment 0ms, prepare 40ms) exit=0 ``` @@ -479,8 +488,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 19ms - × a heartbeat timeout is lease loss rather than a worker body failure 18ms + ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 21ms + × a heartbeat timeout is lease loss rather than a worker body failure 20ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -499,8 +508,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 02:35:49 - Duration 466ms (transform 169ms, setup 17ms, collect 286ms, tests 19ms, environment 0ms, prepare 41ms) + Start at 02:46:58 + Duration 442ms (transform 157ms, setup 13ms, collect 271ms, tests 21ms, environment 0ms, prepare 36ms) exit=1 @@ -509,12 +518,12 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 02:35:50 - Duration 451ms (transform 164ms, setup 17ms, collect 284ms, tests 15ms, environment 0ms, prepare 41ms) + Start at 02:46:59 + Duration 460ms (transform 167ms, setup 17ms, collect 289ms, tests 16ms, environment 0ms, prepare 42ms) exit=0 ``` @@ -541,8 +550,8 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (28 tests | 3 failed | 25 skipped) 25ms - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 16ms + ❯ tests/authored-root.test.ts (28 tests | 3 failed | 25 skipped) 31ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 22ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms @@ -628,8 +637,8 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } Test Files 1 failed (1) Tests 3 failed | 25 skipped (28) - Start at 02:35:50 - Duration 1.28s (transform 669ms, setup 17ms, collect 1.09s, tests 25ms, environment 0ms, prepare 41ms) + Start at 02:47:00 + Duration 1.25s (transform 654ms, setup 16ms, collect 1.06s, tests 31ms, environment 0ms, prepare 30ms) exit=1 @@ -638,12 +647,12 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (28 tests | 25 skipped) 24ms + ✓ tests/authored-root.test.ts (28 tests | 25 skipped) 21ms Test Files 1 passed (1) Tests 3 passed | 25 skipped (28) - Start at 02:35:52 - Duration 1.23s (transform 641ms, setup 16ms, collect 1.06s, tests 24ms, environment 0ms, prepare 40ms) + Start at 02:47:01 + Duration 1.25s (transform 669ms, setup 15ms, collect 1.09s, tests 21ms, environment 0ms, prepare 29ms) exit=0 ``` diff --git a/packages/sdk/src/authored-helper-effect.ts b/packages/sdk/src/authored-helper-effect.ts index f10a7b641..7a627fdc4 100644 --- a/packages/sdk/src/authored-helper-effect.ts +++ b/packages/sdk/src/authored-helper-effect.ts @@ -8,7 +8,7 @@ import type { AuthoredFlowJournalStep } from './authored-flow-executor.js'; import { compileSpec, toKernelSpec } from './compile.js'; import { SPEC_SCHEMA_VERSION, type KernelAgentStep } from './spec.js'; import type { StepDispatchEvent } from './protocol.js'; -import { withWorkerLease } from './worker-lease.js'; +import { isLeaseLost, withWorkerLease } from './worker-lease.js'; import { helperProviders, type HelperCall } from '@relayflows/surface/runtime'; import { helperWriteback, HelperDeliveryError } from './helper-writeback.js'; import { checkSlackHelpers } from './slack-preflight.js'; @@ -101,7 +101,8 @@ async function driveHelperEffect( } await completed; } catch (error) { - if (error instanceof AuthoredFlowExecutionError) throw error; + // A lost lease belongs to the kernel's sweep and retry, not to this step. + if (error instanceof AuthoredFlowExecutionError || isLeaseLost(error)) throw error; throw new AuthoredFlowExecutionError('step_failed', error instanceof Error ? error.message : 'Helper effect failed', 'worker_error', runId); } finally { client.close(); } } diff --git a/packages/sdk/src/communication/worker.ts b/packages/sdk/src/communication/worker.ts index 410af0510..4b67c2f34 100644 --- a/packages/sdk/src/communication/worker.ts +++ b/packages/sdk/src/communication/worker.ts @@ -5,7 +5,7 @@ import { setTimeout as delay } from 'node:timers/promises'; import type { JournalClient } from '../journal-client.js'; import type { StepDispatchEvent } from '../protocol.js'; import type { ResolvedKernelAgentStep } from '../resolved-cli-identity.js'; -import { withWorkerLease } from '../worker-lease.js'; +import { isLeaseLost, withWorkerLease } from '../worker-lease.js'; import { workerInstruction } from '../worker-input.js'; import { resolveCliModel } from '../cli-adapter.js'; import { resolveAgentCwd } from '../agent-cwd.js'; @@ -29,6 +29,8 @@ export async function completeCommunicationDispatch(client: JournalClient, dispa output = await withWorkerLease(client, dispatch, signal => run(client, dispatch, instruction, spec, dataDir, signal, runRoot, environment)); } catch (error) { + // A lost lease belongs to the kernel's sweep and retry: never complete it here. + if (isLeaseLost(error)) throw error; completionReason = 'worker_error'; output = { error: error instanceof Error ? error.message : String(error) }; } diff --git a/packages/sdk/tests/communication-worker.test.ts b/packages/sdk/tests/communication-worker.test.ts index f1956e767..dc379c3e5 100644 --- a/packages/sdk/tests/communication-worker.test.ts +++ b/packages/sdk/tests/communication-worker.test.ts @@ -92,3 +92,19 @@ it('uses the proved identity for a canonical generic communication executable', }), })); }); +it('leaves a heartbeat-lost attempt to the kernel instead of completing it as worker_error', async () => { + const { JournalRequestTimeoutError } = await import('../src/journal-client.js'); + const { isLeaseLost } = await import('../src/worker-lease.js'); + mocks.spawn.mockImplementation(async () => ({ name: 'managed-agent', generation: 'generation', release: mocks.release, waitForReady: mocks.ready })); + const f = fixture(); + f.client.stepHeartbeat.mockImplementation(async () => { throw new JournalRequestTimeoutError('step.heartbeat', 10); }); + (f as unknown as { client: { stepHeartbeat: unknown } }).client.stepHeartbeat = f.client.stepHeartbeat; + const dispatchDeadline = Date.now() + 200; + const error = await completeCommunicationDispatch(f.client as unknown as JournalClient, { + run_id: 'run', step_id: 'agent', attempt: 1, idempotency_key: 'key', pins: {}, lease_id: 'lease', + lease_deadline_ms: dispatchDeadline, spec: { type: 'agent', cli: 'claude', instruction: '' }, + } as StepDispatchEvent, { type: 'relayflows.communication.v1', instruction: 'test', incoming: ['peer'], outgoing: [], timeoutMs: 60_000 }, + '/tmp/data').catch(caught => caught); + expect(isLeaseLost(error)).toBe(true); + expect(f.client.stepComplete).not.toHaveBeenCalled(); +}); diff --git a/packages/sdk/tests/helper-effect-lease-loss.test.ts b/packages/sdk/tests/helper-effect-lease-loss.test.ts new file mode 100644 index 000000000..6ae0fa913 --- /dev/null +++ b/packages/sdk/tests/helper-effect-lease-loss.test.ts @@ -0,0 +1,54 @@ +import { once } from 'node:events'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, expect, it, vi } from 'vitest'; +import { JournalClient } from '../src/journal-client.js'; +import { sendOk, sendResult, sockPath, startLoopback } from './journal-client-loopback.js'; + +vi.mock('../src/worker-lease.js', async original => { + const actual = await original(); + return { ...actual, withWorkerLease: async () => { + throw new actual.WorkerLeaseLostError('renewal_expired', 'journal client: step.heartbeat timed out after 10ms'); + } }; +}); + +const dirs: string[] = []; +afterEach(() => { for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); }); + +it('propagates heartbeat lease loss from a helper effect instead of failing the step', async () => { + const { runHelperEffect } = await import('../src/authored-helper-effect.js'); + const { isLeaseLost } = await import('../src/worker-lease.js'); + const dataDir = mkdtempSync(join(tmpdir(), 'helper-lease-')); + dirs.push(dataDir); + vi.stubEnv('RELAYFLOWS_SLACK_MOCK', '1'); + let completions = 0; + const path = sockPath(); + const server = startLoopback(path, { + hello: sendOk, + 'run.start': ctx => sendResult(ctx, { run_id: 'helper-run', status: 'running', completion_reason: null, completed_steps: 0 }), + // The dispatch goes to the attached worker's own connection, after attach. + 'worker.attach': ctx => { + sendResult(ctx, {}); + setTimeout(() => ctx.send({ event: 'step.dispatch', data: { run_id: 'helper-run', step_id: 'notify', step_type: 'agent', + attempt: 1, idempotency_key: 'key', lease_id: 'lease', lease_deadline_ms: Date.now() + 30_000, pins: {}, + spec: { type: 'agent' } } }), 20); + }, + 'run.resume': ctx => sendResult(ctx, { run_id: 'helper-run', status: 'running', completion_reason: null, completed_steps: 0 }), + 'step.complete': ctx => { completions += 1; sendResult(ctx, {}); }, + }); + await once(server, 'listening'); + const journal = new JournalClient(path); + try { + await journal.connect(); + const error = await runHelperEffect(journal, 'flow', 'notify', + { provider: 'slack', verb: 'post', params: { channel: '#c', text: 'hi' } } as never, dataDir, []) + .catch(caught => caught); + expect(isLeaseLost(error)).toBe(true); + expect(completions).toBe(0); + } finally { + journal.close(); + vi.unstubAllEnvs(); + await new Promise(resolve => server.close(() => resolve())); + } +}); From 7baf1b445a5fdfbf2aa731ed847a8b4022d38924 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 02:57:51 -0700 Subject: [PATCH 17/34] fix(sdk): malformed frames reach a registered watch; flow handshakes get the read budget - JournalClient emits protocol_error on a malformed frame, so a session with nothing pending (a registered completion watch) hears it; the watch fails closed with JournalFrameError, taking precedence over an in-flight snapshot. - A budgeted flow client retries its own hello within the read budget (the Bun verifier and authored child clients included), and an exhausted handshake is a read interruption, so a CPU-delayed handshake parks the root instead of terminalizing it. Evidence re-captured at this tree. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: b8262468-ea26-4a6e-bee9-d193832e77dc --- evidence/run-read-timeout/final-focused.log | 102 +++--- .../mutation-completion-aborts-snapshot.log | 24 +- .../run-read-timeout/mutation-heartbeat.log | 14 +- .../mutation-reader-reconnect.log | 46 +-- .../mutation-reader-setup-retry.log | 22 +- evidence/run-read-timeout/mutation-reader.log | 22 +- .../mutation-root-parking.log | 14 +- .../mutation-snapshot-cancel.log | 24 +- .../mutation-watch-cadence.log | 22 +- evidence/run-read-timeout/resume-live.log | 8 +- evidence/run-read-timeout/verification.md | 298 +++++++++--------- packages/sdk/src/cli/running-step.ts | 9 + packages/sdk/src/journal-budgeted-reads.ts | 6 + packages/sdk/src/journal-client.ts | 10 +- packages/sdk/src/journal-read-policy.ts | 3 +- .../tests/journal-client-read-timeout.test.ts | 13 + packages/sdk/tests/running-step-watch.test.ts | 38 +++ 17 files changed, 376 insertions(+), 299 deletions(-) diff --git a/evidence/run-read-timeout/final-focused.log b/evidence/run-read-timeout/final-focused.log index 99bbb4caa..a3e832c6d 100644 --- a/evidence/run-read-timeout/final-focused.log +++ b/evidence/run-read-timeout/final-focused.log @@ -2,67 +2,69 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5300ms + ✓ tests/cli.test.ts (71 tests) 5320ms + ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 326ms ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2038ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2034ms -(node:3820686) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2037ms +(node:3863527) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (28 tests) 395ms - ✓ tests/journal-client.test.ts (17 tests) 79ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 6970ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 480ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 973ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2720ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 370ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 868ms - ✓ tests/authored-flow-slack.test.ts (7 tests) 1089ms - ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 340ms - ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 410ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2573ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 647ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 615ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 643ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 667ms - ✓ tests/authored-helpers.test.ts (6 tests) 1798ms - ✓ runs every available provider through the real kernel and resumes completed effects without a second write 867ms - ✓ replays after SIGKILL before confirm with the same token and one successful completion 347ms - ✓ replays after SIGKILL before complete with the same token and one successful completion 406ms - ✓ tests/journal-client-read-timeout.test.ts (20 tests) 1411ms - ✓ tests/classify-outcome.test.ts (11 tests) 7426ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2009ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3008ms - ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms - ✓ tests/human-live.test.ts (3 tests) 5563ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 3518ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1228ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 816ms - ✓ tests/running-step-watch.test.ts (8 tests) 8240ms + ✓ tests/authored-root.test.ts (28 tests) 361ms + ✓ tests/journal-client.test.ts (17 tests) 82ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 7494ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 522ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1115ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2804ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 424ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 948ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 1242ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 444ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 424ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2484ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 670ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 501ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 644ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 668ms + ✓ tests/journal-client-read-timeout.test.ts (22 tests) 1729ms + ✓ tests/authored-helpers.test.ts (6 tests) 1856ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 880ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 332ms + ✓ replays after SIGKILL before complete with the same token and one successful completion 415ms + ✓ tests/classify-outcome.test.ts (11 tests) 7430ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2015ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3006ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2002ms + ✓ tests/running-step-watch.test.ts (10 tests) 10266ms ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms ✓ cancels promptly while a lease snapshot read is in flight 2006ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2054ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2055ms ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms - ✓ tests/worker-lease.test.ts (7 tests) 18ms - ✓ tests/communication-worker.test.ts (18 tests) 1720ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 26ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 730ms + ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2004ms + ✓ tests/human-live.test.ts (3 tests) 4601ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2743ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1127ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 730ms + ✓ tests/worker-lease.test.ts (7 tests) 15ms + ✓ tests/communication-worker.test.ts (18 tests) 1711ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 25ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 699ms ✓ tests/memoization.test.ts (58 tests) 66ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2401ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2061ms - ✓ drains read and watch promises before an authored flow completes 340ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2465ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2058ms + ✓ drains read and watch promises before an authored flow completes 406ms ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms - ✓ tests/journal-client-completion.test.ts (6 tests) 99ms - ✓ tests/reuse-summary-interruption.test.ts (5 tests) 4ms + ✓ tests/journal-client-completion.test.ts (6 tests) 101ms + ✓ tests/reuse-summary-interruption.test.ts (5 tests) 3ms ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms - ✓ tests/direct-run-worker-lease.test.ts (3 tests) 9ms - ✓ tests/helper-effect-lease-loss.test.ts (1 test) 237ms - ✓ tests/journal-client-subscriptions.test.ts (1 test) 9ms + ✓ tests/direct-run-worker-lease.test.ts (3 tests) 6ms + ✓ tests/helper-effect-lease-loss.test.ts (1 test) 235ms + ✓ tests/journal-client-subscriptions.test.ts (1 test) 7ms ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 4ms - ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 47ms + ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 31ms ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms Test Files 27 passed (27) - Tests 323 passed (323) - Start at 02:47:03 - Duration 56.48s (transform 1.13s, setup 155ms, collect 6.83s, tests 46.24s, environment 4ms, prepare 986ms) + Tests 327 passed (327) + Start at 02:56:50 + Duration 58.09s (transform 1.10s, setup 154ms, collect 6.42s, tests 48.26s, environment 4ms, prepare 960ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log index fb0b86829..bdd62c0aa 100644 --- a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log +++ b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/cli/running-step.ts -Original SHA256: e535b533e40304d47aeba175c8ac6f6e51808de9fc99d1d0f64648f2b8a39a70 +Original SHA256: 70c81a8ceed373400410f4dcd753ad714b1f86513e9240344de7807162ad3d64 Replaced: snapshotRead?.abort(new Error(`step "${runningStep.id}" completed during the lease snapshot`)); With: @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (8 tests | 1 failed | 7 skipped) 3009ms - × a completion push ends the wait without waiting out an in-flight snapshot 3009ms + ❯ tests/running-step-watch.test.ts (10 tests | 1 failed | 9 skipped) 3012ms + × a completion push ends the wait without waiting out an in-flight snapshot 3012ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -33,23 +33,23 @@ Received: "still waiting" ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 7 skipped (8) - Start at 02:46:51 - Duration 3.45s (transform 183ms, setup 17ms, collect 304ms, tests 3.01s, environment 0ms, prepare 29ms) + Tests 1 failed | 9 skipped (10) + Start at 02:56:35 + Duration 3.47s (transform 183ms, setup 16ms, collect 309ms, tests 3.01s, environment 0ms, prepare 41ms) exit=1 -RESTORED SHA256: e535b533e40304d47aeba175c8ac6f6e51808de9fc99d1d0f64648f2b8a39a70 +RESTORED SHA256: 70c81a8ceed373400410f4dcd753ad714b1f86513e9240344de7807162ad3d64 $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a completion push ends the wait' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (8 tests | 7 skipped) 2062ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2061ms + ✓ tests/running-step-watch.test.ts (10 tests | 9 skipped) 2062ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2062ms Test Files 1 passed (1) - Tests 1 passed | 7 skipped (8) - Start at 02:46:55 - Duration 2.54s (transform 188ms, setup 16ms, collect 318ms, tests 2.06s, environment 0ms, prepare 40ms) + Tests 1 passed | 9 skipped (10) + Start at 02:56:39 + Duration 2.54s (transform 186ms, setup 16ms, collect 312ms, tests 2.06s, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-heartbeat.log b/evidence/run-read-timeout/mutation-heartbeat.log index ad428d77d..6a6d3e7d9 100644 --- a/evidence/run-read-timeout/mutation-heartbeat.log +++ b/evidence/run-read-timeout/mutation-heartbeat.log @@ -13,8 +13,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 21ms - × a heartbeat timeout is lease loss rather than a worker body failure 20ms + ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 19ms + × a heartbeat timeout is lease loss rather than a worker body failure 18ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -33,8 +33,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 02:46:58 - Duration 442ms (transform 157ms, setup 13ms, collect 271ms, tests 21ms, environment 0ms, prepare 36ms) + Start at 02:56:42 + Duration 475ms (transform 170ms, setup 18ms, collect 292ms, tests 19ms, environment 0ms, prepare 41ms) exit=1 @@ -43,11 +43,11 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 02:46:59 - Duration 460ms (transform 167ms, setup 17ms, collect 289ms, tests 16ms, environment 0ms, prepare 42ms) + Start at 02:56:42 + Duration 467ms (transform 169ms, setup 17ms, collect 291ms, tests 15ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-reconnect.log b/evidence/run-read-timeout/mutation-reader-reconnect.log index 0f87af952..14e80c6a7 100644 --- a/evidence/run-read-timeout/mutation-reader-reconnect.log +++ b/evidence/run-read-timeout/mutation-reader-reconnect.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: 247f3340fff1711bea34f2ad3478f080daeb35a0751050ef8624d5c22140f285 +Original SHA256: 0d9162bdc58667fc831e603da1c31a52df2ec4fc8c39cd0383d6f6d9f0fc7bda Replaced: this.drop(reader); @@ -11,28 +11,28 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (20 tests | 1 failed | 19 skipped) 1006ms - × reconnects the reader after it disconnects within the read budget 1006ms - → journal client: run.get read session was interrupted after 82 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ tests/journal-client-read-timeout.test.ts (22 tests | 1 failed | 21 skipped) 1008ms + × reconnects the reader after it disconnects within the read budget 1007ms + → journal client: run.get read session was interrupted after 79 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 82 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed - ❯ exhausted src/journal-read-policy.ts:70:9 - 68| let last: unknown; - 69| const exhausted = () => last instanceof JournalReadInterruptedError - 70| ? new JournalReadInterruptedError(verb, attempts, performance.no… +JournalReadInterruptedError: journal client: run.get read session was interrupted after 79 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ exhausted src/journal-read-policy.ts:71:9 + 69| let last: unknown; + 70| const exhausted = () => last instanceof JournalReadInterruptedError + 71| ? new JournalReadInterruptedError(verb, attempts, performance.no… | ^ - 71| : new JournalRequestTimeoutError(verb, timeoutMs, attempts, perf… - 72| // A caller's cancellation settles this read and drains its queued… - ❯ Timeout. src/journal-read-policy.ts:81:39 + 72| : new JournalRequestTimeoutError(verb, timeoutMs, attempts, perf… + 73| // A caller's cancellation settles this read and drains its queued… + ❯ Timeout. src/journal-read-policy.ts:82:39 Caused by: Error: journal client: not connected (run.get): journal client: connection closed - ❯ src/journal-client.ts:230:16 - ❯ JournalClient.requestOnce src/journal-client.ts:223:12 + ❯ src/journal-client.ts:233:16 + ❯ JournalClient.requestOnce src/journal-client.ts:226:12 ❯ src/journal-budgeted-reads.ts:47:29 - ❯ src/journal-read-policy.ts:93:18 + ❯ src/journal-read-policy.ts:94:18 Caused by: Error: journal client: connection closed ❯ Socket. src/journal-client.ts:134:26 @@ -40,22 +40,22 @@ Caused by: Error: journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 19 skipped (20) - Start at 02:46:35 - Duration 1.44s (transform 164ms, setup 16ms, collect 286ms, tests 1.01s, environment 0ms, prepare 40ms) + Tests 1 failed | 21 skipped (22) + Start at 02:56:19 + Duration 1.45s (transform 164ms, setup 16ms, collect 284ms, tests 1.01s, environment 0ms, prepare 41ms) exit=1 -RESTORED SHA256: 247f3340fff1711bea34f2ad3478f080daeb35a0751050ef8624d5c22140f285 +RESTORED SHA256: 0d9162bdc58667fc831e603da1c31a52df2ec4fc8c39cd0383d6f6d9f0fc7bda $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'reconnects the reader after it disconnects' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (20 tests | 19 skipped) 78ms + ✓ tests/journal-client-read-timeout.test.ts (22 tests | 21 skipped) 92ms Test Files 1 passed (1) - Tests 1 passed | 19 skipped (20) - Start at 02:46:37 - Duration 522ms (transform 168ms, setup 17ms, collect 282ms, tests 78ms, environment 0ms, prepare 40ms) + Tests 1 passed | 21 skipped (22) + Start at 02:56:20 + Duration 543ms (transform 164ms, setup 17ms, collect 286ms, tests 92ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-setup-retry.log b/evidence/run-read-timeout/mutation-reader-setup-retry.log index 5ea22f5bf..8ee80d334 100644 --- a/evidence/run-read-timeout/mutation-reader-setup-retry.log +++ b/evidence/run-read-timeout/mutation-reader-setup-retry.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: 247f3340fff1711bea34f2ad3478f080daeb35a0751050ef8624d5c22140f285 +Original SHA256: 0d9162bdc58667fc831e603da1c31a52df2ec4fc8c39cd0383d6f6d9f0fc7bda Replaced: if (this.reader === reader) { this.reader = undefined; @@ -16,8 +16,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (20 tests | 1 failed | 19 skipped) 83ms - × retries a reader setup that timed out instead of reading on the primary for good 83ms + ❯ tests/journal-client-read-timeout.test.ts (22 tests | 1 failed | 21 skipped) 78ms + × retries a reader setup that timed out instead of reading on the primary for good 77ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -42,22 +42,22 @@ AssertionError: expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 19 skipped (20) - Start at 02:46:38 - Duration 534ms (transform 165ms, setup 16ms, collect 295ms, tests 83ms, environment 0ms, prepare 39ms) + Tests 1 failed | 21 skipped (22) + Start at 02:56:21 + Duration 565ms (transform 179ms, setup 17ms, collect 313ms, tests 78ms, environment 0ms, prepare 47ms) exit=1 -RESTORED SHA256: 247f3340fff1711bea34f2ad3478f080daeb35a0751050ef8624d5c22140f285 +RESTORED SHA256: 0d9162bdc58667fc831e603da1c31a52df2ec4fc8c39cd0383d6f6d9f0fc7bda $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'retries a reader setup that timed out' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (20 tests | 19 skipped) 74ms + ✓ tests/journal-client-read-timeout.test.ts (22 tests | 21 skipped) 67ms Test Files 1 passed (1) - Tests 1 passed | 19 skipped (20) - Start at 02:46:38 - Duration 521ms (transform 165ms, setup 17ms, collect 286ms, tests 74ms, environment 0ms, prepare 40ms) + Tests 1 passed | 21 skipped (22) + Start at 02:56:22 + Duration 519ms (transform 167ms, setup 18ms, collect 290ms, tests 67ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader.log b/evidence/run-read-timeout/mutation-reader.log index f3b4026a3..9c422828f 100644 --- a/evidence/run-read-timeout/mutation-reader.log +++ b/evidence/run-read-timeout/mutation-reader.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: 247f3340fff1711bea34f2ad3478f080daeb35a0751050ef8624d5c22140f285 +Original SHA256: 0d9162bdc58667fc831e603da1c31a52df2ec4fc8c39cd0383d6f6d9f0fc7bda Replaced: return await reader.requestOnce(verb, params, remaining, attemptSignal); With: @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (20 tests | 1 failed | 19 skipped) 113ms - × serves a bounded read while an unbounded command is in flight 112ms + ❯ tests/journal-client-read-timeout.test.ts (22 tests | 1 failed | 21 skipped) 112ms + × serves a bounded read while an unbounded command is in flight 111ms → expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -36,22 +36,22 @@ AssertionError: expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 19 skipped (20) - Start at 02:46:33 - Duration 564ms (transform 166ms, setup 17ms, collect 285ms, tests 113ms, environment 0ms, prepare 42ms) + Tests 1 failed | 21 skipped (22) + Start at 02:56:17 + Duration 565ms (transform 166ms, setup 15ms, collect 298ms, tests 112ms, environment 0ms, prepare 33ms) exit=1 -RESTORED SHA256: 247f3340fff1711bea34f2ad3478f080daeb35a0751050ef8624d5c22140f285 +RESTORED SHA256: 0d9162bdc58667fc831e603da1c31a52df2ec4fc8c39cd0383d6f6d9f0fc7bda $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'serves a bounded read' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (20 tests | 19 skipped) 108ms + ✓ tests/journal-client-read-timeout.test.ts (22 tests | 21 skipped) 107ms Test Files 1 passed (1) - Tests 1 passed | 19 skipped (20) - Start at 02:46:34 - Duration 566ms (transform 168ms, setup 16ms, collect 290ms, tests 108ms, environment 0ms, prepare 40ms) + Tests 1 passed | 21 skipped (22) + Start at 02:56:18 + Duration 566ms (transform 169ms, setup 15ms, collect 293ms, tests 107ms, environment 0ms, prepare 45ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-root-parking.log b/evidence/run-read-timeout/mutation-root-parking.log index 40a6d0bc2..012364b5e 100644 --- a/evidence/run-read-timeout/mutation-root-parking.log +++ b/evidence/run-read-timeout/mutation-root-parking.log @@ -17,14 +17,14 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (28 tests | 3 failed | 25 skipped) 31ms + ❯ tests/authored-root.test.ts (28 tests | 3 failed | 25 skipped) 30ms × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 22ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms → expected AuthoredFlowExecutionError: daemon_unresp… { …(7) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 4ms + × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 3ms → expected JournalReadInterruptedError: journal clie… { …(4) } to match object { code: 'daemon_unresponsive', …(2) } (5 matching properties omitted from actual) @@ -104,8 +104,8 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } Test Files 1 failed (1) Tests 3 failed | 25 skipped (28) - Start at 02:47:00 - Duration 1.25s (transform 654ms, setup 16ms, collect 1.06s, tests 31ms, environment 0ms, prepare 30ms) + Start at 02:56:43 + Duration 1.26s (transform 660ms, setup 16ms, collect 1.07s, tests 30ms, environment 0ms, prepare 41ms) exit=1 @@ -114,11 +114,11 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (28 tests | 25 skipped) 21ms + ✓ tests/authored-root.test.ts (28 tests | 25 skipped) 22ms Test Files 1 passed (1) Tests 3 passed | 25 skipped (28) - Start at 02:47:01 - Duration 1.25s (transform 669ms, setup 15ms, collect 1.09s, tests 21ms, environment 0ms, prepare 29ms) + Start at 02:56:45 + Duration 1.27s (transform 656ms, setup 16ms, collect 1.08s, tests 22ms, environment 0ms, prepare 44ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-snapshot-cancel.log b/evidence/run-read-timeout/mutation-snapshot-cancel.log index a7bdfa079..e0153e0a3 100644 --- a/evidence/run-read-timeout/mutation-snapshot-cancel.log +++ b/evidence/run-read-timeout/mutation-snapshot-cancel.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/cli/running-step.ts -Original SHA256: e535b533e40304d47aeba175c8ac6f6e51808de9fc99d1d0f64648f2b8a39a70 +Original SHA256: 70c81a8ceed373400410f4dcd753ad714b1f86513e9240344de7807162ad3d64 Replaced: const signal = options.signal === undefined ? read.signal : AbortSignal.any([options.signal, read.signal]); @@ -11,8 +11,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (8 tests | 1 failed | 7 skipped) 2217ms - × cancels promptly while a lease snapshot read is in flight 2216ms + ❯ tests/running-step-watch.test.ts (10 tests | 1 failed | 9 skipped) 2216ms + × cancels promptly while a lease snapshot read is in flight 2215ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -30,23 +30,23 @@ AssertionError: expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 7 skipped (8) - Start at 02:46:45 - Duration 2.71s (transform 191ms, setup 16ms, collect 333ms, tests 2.22s, environment 0ms, prepare 42ms) + Tests 1 failed | 9 skipped (10) + Start at 02:56:29 + Duration 2.71s (transform 192ms, setup 18ms, collect 321ms, tests 2.22s, environment 0ms, prepare 45ms) exit=1 -RESTORED SHA256: e535b533e40304d47aeba175c8ac6f6e51808de9fc99d1d0f64648f2b8a39a70 +RESTORED SHA256: 70c81a8ceed373400410f4dcd753ad714b1f86513e9240344de7807162ad3d64 $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels promptly while a lease snapshot read is in flight' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (8 tests | 7 skipped) 2012ms - ✓ cancels promptly while a lease snapshot read is in flight 2011ms + ✓ tests/running-step-watch.test.ts (10 tests | 9 skipped) 2011ms + ✓ cancels promptly while a lease snapshot read is in flight 2010ms Test Files 1 passed (1) - Tests 1 passed | 7 skipped (8) - Start at 02:46:48 - Duration 2.59s (transform 223ms, setup 19ms, collect 389ms, tests 2.01s, environment 0ms, prepare 45ms) + Tests 1 passed | 9 skipped (10) + Start at 02:56:32 + Duration 2.49s (transform 186ms, setup 15ms, collect 313ms, tests 2.01s, environment 0ms, prepare 42ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-watch-cadence.log b/evidence/run-read-timeout/mutation-watch-cadence.log index b376a09a2..96988ba24 100644 --- a/evidence/run-read-timeout/mutation-watch-cadence.log +++ b/evidence/run-read-timeout/mutation-watch-cadence.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/cli/running-step.ts -Original SHA256: e535b533e40304d47aeba175c8ac6f6e51808de9fc99d1d0f64648f2b8a39a70 +Original SHA256: 70c81a8ceed373400410f4dcd753ad714b1f86513e9240344de7807162ad3d64 Replaced: const LEASE_POLL_MS = 2_000; With: @@ -10,7 +10,7 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (8 tests | 1 failed | 7 skipped) 2111ms + ❯ tests/running-step-watch.test.ts (10 tests | 1 failed | 9 skipped) 2111ms × uses pushes for completion with lease-cadence reads and releases its watcher 2110ms → expected 41 to be less than or equal to 1 @@ -29,23 +29,23 @@ AssertionError: expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 7 skipped (8) - Start at 02:46:39 - Duration 2.57s (transform 179ms, setup 12ms, collect 316ms, tests 2.11s, environment 0ms, prepare 36ms) + Tests 1 failed | 9 skipped (10) + Start at 02:56:23 + Duration 2.59s (transform 187ms, setup 17ms, collect 313ms, tests 2.11s, environment 0ms, prepare 41ms) exit=1 -RESTORED SHA256: e535b533e40304d47aeba175c8ac6f6e51808de9fc99d1d0f64648f2b8a39a70 +RESTORED SHA256: 70c81a8ceed373400410f4dcd753ad714b1f86513e9240344de7807162ad3d64 $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pushes for completion' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (8 tests | 7 skipped) 2118ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms + ✓ tests/running-step-watch.test.ts (10 tests | 9 skipped) 2120ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2119ms Test Files 1 passed (1) - Tests 1 passed | 7 skipped (8) - Start at 02:46:42 - Duration 2.63s (transform 205ms, setup 17ms, collect 345ms, tests 2.12s, environment 0ms, prepare 42ms) + Tests 1 passed | 9 skipped (10) + Start at 02:56:26 + Duration 2.59s (transform 187ms, setup 17ms, collect 322ms, tests 2.12s, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/resume-live.log b/evidence/run-read-timeout/resume-live.log index 60d7c3eb2..6a3e329e8 100644 --- a/evidence/run-read-timeout/resume-live.log +++ b/evidence/run-read-timeout/resume-live.log @@ -2,12 +2,12 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 886ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 885ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 935ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 934ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 02:48:00 - Duration 2.16s (transform 630ms, setup 17ms, collect 1.03s, tests 886ms, environment 0ms, prepare 62ms) + Start at 02:57:49 + Duration 2.19s (transform 622ms, setup 20ms, collect 1.02s, tests 935ms, environment 0ms, prepare 85ms) exit=0 diff --git a/evidence/run-read-timeout/verification.md b/evidence/run-read-timeout/verification.md index 068d43b75..e892100c9 100644 --- a/evidence/run-read-timeout/verification.md +++ b/evidence/run-read-timeout/verification.md @@ -15,68 +15,70 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5300ms + ✓ tests/cli.test.ts (71 tests) 5320ms + ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 326ms ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2038ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2034ms -(node:3820686) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2037ms +(node:3863527) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (28 tests) 395ms - ✓ tests/journal-client.test.ts (17 tests) 79ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 6970ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 480ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 973ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2720ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 370ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 868ms - ✓ tests/authored-flow-slack.test.ts (7 tests) 1089ms - ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 340ms - ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 410ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2573ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 647ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 615ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 643ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 667ms - ✓ tests/authored-helpers.test.ts (6 tests) 1798ms - ✓ runs every available provider through the real kernel and resumes completed effects without a second write 867ms - ✓ replays after SIGKILL before confirm with the same token and one successful completion 347ms - ✓ replays after SIGKILL before complete with the same token and one successful completion 406ms - ✓ tests/journal-client-read-timeout.test.ts (20 tests) 1411ms - ✓ tests/classify-outcome.test.ts (11 tests) 7426ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2009ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3008ms - ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms - ✓ tests/human-live.test.ts (3 tests) 5563ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 3518ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1228ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 816ms - ✓ tests/running-step-watch.test.ts (8 tests) 8240ms + ✓ tests/authored-root.test.ts (28 tests) 361ms + ✓ tests/journal-client.test.ts (17 tests) 82ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 7494ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 522ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1115ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2804ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 424ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 948ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 1242ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 444ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 424ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2484ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 670ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 501ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 644ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 668ms + ✓ tests/journal-client-read-timeout.test.ts (22 tests) 1729ms + ✓ tests/authored-helpers.test.ts (6 tests) 1856ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 880ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 332ms + ✓ replays after SIGKILL before complete with the same token and one successful completion 415ms + ✓ tests/classify-outcome.test.ts (11 tests) 7430ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2015ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3006ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2002ms + ✓ tests/running-step-watch.test.ts (10 tests) 10266ms ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms ✓ cancels promptly while a lease snapshot read is in flight 2006ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2054ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2055ms ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms - ✓ tests/worker-lease.test.ts (7 tests) 18ms - ✓ tests/communication-worker.test.ts (18 tests) 1720ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 26ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 730ms + ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2004ms + ✓ tests/human-live.test.ts (3 tests) 4601ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2743ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1127ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 730ms + ✓ tests/worker-lease.test.ts (7 tests) 15ms + ✓ tests/communication-worker.test.ts (18 tests) 1711ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 25ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 699ms ✓ tests/memoization.test.ts (58 tests) 66ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2401ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2061ms - ✓ drains read and watch promises before an authored flow completes 340ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2465ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2058ms + ✓ drains read and watch promises before an authored flow completes 406ms ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms - ✓ tests/journal-client-completion.test.ts (6 tests) 99ms - ✓ tests/reuse-summary-interruption.test.ts (5 tests) 4ms + ✓ tests/journal-client-completion.test.ts (6 tests) 101ms + ✓ tests/reuse-summary-interruption.test.ts (5 tests) 3ms ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms - ✓ tests/direct-run-worker-lease.test.ts (3 tests) 9ms - ✓ tests/helper-effect-lease-loss.test.ts (1 test) 237ms - ✓ tests/journal-client-subscriptions.test.ts (1 test) 9ms + ✓ tests/direct-run-worker-lease.test.ts (3 tests) 6ms + ✓ tests/helper-effect-lease-loss.test.ts (1 test) 235ms + ✓ tests/journal-client-subscriptions.test.ts (1 test) 7ms ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 4ms - ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 47ms + ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 31ms ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms Test Files 27 passed (27) - Tests 323 passed (323) - Start at 02:47:03 - Duration 56.48s (transform 1.13s, setup 155ms, collect 6.83s, tests 46.24s, environment 4ms, prepare 986ms) + Tests 327 passed (327) + Start at 02:56:50 + Duration 58.09s (transform 1.10s, setup 154ms, collect 6.42s, tests 48.26s, environment 4ms, prepare 960ms) exit=0 ``` @@ -88,13 +90,13 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 886ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 885ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 935ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 934ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 02:48:00 - Duration 2.16s (transform 630ms, setup 17ms, collect 1.03s, tests 886ms, environment 0ms, prepare 62ms) + Start at 02:57:49 + Duration 2.19s (transform 622ms, setup 20ms, collect 1.02s, tests 935ms, environment 0ms, prepare 85ms) exit=0 ``` @@ -105,7 +107,7 @@ The mutation sections below were regenerated by `cd packages/sdk && python3 ../. ```text Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: 247f3340fff1711bea34f2ad3478f080daeb35a0751050ef8624d5c22140f285 +Original SHA256: 0d9162bdc58667fc831e603da1c31a52df2ec4fc8c39cd0383d6f6d9f0fc7bda Replaced: return await reader.requestOnce(verb, params, remaining, attemptSignal); With: @@ -116,8 +118,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (20 tests | 1 failed | 19 skipped) 113ms - × serves a bounded read while an unbounded command is in flight 112ms + ❯ tests/journal-client-read-timeout.test.ts (22 tests | 1 failed | 21 skipped) 112ms + × serves a bounded read while an unbounded command is in flight 111ms → expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -142,23 +144,23 @@ AssertionError: expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 19 skipped (20) - Start at 02:46:33 - Duration 564ms (transform 166ms, setup 17ms, collect 285ms, tests 113ms, environment 0ms, prepare 42ms) + Tests 1 failed | 21 skipped (22) + Start at 02:56:17 + Duration 565ms (transform 166ms, setup 15ms, collect 298ms, tests 112ms, environment 0ms, prepare 33ms) exit=1 -RESTORED SHA256: 247f3340fff1711bea34f2ad3478f080daeb35a0751050ef8624d5c22140f285 +RESTORED SHA256: 0d9162bdc58667fc831e603da1c31a52df2ec4fc8c39cd0383d6f6d9f0fc7bda $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'serves a bounded read' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (20 tests | 19 skipped) 108ms + ✓ tests/journal-client-read-timeout.test.ts (22 tests | 21 skipped) 107ms Test Files 1 passed (1) - Tests 1 passed | 19 skipped (20) - Start at 02:46:34 - Duration 566ms (transform 168ms, setup 16ms, collect 290ms, tests 108ms, environment 0ms, prepare 40ms) + Tests 1 passed | 21 skipped (22) + Start at 02:56:18 + Duration 566ms (transform 169ms, setup 15ms, collect 293ms, tests 107ms, environment 0ms, prepare 45ms) exit=0 ``` @@ -167,7 +169,7 @@ exit=0 ```text Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: 247f3340fff1711bea34f2ad3478f080daeb35a0751050ef8624d5c22140f285 +Original SHA256: 0d9162bdc58667fc831e603da1c31a52df2ec4fc8c39cd0383d6f6d9f0fc7bda Replaced: this.drop(reader); @@ -179,28 +181,28 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (20 tests | 1 failed | 19 skipped) 1006ms - × reconnects the reader after it disconnects within the read budget 1006ms - → journal client: run.get read session was interrupted after 82 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ tests/journal-client-read-timeout.test.ts (22 tests | 1 failed | 21 skipped) 1008ms + × reconnects the reader after it disconnects within the read budget 1007ms + → journal client: run.get read session was interrupted after 79 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 82 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed - ❯ exhausted src/journal-read-policy.ts:70:9 - 68| let last: unknown; - 69| const exhausted = () => last instanceof JournalReadInterruptedError - 70| ? new JournalReadInterruptedError(verb, attempts, performance.no… +JournalReadInterruptedError: journal client: run.get read session was interrupted after 79 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ exhausted src/journal-read-policy.ts:71:9 + 69| let last: unknown; + 70| const exhausted = () => last instanceof JournalReadInterruptedError + 71| ? new JournalReadInterruptedError(verb, attempts, performance.no… | ^ - 71| : new JournalRequestTimeoutError(verb, timeoutMs, attempts, perf… - 72| // A caller's cancellation settles this read and drains its queued… - ❯ Timeout. src/journal-read-policy.ts:81:39 + 72| : new JournalRequestTimeoutError(verb, timeoutMs, attempts, perf… + 73| // A caller's cancellation settles this read and drains its queued… + ❯ Timeout. src/journal-read-policy.ts:82:39 Caused by: Error: journal client: not connected (run.get): journal client: connection closed - ❯ src/journal-client.ts:230:16 - ❯ JournalClient.requestOnce src/journal-client.ts:223:12 + ❯ src/journal-client.ts:233:16 + ❯ JournalClient.requestOnce src/journal-client.ts:226:12 ❯ src/journal-budgeted-reads.ts:47:29 - ❯ src/journal-read-policy.ts:93:18 + ❯ src/journal-read-policy.ts:94:18 Caused by: Error: journal client: connection closed ❯ Socket. src/journal-client.ts:134:26 @@ -208,23 +210,23 @@ Caused by: Error: journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 19 skipped (20) - Start at 02:46:35 - Duration 1.44s (transform 164ms, setup 16ms, collect 286ms, tests 1.01s, environment 0ms, prepare 40ms) + Tests 1 failed | 21 skipped (22) + Start at 02:56:19 + Duration 1.45s (transform 164ms, setup 16ms, collect 284ms, tests 1.01s, environment 0ms, prepare 41ms) exit=1 -RESTORED SHA256: 247f3340fff1711bea34f2ad3478f080daeb35a0751050ef8624d5c22140f285 +RESTORED SHA256: 0d9162bdc58667fc831e603da1c31a52df2ec4fc8c39cd0383d6f6d9f0fc7bda $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'reconnects the reader after it disconnects' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (20 tests | 19 skipped) 78ms + ✓ tests/journal-client-read-timeout.test.ts (22 tests | 21 skipped) 92ms Test Files 1 passed (1) - Tests 1 passed | 19 skipped (20) - Start at 02:46:37 - Duration 522ms (transform 168ms, setup 17ms, collect 282ms, tests 78ms, environment 0ms, prepare 40ms) + Tests 1 passed | 21 skipped (22) + Start at 02:56:20 + Duration 543ms (transform 164ms, setup 17ms, collect 286ms, tests 92ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -233,7 +235,7 @@ exit=0 ```text Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: 247f3340fff1711bea34f2ad3478f080daeb35a0751050ef8624d5c22140f285 +Original SHA256: 0d9162bdc58667fc831e603da1c31a52df2ec4fc8c39cd0383d6f6d9f0fc7bda Replaced: if (this.reader === reader) { this.reader = undefined; @@ -250,8 +252,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (20 tests | 1 failed | 19 skipped) 83ms - × retries a reader setup that timed out instead of reading on the primary for good 83ms + ❯ tests/journal-client-read-timeout.test.ts (22 tests | 1 failed | 21 skipped) 78ms + × retries a reader setup that timed out instead of reading on the primary for good 77ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -276,23 +278,23 @@ AssertionError: expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 19 skipped (20) - Start at 02:46:38 - Duration 534ms (transform 165ms, setup 16ms, collect 295ms, tests 83ms, environment 0ms, prepare 39ms) + Tests 1 failed | 21 skipped (22) + Start at 02:56:21 + Duration 565ms (transform 179ms, setup 17ms, collect 313ms, tests 78ms, environment 0ms, prepare 47ms) exit=1 -RESTORED SHA256: 247f3340fff1711bea34f2ad3478f080daeb35a0751050ef8624d5c22140f285 +RESTORED SHA256: 0d9162bdc58667fc831e603da1c31a52df2ec4fc8c39cd0383d6f6d9f0fc7bda $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'retries a reader setup that timed out' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (20 tests | 19 skipped) 74ms + ✓ tests/journal-client-read-timeout.test.ts (22 tests | 21 skipped) 67ms Test Files 1 passed (1) - Tests 1 passed | 19 skipped (20) - Start at 02:46:38 - Duration 521ms (transform 165ms, setup 17ms, collect 286ms, tests 74ms, environment 0ms, prepare 40ms) + Tests 1 passed | 21 skipped (22) + Start at 02:56:22 + Duration 519ms (transform 167ms, setup 18ms, collect 290ms, tests 67ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -301,7 +303,7 @@ exit=0 ```text Source: packages/sdk/src/cli/running-step.ts -Original SHA256: e535b533e40304d47aeba175c8ac6f6e51808de9fc99d1d0f64648f2b8a39a70 +Original SHA256: 70c81a8ceed373400410f4dcd753ad714b1f86513e9240344de7807162ad3d64 Replaced: const LEASE_POLL_MS = 2_000; With: @@ -312,7 +314,7 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (8 tests | 1 failed | 7 skipped) 2111ms + ❯ tests/running-step-watch.test.ts (10 tests | 1 failed | 9 skipped) 2111ms × uses pushes for completion with lease-cadence reads and releases its watcher 2110ms → expected 41 to be less than or equal to 1 @@ -331,24 +333,24 @@ AssertionError: expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 7 skipped (8) - Start at 02:46:39 - Duration 2.57s (transform 179ms, setup 12ms, collect 316ms, tests 2.11s, environment 0ms, prepare 36ms) + Tests 1 failed | 9 skipped (10) + Start at 02:56:23 + Duration 2.59s (transform 187ms, setup 17ms, collect 313ms, tests 2.11s, environment 0ms, prepare 41ms) exit=1 -RESTORED SHA256: e535b533e40304d47aeba175c8ac6f6e51808de9fc99d1d0f64648f2b8a39a70 +RESTORED SHA256: 70c81a8ceed373400410f4dcd753ad714b1f86513e9240344de7807162ad3d64 $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pushes for completion' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (8 tests | 7 skipped) 2118ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms + ✓ tests/running-step-watch.test.ts (10 tests | 9 skipped) 2120ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2119ms Test Files 1 passed (1) - Tests 1 passed | 7 skipped (8) - Start at 02:46:42 - Duration 2.63s (transform 205ms, setup 17ms, collect 345ms, tests 2.12s, environment 0ms, prepare 42ms) + Tests 1 passed | 9 skipped (10) + Start at 02:56:26 + Duration 2.59s (transform 187ms, setup 17ms, collect 322ms, tests 2.12s, environment 0ms, prepare 41ms) exit=0 ``` @@ -357,7 +359,7 @@ exit=0 ```text Source: packages/sdk/src/cli/running-step.ts -Original SHA256: e535b533e40304d47aeba175c8ac6f6e51808de9fc99d1d0f64648f2b8a39a70 +Original SHA256: 70c81a8ceed373400410f4dcd753ad714b1f86513e9240344de7807162ad3d64 Replaced: const signal = options.signal === undefined ? read.signal : AbortSignal.any([options.signal, read.signal]); @@ -369,8 +371,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (8 tests | 1 failed | 7 skipped) 2217ms - × cancels promptly while a lease snapshot read is in flight 2216ms + ❯ tests/running-step-watch.test.ts (10 tests | 1 failed | 9 skipped) 2216ms + × cancels promptly while a lease snapshot read is in flight 2215ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -388,24 +390,24 @@ AssertionError: expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 7 skipped (8) - Start at 02:46:45 - Duration 2.71s (transform 191ms, setup 16ms, collect 333ms, tests 2.22s, environment 0ms, prepare 42ms) + Tests 1 failed | 9 skipped (10) + Start at 02:56:29 + Duration 2.71s (transform 192ms, setup 18ms, collect 321ms, tests 2.22s, environment 0ms, prepare 45ms) exit=1 -RESTORED SHA256: e535b533e40304d47aeba175c8ac6f6e51808de9fc99d1d0f64648f2b8a39a70 +RESTORED SHA256: 70c81a8ceed373400410f4dcd753ad714b1f86513e9240344de7807162ad3d64 $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels promptly while a lease snapshot read is in flight' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (8 tests | 7 skipped) 2012ms - ✓ cancels promptly while a lease snapshot read is in flight 2011ms + ✓ tests/running-step-watch.test.ts (10 tests | 9 skipped) 2011ms + ✓ cancels promptly while a lease snapshot read is in flight 2010ms Test Files 1 passed (1) - Tests 1 passed | 7 skipped (8) - Start at 02:46:48 - Duration 2.59s (transform 223ms, setup 19ms, collect 389ms, tests 2.01s, environment 0ms, prepare 45ms) + Tests 1 passed | 9 skipped (10) + Start at 02:56:32 + Duration 2.49s (transform 186ms, setup 15ms, collect 313ms, tests 2.01s, environment 0ms, prepare 42ms) exit=0 ``` @@ -414,7 +416,7 @@ exit=0 ```text Source: packages/sdk/src/cli/running-step.ts -Original SHA256: e535b533e40304d47aeba175c8ac6f6e51808de9fc99d1d0f64648f2b8a39a70 +Original SHA256: 70c81a8ceed373400410f4dcd753ad714b1f86513e9240344de7807162ad3d64 Replaced: snapshotRead?.abort(new Error(`step "${runningStep.id}" completed during the lease snapshot`)); With: @@ -425,8 +427,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (8 tests | 1 failed | 7 skipped) 3009ms - × a completion push ends the wait without waiting out an in-flight snapshot 3009ms + ❯ tests/running-step-watch.test.ts (10 tests | 1 failed | 9 skipped) 3012ms + × a completion push ends the wait without waiting out an in-flight snapshot 3012ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -448,24 +450,24 @@ Received: "still waiting" ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 7 skipped (8) - Start at 02:46:51 - Duration 3.45s (transform 183ms, setup 17ms, collect 304ms, tests 3.01s, environment 0ms, prepare 29ms) + Tests 1 failed | 9 skipped (10) + Start at 02:56:35 + Duration 3.47s (transform 183ms, setup 16ms, collect 309ms, tests 3.01s, environment 0ms, prepare 41ms) exit=1 -RESTORED SHA256: e535b533e40304d47aeba175c8ac6f6e51808de9fc99d1d0f64648f2b8a39a70 +RESTORED SHA256: 70c81a8ceed373400410f4dcd753ad714b1f86513e9240344de7807162ad3d64 $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a completion push ends the wait' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (8 tests | 7 skipped) 2062ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2061ms + ✓ tests/running-step-watch.test.ts (10 tests | 9 skipped) 2062ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2062ms Test Files 1 passed (1) - Tests 1 passed | 7 skipped (8) - Start at 02:46:55 - Duration 2.54s (transform 188ms, setup 16ms, collect 318ms, tests 2.06s, environment 0ms, prepare 40ms) + Tests 1 passed | 9 skipped (10) + Start at 02:56:39 + Duration 2.54s (transform 186ms, setup 16ms, collect 312ms, tests 2.06s, environment 0ms, prepare 40ms) exit=0 ``` @@ -488,8 +490,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 21ms - × a heartbeat timeout is lease loss rather than a worker body failure 20ms + ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 19ms + × a heartbeat timeout is lease loss rather than a worker body failure 18ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -508,8 +510,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 02:46:58 - Duration 442ms (transform 157ms, setup 13ms, collect 271ms, tests 21ms, environment 0ms, prepare 36ms) + Start at 02:56:42 + Duration 475ms (transform 170ms, setup 18ms, collect 292ms, tests 19ms, environment 0ms, prepare 41ms) exit=1 @@ -518,12 +520,12 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 02:46:59 - Duration 460ms (transform 167ms, setup 17ms, collect 289ms, tests 16ms, environment 0ms, prepare 42ms) + Start at 02:56:42 + Duration 467ms (transform 169ms, setup 17ms, collect 291ms, tests 15ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -550,14 +552,14 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (28 tests | 3 failed | 25 skipped) 31ms + ❯ tests/authored-root.test.ts (28 tests | 3 failed | 25 skipped) 30ms × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 22ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms → expected AuthoredFlowExecutionError: daemon_unresp… { …(7) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 4ms + × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 3ms → expected JournalReadInterruptedError: journal clie… { …(4) } to match object { code: 'daemon_unresponsive', …(2) } (5 matching properties omitted from actual) @@ -637,8 +639,8 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } Test Files 1 failed (1) Tests 3 failed | 25 skipped (28) - Start at 02:47:00 - Duration 1.25s (transform 654ms, setup 16ms, collect 1.06s, tests 31ms, environment 0ms, prepare 30ms) + Start at 02:56:43 + Duration 1.26s (transform 660ms, setup 16ms, collect 1.07s, tests 30ms, environment 0ms, prepare 41ms) exit=1 @@ -647,12 +649,12 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (28 tests | 25 skipped) 21ms + ✓ tests/authored-root.test.ts (28 tests | 25 skipped) 22ms Test Files 1 passed (1) Tests 3 passed | 25 skipped (28) - Start at 02:47:01 - Duration 1.25s (transform 669ms, setup 15ms, collect 1.09s, tests 21ms, environment 0ms, prepare 29ms) + Start at 02:56:45 + Duration 1.27s (transform 656ms, setup 16ms, collect 1.08s, tests 22ms, environment 0ms, prepare 44ms) exit=0 ``` diff --git a/packages/sdk/src/cli/running-step.ts b/packages/sdk/src/cli/running-step.ts index bd5736a89..c23207f84 100644 --- a/packages/sdk/src/cli/running-step.ts +++ b/packages/sdk/src/cli/running-step.ts @@ -26,10 +26,12 @@ export async function waitForRunningStep(client: JournalClient, runId: string, let ready = false; let finish: () => void = () => {}; let cancel: () => void = () => {}; + let fail: (error: unknown) => void = () => {}; const canceled = () => new Error(`worker wait for step "${runningStep.id}" was canceled`); const completed = new Promise((resolve, reject) => { finish = resolve; cancel = () => reject(canceled()); + fail = reject; }); // The subscription handshake itself may be pending when cancellation arrives. void completed.catch(() => {}); @@ -49,6 +51,10 @@ export async function waitForRunningStep(client: JournalClient, runId: string, } }; watch.on('entry', onEntry); + // A malformed frame after registration has no pending request to reject: fail closed here. + let protocolFailure: unknown; + const onProtocolError = (error: unknown) => { protocolFailure ??= error; fail(error); snapshotRead?.abort(error); }; + watch.on('protocol_error', onProtocolError); options.signal?.addEventListener('abort', cancel, { once: true }); if (options.signal?.aborted) cancel(); try { @@ -82,6 +88,7 @@ export async function waitForRunningStep(client: JournalClient, runId: string, const snapshot = await client.runGet(runId, { signal }) .catch(error => { if (options.signal?.aborted) throw canceled(); + if (protocolFailure !== undefined) throw protocolFailure; // A completion push aborted this read: completion was observed, and // finish() is one-shot, even if a retry attempt started right after. if (settled || read.signal.aborted) return undefined; @@ -89,6 +96,7 @@ export async function waitForRunningStep(client: JournalClient, runId: string, }) .finally(() => { snapshotRead = undefined; }); if (options.signal?.aborted) throw canceled(); + if (protocolFailure !== undefined) throw protocolFailure; if (settled || snapshot === undefined || snapshot.steps[runningStep.id]?.state !== 'running') return; const deadline = snapshot.steps[runningStep.id]?.lease_deadline_ms; if (deadline !== undefined && deadline !== leaseDeadlineMs) { @@ -105,6 +113,7 @@ export async function waitForRunningStep(client: JournalClient, runId: string, finish(); options.signal?.removeEventListener('abort', cancel); watch.off('entry', onEntry); + watch.off('protocol_error', onProtocolError); watch.close(); } } diff --git a/packages/sdk/src/journal-budgeted-reads.ts b/packages/sdk/src/journal-budgeted-reads.ts index b10e06deb..272d14d62 100644 --- a/packages/sdk/src/journal-budgeted-reads.ts +++ b/packages/sdk/src/journal-budgeted-reads.ts @@ -55,6 +55,12 @@ export class BudgetedReads { }, combined); } + /** The primary session's own `hello`, retried within the same budget as reads. */ + handshake(params: VerbContract['hello']['params'], timeoutMs: number): Promise { + return this.policy.read('hello', timeoutMs, this.budgetMs, + (bound, attemptSignal) => this.primary.requestOnce('hello', params, bound, attemptSignal), this.lifecycle); + } + private session(): Promise { if (this.readerReady === undefined) { const reader = this.primary.createPeer(); diff --git a/packages/sdk/src/journal-client.ts b/packages/sdk/src/journal-client.ts index a1a6fd617..96e4cd42b 100644 --- a/packages/sdk/src/journal-client.ts +++ b/packages/sdk/src/journal-client.ts @@ -178,7 +178,10 @@ export class JournalClient extends EventEmitter { msg = JSON.parse(line) as Response | ServerEvent; } catch { // A malformed frame is a protocol violation; fail closed. - this.failAll(new JournalFrameError()); + const error = new JournalFrameError(); + this.failAll(error); + // Sessions with nothing pending (a registered watch) must hear it too. + this.emit('protocol_error', error); return; } @@ -271,7 +274,10 @@ export class JournalClient extends EventEmitter { /** Handshake; version mismatch is a hard error. Records the daemon's additive features. */ async hello(client: string): Promise { - const result = await this.request('hello', { protocol: PROTOCOL_VERSION, client }); + const params = { protocol: PROTOCOL_VERSION, client }; + // A flow client's handshake meets the same CPU load as its reads. + const result = this.budgeted === undefined ? await this.request('hello', params) + : await this.budgeted.handshake(params, this.requestTimeoutMs); this.features = new Set(Array.isArray(result.features) ? result.features.filter(f => typeof f === 'string') : []); return result; } diff --git a/packages/sdk/src/journal-read-policy.ts b/packages/sdk/src/journal-read-policy.ts index c7f8f4526..9c17f8156 100644 --- a/packages/sdk/src/journal-read-policy.ts +++ b/packages/sdk/src/journal-read-policy.ts @@ -44,7 +44,8 @@ export class JournalReadInterruptedError extends Error { /** A read that could not be answered — by timeout or lost read transport — not a run failure. */ export function isReadInterruptionError(error: unknown): error is JournalRequestTimeoutError | JournalReadInterruptedError { - return (error instanceof JournalRequestTimeoutError && (READ_ONLY_VERBS.has(error.verb) || error.verb === 'run.watch')) + return (error instanceof JournalRequestTimeoutError + && (READ_ONLY_VERBS.has(error.verb) || error.verb === 'run.watch' || error.verb === 'hello')) || error instanceof JournalReadInterruptedError; } diff --git a/packages/sdk/tests/journal-client-read-timeout.test.ts b/packages/sdk/tests/journal-client-read-timeout.test.ts index 81e059e90..16fbe9734 100644 --- a/packages/sdk/tests/journal-client-read-timeout.test.ts +++ b/packages/sdk/tests/journal-client-read-timeout.test.ts @@ -245,3 +245,16 @@ it('fails closed on a malformed frame during reader setup', async () => { expect(hellos).toBe(1); expect(reads).toBe(0); }); + +it('retries a delayed handshake within the read budget on a budgeted flow client', async () => { + let hellos = 0; + const client = await setup({ hello: ctx => { if (++hellos > 1) sendOk(ctx); } }, 5_000); + await expect(client.hello('flows-authored-node')).resolves.toBeDefined(); + expect(hellos).toBe(2); +}); + +it('an exhausted handshake budget is a read interruption, not a body failure', async () => { + const client = await setup({ hello: () => {} }, 250); + const error = await client.hello('flows-authored-node').catch(caught => caught); + expect(isReadInterruptionError(error)).toBe(true); +}); diff --git a/packages/sdk/tests/running-step-watch.test.ts b/packages/sdk/tests/running-step-watch.test.ts index 9c8ac3b7a..6857e81ef 100644 --- a/packages/sdk/tests/running-step-watch.test.ts +++ b/packages/sdk/tests/running-step-watch.test.ts @@ -192,3 +192,41 @@ it('fails closed on a malformed frame during watch setup', async () => { expect(isReadInterruption(error)).toBe(false); } finally { client.close(); await new Promise(resolve => server.close(() => resolve())); } }); + +it('fails closed on a malformed frame after the watch is registered', async () => { + const path = sockPath(); + const server = startLoopback(path, { hello: sendOk, 'run.watch': ctx => { + sendResult(ctx, {}); + setTimeout(() => ctx.socket.write('{not json\n'), 20); + } }); + await once(server, 'listening'); + const client = new JournalClient(path); + try { + await client.connect(); + const outcome = await Promise.race([ + waitForRunningStep(client, 'run', { id: 'step', type: 'agent', leaseDeadlineMs: Date.now() + 30_000 }, {}).then(() => 'returned', error => error), + sleep(1_000).then(() => 'still waiting'), + ]); + expect(outcome).toBeInstanceOf(JournalFrameError); + } finally { client.close(); await new Promise(resolve => server.close(() => resolve())); } +}); + +it('a malformed watch frame during an in-flight snapshot is an error, not a completion', async () => { + const path = sockPath(); + let corrupt!: () => void; + const server = startLoopback(path, { + hello: sendOk, + 'run.watch': ctx => { sendResult(ctx, {}); corrupt = () => ctx.socket.write('{not json\n'); }, + 'run.get': () => corrupt(), // the frame breaks while the snapshot is in flight + }); + await once(server, 'listening'); + const client = new JournalClient(path, { readBudgetMs: 300_000 }); + try { + await client.connect(); + const outcome = await Promise.race([ + waitForRunningStep(client, 'run', { id: 'step', type: 'agent', leaseDeadlineMs: Date.now() + 30_000 }, {}).then(() => 'returned', error => error), + sleep(4_000).then(() => 'still waiting'), + ]); + expect(outcome).toBeInstanceOf(JournalFrameError); + } finally { client.close(); await new Promise(resolve => server.close(() => resolve())); } +}); From d4a56c73a82348972988d5cd9c0b166a4648dce3 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 03:08:35 -0700 Subject: [PATCH 18/34] fix(sdk): a watch dropped after registration is a read interruption A registered completion watch has no pending request, so a dropped socket rejected nothing and the wait kept polling while completion pushes were lost. JournalClient now emits 'disconnected' for a drop the caller did not ask for, and waitForRunningStep turns it into a resumable daemon_unresponsive. Evidence re-captured at this tree. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: b8262468-ea26-4a6e-bee9-d193832e77dc --- evidence/run-read-timeout/final-focused.log | 102 +++---- .../mutation-completion-aborts-snapshot.log | 24 +- .../run-read-timeout/mutation-heartbeat.log | 14 +- .../mutation-reader-reconnect.log | 22 +- .../mutation-reader-setup-retry.log | 14 +- evidence/run-read-timeout/mutation-reader.log | 12 +- .../mutation-root-parking.log | 16 +- .../mutation-snapshot-cancel.log | 22 +- .../mutation-watch-cadence.log | 24 +- evidence/run-read-timeout/resume-live.log | 8 +- evidence/run-read-timeout/verification.md | 258 +++++++++--------- packages/sdk/src/cli/running-step.ts | 7 + packages/sdk/src/journal-client.ts | 3 + packages/sdk/tests/running-step-watch.test.ts | 19 ++ 14 files changed, 287 insertions(+), 258 deletions(-) diff --git a/evidence/run-read-timeout/final-focused.log b/evidence/run-read-timeout/final-focused.log index a3e832c6d..605148a69 100644 --- a/evidence/run-read-timeout/final-focused.log +++ b/evidence/run-read-timeout/final-focused.log @@ -5,66 +5,66 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo ✓ tests/cli.test.ts (71 tests) 5320ms ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 326ms ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2038ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2037ms -(node:3863527) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2046ms +(node:3908233) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (28 tests) 361ms - ✓ tests/journal-client.test.ts (17 tests) 82ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 7494ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 522ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1115ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2804ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 424ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 948ms - ✓ tests/authored-flow-slack.test.ts (7 tests) 1242ms - ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 444ms - ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 424ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2484ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 670ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 501ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 644ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 668ms - ✓ tests/journal-client-read-timeout.test.ts (22 tests) 1729ms - ✓ tests/authored-helpers.test.ts (6 tests) 1856ms - ✓ runs every available provider through the real kernel and resumes completed effects without a second write 880ms - ✓ replays after SIGKILL before confirm with the same token and one successful completion 332ms - ✓ replays after SIGKILL before complete with the same token and one successful completion 415ms - ✓ tests/classify-outcome.test.ts (11 tests) 7430ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2015ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3006ms - ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2002ms - ✓ tests/running-step-watch.test.ts (10 tests) 10266ms + ✓ tests/authored-root.test.ts (28 tests) 363ms + ✓ tests/journal-client.test.ts (17 tests) 80ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 6984ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 467ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1065ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2779ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 375ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 751ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 1188ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 433ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 417ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2530ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 580ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 627ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 703ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 618ms + ✓ tests/journal-client-read-timeout.test.ts (22 tests) 1730ms + ✓ tests/running-step-watch.test.ts (11 tests) 10283ms ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms ✓ cancels promptly while a lease snapshot read is in flight 2006ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2055ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2053ms ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms - ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2004ms - ✓ tests/human-live.test.ts (3 tests) 4601ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2743ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1127ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 730ms - ✓ tests/worker-lease.test.ts (7 tests) 15ms - ✓ tests/communication-worker.test.ts (18 tests) 1711ms + ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2002ms + ✓ tests/authored-helpers.test.ts (6 tests) 1748ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 760ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 379ms + ✓ replays after SIGKILL before complete with the same token and one successful completion 393ms + ✓ tests/classify-outcome.test.ts (11 tests) 7434ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2017ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3008ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms + ✓ tests/human-live.test.ts (3 tests) 4717ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2802ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1179ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 736ms + ✓ tests/worker-lease.test.ts (7 tests) 19ms + ✓ tests/communication-worker.test.ts (18 tests) 1714ms ✓ tests/worker-lease-lost.test.ts (17 tests) 25ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 699ms - ✓ tests/memoization.test.ts (58 tests) 66ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2465ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 672ms + ✓ tests/memoization.test.ts (58 tests) 65ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2419ms ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2058ms - ✓ drains read and watch promises before an authored flow completes 406ms + ✓ drains read and watch promises before an authored flow completes 360ms ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms - ✓ tests/journal-client-completion.test.ts (6 tests) 101ms - ✓ tests/reuse-summary-interruption.test.ts (5 tests) 3ms - ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms - ✓ tests/direct-run-worker-lease.test.ts (3 tests) 6ms - ✓ tests/helper-effect-lease-loss.test.ts (1 test) 235ms - ✓ tests/journal-client-subscriptions.test.ts (1 test) 7ms + ✓ tests/journal-client-completion.test.ts (6 tests) 104ms + ✓ tests/reuse-summary-interruption.test.ts (5 tests) 4ms + ✓ tests/resume-worker-lease.test.ts (3 tests) 8ms + ✓ tests/direct-run-worker-lease.test.ts (3 tests) 9ms + ✓ tests/helper-effect-lease-loss.test.ts (1 test) 217ms + ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 4ms - ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 31ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms + ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 45ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms Test Files 27 passed (27) - Tests 327 passed (327) - Start at 02:56:50 - Duration 58.09s (transform 1.10s, setup 154ms, collect 6.42s, tests 48.26s, environment 4ms, prepare 960ms) + Tests 328 passed (328) + Start at 03:07:34 + Duration 57.70s (transform 1.11s, setup 156ms, collect 6.57s, tests 47.71s, environment 4ms, prepare 978ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log index bdd62c0aa..31fe22e1d 100644 --- a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log +++ b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/cli/running-step.ts -Original SHA256: 70c81a8ceed373400410f4dcd753ad714b1f86513e9240344de7807162ad3d64 +Original SHA256: 34223d566de94486b7039f5eb783ddacbe62ccc3cca63c2b5278d72c4de59c38 Replaced: snapshotRead?.abort(new Error(`step "${runningStep.id}" completed during the lease snapshot`)); With: @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (10 tests | 1 failed | 9 skipped) 3012ms - × a completion push ends the wait without waiting out an in-flight snapshot 3012ms + ❯ tests/running-step-watch.test.ts (11 tests | 1 failed | 10 skipped) 3014ms + × a completion push ends the wait without waiting out an in-flight snapshot 3013ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -33,23 +33,23 @@ Received: "still waiting" ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 9 skipped (10) - Start at 02:56:35 - Duration 3.47s (transform 183ms, setup 16ms, collect 309ms, tests 3.01s, environment 0ms, prepare 41ms) + Tests 1 failed | 10 skipped (11) + Start at 03:07:18 + Duration 3.51s (transform 198ms, setup 17ms, collect 331ms, tests 3.01s, environment 0ms, prepare 42ms) exit=1 -RESTORED SHA256: 70c81a8ceed373400410f4dcd753ad714b1f86513e9240344de7807162ad3d64 +RESTORED SHA256: 34223d566de94486b7039f5eb783ddacbe62ccc3cca63c2b5278d72c4de59c38 $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a completion push ends the wait' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (10 tests | 9 skipped) 2062ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2062ms + ✓ tests/running-step-watch.test.ts (11 tests | 10 skipped) 2062ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2061ms Test Files 1 passed (1) - Tests 1 passed | 9 skipped (10) - Start at 02:56:39 - Duration 2.54s (transform 186ms, setup 16ms, collect 312ms, tests 2.06s, environment 0ms, prepare 40ms) + Tests 1 passed | 10 skipped (11) + Start at 03:07:22 + Duration 2.57s (transform 199ms, setup 16ms, collect 343ms, tests 2.06s, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-heartbeat.log b/evidence/run-read-timeout/mutation-heartbeat.log index 6a6d3e7d9..bf3a7f18b 100644 --- a/evidence/run-read-timeout/mutation-heartbeat.log +++ b/evidence/run-read-timeout/mutation-heartbeat.log @@ -13,8 +13,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 19ms - × a heartbeat timeout is lease loss rather than a worker body failure 18ms + ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 17ms + × a heartbeat timeout is lease loss rather than a worker body failure 17ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -33,8 +33,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 02:56:42 - Duration 475ms (transform 170ms, setup 18ms, collect 292ms, tests 19ms, environment 0ms, prepare 41ms) + Start at 03:07:25 + Duration 437ms (transform 169ms, setup 17ms, collect 278ms, tests 17ms, environment 0ms, prepare 33ms) exit=1 @@ -43,11 +43,11 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 02:56:42 - Duration 467ms (transform 169ms, setup 17ms, collect 291ms, tests 15ms, environment 0ms, prepare 41ms) + Start at 03:07:26 + Duration 552ms (transform 195ms, setup 21ms, collect 352ms, tests 16ms, environment 0ms, prepare 45ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-reconnect.log b/evidence/run-read-timeout/mutation-reader-reconnect.log index 14e80c6a7..792d4c618 100644 --- a/evidence/run-read-timeout/mutation-reader-reconnect.log +++ b/evidence/run-read-timeout/mutation-reader-reconnect.log @@ -11,14 +11,14 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (22 tests | 1 failed | 21 skipped) 1008ms - × reconnects the reader after it disconnects within the read budget 1007ms - → journal client: run.get read session was interrupted after 79 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ tests/journal-client-read-timeout.test.ts (22 tests | 1 failed | 21 skipped) 1010ms + × reconnects the reader after it disconnects within the read budget 1009ms + → journal client: run.get read session was interrupted after 77 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 79 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 77 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:71:9 69| let last: unknown; 70| const exhausted = () => last instanceof JournalReadInterruptedError @@ -29,8 +29,8 @@ JournalReadInterruptedError: journal client: run.get read session was interrupte ❯ Timeout. src/journal-read-policy.ts:82:39 Caused by: Error: journal client: not connected (run.get): journal client: connection closed - ❯ src/journal-client.ts:233:16 - ❯ JournalClient.requestOnce src/journal-client.ts:226:12 + ❯ src/journal-client.ts:236:16 + ❯ JournalClient.requestOnce src/journal-client.ts:229:12 ❯ src/journal-budgeted-reads.ts:47:29 ❯ src/journal-read-policy.ts:94:18 @@ -41,8 +41,8 @@ Caused by: Error: journal client: connection closed Test Files 1 failed (1) Tests 1 failed | 21 skipped (22) - Start at 02:56:19 - Duration 1.45s (transform 164ms, setup 16ms, collect 284ms, tests 1.01s, environment 0ms, prepare 41ms) + Start at 03:07:01 + Duration 1.46s (transform 169ms, setup 16ms, collect 291ms, tests 1.01s, environment 0ms, prepare 43ms) exit=1 @@ -51,11 +51,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (22 tests | 21 skipped) 92ms + ✓ tests/journal-client-read-timeout.test.ts (22 tests | 21 skipped) 84ms Test Files 1 passed (1) Tests 1 passed | 21 skipped (22) - Start at 02:56:20 - Duration 543ms (transform 164ms, setup 17ms, collect 286ms, tests 92ms, environment 0ms, prepare 41ms) + Start at 03:07:03 + Duration 732ms (transform 213ms, setup 24ms, collect 411ms, tests 84ms, environment 0ms, prepare 65ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-setup-retry.log b/evidence/run-read-timeout/mutation-reader-setup-retry.log index 8ee80d334..7dba81b7e 100644 --- a/evidence/run-read-timeout/mutation-reader-setup-retry.log +++ b/evidence/run-read-timeout/mutation-reader-setup-retry.log @@ -16,8 +16,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (22 tests | 1 failed | 21 skipped) 78ms - × retries a reader setup that timed out instead of reading on the primary for good 77ms + ❯ tests/journal-client-read-timeout.test.ts (22 tests | 1 failed | 21 skipped) 74ms + × retries a reader setup that timed out instead of reading on the primary for good 73ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -43,8 +43,8 @@ AssertionError: expected 1 to be 2 // Object.is equality Test Files 1 failed (1) Tests 1 failed | 21 skipped (22) - Start at 02:56:21 - Duration 565ms (transform 179ms, setup 17ms, collect 313ms, tests 78ms, environment 0ms, prepare 47ms) + Start at 03:07:04 + Duration 530ms (transform 169ms, setup 16ms, collect 294ms, tests 74ms, environment 0ms, prepare 43ms) exit=1 @@ -53,11 +53,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (22 tests | 21 skipped) 67ms + ✓ tests/journal-client-read-timeout.test.ts (22 tests | 21 skipped) 71ms Test Files 1 passed (1) Tests 1 passed | 21 skipped (22) - Start at 02:56:22 - Duration 519ms (transform 167ms, setup 18ms, collect 290ms, tests 67ms, environment 0ms, prepare 41ms) + Start at 03:07:05 + Duration 525ms (transform 168ms, setup 17ms, collect 290ms, tests 71ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader.log b/evidence/run-read-timeout/mutation-reader.log index 9c422828f..b34b7c95b 100644 --- a/evidence/run-read-timeout/mutation-reader.log +++ b/evidence/run-read-timeout/mutation-reader.log @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (22 tests | 1 failed | 21 skipped) 112ms - × serves a bounded read while an unbounded command is in flight 111ms + ❯ tests/journal-client-read-timeout.test.ts (22 tests | 1 failed | 21 skipped) 113ms + × serves a bounded read while an unbounded command is in flight 112ms → expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -37,8 +37,8 @@ AssertionError: expected true to be false // Object.is equality Test Files 1 failed (1) Tests 1 failed | 21 skipped (22) - Start at 02:56:17 - Duration 565ms (transform 166ms, setup 15ms, collect 298ms, tests 112ms, environment 0ms, prepare 33ms) + Start at 03:06:59 + Duration 555ms (transform 164ms, setup 16ms, collect 290ms, tests 113ms, environment 0ms, prepare 40ms) exit=1 @@ -51,7 +51,7 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t Test Files 1 passed (1) Tests 1 passed | 21 skipped (22) - Start at 02:56:18 - Duration 566ms (transform 169ms, setup 15ms, collect 293ms, tests 107ms, environment 0ms, prepare 45ms) + Start at 03:07:00 + Duration 560ms (transform 166ms, setup 17ms, collect 290ms, tests 107ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-root-parking.log b/evidence/run-read-timeout/mutation-root-parking.log index 012364b5e..b62c3a044 100644 --- a/evidence/run-read-timeout/mutation-root-parking.log +++ b/evidence/run-read-timeout/mutation-root-parking.log @@ -17,14 +17,14 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (28 tests | 3 failed | 25 skipped) 30ms - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 22ms + ❯ tests/authored-root.test.ts (28 tests | 3 failed | 25 skipped) 37ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 28ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms → expected AuthoredFlowExecutionError: daemon_unresp… { …(7) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 3ms + × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 4ms → expected JournalReadInterruptedError: journal clie… { …(4) } to match object { code: 'daemon_unresponsive', …(2) } (5 matching properties omitted from actual) @@ -104,8 +104,8 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } Test Files 1 failed (1) Tests 3 failed | 25 skipped (28) - Start at 02:56:43 - Duration 1.26s (transform 660ms, setup 16ms, collect 1.07s, tests 30ms, environment 0ms, prepare 41ms) + Start at 03:07:27 + Duration 1.54s (transform 769ms, setup 18ms, collect 1.31s, tests 37ms, environment 0ms, prepare 53ms) exit=1 @@ -114,11 +114,11 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (28 tests | 25 skipped) 22ms + ✓ tests/authored-root.test.ts (28 tests | 25 skipped) 16ms Test Files 1 passed (1) Tests 3 passed | 25 skipped (28) - Start at 02:56:45 - Duration 1.27s (transform 656ms, setup 16ms, collect 1.08s, tests 22ms, environment 0ms, prepare 44ms) + Start at 03:07:29 + Duration 1.31s (transform 678ms, setup 19ms, collect 1.12s, tests 16ms, environment 0ms, prepare 42ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-snapshot-cancel.log b/evidence/run-read-timeout/mutation-snapshot-cancel.log index e0153e0a3..2a3619a8c 100644 --- a/evidence/run-read-timeout/mutation-snapshot-cancel.log +++ b/evidence/run-read-timeout/mutation-snapshot-cancel.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/cli/running-step.ts -Original SHA256: 70c81a8ceed373400410f4dcd753ad714b1f86513e9240344de7807162ad3d64 +Original SHA256: 34223d566de94486b7039f5eb783ddacbe62ccc3cca63c2b5278d72c4de59c38 Replaced: const signal = options.signal === undefined ? read.signal : AbortSignal.any([options.signal, read.signal]); @@ -11,8 +11,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (10 tests | 1 failed | 9 skipped) 2216ms - × cancels promptly while a lease snapshot read is in flight 2215ms + ❯ tests/running-step-watch.test.ts (11 tests | 1 failed | 10 skipped) 2213ms + × cancels promptly while a lease snapshot read is in flight 2212ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -30,23 +30,23 @@ AssertionError: expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 9 skipped (10) - Start at 02:56:29 - Duration 2.71s (transform 192ms, setup 18ms, collect 321ms, tests 2.22s, environment 0ms, prepare 45ms) + Tests 1 failed | 10 skipped (11) + Start at 03:07:12 + Duration 2.69s (transform 192ms, setup 16ms, collect 324ms, tests 2.21s, environment 0ms, prepare 40ms) exit=1 -RESTORED SHA256: 70c81a8ceed373400410f4dcd753ad714b1f86513e9240344de7807162ad3d64 +RESTORED SHA256: 34223d566de94486b7039f5eb783ddacbe62ccc3cca63c2b5278d72c4de59c38 $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels promptly while a lease snapshot read is in flight' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (10 tests | 9 skipped) 2011ms + ✓ tests/running-step-watch.test.ts (11 tests | 10 skipped) 2011ms ✓ cancels promptly while a lease snapshot read is in flight 2010ms Test Files 1 passed (1) - Tests 1 passed | 9 skipped (10) - Start at 02:56:32 - Duration 2.49s (transform 186ms, setup 15ms, collect 313ms, tests 2.01s, environment 0ms, prepare 42ms) + Tests 1 passed | 10 skipped (11) + Start at 03:07:15 + Duration 2.48s (transform 188ms, setup 16ms, collect 309ms, tests 2.01s, environment 0ms, prepare 44ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-watch-cadence.log b/evidence/run-read-timeout/mutation-watch-cadence.log index 96988ba24..db866d6ed 100644 --- a/evidence/run-read-timeout/mutation-watch-cadence.log +++ b/evidence/run-read-timeout/mutation-watch-cadence.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/cli/running-step.ts -Original SHA256: 70c81a8ceed373400410f4dcd753ad714b1f86513e9240344de7807162ad3d64 +Original SHA256: 34223d566de94486b7039f5eb783ddacbe62ccc3cca63c2b5278d72c4de59c38 Replaced: const LEASE_POLL_MS = 2_000; With: @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (10 tests | 1 failed | 9 skipped) 2111ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2110ms + ❯ tests/running-step-watch.test.ts (11 tests | 1 failed | 10 skipped) 2108ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2107ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -29,23 +29,23 @@ AssertionError: expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 9 skipped (10) - Start at 02:56:23 - Duration 2.59s (transform 187ms, setup 17ms, collect 313ms, tests 2.11s, environment 0ms, prepare 41ms) + Tests 1 failed | 10 skipped (11) + Start at 03:07:06 + Duration 2.60s (transform 189ms, setup 17ms, collect 330ms, tests 2.11s, environment 0ms, prepare 41ms) exit=1 -RESTORED SHA256: 70c81a8ceed373400410f4dcd753ad714b1f86513e9240344de7807162ad3d64 +RESTORED SHA256: 34223d566de94486b7039f5eb783ddacbe62ccc3cca63c2b5278d72c4de59c38 $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pushes for completion' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (10 tests | 9 skipped) 2120ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2119ms + ✓ tests/running-step-watch.test.ts (11 tests | 10 skipped) 2118ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms Test Files 1 passed (1) - Tests 1 passed | 9 skipped (10) - Start at 02:56:26 - Duration 2.59s (transform 187ms, setup 17ms, collect 322ms, tests 2.12s, environment 0ms, prepare 41ms) + Tests 1 passed | 10 skipped (11) + Start at 03:07:09 + Duration 2.56s (transform 182ms, setup 17ms, collect 291ms, tests 2.12s, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/resume-live.log b/evidence/run-read-timeout/resume-live.log index 6a3e329e8..1148ee31f 100644 --- a/evidence/run-read-timeout/resume-live.log +++ b/evidence/run-read-timeout/resume-live.log @@ -2,12 +2,12 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 935ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 934ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 912ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 911ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 02:57:49 - Duration 2.19s (transform 622ms, setup 20ms, collect 1.02s, tests 935ms, environment 0ms, prepare 85ms) + Start at 03:08:32 + Duration 2.19s (transform 626ms, setup 19ms, collect 1.04s, tests 912ms, environment 0ms, prepare 53ms) exit=0 diff --git a/evidence/run-read-timeout/verification.md b/evidence/run-read-timeout/verification.md index e892100c9..d3164ce89 100644 --- a/evidence/run-read-timeout/verification.md +++ b/evidence/run-read-timeout/verification.md @@ -18,67 +18,67 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo ✓ tests/cli.test.ts (71 tests) 5320ms ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 326ms ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2038ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2037ms -(node:3863527) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2046ms +(node:3908233) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (28 tests) 361ms - ✓ tests/journal-client.test.ts (17 tests) 82ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 7494ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 522ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1115ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2804ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 424ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 948ms - ✓ tests/authored-flow-slack.test.ts (7 tests) 1242ms - ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 444ms - ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 424ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2484ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 670ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 501ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 644ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 668ms - ✓ tests/journal-client-read-timeout.test.ts (22 tests) 1729ms - ✓ tests/authored-helpers.test.ts (6 tests) 1856ms - ✓ runs every available provider through the real kernel and resumes completed effects without a second write 880ms - ✓ replays after SIGKILL before confirm with the same token and one successful completion 332ms - ✓ replays after SIGKILL before complete with the same token and one successful completion 415ms - ✓ tests/classify-outcome.test.ts (11 tests) 7430ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2015ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3006ms - ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2002ms - ✓ tests/running-step-watch.test.ts (10 tests) 10266ms + ✓ tests/authored-root.test.ts (28 tests) 363ms + ✓ tests/journal-client.test.ts (17 tests) 80ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 6984ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 467ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1065ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2779ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 375ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 751ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 1188ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 433ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 417ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2530ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 580ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 627ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 703ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 618ms + ✓ tests/journal-client-read-timeout.test.ts (22 tests) 1730ms + ✓ tests/running-step-watch.test.ts (11 tests) 10283ms ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms ✓ cancels promptly while a lease snapshot read is in flight 2006ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2055ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2053ms ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms - ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2004ms - ✓ tests/human-live.test.ts (3 tests) 4601ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2743ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1127ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 730ms - ✓ tests/worker-lease.test.ts (7 tests) 15ms - ✓ tests/communication-worker.test.ts (18 tests) 1711ms + ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2002ms + ✓ tests/authored-helpers.test.ts (6 tests) 1748ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 760ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 379ms + ✓ replays after SIGKILL before complete with the same token and one successful completion 393ms + ✓ tests/classify-outcome.test.ts (11 tests) 7434ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2017ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3008ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms + ✓ tests/human-live.test.ts (3 tests) 4717ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2802ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1179ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 736ms + ✓ tests/worker-lease.test.ts (7 tests) 19ms + ✓ tests/communication-worker.test.ts (18 tests) 1714ms ✓ tests/worker-lease-lost.test.ts (17 tests) 25ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 699ms - ✓ tests/memoization.test.ts (58 tests) 66ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2465ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 672ms + ✓ tests/memoization.test.ts (58 tests) 65ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2419ms ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2058ms - ✓ drains read and watch promises before an authored flow completes 406ms + ✓ drains read and watch promises before an authored flow completes 360ms ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms - ✓ tests/journal-client-completion.test.ts (6 tests) 101ms - ✓ tests/reuse-summary-interruption.test.ts (5 tests) 3ms - ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms - ✓ tests/direct-run-worker-lease.test.ts (3 tests) 6ms - ✓ tests/helper-effect-lease-loss.test.ts (1 test) 235ms - ✓ tests/journal-client-subscriptions.test.ts (1 test) 7ms + ✓ tests/journal-client-completion.test.ts (6 tests) 104ms + ✓ tests/reuse-summary-interruption.test.ts (5 tests) 4ms + ✓ tests/resume-worker-lease.test.ts (3 tests) 8ms + ✓ tests/direct-run-worker-lease.test.ts (3 tests) 9ms + ✓ tests/helper-effect-lease-loss.test.ts (1 test) 217ms + ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 4ms - ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 31ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms + ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 45ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms Test Files 27 passed (27) - Tests 327 passed (327) - Start at 02:56:50 - Duration 58.09s (transform 1.10s, setup 154ms, collect 6.42s, tests 48.26s, environment 4ms, prepare 960ms) + Tests 328 passed (328) + Start at 03:07:34 + Duration 57.70s (transform 1.11s, setup 156ms, collect 6.57s, tests 47.71s, environment 4ms, prepare 978ms) exit=0 ``` @@ -90,13 +90,13 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 935ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 934ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 912ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 911ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 02:57:49 - Duration 2.19s (transform 622ms, setup 20ms, collect 1.02s, tests 935ms, environment 0ms, prepare 85ms) + Start at 03:08:32 + Duration 2.19s (transform 626ms, setup 19ms, collect 1.04s, tests 912ms, environment 0ms, prepare 53ms) exit=0 ``` @@ -118,8 +118,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (22 tests | 1 failed | 21 skipped) 112ms - × serves a bounded read while an unbounded command is in flight 111ms + ❯ tests/journal-client-read-timeout.test.ts (22 tests | 1 failed | 21 skipped) 113ms + × serves a bounded read while an unbounded command is in flight 112ms → expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -145,8 +145,8 @@ AssertionError: expected true to be false // Object.is equality Test Files 1 failed (1) Tests 1 failed | 21 skipped (22) - Start at 02:56:17 - Duration 565ms (transform 166ms, setup 15ms, collect 298ms, tests 112ms, environment 0ms, prepare 33ms) + Start at 03:06:59 + Duration 555ms (transform 164ms, setup 16ms, collect 290ms, tests 113ms, environment 0ms, prepare 40ms) exit=1 @@ -159,8 +159,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t Test Files 1 passed (1) Tests 1 passed | 21 skipped (22) - Start at 02:56:18 - Duration 566ms (transform 169ms, setup 15ms, collect 293ms, tests 107ms, environment 0ms, prepare 45ms) + Start at 03:07:00 + Duration 560ms (transform 166ms, setup 17ms, collect 290ms, tests 107ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -181,14 +181,14 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (22 tests | 1 failed | 21 skipped) 1008ms - × reconnects the reader after it disconnects within the read budget 1007ms - → journal client: run.get read session was interrupted after 79 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ tests/journal-client-read-timeout.test.ts (22 tests | 1 failed | 21 skipped) 1010ms + × reconnects the reader after it disconnects within the read budget 1009ms + → journal client: run.get read session was interrupted after 77 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 79 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 77 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:71:9 69| let last: unknown; 70| const exhausted = () => last instanceof JournalReadInterruptedError @@ -199,8 +199,8 @@ JournalReadInterruptedError: journal client: run.get read session was interrupte ❯ Timeout. src/journal-read-policy.ts:82:39 Caused by: Error: journal client: not connected (run.get): journal client: connection closed - ❯ src/journal-client.ts:233:16 - ❯ JournalClient.requestOnce src/journal-client.ts:226:12 + ❯ src/journal-client.ts:236:16 + ❯ JournalClient.requestOnce src/journal-client.ts:229:12 ❯ src/journal-budgeted-reads.ts:47:29 ❯ src/journal-read-policy.ts:94:18 @@ -211,8 +211,8 @@ Caused by: Error: journal client: connection closed Test Files 1 failed (1) Tests 1 failed | 21 skipped (22) - Start at 02:56:19 - Duration 1.45s (transform 164ms, setup 16ms, collect 284ms, tests 1.01s, environment 0ms, prepare 41ms) + Start at 03:07:01 + Duration 1.46s (transform 169ms, setup 16ms, collect 291ms, tests 1.01s, environment 0ms, prepare 43ms) exit=1 @@ -221,12 +221,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (22 tests | 21 skipped) 92ms + ✓ tests/journal-client-read-timeout.test.ts (22 tests | 21 skipped) 84ms Test Files 1 passed (1) Tests 1 passed | 21 skipped (22) - Start at 02:56:20 - Duration 543ms (transform 164ms, setup 17ms, collect 286ms, tests 92ms, environment 0ms, prepare 41ms) + Start at 03:07:03 + Duration 732ms (transform 213ms, setup 24ms, collect 411ms, tests 84ms, environment 0ms, prepare 65ms) exit=0 ``` @@ -252,8 +252,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (22 tests | 1 failed | 21 skipped) 78ms - × retries a reader setup that timed out instead of reading on the primary for good 77ms + ❯ tests/journal-client-read-timeout.test.ts (22 tests | 1 failed | 21 skipped) 74ms + × retries a reader setup that timed out instead of reading on the primary for good 73ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -279,8 +279,8 @@ AssertionError: expected 1 to be 2 // Object.is equality Test Files 1 failed (1) Tests 1 failed | 21 skipped (22) - Start at 02:56:21 - Duration 565ms (transform 179ms, setup 17ms, collect 313ms, tests 78ms, environment 0ms, prepare 47ms) + Start at 03:07:04 + Duration 530ms (transform 169ms, setup 16ms, collect 294ms, tests 74ms, environment 0ms, prepare 43ms) exit=1 @@ -289,12 +289,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (22 tests | 21 skipped) 67ms + ✓ tests/journal-client-read-timeout.test.ts (22 tests | 21 skipped) 71ms Test Files 1 passed (1) Tests 1 passed | 21 skipped (22) - Start at 02:56:22 - Duration 519ms (transform 167ms, setup 18ms, collect 290ms, tests 67ms, environment 0ms, prepare 41ms) + Start at 03:07:05 + Duration 525ms (transform 168ms, setup 17ms, collect 290ms, tests 71ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -303,7 +303,7 @@ exit=0 ```text Source: packages/sdk/src/cli/running-step.ts -Original SHA256: 70c81a8ceed373400410f4dcd753ad714b1f86513e9240344de7807162ad3d64 +Original SHA256: 34223d566de94486b7039f5eb783ddacbe62ccc3cca63c2b5278d72c4de59c38 Replaced: const LEASE_POLL_MS = 2_000; With: @@ -314,8 +314,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (10 tests | 1 failed | 9 skipped) 2111ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2110ms + ❯ tests/running-step-watch.test.ts (11 tests | 1 failed | 10 skipped) 2108ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2107ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -333,24 +333,24 @@ AssertionError: expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 9 skipped (10) - Start at 02:56:23 - Duration 2.59s (transform 187ms, setup 17ms, collect 313ms, tests 2.11s, environment 0ms, prepare 41ms) + Tests 1 failed | 10 skipped (11) + Start at 03:07:06 + Duration 2.60s (transform 189ms, setup 17ms, collect 330ms, tests 2.11s, environment 0ms, prepare 41ms) exit=1 -RESTORED SHA256: 70c81a8ceed373400410f4dcd753ad714b1f86513e9240344de7807162ad3d64 +RESTORED SHA256: 34223d566de94486b7039f5eb783ddacbe62ccc3cca63c2b5278d72c4de59c38 $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pushes for completion' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (10 tests | 9 skipped) 2120ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2119ms + ✓ tests/running-step-watch.test.ts (11 tests | 10 skipped) 2118ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms Test Files 1 passed (1) - Tests 1 passed | 9 skipped (10) - Start at 02:56:26 - Duration 2.59s (transform 187ms, setup 17ms, collect 322ms, tests 2.12s, environment 0ms, prepare 41ms) + Tests 1 passed | 10 skipped (11) + Start at 03:07:09 + Duration 2.56s (transform 182ms, setup 17ms, collect 291ms, tests 2.12s, environment 0ms, prepare 41ms) exit=0 ``` @@ -359,7 +359,7 @@ exit=0 ```text Source: packages/sdk/src/cli/running-step.ts -Original SHA256: 70c81a8ceed373400410f4dcd753ad714b1f86513e9240344de7807162ad3d64 +Original SHA256: 34223d566de94486b7039f5eb783ddacbe62ccc3cca63c2b5278d72c4de59c38 Replaced: const signal = options.signal === undefined ? read.signal : AbortSignal.any([options.signal, read.signal]); @@ -371,8 +371,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (10 tests | 1 failed | 9 skipped) 2216ms - × cancels promptly while a lease snapshot read is in flight 2215ms + ❯ tests/running-step-watch.test.ts (11 tests | 1 failed | 10 skipped) 2213ms + × cancels promptly while a lease snapshot read is in flight 2212ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -390,24 +390,24 @@ AssertionError: expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 9 skipped (10) - Start at 02:56:29 - Duration 2.71s (transform 192ms, setup 18ms, collect 321ms, tests 2.22s, environment 0ms, prepare 45ms) + Tests 1 failed | 10 skipped (11) + Start at 03:07:12 + Duration 2.69s (transform 192ms, setup 16ms, collect 324ms, tests 2.21s, environment 0ms, prepare 40ms) exit=1 -RESTORED SHA256: 70c81a8ceed373400410f4dcd753ad714b1f86513e9240344de7807162ad3d64 +RESTORED SHA256: 34223d566de94486b7039f5eb783ddacbe62ccc3cca63c2b5278d72c4de59c38 $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels promptly while a lease snapshot read is in flight' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (10 tests | 9 skipped) 2011ms + ✓ tests/running-step-watch.test.ts (11 tests | 10 skipped) 2011ms ✓ cancels promptly while a lease snapshot read is in flight 2010ms Test Files 1 passed (1) - Tests 1 passed | 9 skipped (10) - Start at 02:56:32 - Duration 2.49s (transform 186ms, setup 15ms, collect 313ms, tests 2.01s, environment 0ms, prepare 42ms) + Tests 1 passed | 10 skipped (11) + Start at 03:07:15 + Duration 2.48s (transform 188ms, setup 16ms, collect 309ms, tests 2.01s, environment 0ms, prepare 44ms) exit=0 ``` @@ -416,7 +416,7 @@ exit=0 ```text Source: packages/sdk/src/cli/running-step.ts -Original SHA256: 70c81a8ceed373400410f4dcd753ad714b1f86513e9240344de7807162ad3d64 +Original SHA256: 34223d566de94486b7039f5eb783ddacbe62ccc3cca63c2b5278d72c4de59c38 Replaced: snapshotRead?.abort(new Error(`step "${runningStep.id}" completed during the lease snapshot`)); With: @@ -427,8 +427,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (10 tests | 1 failed | 9 skipped) 3012ms - × a completion push ends the wait without waiting out an in-flight snapshot 3012ms + ❯ tests/running-step-watch.test.ts (11 tests | 1 failed | 10 skipped) 3014ms + × a completion push ends the wait without waiting out an in-flight snapshot 3013ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -450,24 +450,24 @@ Received: "still waiting" ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 9 skipped (10) - Start at 02:56:35 - Duration 3.47s (transform 183ms, setup 16ms, collect 309ms, tests 3.01s, environment 0ms, prepare 41ms) + Tests 1 failed | 10 skipped (11) + Start at 03:07:18 + Duration 3.51s (transform 198ms, setup 17ms, collect 331ms, tests 3.01s, environment 0ms, prepare 42ms) exit=1 -RESTORED SHA256: 70c81a8ceed373400410f4dcd753ad714b1f86513e9240344de7807162ad3d64 +RESTORED SHA256: 34223d566de94486b7039f5eb783ddacbe62ccc3cca63c2b5278d72c4de59c38 $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a completion push ends the wait' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (10 tests | 9 skipped) 2062ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2062ms + ✓ tests/running-step-watch.test.ts (11 tests | 10 skipped) 2062ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2061ms Test Files 1 passed (1) - Tests 1 passed | 9 skipped (10) - Start at 02:56:39 - Duration 2.54s (transform 186ms, setup 16ms, collect 312ms, tests 2.06s, environment 0ms, prepare 40ms) + Tests 1 passed | 10 skipped (11) + Start at 03:07:22 + Duration 2.57s (transform 199ms, setup 16ms, collect 343ms, tests 2.06s, environment 0ms, prepare 41ms) exit=0 ``` @@ -490,8 +490,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 19ms - × a heartbeat timeout is lease loss rather than a worker body failure 18ms + ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 17ms + × a heartbeat timeout is lease loss rather than a worker body failure 17ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -510,8 +510,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 02:56:42 - Duration 475ms (transform 170ms, setup 18ms, collect 292ms, tests 19ms, environment 0ms, prepare 41ms) + Start at 03:07:25 + Duration 437ms (transform 169ms, setup 17ms, collect 278ms, tests 17ms, environment 0ms, prepare 33ms) exit=1 @@ -520,12 +520,12 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 02:56:42 - Duration 467ms (transform 169ms, setup 17ms, collect 291ms, tests 15ms, environment 0ms, prepare 41ms) + Start at 03:07:26 + Duration 552ms (transform 195ms, setup 21ms, collect 352ms, tests 16ms, environment 0ms, prepare 45ms) exit=0 ``` @@ -552,14 +552,14 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (28 tests | 3 failed | 25 skipped) 30ms - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 22ms + ❯ tests/authored-root.test.ts (28 tests | 3 failed | 25 skipped) 37ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 28ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms → expected AuthoredFlowExecutionError: daemon_unresp… { …(7) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 3ms + × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 4ms → expected JournalReadInterruptedError: journal clie… { …(4) } to match object { code: 'daemon_unresponsive', …(2) } (5 matching properties omitted from actual) @@ -639,8 +639,8 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } Test Files 1 failed (1) Tests 3 failed | 25 skipped (28) - Start at 02:56:43 - Duration 1.26s (transform 660ms, setup 16ms, collect 1.07s, tests 30ms, environment 0ms, prepare 41ms) + Start at 03:07:27 + Duration 1.54s (transform 769ms, setup 18ms, collect 1.31s, tests 37ms, environment 0ms, prepare 53ms) exit=1 @@ -649,12 +649,12 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (28 tests | 25 skipped) 22ms + ✓ tests/authored-root.test.ts (28 tests | 25 skipped) 16ms Test Files 1 passed (1) Tests 3 passed | 25 skipped (28) - Start at 02:56:45 - Duration 1.27s (transform 656ms, setup 16ms, collect 1.08s, tests 22ms, environment 0ms, prepare 44ms) + Start at 03:07:29 + Duration 1.31s (transform 678ms, setup 19ms, collect 1.12s, tests 16ms, environment 0ms, prepare 42ms) exit=0 ``` diff --git a/packages/sdk/src/cli/running-step.ts b/packages/sdk/src/cli/running-step.ts index c23207f84..0ae332e72 100644 --- a/packages/sdk/src/cli/running-step.ts +++ b/packages/sdk/src/cli/running-step.ts @@ -55,6 +55,12 @@ export async function waitForRunningStep(client: JournalClient, runId: string, let protocolFailure: unknown; const onProtocolError = (error: unknown) => { protocolFailure ??= error; fail(error); snapshotRead?.abort(error); }; watch.on('protocol_error', onProtocolError); + // A dropped watch loses its completion pushes: a resumable read interruption. + const onDisconnected = (error: Error) => { + if (!settled) fail(new AuthoredFlowExecutionError('daemon_unresponsive', + `the completion watch for step "${runningStep.id}" disconnected: ${error.message}`)); + }; + watch.on('disconnected', onDisconnected); options.signal?.addEventListener('abort', cancel, { once: true }); if (options.signal?.aborted) cancel(); try { @@ -114,6 +120,7 @@ export async function waitForRunningStep(client: JournalClient, runId: string, options.signal?.removeEventListener('abort', cancel); watch.off('entry', onEntry); watch.off('protocol_error', onProtocolError); + watch.off('disconnected', onDisconnected); watch.close(); } } diff --git a/packages/sdk/src/journal-client.ts b/packages/sdk/src/journal-client.ts index 96e4cd42b..af014e29f 100644 --- a/packages/sdk/src/journal-client.ts +++ b/packages/sdk/src/journal-client.ts @@ -135,6 +135,9 @@ export class JournalClient extends EventEmitter { this.budgeted?.close(closed); this.disconnectCause ??= closed; this.failAll(closed); + // Only a drop the caller did not ask for; close() clears this.socket first. + // Sessions with nothing pending (a registered watch) must hear it too. + if (this.socket === socket) this.emit('disconnected', closed); }); this.disconnectCause = undefined; this.connectingSocket = undefined; diff --git a/packages/sdk/tests/running-step-watch.test.ts b/packages/sdk/tests/running-step-watch.test.ts index 6857e81ef..ecc7a654a 100644 --- a/packages/sdk/tests/running-step-watch.test.ts +++ b/packages/sdk/tests/running-step-watch.test.ts @@ -230,3 +230,22 @@ it('a malformed watch frame during an in-flight snapshot is an error, not a comp expect(outcome).toBeInstanceOf(JournalFrameError); } finally { client.close(); await new Promise(resolve => server.close(() => resolve())); } }); + +it('a watch connection lost after registration is a read interruption, not a silent wait', async () => { + const path = sockPath(); + const server = startLoopback(path, { hello: sendOk, 'run.watch': ctx => { + sendResult(ctx, {}); + setTimeout(() => ctx.socket.destroy(), 20); + } }); + await once(server, 'listening'); + const client = new JournalClient(path); + try { + await client.connect(); + const outcome = await Promise.race([ + waitForRunningStep(client, 'run', { id: 'step', type: 'agent', leaseDeadlineMs: Date.now() + 30_000 }, {}).then(() => 'returned', error => error), + sleep(1_000).then(() => 'still waiting'), + ]); + expect(outcome).toBeInstanceOf(AuthoredFlowExecutionError); + expect(isReadInterruption(outcome)).toBe(true); + } finally { client.close(); await new Promise(resolve => server.close(() => resolve())); } +}); From 89b2c69d7bf3772191767dcb4f433a381d9eaddb Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 03:16:55 -0700 Subject: [PATCH 19/34] fix(sdk): budget the handshake only for post-admission sessions Every top-level flows run/resume/direct-run client carries a read budget, so budgeting hello there turned the initial compatibility check from a 30 s refusal into up to 5 minutes of retries, contrary to SURFACE.md and connect's contract. The handshake budget is now an explicit budgetHandshake option, set only by the authored child (authored-node-entry) and its verifier. Evidence re-captured at this tree. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: b8262468-ea26-4a6e-bee9-d193832e77dc --- evidence/run-read-timeout/final-focused.log | 102 ++++---- .../mutation-completion-aborts-snapshot.log | 16 +- .../run-read-timeout/mutation-heartbeat.log | 12 +- .../mutation-reader-reconnect.log | 28 +- .../mutation-reader-setup-retry.log | 18 +- evidence/run-read-timeout/mutation-reader.log | 16 +- .../mutation-root-parking.log | 16 +- .../mutation-snapshot-cancel.log | 14 +- .../mutation-watch-cadence.log | 16 +- evidence/run-read-timeout/resume-live.log | 8 +- evidence/run-read-timeout/verification.md | 246 +++++++++--------- packages/sdk/src/authored-node-entry.ts | 2 +- packages/sdk/src/authored-node-runner.ts | 2 +- packages/sdk/src/journal-client.ts | 10 +- .../authored-verifier-read-budget.test.ts | 2 +- .../tests/journal-client-read-timeout.test.ts | 26 +- 16 files changed, 280 insertions(+), 254 deletions(-) diff --git a/evidence/run-read-timeout/final-focused.log b/evidence/run-read-timeout/final-focused.log index 605148a69..b84bcfdb1 100644 --- a/evidence/run-read-timeout/final-focused.log +++ b/evidence/run-read-timeout/final-focused.log @@ -2,69 +2,69 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5320ms - ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 326ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2038ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2046ms -(node:3908233) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5293ms + ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 312ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2036ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2038ms +(node:3978255) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (28 tests) 363ms - ✓ tests/journal-client.test.ts (17 tests) 80ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 6984ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 467ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1065ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2779ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 375ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 751ms - ✓ tests/authored-flow-slack.test.ts (7 tests) 1188ms - ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 433ms - ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 417ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2530ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 580ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 627ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 703ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 618ms - ✓ tests/journal-client-read-timeout.test.ts (22 tests) 1730ms - ✓ tests/running-step-watch.test.ts (11 tests) 10283ms + ✓ tests/authored-root.test.ts (28 tests) 391ms + ✓ tests/journal-client.test.ts (17 tests) 78ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 6906ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 488ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1069ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2720ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 344ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 848ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 1174ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 411ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 418ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2558ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 627ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 623ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 656ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 651ms + ✓ tests/journal-client-read-timeout.test.ts (23 tests) 1726ms + ✓ tests/running-step-watch.test.ts (11 tests) 10286ms ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms - ✓ cancels promptly while a lease snapshot read is in flight 2006ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2053ms + ✓ cancels promptly while a lease snapshot read is in flight 2004ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2054ms ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms - ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2002ms - ✓ tests/authored-helpers.test.ts (6 tests) 1748ms - ✓ runs every available provider through the real kernel and resumes completed effects without a second write 760ms - ✓ replays after SIGKILL before confirm with the same token and one successful completion 379ms - ✓ replays after SIGKILL before complete with the same token and one successful completion 393ms - ✓ tests/classify-outcome.test.ts (11 tests) 7434ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2017ms + ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2004ms + ✓ tests/authored-helpers.test.ts (6 tests) 1949ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 885ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 421ms + ✓ replays after SIGKILL before complete with the same token and one successful completion 418ms + ✓ tests/classify-outcome.test.ts (11 tests) 7435ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2019ms ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3008ms ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms - ✓ tests/human-live.test.ts (3 tests) 4717ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2802ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1179ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 736ms + ✓ tests/human-live.test.ts (3 tests) 4487ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2652ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1151ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 683ms ✓ tests/worker-lease.test.ts (7 tests) 19ms - ✓ tests/communication-worker.test.ts (18 tests) 1714ms + ✓ tests/communication-worker.test.ts (18 tests) 1706ms ✓ tests/worker-lease-lost.test.ts (17 tests) 25ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 672ms - ✓ tests/memoization.test.ts (58 tests) 65ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2419ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2058ms - ✓ drains read and watch promises before an authored flow completes 360ms - ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms - ✓ tests/journal-client-completion.test.ts (6 tests) 104ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 705ms + ✓ tests/memoization.test.ts (58 tests) 62ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2469ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2055ms + ✓ drains read and watch promises before an authored flow completes 413ms + ✓ tests/worker-lease-sweep.test.ts (4 tests) 6ms + ✓ tests/journal-client-completion.test.ts (6 tests) 102ms ✓ tests/reuse-summary-interruption.test.ts (5 tests) 4ms - ✓ tests/resume-worker-lease.test.ts (3 tests) 8ms + ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms ✓ tests/direct-run-worker-lease.test.ts (3 tests) 9ms - ✓ tests/helper-effect-lease-loss.test.ts (1 test) 217ms + ✓ tests/helper-effect-lease-loss.test.ts (1 test) 204ms ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 4ms - ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 45ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms + ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 46ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 17ms Test Files 27 passed (27) - Tests 328 passed (328) - Start at 03:07:34 - Duration 57.70s (transform 1.11s, setup 156ms, collect 6.57s, tests 47.71s, environment 4ms, prepare 978ms) + Tests 329 passed (329) + Start at 03:15:44 + Duration 57.61s (transform 1.13s, setup 149ms, collect 6.59s, tests 47.68s, environment 4ms, prepare 953ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log index 31fe22e1d..004ef96b9 100644 --- a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log +++ b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (11 tests | 1 failed | 10 skipped) 3014ms - × a completion push ends the wait without waiting out an in-flight snapshot 3013ms + ❯ tests/running-step-watch.test.ts (11 tests | 1 failed | 10 skipped) 3012ms + × a completion push ends the wait without waiting out an in-flight snapshot 3012ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -34,8 +34,8 @@ Received: "still waiting" Test Files 1 failed (1) Tests 1 failed | 10 skipped (11) - Start at 03:07:18 - Duration 3.51s (transform 198ms, setup 17ms, collect 331ms, tests 3.01s, environment 0ms, prepare 42ms) + Start at 03:15:29 + Duration 3.49s (transform 187ms, setup 17ms, collect 317ms, tests 3.01s, environment 0ms, prepare 40ms) exit=1 @@ -44,12 +44,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (11 tests | 10 skipped) 2062ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2061ms + ✓ tests/running-step-watch.test.ts (11 tests | 10 skipped) 2059ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2059ms Test Files 1 passed (1) Tests 1 passed | 10 skipped (11) - Start at 03:07:22 - Duration 2.57s (transform 199ms, setup 16ms, collect 343ms, tests 2.06s, environment 0ms, prepare 41ms) + Start at 03:15:33 + Duration 2.51s (transform 180ms, setup 16ms, collect 300ms, tests 2.06s, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-heartbeat.log b/evidence/run-read-timeout/mutation-heartbeat.log index bf3a7f18b..ed865067a 100644 --- a/evidence/run-read-timeout/mutation-heartbeat.log +++ b/evidence/run-read-timeout/mutation-heartbeat.log @@ -13,8 +13,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 17ms - × a heartbeat timeout is lease loss rather than a worker body failure 17ms + ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 19ms + × a heartbeat timeout is lease loss rather than a worker body failure 19ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -33,8 +33,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 03:07:25 - Duration 437ms (transform 169ms, setup 17ms, collect 278ms, tests 17ms, environment 0ms, prepare 33ms) + Start at 03:15:36 + Duration 473ms (transform 168ms, setup 16ms, collect 291ms, tests 19ms, environment 0ms, prepare 40ms) exit=1 @@ -47,7 +47,7 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb Test Files 1 passed (1) Tests 1 passed (1) - Start at 03:07:26 - Duration 552ms (transform 195ms, setup 21ms, collect 352ms, tests 16ms, environment 0ms, prepare 45ms) + Start at 03:15:37 + Duration 455ms (transform 163ms, setup 16ms, collect 280ms, tests 16ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-reconnect.log b/evidence/run-read-timeout/mutation-reader-reconnect.log index 792d4c618..84b1d5ce2 100644 --- a/evidence/run-read-timeout/mutation-reader-reconnect.log +++ b/evidence/run-read-timeout/mutation-reader-reconnect.log @@ -11,14 +11,14 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (22 tests | 1 failed | 21 skipped) 1010ms - × reconnects the reader after it disconnects within the read budget 1009ms - → journal client: run.get read session was interrupted after 77 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ tests/journal-client-read-timeout.test.ts (23 tests | 1 failed | 22 skipped) 1009ms + × reconnects the reader after it disconnects within the read budget 1008ms + → journal client: run.get read session was interrupted after 78 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 77 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 78 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:71:9 69| let last: unknown; 70| const exhausted = () => last instanceof JournalReadInterruptedError @@ -29,20 +29,20 @@ JournalReadInterruptedError: journal client: run.get read session was interrupte ❯ Timeout. src/journal-read-policy.ts:82:39 Caused by: Error: journal client: not connected (run.get): journal client: connection closed - ❯ src/journal-client.ts:236:16 - ❯ JournalClient.requestOnce src/journal-client.ts:229:12 + ❯ src/journal-client.ts:244:16 + ❯ JournalClient.requestOnce src/journal-client.ts:237:12 ❯ src/journal-budgeted-reads.ts:47:29 ❯ src/journal-read-policy.ts:94:18 Caused by: Error: journal client: connection closed - ❯ Socket. src/journal-client.ts:134:26 + ❯ Socket. src/journal-client.ts:142:26 ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 21 skipped (22) - Start at 03:07:01 - Duration 1.46s (transform 169ms, setup 16ms, collect 291ms, tests 1.01s, environment 0ms, prepare 43ms) + Tests 1 failed | 22 skipped (23) + Start at 03:15:13 + Duration 1.46s (transform 173ms, setup 16ms, collect 299ms, tests 1.01s, environment 0ms, prepare 41ms) exit=1 @@ -51,11 +51,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (22 tests | 21 skipped) 84ms + ✓ tests/journal-client-read-timeout.test.ts (23 tests | 22 skipped) 94ms Test Files 1 passed (1) - Tests 1 passed | 21 skipped (22) - Start at 03:07:03 - Duration 732ms (transform 213ms, setup 24ms, collect 411ms, tests 84ms, environment 0ms, prepare 65ms) + Tests 1 passed | 22 skipped (23) + Start at 03:15:15 + Duration 504ms (transform 165ms, setup 17ms, collect 276ms, tests 94ms, environment 0ms, prepare 30ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-setup-retry.log b/evidence/run-read-timeout/mutation-reader-setup-retry.log index 7dba81b7e..cf3cf4398 100644 --- a/evidence/run-read-timeout/mutation-reader-setup-retry.log +++ b/evidence/run-read-timeout/mutation-reader-setup-retry.log @@ -16,8 +16,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (22 tests | 1 failed | 21 skipped) 74ms - × retries a reader setup that timed out instead of reading on the primary for good 73ms + ❯ tests/journal-client-read-timeout.test.ts (23 tests | 1 failed | 22 skipped) 81ms + × retries a reader setup that timed out instead of reading on the primary for good 80ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -42,9 +42,9 @@ AssertionError: expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 21 skipped (22) - Start at 03:07:04 - Duration 530ms (transform 169ms, setup 16ms, collect 294ms, tests 74ms, environment 0ms, prepare 43ms) + Tests 1 failed | 22 skipped (23) + Start at 03:15:16 + Duration 515ms (transform 166ms, setup 16ms, collect 291ms, tests 81ms, environment 0ms, prepare 31ms) exit=1 @@ -53,11 +53,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (22 tests | 21 skipped) 71ms + ✓ tests/journal-client-read-timeout.test.ts (23 tests | 22 skipped) 69ms Test Files 1 passed (1) - Tests 1 passed | 21 skipped (22) - Start at 03:07:05 - Duration 525ms (transform 168ms, setup 17ms, collect 290ms, tests 71ms, environment 0ms, prepare 41ms) + Tests 1 passed | 22 skipped (23) + Start at 03:15:17 + Duration 532ms (transform 170ms, setup 17ms, collect 299ms, tests 69ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader.log b/evidence/run-read-timeout/mutation-reader.log index b34b7c95b..d69f668eb 100644 --- a/evidence/run-read-timeout/mutation-reader.log +++ b/evidence/run-read-timeout/mutation-reader.log @@ -10,7 +10,7 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (22 tests | 1 failed | 21 skipped) 113ms + ❯ tests/journal-client-read-timeout.test.ts (23 tests | 1 failed | 22 skipped) 113ms × serves a bounded read while an unbounded command is in flight 112ms → expected true to be false // Object.is equality @@ -36,9 +36,9 @@ AssertionError: expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 21 skipped (22) - Start at 03:06:59 - Duration 555ms (transform 164ms, setup 16ms, collect 290ms, tests 113ms, environment 0ms, prepare 40ms) + Tests 1 failed | 22 skipped (23) + Start at 03:15:11 + Duration 592ms (transform 183ms, setup 17ms, collect 325ms, tests 113ms, environment 0ms, prepare 41ms) exit=1 @@ -47,11 +47,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (22 tests | 21 skipped) 107ms + ✓ tests/journal-client-read-timeout.test.ts (23 tests | 22 skipped) 107ms Test Files 1 passed (1) - Tests 1 passed | 21 skipped (22) - Start at 03:07:00 - Duration 560ms (transform 166ms, setup 17ms, collect 290ms, tests 107ms, environment 0ms, prepare 41ms) + Tests 1 passed | 22 skipped (23) + Start at 03:15:12 + Duration 556ms (transform 166ms, setup 16ms, collect 289ms, tests 107ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-root-parking.log b/evidence/run-read-timeout/mutation-root-parking.log index b62c3a044..243dff4a7 100644 --- a/evidence/run-read-timeout/mutation-root-parking.log +++ b/evidence/run-read-timeout/mutation-root-parking.log @@ -17,14 +17,14 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (28 tests | 3 failed | 25 skipped) 37ms - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 28ms + ❯ tests/authored-root.test.ts (28 tests | 3 failed | 25 skipped) 21ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 15ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 3ms → expected AuthoredFlowExecutionError: daemon_unresp… { …(7) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 4ms + × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 2ms → expected JournalReadInterruptedError: journal clie… { …(4) } to match object { code: 'daemon_unresponsive', …(2) } (5 matching properties omitted from actual) @@ -104,8 +104,8 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } Test Files 1 failed (1) Tests 3 failed | 25 skipped (28) - Start at 03:07:27 - Duration 1.54s (transform 769ms, setup 18ms, collect 1.31s, tests 37ms, environment 0ms, prepare 53ms) + Start at 03:15:38 + Duration 1.22s (transform 643ms, setup 14ms, collect 1.04s, tests 21ms, environment 0ms, prepare 41ms) exit=1 @@ -118,7 +118,7 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r Test Files 1 passed (1) Tests 3 passed | 25 skipped (28) - Start at 03:07:29 - Duration 1.31s (transform 678ms, setup 19ms, collect 1.12s, tests 16ms, environment 0ms, prepare 42ms) + Start at 03:15:39 + Duration 1.22s (transform 650ms, setup 16ms, collect 1.06s, tests 16ms, environment 0ms, prepare 30ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-snapshot-cancel.log b/evidence/run-read-timeout/mutation-snapshot-cancel.log index 2a3619a8c..f4b49237d 100644 --- a/evidence/run-read-timeout/mutation-snapshot-cancel.log +++ b/evidence/run-read-timeout/mutation-snapshot-cancel.log @@ -11,8 +11,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (11 tests | 1 failed | 10 skipped) 2213ms - × cancels promptly while a lease snapshot read is in flight 2212ms + ❯ tests/running-step-watch.test.ts (11 tests | 1 failed | 10 skipped) 2216ms + × cancels promptly while a lease snapshot read is in flight 2215ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -31,8 +31,8 @@ AssertionError: expected 'still waiting' to be an instance of Error Test Files 1 failed (1) Tests 1 failed | 10 skipped (11) - Start at 03:07:12 - Duration 2.69s (transform 192ms, setup 16ms, collect 324ms, tests 2.21s, environment 0ms, prepare 40ms) + Start at 03:15:24 + Duration 2.70s (transform 188ms, setup 17ms, collect 317ms, tests 2.22s, environment 0ms, prepare 41ms) exit=1 @@ -42,11 +42,11 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk ✓ tests/running-step-watch.test.ts (11 tests | 10 skipped) 2011ms - ✓ cancels promptly while a lease snapshot read is in flight 2010ms + ✓ cancels promptly while a lease snapshot read is in flight 2011ms Test Files 1 passed (1) Tests 1 passed | 10 skipped (11) - Start at 03:07:15 - Duration 2.48s (transform 188ms, setup 16ms, collect 309ms, tests 2.01s, environment 0ms, prepare 44ms) + Start at 03:15:27 + Duration 2.48s (transform 185ms, setup 17ms, collect 309ms, tests 2.01s, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-watch-cadence.log b/evidence/run-read-timeout/mutation-watch-cadence.log index db866d6ed..d8d2e7539 100644 --- a/evidence/run-read-timeout/mutation-watch-cadence.log +++ b/evidence/run-read-timeout/mutation-watch-cadence.log @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (11 tests | 1 failed | 10 skipped) 2108ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2107ms + ❯ tests/running-step-watch.test.ts (11 tests | 1 failed | 10 skipped) 2109ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2108ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -30,8 +30,8 @@ AssertionError: expected 41 to be less than or equal to 1 Test Files 1 failed (1) Tests 1 failed | 10 skipped (11) - Start at 03:07:06 - Duration 2.60s (transform 189ms, setup 17ms, collect 330ms, tests 2.11s, environment 0ms, prepare 41ms) + Start at 03:15:18 + Duration 2.58s (transform 186ms, setup 16ms, collect 321ms, tests 2.11s, environment 0ms, prepare 41ms) exit=1 @@ -40,12 +40,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (11 tests | 10 skipped) 2118ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms + ✓ tests/running-step-watch.test.ts (11 tests | 10 skipped) 2117ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2116ms Test Files 1 passed (1) Tests 1 passed | 10 skipped (11) - Start at 03:07:09 - Duration 2.56s (transform 182ms, setup 17ms, collect 291ms, tests 2.12s, environment 0ms, prepare 41ms) + Start at 03:15:21 + Duration 2.58s (transform 184ms, setup 16ms, collect 310ms, tests 2.12s, environment 0ms, prepare 42ms) exit=0 diff --git a/evidence/run-read-timeout/resume-live.log b/evidence/run-read-timeout/resume-live.log index 1148ee31f..da4ddd84a 100644 --- a/evidence/run-read-timeout/resume-live.log +++ b/evidence/run-read-timeout/resume-live.log @@ -2,12 +2,12 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 912ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 911ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 928ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 927ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 03:08:32 - Duration 2.19s (transform 626ms, setup 19ms, collect 1.04s, tests 912ms, environment 0ms, prepare 53ms) + Start at 03:16:42 + Duration 2.23s (transform 626ms, setup 18ms, collect 1.06s, tests 928ms, environment 0ms, prepare 68ms) exit=0 diff --git a/evidence/run-read-timeout/verification.md b/evidence/run-read-timeout/verification.md index d3164ce89..d13535f7a 100644 --- a/evidence/run-read-timeout/verification.md +++ b/evidence/run-read-timeout/verification.md @@ -15,70 +15,70 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5320ms - ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 326ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2038ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2046ms -(node:3908233) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5293ms + ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 312ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2036ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2038ms +(node:3978255) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (28 tests) 363ms - ✓ tests/journal-client.test.ts (17 tests) 80ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 6984ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 467ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1065ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2779ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 375ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 751ms - ✓ tests/authored-flow-slack.test.ts (7 tests) 1188ms - ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 433ms - ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 417ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2530ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 580ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 627ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 703ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 618ms - ✓ tests/journal-client-read-timeout.test.ts (22 tests) 1730ms - ✓ tests/running-step-watch.test.ts (11 tests) 10283ms + ✓ tests/authored-root.test.ts (28 tests) 391ms + ✓ tests/journal-client.test.ts (17 tests) 78ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 6906ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 488ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1069ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2720ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 344ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 848ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 1174ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 411ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 418ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2558ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 627ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 623ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 656ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 651ms + ✓ tests/journal-client-read-timeout.test.ts (23 tests) 1726ms + ✓ tests/running-step-watch.test.ts (11 tests) 10286ms ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms - ✓ cancels promptly while a lease snapshot read is in flight 2006ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2053ms + ✓ cancels promptly while a lease snapshot read is in flight 2004ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2054ms ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms - ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2002ms - ✓ tests/authored-helpers.test.ts (6 tests) 1748ms - ✓ runs every available provider through the real kernel and resumes completed effects without a second write 760ms - ✓ replays after SIGKILL before confirm with the same token and one successful completion 379ms - ✓ replays after SIGKILL before complete with the same token and one successful completion 393ms - ✓ tests/classify-outcome.test.ts (11 tests) 7434ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2017ms + ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2004ms + ✓ tests/authored-helpers.test.ts (6 tests) 1949ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 885ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 421ms + ✓ replays after SIGKILL before complete with the same token and one successful completion 418ms + ✓ tests/classify-outcome.test.ts (11 tests) 7435ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2019ms ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3008ms ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms - ✓ tests/human-live.test.ts (3 tests) 4717ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2802ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1179ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 736ms + ✓ tests/human-live.test.ts (3 tests) 4487ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2652ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1151ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 683ms ✓ tests/worker-lease.test.ts (7 tests) 19ms - ✓ tests/communication-worker.test.ts (18 tests) 1714ms + ✓ tests/communication-worker.test.ts (18 tests) 1706ms ✓ tests/worker-lease-lost.test.ts (17 tests) 25ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 672ms - ✓ tests/memoization.test.ts (58 tests) 65ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2419ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2058ms - ✓ drains read and watch promises before an authored flow completes 360ms - ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms - ✓ tests/journal-client-completion.test.ts (6 tests) 104ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 705ms + ✓ tests/memoization.test.ts (58 tests) 62ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2469ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2055ms + ✓ drains read and watch promises before an authored flow completes 413ms + ✓ tests/worker-lease-sweep.test.ts (4 tests) 6ms + ✓ tests/journal-client-completion.test.ts (6 tests) 102ms ✓ tests/reuse-summary-interruption.test.ts (5 tests) 4ms - ✓ tests/resume-worker-lease.test.ts (3 tests) 8ms + ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms ✓ tests/direct-run-worker-lease.test.ts (3 tests) 9ms - ✓ tests/helper-effect-lease-loss.test.ts (1 test) 217ms + ✓ tests/helper-effect-lease-loss.test.ts (1 test) 204ms ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 4ms - ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 45ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms + ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 46ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 17ms Test Files 27 passed (27) - Tests 328 passed (328) - Start at 03:07:34 - Duration 57.70s (transform 1.11s, setup 156ms, collect 6.57s, tests 47.71s, environment 4ms, prepare 978ms) + Tests 329 passed (329) + Start at 03:15:44 + Duration 57.61s (transform 1.13s, setup 149ms, collect 6.59s, tests 47.68s, environment 4ms, prepare 953ms) exit=0 ``` @@ -90,13 +90,13 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 912ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 911ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 928ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 927ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 03:08:32 - Duration 2.19s (transform 626ms, setup 19ms, collect 1.04s, tests 912ms, environment 0ms, prepare 53ms) + Start at 03:16:42 + Duration 2.23s (transform 626ms, setup 18ms, collect 1.06s, tests 928ms, environment 0ms, prepare 68ms) exit=0 ``` @@ -118,7 +118,7 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (22 tests | 1 failed | 21 skipped) 113ms + ❯ tests/journal-client-read-timeout.test.ts (23 tests | 1 failed | 22 skipped) 113ms × serves a bounded read while an unbounded command is in flight 112ms → expected true to be false // Object.is equality @@ -144,9 +144,9 @@ AssertionError: expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 21 skipped (22) - Start at 03:06:59 - Duration 555ms (transform 164ms, setup 16ms, collect 290ms, tests 113ms, environment 0ms, prepare 40ms) + Tests 1 failed | 22 skipped (23) + Start at 03:15:11 + Duration 592ms (transform 183ms, setup 17ms, collect 325ms, tests 113ms, environment 0ms, prepare 41ms) exit=1 @@ -155,12 +155,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (22 tests | 21 skipped) 107ms + ✓ tests/journal-client-read-timeout.test.ts (23 tests | 22 skipped) 107ms Test Files 1 passed (1) - Tests 1 passed | 21 skipped (22) - Start at 03:07:00 - Duration 560ms (transform 166ms, setup 17ms, collect 290ms, tests 107ms, environment 0ms, prepare 41ms) + Tests 1 passed | 22 skipped (23) + Start at 03:15:12 + Duration 556ms (transform 166ms, setup 16ms, collect 289ms, tests 107ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -181,14 +181,14 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (22 tests | 1 failed | 21 skipped) 1010ms - × reconnects the reader after it disconnects within the read budget 1009ms - → journal client: run.get read session was interrupted after 77 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ tests/journal-client-read-timeout.test.ts (23 tests | 1 failed | 22 skipped) 1009ms + × reconnects the reader after it disconnects within the read budget 1008ms + → journal client: run.get read session was interrupted after 78 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 77 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 78 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:71:9 69| let last: unknown; 70| const exhausted = () => last instanceof JournalReadInterruptedError @@ -199,20 +199,20 @@ JournalReadInterruptedError: journal client: run.get read session was interrupte ❯ Timeout. src/journal-read-policy.ts:82:39 Caused by: Error: journal client: not connected (run.get): journal client: connection closed - ❯ src/journal-client.ts:236:16 - ❯ JournalClient.requestOnce src/journal-client.ts:229:12 + ❯ src/journal-client.ts:244:16 + ❯ JournalClient.requestOnce src/journal-client.ts:237:12 ❯ src/journal-budgeted-reads.ts:47:29 ❯ src/journal-read-policy.ts:94:18 Caused by: Error: journal client: connection closed - ❯ Socket. src/journal-client.ts:134:26 + ❯ Socket. src/journal-client.ts:142:26 ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 21 skipped (22) - Start at 03:07:01 - Duration 1.46s (transform 169ms, setup 16ms, collect 291ms, tests 1.01s, environment 0ms, prepare 43ms) + Tests 1 failed | 22 skipped (23) + Start at 03:15:13 + Duration 1.46s (transform 173ms, setup 16ms, collect 299ms, tests 1.01s, environment 0ms, prepare 41ms) exit=1 @@ -221,12 +221,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (22 tests | 21 skipped) 84ms + ✓ tests/journal-client-read-timeout.test.ts (23 tests | 22 skipped) 94ms Test Files 1 passed (1) - Tests 1 passed | 21 skipped (22) - Start at 03:07:03 - Duration 732ms (transform 213ms, setup 24ms, collect 411ms, tests 84ms, environment 0ms, prepare 65ms) + Tests 1 passed | 22 skipped (23) + Start at 03:15:15 + Duration 504ms (transform 165ms, setup 17ms, collect 276ms, tests 94ms, environment 0ms, prepare 30ms) exit=0 ``` @@ -252,8 +252,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (22 tests | 1 failed | 21 skipped) 74ms - × retries a reader setup that timed out instead of reading on the primary for good 73ms + ❯ tests/journal-client-read-timeout.test.ts (23 tests | 1 failed | 22 skipped) 81ms + × retries a reader setup that timed out instead of reading on the primary for good 80ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -278,9 +278,9 @@ AssertionError: expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 21 skipped (22) - Start at 03:07:04 - Duration 530ms (transform 169ms, setup 16ms, collect 294ms, tests 74ms, environment 0ms, prepare 43ms) + Tests 1 failed | 22 skipped (23) + Start at 03:15:16 + Duration 515ms (transform 166ms, setup 16ms, collect 291ms, tests 81ms, environment 0ms, prepare 31ms) exit=1 @@ -289,12 +289,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (22 tests | 21 skipped) 71ms + ✓ tests/journal-client-read-timeout.test.ts (23 tests | 22 skipped) 69ms Test Files 1 passed (1) - Tests 1 passed | 21 skipped (22) - Start at 03:07:05 - Duration 525ms (transform 168ms, setup 17ms, collect 290ms, tests 71ms, environment 0ms, prepare 41ms) + Tests 1 passed | 22 skipped (23) + Start at 03:15:17 + Duration 532ms (transform 170ms, setup 17ms, collect 299ms, tests 69ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -314,8 +314,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (11 tests | 1 failed | 10 skipped) 2108ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2107ms + ❯ tests/running-step-watch.test.ts (11 tests | 1 failed | 10 skipped) 2109ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2108ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -334,8 +334,8 @@ AssertionError: expected 41 to be less than or equal to 1 Test Files 1 failed (1) Tests 1 failed | 10 skipped (11) - Start at 03:07:06 - Duration 2.60s (transform 189ms, setup 17ms, collect 330ms, tests 2.11s, environment 0ms, prepare 41ms) + Start at 03:15:18 + Duration 2.58s (transform 186ms, setup 16ms, collect 321ms, tests 2.11s, environment 0ms, prepare 41ms) exit=1 @@ -344,13 +344,13 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (11 tests | 10 skipped) 2118ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms + ✓ tests/running-step-watch.test.ts (11 tests | 10 skipped) 2117ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2116ms Test Files 1 passed (1) Tests 1 passed | 10 skipped (11) - Start at 03:07:09 - Duration 2.56s (transform 182ms, setup 17ms, collect 291ms, tests 2.12s, environment 0ms, prepare 41ms) + Start at 03:15:21 + Duration 2.58s (transform 184ms, setup 16ms, collect 310ms, tests 2.12s, environment 0ms, prepare 42ms) exit=0 ``` @@ -371,8 +371,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (11 tests | 1 failed | 10 skipped) 2213ms - × cancels promptly while a lease snapshot read is in flight 2212ms + ❯ tests/running-step-watch.test.ts (11 tests | 1 failed | 10 skipped) 2216ms + × cancels promptly while a lease snapshot read is in flight 2215ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -391,8 +391,8 @@ AssertionError: expected 'still waiting' to be an instance of Error Test Files 1 failed (1) Tests 1 failed | 10 skipped (11) - Start at 03:07:12 - Duration 2.69s (transform 192ms, setup 16ms, collect 324ms, tests 2.21s, environment 0ms, prepare 40ms) + Start at 03:15:24 + Duration 2.70s (transform 188ms, setup 17ms, collect 317ms, tests 2.22s, environment 0ms, prepare 41ms) exit=1 @@ -402,12 +402,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk ✓ tests/running-step-watch.test.ts (11 tests | 10 skipped) 2011ms - ✓ cancels promptly while a lease snapshot read is in flight 2010ms + ✓ cancels promptly while a lease snapshot read is in flight 2011ms Test Files 1 passed (1) Tests 1 passed | 10 skipped (11) - Start at 03:07:15 - Duration 2.48s (transform 188ms, setup 16ms, collect 309ms, tests 2.01s, environment 0ms, prepare 44ms) + Start at 03:15:27 + Duration 2.48s (transform 185ms, setup 17ms, collect 309ms, tests 2.01s, environment 0ms, prepare 40ms) exit=0 ``` @@ -427,8 +427,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (11 tests | 1 failed | 10 skipped) 3014ms - × a completion push ends the wait without waiting out an in-flight snapshot 3013ms + ❯ tests/running-step-watch.test.ts (11 tests | 1 failed | 10 skipped) 3012ms + × a completion push ends the wait without waiting out an in-flight snapshot 3012ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -451,8 +451,8 @@ Received: "still waiting" Test Files 1 failed (1) Tests 1 failed | 10 skipped (11) - Start at 03:07:18 - Duration 3.51s (transform 198ms, setup 17ms, collect 331ms, tests 3.01s, environment 0ms, prepare 42ms) + Start at 03:15:29 + Duration 3.49s (transform 187ms, setup 17ms, collect 317ms, tests 3.01s, environment 0ms, prepare 40ms) exit=1 @@ -461,13 +461,13 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (11 tests | 10 skipped) 2062ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2061ms + ✓ tests/running-step-watch.test.ts (11 tests | 10 skipped) 2059ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2059ms Test Files 1 passed (1) Tests 1 passed | 10 skipped (11) - Start at 03:07:22 - Duration 2.57s (transform 199ms, setup 16ms, collect 343ms, tests 2.06s, environment 0ms, prepare 41ms) + Start at 03:15:33 + Duration 2.51s (transform 180ms, setup 16ms, collect 300ms, tests 2.06s, environment 0ms, prepare 41ms) exit=0 ``` @@ -490,8 +490,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 17ms - × a heartbeat timeout is lease loss rather than a worker body failure 17ms + ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 19ms + × a heartbeat timeout is lease loss rather than a worker body failure 19ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -510,8 +510,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 03:07:25 - Duration 437ms (transform 169ms, setup 17ms, collect 278ms, tests 17ms, environment 0ms, prepare 33ms) + Start at 03:15:36 + Duration 473ms (transform 168ms, setup 16ms, collect 291ms, tests 19ms, environment 0ms, prepare 40ms) exit=1 @@ -524,8 +524,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb Test Files 1 passed (1) Tests 1 passed (1) - Start at 03:07:26 - Duration 552ms (transform 195ms, setup 21ms, collect 352ms, tests 16ms, environment 0ms, prepare 45ms) + Start at 03:15:37 + Duration 455ms (transform 163ms, setup 16ms, collect 280ms, tests 16ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -552,14 +552,14 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (28 tests | 3 failed | 25 skipped) 37ms - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 28ms + ❯ tests/authored-root.test.ts (28 tests | 3 failed | 25 skipped) 21ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 15ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 3ms → expected AuthoredFlowExecutionError: daemon_unresp… { …(7) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 4ms + × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 2ms → expected JournalReadInterruptedError: journal clie… { …(4) } to match object { code: 'daemon_unresponsive', …(2) } (5 matching properties omitted from actual) @@ -639,8 +639,8 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } Test Files 1 failed (1) Tests 3 failed | 25 skipped (28) - Start at 03:07:27 - Duration 1.54s (transform 769ms, setup 18ms, collect 1.31s, tests 37ms, environment 0ms, prepare 53ms) + Start at 03:15:38 + Duration 1.22s (transform 643ms, setup 14ms, collect 1.04s, tests 21ms, environment 0ms, prepare 41ms) exit=1 @@ -653,8 +653,8 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r Test Files 1 passed (1) Tests 3 passed | 25 skipped (28) - Start at 03:07:29 - Duration 1.31s (transform 678ms, setup 19ms, collect 1.12s, tests 16ms, environment 0ms, prepare 42ms) + Start at 03:15:39 + Duration 1.22s (transform 650ms, setup 16ms, collect 1.06s, tests 16ms, environment 0ms, prepare 30ms) exit=0 ``` diff --git a/packages/sdk/src/authored-node-entry.ts b/packages/sdk/src/authored-node-entry.ts index 978f59088..1ecd4a265 100644 --- a/packages/sdk/src/authored-node-entry.ts +++ b/packages/sdk/src/authored-node-entry.ts @@ -75,7 +75,7 @@ try { const loaded = await loadPinnedAuthoredSource(request.metadata, true); if (request.localAgentStream !== request.metadata.localAgentStream) throw new Error('authored root local agent surface mismatch'); if (request.workerCapacity !== undefined && !isAgentCapacity(request.workerCapacity)) throw new Error('invalid authored worker capacity'); - client = new JournalClient(request.socketPath, { readBudgetMs: FLOW_READ_BUDGET_MS }); + client = new JournalClient(request.socketPath, { readBudgetMs: FLOW_READ_BUDGET_MS, budgetHandshake: true }); await client.connect(); await client.hello('flows-authored-node'); const result = await executeAuthoredFlow(loaded.handle, client, request.metadata.inputPresent ? request.metadata.input : undefined, { diff --git a/packages/sdk/src/authored-node-runner.ts b/packages/sdk/src/authored-node-runner.ts index 8c3c893c5..a530b4a1b 100644 --- a/packages/sdk/src/authored-node-runner.ts +++ b/packages/sdk/src/authored-node-runner.ts @@ -356,7 +356,7 @@ export async function verifyAuthoredNodeResult( } const runs = new Set(); // The verifier reads the same journal under the same CPU load as the body. - const journal = new JournalClient(socketPath, { readBudgetMs: FLOW_READ_BUDGET_MS, + const journal = new JournalClient(socketPath, { readBudgetMs: FLOW_READ_BUDGET_MS, budgetHandshake: true, ...(signal === undefined ? {} : { readSignal: signal }) }); await journal.connect(); try { diff --git a/packages/sdk/src/journal-client.ts b/packages/sdk/src/journal-client.ts index af014e29f..3ed235e67 100644 --- a/packages/sdk/src/journal-client.ts +++ b/packages/sdk/src/journal-client.ts @@ -34,6 +34,12 @@ export interface JournalClientOptions { readBudgetMs?: number; /** Cancels every budgeted read, so a run's lifecycle abort never waits out the budget. */ readSignal?: AbortSignal; + /** + * Retry this session's own `hello` within the read budget. Only for sessions + * opened after a run is admitted (an authored child, its verifier); a + * top-level command's handshake stays a single-shot compatibility check. + */ + budgetHandshake?: boolean; /** Override the timeout for bounded protocol requests (ms). Default 30000. */ requestTimeoutMs?: number; /** @@ -77,6 +83,7 @@ export class JournalFrameError extends Error { export class JournalClient extends EventEmitter { private readonly budgeted: BudgetedReads | undefined; + private readonly budgetHandshake: boolean; /** Additive request capabilities the daemon advertised at `hello`; none until then. */ private features: ReadonlySet = new Set(); private socket: Socket | null = null; @@ -93,6 +100,7 @@ export class JournalClient extends EventEmitter { options: JournalClientOptions = {}, ) { super(); + this.budgetHandshake = options.budgetHandshake === true; this.budgeted = options.readBudgetMs === undefined ? undefined : new BudgetedReads(this, options.readBudgetMs, options.readSignal); this.requestTimeoutMs = options.requestTimeoutMs ?? 30_000; @@ -279,7 +287,7 @@ export class JournalClient extends EventEmitter { async hello(client: string): Promise { const params = { protocol: PROTOCOL_VERSION, client }; // A flow client's handshake meets the same CPU load as its reads. - const result = this.budgeted === undefined ? await this.request('hello', params) + const result = this.budgeted === undefined || !this.budgetHandshake ? await this.request('hello', params) : await this.budgeted.handshake(params, this.requestTimeoutMs); this.features = new Set(Array.isArray(result.features) ? result.features.filter(f => typeof f === 'string') : []); return result; diff --git a/packages/sdk/tests/authored-verifier-read-budget.test.ts b/packages/sdk/tests/authored-verifier-read-budget.test.ts index de6028b00..b26ab1fe9 100644 --- a/packages/sdk/tests/authored-verifier-read-budget.test.ts +++ b/packages/sdk/tests/authored-verifier-read-budget.test.ts @@ -19,7 +19,7 @@ it('verifies a Bun-run authored result with the flow read budget, not single-sho const result = { rootRunId: 'root', name: 'flow', completionReason: 'success', journalSteps: [{ id: 'complete-1' }] }; await verifyAuthoredNodeResult(result as never, { flowName: 'flow' } as never, 'root', '/unused.sock').catch(() => undefined); - expect(constructed).toContainEqual(expect.objectContaining({ readBudgetMs: FLOW_READ_BUDGET_MS })); + expect(constructed).toContainEqual(expect.objectContaining({ readBudgetMs: FLOW_READ_BUDGET_MS, budgetHandshake: true })); }); it('hands the lifecycle signal to the verifier so cancellation stops its reads', async () => { diff --git a/packages/sdk/tests/journal-client-read-timeout.test.ts b/packages/sdk/tests/journal-client-read-timeout.test.ts index 16fbe9734..ae31fb274 100644 --- a/packages/sdk/tests/journal-client-read-timeout.test.ts +++ b/packages/sdk/tests/journal-client-read-timeout.test.ts @@ -246,15 +246,33 @@ it('fails closed on a malformed frame during reader setup', async () => { expect(reads).toBe(0); }); -it('retries a delayed handshake within the read budget on a budgeted flow client', async () => { +async function handshakeClient(handlers: LoopbackHandlers, budget: number) { + const path = sockPath(); + const server = startLoopback(path, { hello: sendOk, ...handlers }, { serialize: true }); + servers.push(server); + await once(server, 'listening'); + const client = new JournalClient(path, { requestTimeoutMs: 50, readBudgetMs: budget, budgetHandshake: true }); + clients.push(client); + await client.connect(); + return client; +} + +it('keeps a pre-admission handshake single-shot even on a budgeted flow client', async () => { + let hellos = 0; + const client = await setup({ hello: () => { hellos += 1; } }, 5_000); + await expect(client.hello('flows-run')).rejects.toMatchObject({ verb: 'hello', attempts: 1 }); + expect(hellos).toBe(1); +}); + +it('retries a delayed post-admission handshake within the read budget', async () => { let hellos = 0; - const client = await setup({ hello: ctx => { if (++hellos > 1) sendOk(ctx); } }, 5_000); + const client = await handshakeClient({ hello: ctx => { if (++hellos > 1) sendOk(ctx); } }, 5_000); await expect(client.hello('flows-authored-node')).resolves.toBeDefined(); expect(hellos).toBe(2); }); -it('an exhausted handshake budget is a read interruption, not a body failure', async () => { - const client = await setup({ hello: () => {} }, 250); +it('an exhausted post-admission handshake budget is a read interruption, not a body failure', async () => { + const client = await handshakeClient({ hello: () => {} }, 250); const error = await client.hello('flows-authored-node').catch(caught => caught); expect(isReadInterruptionError(error)).toBe(true); }); From 65a62fe15f788bc8e6453bbe6ee11e9581c3d011 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 03:27:52 -0700 Subject: [PATCH 20/34] fix(sdk): the root lease and a dropped watch both stop in-flight budgeted reads - JournalClient.scopeReads(signal) adds an abort source to budgeted reads until released; driveRoot scopes each root attempt's lease signal over the body, so a lost lease stops reads (waitForTerminalChildRun and others) instead of letting them run out the 300 s budget while re-dispatched attempts expire. - A watch that drops during an in-flight snapshot aborts the snapshot and surfaces the resumable interruption at once. Evidence re-captured at this tree. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: b8262468-ea26-4a6e-bee9-d193832e77dc --- evidence/run-read-timeout/final-focused.log | 111 +++---- .../mutation-completion-aborts-snapshot.log | 22 +- .../run-read-timeout/mutation-heartbeat.log | 8 +- .../mutation-reader-reconnect.log | 30 +- .../mutation-reader-setup-retry.log | 22 +- evidence/run-read-timeout/mutation-reader.log | 22 +- .../mutation-root-parking.log | 26 +- .../mutation-snapshot-cancel.log | 22 +- .../mutation-watch-cadence.log | 24 +- evidence/run-read-timeout/resume-live.log | 8 +- evidence/run-read-timeout/verification.md | 295 +++++++++--------- packages/sdk/src/authored-root.ts | 68 ++-- packages/sdk/src/cli/running-step.ts | 8 +- packages/sdk/src/journal-budgeted-reads.ts | 11 +- packages/sdk/src/journal-client.ts | 8 + packages/sdk/tests/authored-root.test.ts | 13 + .../tests/journal-client-read-timeout.test.ts | 15 + packages/sdk/tests/running-step-watch.test.ts | 21 ++ 18 files changed, 404 insertions(+), 330 deletions(-) diff --git a/evidence/run-read-timeout/final-focused.log b/evidence/run-read-timeout/final-focused.log index b84bcfdb1..45a13593c 100644 --- a/evidence/run-read-timeout/final-focused.log +++ b/evidence/run-read-timeout/final-focused.log @@ -2,69 +2,70 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5293ms - ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 312ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2036ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2038ms -(node:3978255) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5327ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2039ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2045ms +(node:4039318) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (28 tests) 391ms - ✓ tests/journal-client.test.ts (17 tests) 78ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 6906ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 488ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1069ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2720ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 344ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 848ms - ✓ tests/authored-flow-slack.test.ts (7 tests) 1174ms - ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 411ms - ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 418ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2558ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 627ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 623ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 656ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 651ms - ✓ tests/journal-client-read-timeout.test.ts (23 tests) 1726ms - ✓ tests/running-step-watch.test.ts (11 tests) 10286ms + ✓ tests/authored-root.test.ts (29 tests) 399ms + ✓ tests/journal-client.test.ts (17 tests) 70ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 6998ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 450ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1015ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2825ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 388ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 865ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 1252ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 437ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 434ms + ✓ tests/journal-client-read-timeout.test.ts (24 tests) 1877ms + ✓ a recovered read timeout does not become an authored callback failure 363ms + ✓ tests/running-step-watch.test.ts (12 tests) 12287ms ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms - ✓ cancels promptly while a lease snapshot read is in flight 2004ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2054ms + ✓ cancels promptly while a lease snapshot read is in flight 2006ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2053ms ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms - ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2004ms - ✓ tests/authored-helpers.test.ts (6 tests) 1949ms - ✓ runs every available provider through the real kernel and resumes completed effects without a second write 885ms - ✓ replays after SIGKILL before confirm with the same token and one successful completion 421ms - ✓ replays after SIGKILL before complete with the same token and one successful completion 418ms - ✓ tests/classify-outcome.test.ts (11 tests) 7435ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2019ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3008ms - ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms - ✓ tests/human-live.test.ts (3 tests) 4487ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2652ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1151ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 683ms + ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2003ms + ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2002ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2582ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 692ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 645ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 660ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 585ms + ✓ tests/authored-helpers.test.ts (6 tests) 1962ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 904ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 402ms + ✓ replays after SIGKILL before complete with the same token and one successful completion 427ms + ✓ tests/classify-outcome.test.ts (11 tests) 7429ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2015ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3006ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2002ms + ✓ tests/human-live.test.ts (3 tests) 4589ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2702ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1114ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 772ms ✓ tests/worker-lease.test.ts (7 tests) 19ms - ✓ tests/communication-worker.test.ts (18 tests) 1706ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 25ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 705ms - ✓ tests/memoization.test.ts (58 tests) 62ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2469ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2055ms - ✓ drains read and watch promises before an authored flow completes 413ms - ✓ tests/worker-lease-sweep.test.ts (4 tests) 6ms - ✓ tests/journal-client-completion.test.ts (6 tests) 102ms + ✓ tests/communication-worker.test.ts (18 tests) 1722ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 24ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 733ms + ✓ tests/memoization.test.ts (58 tests) 67ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2475ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2056ms + ✓ drains read and watch promises before an authored flow completes 418ms + ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms + ✓ tests/journal-client-completion.test.ts (6 tests) 100ms ✓ tests/reuse-summary-interruption.test.ts (5 tests) 4ms - ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms + ✓ tests/resume-worker-lease.test.ts (3 tests) 8ms ✓ tests/direct-run-worker-lease.test.ts (3 tests) 9ms - ✓ tests/helper-effect-lease-loss.test.ts (1 test) 204ms - ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms + ✓ tests/helper-effect-lease-loss.test.ts (1 test) 229ms + ✓ tests/journal-client-subscriptions.test.ts (1 test) 6ms ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 4ms - ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 46ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 17ms + ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 47ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms Test Files 27 passed (27) - Tests 329 passed (329) - Start at 03:15:44 - Duration 57.61s (transform 1.13s, setup 149ms, collect 6.59s, tests 47.68s, environment 4ms, prepare 953ms) + Tests 332 passed (332) + Start at 03:26:49 + Duration 60.48s (transform 1.20s, setup 151ms, collect 6.79s, tests 50.24s, environment 4ms, prepare 970ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log index 004ef96b9..e3ff1b5b6 100644 --- a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log +++ b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/cli/running-step.ts -Original SHA256: 34223d566de94486b7039f5eb783ddacbe62ccc3cca63c2b5278d72c4de59c38 +Original SHA256: 2d95d1b21da7acb5e91e79236cb099604beabaed7e1e0443c96aeccf4c704c3e Replaced: snapshotRead?.abort(new Error(`step "${runningStep.id}" completed during the lease snapshot`)); With: @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (11 tests | 1 failed | 10 skipped) 3012ms - × a completion push ends the wait without waiting out an in-flight snapshot 3012ms + ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 3012ms + × a completion push ends the wait without waiting out an in-flight snapshot 3011ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -33,23 +33,23 @@ Received: "still waiting" ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 10 skipped (11) - Start at 03:15:29 - Duration 3.49s (transform 187ms, setup 17ms, collect 317ms, tests 3.01s, environment 0ms, prepare 40ms) + Tests 1 failed | 11 skipped (12) + Start at 03:26:34 + Duration 3.49s (transform 188ms, setup 16ms, collect 318ms, tests 3.01s, environment 0ms, prepare 41ms) exit=1 -RESTORED SHA256: 34223d566de94486b7039f5eb783ddacbe62ccc3cca63c2b5278d72c4de59c38 +RESTORED SHA256: 2d95d1b21da7acb5e91e79236cb099604beabaed7e1e0443c96aeccf4c704c3e $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a completion push ends the wait' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (11 tests | 10 skipped) 2059ms + ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2060ms ✓ a completion push ends the wait without waiting out an in-flight snapshot 2059ms Test Files 1 passed (1) - Tests 1 passed | 10 skipped (11) - Start at 03:15:33 - Duration 2.51s (transform 180ms, setup 16ms, collect 300ms, tests 2.06s, environment 0ms, prepare 41ms) + Tests 1 passed | 11 skipped (12) + Start at 03:26:38 + Duration 2.54s (transform 188ms, setup 17ms, collect 314ms, tests 2.06s, environment 0ms, prepare 42ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-heartbeat.log b/evidence/run-read-timeout/mutation-heartbeat.log index ed865067a..8e35ca69e 100644 --- a/evidence/run-read-timeout/mutation-heartbeat.log +++ b/evidence/run-read-timeout/mutation-heartbeat.log @@ -33,8 +33,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 03:15:36 - Duration 473ms (transform 168ms, setup 16ms, collect 291ms, tests 19ms, environment 0ms, prepare 40ms) + Start at 03:26:41 + Duration 455ms (transform 165ms, setup 16ms, collect 289ms, tests 19ms, environment 0ms, prepare 41ms) exit=1 @@ -47,7 +47,7 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb Test Files 1 passed (1) Tests 1 passed (1) - Start at 03:15:37 - Duration 455ms (transform 163ms, setup 16ms, collect 280ms, tests 16ms, environment 0ms, prepare 41ms) + Start at 03:26:42 + Duration 445ms (transform 163ms, setup 16ms, collect 275ms, tests 16ms, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-reconnect.log b/evidence/run-read-timeout/mutation-reader-reconnect.log index 84b1d5ce2..8a0792eb2 100644 --- a/evidence/run-read-timeout/mutation-reader-reconnect.log +++ b/evidence/run-read-timeout/mutation-reader-reconnect.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: 0d9162bdc58667fc831e603da1c31a52df2ec4fc8c39cd0383d6f6d9f0fc7bda +Original SHA256: efc2e4cdfdfba782ea0732b5573f4190ae02c486b7de4731a33f280f7bce6fc1 Replaced: this.drop(reader); @@ -11,14 +11,14 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (23 tests | 1 failed | 22 skipped) 1009ms + ❯ tests/journal-client-read-timeout.test.ts (24 tests | 1 failed | 23 skipped) 1009ms × reconnects the reader after it disconnects within the read budget 1008ms - → journal client: run.get read session was interrupted after 78 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + → journal client: run.get read session was interrupted after 80 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 78 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 80 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:71:9 69| let last: unknown; 70| const exhausted = () => last instanceof JournalReadInterruptedError @@ -29,9 +29,9 @@ JournalReadInterruptedError: journal client: run.get read session was interrupte ❯ Timeout. src/journal-read-policy.ts:82:39 Caused by: Error: journal client: not connected (run.get): journal client: connection closed - ❯ src/journal-client.ts:244:16 - ❯ JournalClient.requestOnce src/journal-client.ts:237:12 - ❯ src/journal-budgeted-reads.ts:47:29 + ❯ src/journal-client.ts:252:16 + ❯ JournalClient.requestOnce src/journal-client.ts:245:12 + ❯ src/journal-budgeted-reads.ts:54:29 ❯ src/journal-read-policy.ts:94:18 Caused by: Error: journal client: connection closed @@ -40,22 +40,22 @@ Caused by: Error: journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 22 skipped (23) - Start at 03:15:13 - Duration 1.46s (transform 173ms, setup 16ms, collect 299ms, tests 1.01s, environment 0ms, prepare 41ms) + Tests 1 failed | 23 skipped (24) + Start at 03:26:18 + Duration 1.52s (transform 182ms, setup 18ms, collect 325ms, tests 1.01s, environment 0ms, prepare 46ms) exit=1 -RESTORED SHA256: 0d9162bdc58667fc831e603da1c31a52df2ec4fc8c39cd0383d6f6d9f0fc7bda +RESTORED SHA256: efc2e4cdfdfba782ea0732b5573f4190ae02c486b7de4731a33f280f7bce6fc1 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'reconnects the reader after it disconnects' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (23 tests | 22 skipped) 94ms + ✓ tests/journal-client-read-timeout.test.ts (24 tests | 23 skipped) 86ms Test Files 1 passed (1) - Tests 1 passed | 22 skipped (23) - Start at 03:15:15 - Duration 504ms (transform 165ms, setup 17ms, collect 276ms, tests 94ms, environment 0ms, prepare 30ms) + Tests 1 passed | 23 skipped (24) + Start at 03:26:20 + Duration 517ms (transform 160ms, setup 17ms, collect 283ms, tests 86ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-setup-retry.log b/evidence/run-read-timeout/mutation-reader-setup-retry.log index cf3cf4398..6bc70f966 100644 --- a/evidence/run-read-timeout/mutation-reader-setup-retry.log +++ b/evidence/run-read-timeout/mutation-reader-setup-retry.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: 0d9162bdc58667fc831e603da1c31a52df2ec4fc8c39cd0383d6f6d9f0fc7bda +Original SHA256: efc2e4cdfdfba782ea0732b5573f4190ae02c486b7de4731a33f280f7bce6fc1 Replaced: if (this.reader === reader) { this.reader = undefined; @@ -16,8 +16,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (23 tests | 1 failed | 22 skipped) 81ms - × retries a reader setup that timed out instead of reading on the primary for good 80ms + ❯ tests/journal-client-read-timeout.test.ts (24 tests | 1 failed | 23 skipped) 76ms + × retries a reader setup that timed out instead of reading on the primary for good 76ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -42,22 +42,22 @@ AssertionError: expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 22 skipped (23) - Start at 03:15:16 - Duration 515ms (transform 166ms, setup 16ms, collect 291ms, tests 81ms, environment 0ms, prepare 31ms) + Tests 1 failed | 23 skipped (24) + Start at 03:26:21 + Duration 518ms (transform 160ms, setup 16ms, collect 281ms, tests 76ms, environment 0ms, prepare 40ms) exit=1 -RESTORED SHA256: 0d9162bdc58667fc831e603da1c31a52df2ec4fc8c39cd0383d6f6d9f0fc7bda +RESTORED SHA256: efc2e4cdfdfba782ea0732b5573f4190ae02c486b7de4731a33f280f7bce6fc1 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'retries a reader setup that timed out' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (23 tests | 22 skipped) 69ms + ✓ tests/journal-client-read-timeout.test.ts (24 tests | 23 skipped) 70ms Test Files 1 passed (1) - Tests 1 passed | 22 skipped (23) - Start at 03:15:17 - Duration 532ms (transform 170ms, setup 17ms, collect 299ms, tests 69ms, environment 0ms, prepare 41ms) + Tests 1 passed | 23 skipped (24) + Start at 03:26:22 + Duration 518ms (transform 167ms, setup 16ms, collect 293ms, tests 70ms, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader.log b/evidence/run-read-timeout/mutation-reader.log index d69f668eb..bd205012a 100644 --- a/evidence/run-read-timeout/mutation-reader.log +++ b/evidence/run-read-timeout/mutation-reader.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: 0d9162bdc58667fc831e603da1c31a52df2ec4fc8c39cd0383d6f6d9f0fc7bda +Original SHA256: efc2e4cdfdfba782ea0732b5573f4190ae02c486b7de4731a33f280f7bce6fc1 Replaced: return await reader.requestOnce(verb, params, remaining, attemptSignal); With: @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (23 tests | 1 failed | 22 skipped) 113ms - × serves a bounded read while an unbounded command is in flight 112ms + ❯ tests/journal-client-read-timeout.test.ts (24 tests | 1 failed | 23 skipped) 116ms + × serves a bounded read while an unbounded command is in flight 116ms → expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -36,22 +36,22 @@ AssertionError: expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 22 skipped (23) - Start at 03:15:11 - Duration 592ms (transform 183ms, setup 17ms, collect 325ms, tests 113ms, environment 0ms, prepare 41ms) + Tests 1 failed | 23 skipped (24) + Start at 03:26:16 + Duration 626ms (transform 199ms, setup 14ms, collect 339ms, tests 116ms, environment 0ms, prepare 43ms) exit=1 -RESTORED SHA256: 0d9162bdc58667fc831e603da1c31a52df2ec4fc8c39cd0383d6f6d9f0fc7bda +RESTORED SHA256: efc2e4cdfdfba782ea0732b5573f4190ae02c486b7de4731a33f280f7bce6fc1 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'serves a bounded read' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (23 tests | 22 skipped) 107ms + ✓ tests/journal-client-read-timeout.test.ts (24 tests | 23 skipped) 106ms Test Files 1 passed (1) - Tests 1 passed | 22 skipped (23) - Start at 03:15:12 - Duration 556ms (transform 166ms, setup 16ms, collect 289ms, tests 107ms, environment 0ms, prepare 41ms) + Tests 1 passed | 23 skipped (24) + Start at 03:26:17 + Duration 639ms (transform 188ms, setup 21ms, collect 312ms, tests 106ms, environment 0ms, prepare 65ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-root-parking.log b/evidence/run-read-timeout/mutation-root-parking.log index 243dff4a7..a604e58ff 100644 --- a/evidence/run-read-timeout/mutation-root-parking.log +++ b/evidence/run-read-timeout/mutation-root-parking.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/authored-root.ts -Original SHA256: 79b644e3c53dc5ec73fa859720e6da87008efb4ce9c4dd1447aa362729624dc8 +Original SHA256: 79f49acf9e87f35a2ff6a3b757c2d193e4e821eadfd28f96181034f550b01dfa Replaced: if (isReadInterruptionError(error) || (error instanceof AuthoredFlowExecutionError && error.code === 'daemon_unresponsive')) { @@ -17,14 +17,14 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (28 tests | 3 failed | 25 skipped) 21ms - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 15ms + ❯ tests/authored-root.test.ts (29 tests | 3 failed | 26 skipped) 32ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 22ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 3ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms → expected AuthoredFlowExecutionError: daemon_unresp… { …(7) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 2ms + × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 4ms → expected JournalReadInterruptedError: journal clie… { …(4) } to match object { code: 'daemon_unresponsive', …(2) } (5 matching properties omitted from actual) @@ -103,22 +103,22 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[3/3]⎯ Test Files 1 failed (1) - Tests 3 failed | 25 skipped (28) - Start at 03:15:38 - Duration 1.22s (transform 643ms, setup 14ms, collect 1.04s, tests 21ms, environment 0ms, prepare 41ms) + Tests 3 failed | 26 skipped (29) + Start at 03:26:42 + Duration 1.29s (transform 668ms, setup 17ms, collect 1.09s, tests 32ms, environment 0ms, prepare 42ms) exit=1 -RESTORED SHA256: 79b644e3c53dc5ec73fa859720e6da87008efb4ce9c4dd1447aa362729624dc8 +RESTORED SHA256: 79f49acf9e87f35a2ff6a3b757c2d193e4e821eadfd28f96181034f550b01dfa $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the root resumable' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (28 tests | 25 skipped) 16ms + ✓ tests/authored-root.test.ts (29 tests | 26 skipped) 24ms Test Files 1 passed (1) - Tests 3 passed | 25 skipped (28) - Start at 03:15:39 - Duration 1.22s (transform 650ms, setup 16ms, collect 1.06s, tests 16ms, environment 0ms, prepare 30ms) + Tests 3 passed | 26 skipped (29) + Start at 03:26:44 + Duration 1.24s (transform 644ms, setup 16ms, collect 1.05s, tests 24ms, environment 0ms, prepare 42ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-snapshot-cancel.log b/evidence/run-read-timeout/mutation-snapshot-cancel.log index f4b49237d..71aef5ff8 100644 --- a/evidence/run-read-timeout/mutation-snapshot-cancel.log +++ b/evidence/run-read-timeout/mutation-snapshot-cancel.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/cli/running-step.ts -Original SHA256: 34223d566de94486b7039f5eb783ddacbe62ccc3cca63c2b5278d72c4de59c38 +Original SHA256: 2d95d1b21da7acb5e91e79236cb099604beabaed7e1e0443c96aeccf4c704c3e Replaced: const signal = options.signal === undefined ? read.signal : AbortSignal.any([options.signal, read.signal]); @@ -11,7 +11,7 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (11 tests | 1 failed | 10 skipped) 2216ms + ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2216ms × cancels promptly while a lease snapshot read is in flight 2215ms → expected 'still waiting' to be an instance of Error @@ -30,23 +30,23 @@ AssertionError: expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 10 skipped (11) - Start at 03:15:24 - Duration 2.70s (transform 188ms, setup 17ms, collect 317ms, tests 2.22s, environment 0ms, prepare 41ms) + Tests 1 failed | 11 skipped (12) + Start at 03:26:28 + Duration 2.70s (transform 192ms, setup 17ms, collect 324ms, tests 2.22s, environment 0ms, prepare 40ms) exit=1 -RESTORED SHA256: 34223d566de94486b7039f5eb783ddacbe62ccc3cca63c2b5278d72c4de59c38 +RESTORED SHA256: 2d95d1b21da7acb5e91e79236cb099604beabaed7e1e0443c96aeccf4c704c3e $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels promptly while a lease snapshot read is in flight' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (11 tests | 10 skipped) 2011ms - ✓ cancels promptly while a lease snapshot read is in flight 2011ms + ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2011ms + ✓ cancels promptly while a lease snapshot read is in flight 2010ms Test Files 1 passed (1) - Tests 1 passed | 10 skipped (11) - Start at 03:15:27 - Duration 2.48s (transform 185ms, setup 17ms, collect 309ms, tests 2.01s, environment 0ms, prepare 40ms) + Tests 1 passed | 11 skipped (12) + Start at 03:26:31 + Duration 2.49s (transform 184ms, setup 17ms, collect 305ms, tests 2.01s, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-watch-cadence.log b/evidence/run-read-timeout/mutation-watch-cadence.log index d8d2e7539..7a82e5803 100644 --- a/evidence/run-read-timeout/mutation-watch-cadence.log +++ b/evidence/run-read-timeout/mutation-watch-cadence.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/cli/running-step.ts -Original SHA256: 34223d566de94486b7039f5eb783ddacbe62ccc3cca63c2b5278d72c4de59c38 +Original SHA256: 2d95d1b21da7acb5e91e79236cb099604beabaed7e1e0443c96aeccf4c704c3e Replaced: const LEASE_POLL_MS = 2_000; With: @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (11 tests | 1 failed | 10 skipped) 2109ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2108ms + ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2111ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2111ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -29,23 +29,23 @@ AssertionError: expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 10 skipped (11) - Start at 03:15:18 - Duration 2.58s (transform 186ms, setup 16ms, collect 321ms, tests 2.11s, environment 0ms, prepare 41ms) + Tests 1 failed | 11 skipped (12) + Start at 03:26:23 + Duration 2.58s (transform 185ms, setup 15ms, collect 316ms, tests 2.11s, environment 0ms, prepare 42ms) exit=1 -RESTORED SHA256: 34223d566de94486b7039f5eb783ddacbe62ccc3cca63c2b5278d72c4de59c38 +RESTORED SHA256: 2d95d1b21da7acb5e91e79236cb099604beabaed7e1e0443c96aeccf4c704c3e $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pushes for completion' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (11 tests | 10 skipped) 2117ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2116ms + ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2118ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms Test Files 1 passed (1) - Tests 1 passed | 10 skipped (11) - Start at 03:15:21 - Duration 2.58s (transform 184ms, setup 16ms, collect 310ms, tests 2.12s, environment 0ms, prepare 42ms) + Tests 1 passed | 11 skipped (12) + Start at 03:26:25 + Duration 2.61s (transform 190ms, setup 17ms, collect 324ms, tests 2.12s, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/resume-live.log b/evidence/run-read-timeout/resume-live.log index da4ddd84a..19a7a6043 100644 --- a/evidence/run-read-timeout/resume-live.log +++ b/evidence/run-read-timeout/resume-live.log @@ -2,12 +2,12 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 928ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 927ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 944ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 943ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 03:16:42 - Duration 2.23s (transform 626ms, setup 18ms, collect 1.06s, tests 928ms, environment 0ms, prepare 68ms) + Start at 03:27:50 + Duration 2.22s (transform 636ms, setup 21ms, collect 1.04s, tests 944ms, environment 0ms, prepare 85ms) exit=0 diff --git a/evidence/run-read-timeout/verification.md b/evidence/run-read-timeout/verification.md index d13535f7a..ed37579c1 100644 --- a/evidence/run-read-timeout/verification.md +++ b/evidence/run-read-timeout/verification.md @@ -15,70 +15,71 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5293ms - ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 312ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2036ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2038ms -(node:3978255) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5327ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2039ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2045ms +(node:4039318) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (28 tests) 391ms - ✓ tests/journal-client.test.ts (17 tests) 78ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 6906ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 488ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1069ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2720ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 344ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 848ms - ✓ tests/authored-flow-slack.test.ts (7 tests) 1174ms - ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 411ms - ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 418ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2558ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 627ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 623ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 656ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 651ms - ✓ tests/journal-client-read-timeout.test.ts (23 tests) 1726ms - ✓ tests/running-step-watch.test.ts (11 tests) 10286ms + ✓ tests/authored-root.test.ts (29 tests) 399ms + ✓ tests/journal-client.test.ts (17 tests) 70ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 6998ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 450ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1015ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2825ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 388ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 865ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 1252ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 437ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 434ms + ✓ tests/journal-client-read-timeout.test.ts (24 tests) 1877ms + ✓ a recovered read timeout does not become an authored callback failure 363ms + ✓ tests/running-step-watch.test.ts (12 tests) 12287ms ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms - ✓ cancels promptly while a lease snapshot read is in flight 2004ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2054ms + ✓ cancels promptly while a lease snapshot read is in flight 2006ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2053ms ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms - ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2004ms - ✓ tests/authored-helpers.test.ts (6 tests) 1949ms - ✓ runs every available provider through the real kernel and resumes completed effects without a second write 885ms - ✓ replays after SIGKILL before confirm with the same token and one successful completion 421ms - ✓ replays after SIGKILL before complete with the same token and one successful completion 418ms - ✓ tests/classify-outcome.test.ts (11 tests) 7435ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2019ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3008ms - ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms - ✓ tests/human-live.test.ts (3 tests) 4487ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2652ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1151ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 683ms + ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2003ms + ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2002ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2582ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 692ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 645ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 660ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 585ms + ✓ tests/authored-helpers.test.ts (6 tests) 1962ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 904ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 402ms + ✓ replays after SIGKILL before complete with the same token and one successful completion 427ms + ✓ tests/classify-outcome.test.ts (11 tests) 7429ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2015ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3006ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2002ms + ✓ tests/human-live.test.ts (3 tests) 4589ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2702ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1114ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 772ms ✓ tests/worker-lease.test.ts (7 tests) 19ms - ✓ tests/communication-worker.test.ts (18 tests) 1706ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 25ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 705ms - ✓ tests/memoization.test.ts (58 tests) 62ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2469ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2055ms - ✓ drains read and watch promises before an authored flow completes 413ms - ✓ tests/worker-lease-sweep.test.ts (4 tests) 6ms - ✓ tests/journal-client-completion.test.ts (6 tests) 102ms + ✓ tests/communication-worker.test.ts (18 tests) 1722ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 24ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 733ms + ✓ tests/memoization.test.ts (58 tests) 67ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2475ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2056ms + ✓ drains read and watch promises before an authored flow completes 418ms + ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms + ✓ tests/journal-client-completion.test.ts (6 tests) 100ms ✓ tests/reuse-summary-interruption.test.ts (5 tests) 4ms - ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms + ✓ tests/resume-worker-lease.test.ts (3 tests) 8ms ✓ tests/direct-run-worker-lease.test.ts (3 tests) 9ms - ✓ tests/helper-effect-lease-loss.test.ts (1 test) 204ms - ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms + ✓ tests/helper-effect-lease-loss.test.ts (1 test) 229ms + ✓ tests/journal-client-subscriptions.test.ts (1 test) 6ms ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 4ms - ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 46ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 17ms + ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 47ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms Test Files 27 passed (27) - Tests 329 passed (329) - Start at 03:15:44 - Duration 57.61s (transform 1.13s, setup 149ms, collect 6.59s, tests 47.68s, environment 4ms, prepare 953ms) + Tests 332 passed (332) + Start at 03:26:49 + Duration 60.48s (transform 1.20s, setup 151ms, collect 6.79s, tests 50.24s, environment 4ms, prepare 970ms) exit=0 ``` @@ -90,13 +91,13 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 928ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 927ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 944ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 943ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 03:16:42 - Duration 2.23s (transform 626ms, setup 18ms, collect 1.06s, tests 928ms, environment 0ms, prepare 68ms) + Start at 03:27:50 + Duration 2.22s (transform 636ms, setup 21ms, collect 1.04s, tests 944ms, environment 0ms, prepare 85ms) exit=0 ``` @@ -107,7 +108,7 @@ The mutation sections below were regenerated by `cd packages/sdk && python3 ../. ```text Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: 0d9162bdc58667fc831e603da1c31a52df2ec4fc8c39cd0383d6f6d9f0fc7bda +Original SHA256: efc2e4cdfdfba782ea0732b5573f4190ae02c486b7de4731a33f280f7bce6fc1 Replaced: return await reader.requestOnce(verb, params, remaining, attemptSignal); With: @@ -118,8 +119,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (23 tests | 1 failed | 22 skipped) 113ms - × serves a bounded read while an unbounded command is in flight 112ms + ❯ tests/journal-client-read-timeout.test.ts (24 tests | 1 failed | 23 skipped) 116ms + × serves a bounded read while an unbounded command is in flight 116ms → expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -144,23 +145,23 @@ AssertionError: expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 22 skipped (23) - Start at 03:15:11 - Duration 592ms (transform 183ms, setup 17ms, collect 325ms, tests 113ms, environment 0ms, prepare 41ms) + Tests 1 failed | 23 skipped (24) + Start at 03:26:16 + Duration 626ms (transform 199ms, setup 14ms, collect 339ms, tests 116ms, environment 0ms, prepare 43ms) exit=1 -RESTORED SHA256: 0d9162bdc58667fc831e603da1c31a52df2ec4fc8c39cd0383d6f6d9f0fc7bda +RESTORED SHA256: efc2e4cdfdfba782ea0732b5573f4190ae02c486b7de4731a33f280f7bce6fc1 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'serves a bounded read' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (23 tests | 22 skipped) 107ms + ✓ tests/journal-client-read-timeout.test.ts (24 tests | 23 skipped) 106ms Test Files 1 passed (1) - Tests 1 passed | 22 skipped (23) - Start at 03:15:12 - Duration 556ms (transform 166ms, setup 16ms, collect 289ms, tests 107ms, environment 0ms, prepare 41ms) + Tests 1 passed | 23 skipped (24) + Start at 03:26:17 + Duration 639ms (transform 188ms, setup 21ms, collect 312ms, tests 106ms, environment 0ms, prepare 65ms) exit=0 ``` @@ -169,7 +170,7 @@ exit=0 ```text Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: 0d9162bdc58667fc831e603da1c31a52df2ec4fc8c39cd0383d6f6d9f0fc7bda +Original SHA256: efc2e4cdfdfba782ea0732b5573f4190ae02c486b7de4731a33f280f7bce6fc1 Replaced: this.drop(reader); @@ -181,14 +182,14 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (23 tests | 1 failed | 22 skipped) 1009ms + ❯ tests/journal-client-read-timeout.test.ts (24 tests | 1 failed | 23 skipped) 1009ms × reconnects the reader after it disconnects within the read budget 1008ms - → journal client: run.get read session was interrupted after 78 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + → journal client: run.get read session was interrupted after 80 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 78 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 80 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:71:9 69| let last: unknown; 70| const exhausted = () => last instanceof JournalReadInterruptedError @@ -199,9 +200,9 @@ JournalReadInterruptedError: journal client: run.get read session was interrupte ❯ Timeout. src/journal-read-policy.ts:82:39 Caused by: Error: journal client: not connected (run.get): journal client: connection closed - ❯ src/journal-client.ts:244:16 - ❯ JournalClient.requestOnce src/journal-client.ts:237:12 - ❯ src/journal-budgeted-reads.ts:47:29 + ❯ src/journal-client.ts:252:16 + ❯ JournalClient.requestOnce src/journal-client.ts:245:12 + ❯ src/journal-budgeted-reads.ts:54:29 ❯ src/journal-read-policy.ts:94:18 Caused by: Error: journal client: connection closed @@ -210,23 +211,23 @@ Caused by: Error: journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 22 skipped (23) - Start at 03:15:13 - Duration 1.46s (transform 173ms, setup 16ms, collect 299ms, tests 1.01s, environment 0ms, prepare 41ms) + Tests 1 failed | 23 skipped (24) + Start at 03:26:18 + Duration 1.52s (transform 182ms, setup 18ms, collect 325ms, tests 1.01s, environment 0ms, prepare 46ms) exit=1 -RESTORED SHA256: 0d9162bdc58667fc831e603da1c31a52df2ec4fc8c39cd0383d6f6d9f0fc7bda +RESTORED SHA256: efc2e4cdfdfba782ea0732b5573f4190ae02c486b7de4731a33f280f7bce6fc1 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'reconnects the reader after it disconnects' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (23 tests | 22 skipped) 94ms + ✓ tests/journal-client-read-timeout.test.ts (24 tests | 23 skipped) 86ms Test Files 1 passed (1) - Tests 1 passed | 22 skipped (23) - Start at 03:15:15 - Duration 504ms (transform 165ms, setup 17ms, collect 276ms, tests 94ms, environment 0ms, prepare 30ms) + Tests 1 passed | 23 skipped (24) + Start at 03:26:20 + Duration 517ms (transform 160ms, setup 17ms, collect 283ms, tests 86ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -235,7 +236,7 @@ exit=0 ```text Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: 0d9162bdc58667fc831e603da1c31a52df2ec4fc8c39cd0383d6f6d9f0fc7bda +Original SHA256: efc2e4cdfdfba782ea0732b5573f4190ae02c486b7de4731a33f280f7bce6fc1 Replaced: if (this.reader === reader) { this.reader = undefined; @@ -252,8 +253,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (23 tests | 1 failed | 22 skipped) 81ms - × retries a reader setup that timed out instead of reading on the primary for good 80ms + ❯ tests/journal-client-read-timeout.test.ts (24 tests | 1 failed | 23 skipped) 76ms + × retries a reader setup that timed out instead of reading on the primary for good 76ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -278,23 +279,23 @@ AssertionError: expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 22 skipped (23) - Start at 03:15:16 - Duration 515ms (transform 166ms, setup 16ms, collect 291ms, tests 81ms, environment 0ms, prepare 31ms) + Tests 1 failed | 23 skipped (24) + Start at 03:26:21 + Duration 518ms (transform 160ms, setup 16ms, collect 281ms, tests 76ms, environment 0ms, prepare 40ms) exit=1 -RESTORED SHA256: 0d9162bdc58667fc831e603da1c31a52df2ec4fc8c39cd0383d6f6d9f0fc7bda +RESTORED SHA256: efc2e4cdfdfba782ea0732b5573f4190ae02c486b7de4731a33f280f7bce6fc1 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'retries a reader setup that timed out' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (23 tests | 22 skipped) 69ms + ✓ tests/journal-client-read-timeout.test.ts (24 tests | 23 skipped) 70ms Test Files 1 passed (1) - Tests 1 passed | 22 skipped (23) - Start at 03:15:17 - Duration 532ms (transform 170ms, setup 17ms, collect 299ms, tests 69ms, environment 0ms, prepare 41ms) + Tests 1 passed | 23 skipped (24) + Start at 03:26:22 + Duration 518ms (transform 167ms, setup 16ms, collect 293ms, tests 70ms, environment 0ms, prepare 40ms) exit=0 ``` @@ -303,7 +304,7 @@ exit=0 ```text Source: packages/sdk/src/cli/running-step.ts -Original SHA256: 34223d566de94486b7039f5eb783ddacbe62ccc3cca63c2b5278d72c4de59c38 +Original SHA256: 2d95d1b21da7acb5e91e79236cb099604beabaed7e1e0443c96aeccf4c704c3e Replaced: const LEASE_POLL_MS = 2_000; With: @@ -314,8 +315,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (11 tests | 1 failed | 10 skipped) 2109ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2108ms + ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2111ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2111ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -333,24 +334,24 @@ AssertionError: expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 10 skipped (11) - Start at 03:15:18 - Duration 2.58s (transform 186ms, setup 16ms, collect 321ms, tests 2.11s, environment 0ms, prepare 41ms) + Tests 1 failed | 11 skipped (12) + Start at 03:26:23 + Duration 2.58s (transform 185ms, setup 15ms, collect 316ms, tests 2.11s, environment 0ms, prepare 42ms) exit=1 -RESTORED SHA256: 34223d566de94486b7039f5eb783ddacbe62ccc3cca63c2b5278d72c4de59c38 +RESTORED SHA256: 2d95d1b21da7acb5e91e79236cb099604beabaed7e1e0443c96aeccf4c704c3e $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pushes for completion' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (11 tests | 10 skipped) 2117ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2116ms + ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2118ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms Test Files 1 passed (1) - Tests 1 passed | 10 skipped (11) - Start at 03:15:21 - Duration 2.58s (transform 184ms, setup 16ms, collect 310ms, tests 2.12s, environment 0ms, prepare 42ms) + Tests 1 passed | 11 skipped (12) + Start at 03:26:25 + Duration 2.61s (transform 190ms, setup 17ms, collect 324ms, tests 2.12s, environment 0ms, prepare 40ms) exit=0 ``` @@ -359,7 +360,7 @@ exit=0 ```text Source: packages/sdk/src/cli/running-step.ts -Original SHA256: 34223d566de94486b7039f5eb783ddacbe62ccc3cca63c2b5278d72c4de59c38 +Original SHA256: 2d95d1b21da7acb5e91e79236cb099604beabaed7e1e0443c96aeccf4c704c3e Replaced: const signal = options.signal === undefined ? read.signal : AbortSignal.any([options.signal, read.signal]); @@ -371,7 +372,7 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (11 tests | 1 failed | 10 skipped) 2216ms + ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2216ms × cancels promptly while a lease snapshot read is in flight 2215ms → expected 'still waiting' to be an instance of Error @@ -390,24 +391,24 @@ AssertionError: expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 10 skipped (11) - Start at 03:15:24 - Duration 2.70s (transform 188ms, setup 17ms, collect 317ms, tests 2.22s, environment 0ms, prepare 41ms) + Tests 1 failed | 11 skipped (12) + Start at 03:26:28 + Duration 2.70s (transform 192ms, setup 17ms, collect 324ms, tests 2.22s, environment 0ms, prepare 40ms) exit=1 -RESTORED SHA256: 34223d566de94486b7039f5eb783ddacbe62ccc3cca63c2b5278d72c4de59c38 +RESTORED SHA256: 2d95d1b21da7acb5e91e79236cb099604beabaed7e1e0443c96aeccf4c704c3e $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels promptly while a lease snapshot read is in flight' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (11 tests | 10 skipped) 2011ms - ✓ cancels promptly while a lease snapshot read is in flight 2011ms + ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2011ms + ✓ cancels promptly while a lease snapshot read is in flight 2010ms Test Files 1 passed (1) - Tests 1 passed | 10 skipped (11) - Start at 03:15:27 - Duration 2.48s (transform 185ms, setup 17ms, collect 309ms, tests 2.01s, environment 0ms, prepare 40ms) + Tests 1 passed | 11 skipped (12) + Start at 03:26:31 + Duration 2.49s (transform 184ms, setup 17ms, collect 305ms, tests 2.01s, environment 0ms, prepare 40ms) exit=0 ``` @@ -416,7 +417,7 @@ exit=0 ```text Source: packages/sdk/src/cli/running-step.ts -Original SHA256: 34223d566de94486b7039f5eb783ddacbe62ccc3cca63c2b5278d72c4de59c38 +Original SHA256: 2d95d1b21da7acb5e91e79236cb099604beabaed7e1e0443c96aeccf4c704c3e Replaced: snapshotRead?.abort(new Error(`step "${runningStep.id}" completed during the lease snapshot`)); With: @@ -427,8 +428,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (11 tests | 1 failed | 10 skipped) 3012ms - × a completion push ends the wait without waiting out an in-flight snapshot 3012ms + ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 3012ms + × a completion push ends the wait without waiting out an in-flight snapshot 3011ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -450,24 +451,24 @@ Received: "still waiting" ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 10 skipped (11) - Start at 03:15:29 - Duration 3.49s (transform 187ms, setup 17ms, collect 317ms, tests 3.01s, environment 0ms, prepare 40ms) + Tests 1 failed | 11 skipped (12) + Start at 03:26:34 + Duration 3.49s (transform 188ms, setup 16ms, collect 318ms, tests 3.01s, environment 0ms, prepare 41ms) exit=1 -RESTORED SHA256: 34223d566de94486b7039f5eb783ddacbe62ccc3cca63c2b5278d72c4de59c38 +RESTORED SHA256: 2d95d1b21da7acb5e91e79236cb099604beabaed7e1e0443c96aeccf4c704c3e $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a completion push ends the wait' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (11 tests | 10 skipped) 2059ms + ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2060ms ✓ a completion push ends the wait without waiting out an in-flight snapshot 2059ms Test Files 1 passed (1) - Tests 1 passed | 10 skipped (11) - Start at 03:15:33 - Duration 2.51s (transform 180ms, setup 16ms, collect 300ms, tests 2.06s, environment 0ms, prepare 41ms) + Tests 1 passed | 11 skipped (12) + Start at 03:26:38 + Duration 2.54s (transform 188ms, setup 17ms, collect 314ms, tests 2.06s, environment 0ms, prepare 42ms) exit=0 ``` @@ -510,8 +511,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 03:15:36 - Duration 473ms (transform 168ms, setup 16ms, collect 291ms, tests 19ms, environment 0ms, prepare 40ms) + Start at 03:26:41 + Duration 455ms (transform 165ms, setup 16ms, collect 289ms, tests 19ms, environment 0ms, prepare 41ms) exit=1 @@ -524,8 +525,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb Test Files 1 passed (1) Tests 1 passed (1) - Start at 03:15:37 - Duration 455ms (transform 163ms, setup 16ms, collect 280ms, tests 16ms, environment 0ms, prepare 41ms) + Start at 03:26:42 + Duration 445ms (transform 163ms, setup 16ms, collect 275ms, tests 16ms, environment 0ms, prepare 40ms) exit=0 ``` @@ -534,7 +535,7 @@ exit=0 ```text Source: packages/sdk/src/authored-root.ts -Original SHA256: 79b644e3c53dc5ec73fa859720e6da87008efb4ce9c4dd1447aa362729624dc8 +Original SHA256: 79f49acf9e87f35a2ff6a3b757c2d193e4e821eadfd28f96181034f550b01dfa Replaced: if (isReadInterruptionError(error) || (error instanceof AuthoredFlowExecutionError && error.code === 'daemon_unresponsive')) { @@ -552,14 +553,14 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (28 tests | 3 failed | 25 skipped) 21ms - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 15ms + ❯ tests/authored-root.test.ts (29 tests | 3 failed | 26 skipped) 32ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 22ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 3ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms → expected AuthoredFlowExecutionError: daemon_unresp… { …(7) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 2ms + × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 4ms → expected JournalReadInterruptedError: journal clie… { …(4) } to match object { code: 'daemon_unresponsive', …(2) } (5 matching properties omitted from actual) @@ -638,23 +639,23 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[3/3]⎯ Test Files 1 failed (1) - Tests 3 failed | 25 skipped (28) - Start at 03:15:38 - Duration 1.22s (transform 643ms, setup 14ms, collect 1.04s, tests 21ms, environment 0ms, prepare 41ms) + Tests 3 failed | 26 skipped (29) + Start at 03:26:42 + Duration 1.29s (transform 668ms, setup 17ms, collect 1.09s, tests 32ms, environment 0ms, prepare 42ms) exit=1 -RESTORED SHA256: 79b644e3c53dc5ec73fa859720e6da87008efb4ce9c4dd1447aa362729624dc8 +RESTORED SHA256: 79f49acf9e87f35a2ff6a3b757c2d193e4e821eadfd28f96181034f550b01dfa $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the root resumable' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (28 tests | 25 skipped) 16ms + ✓ tests/authored-root.test.ts (29 tests | 26 skipped) 24ms Test Files 1 passed (1) - Tests 3 passed | 25 skipped (28) - Start at 03:15:39 - Duration 1.22s (transform 650ms, setup 16ms, collect 1.06s, tests 16ms, environment 0ms, prepare 30ms) + Tests 3 passed | 26 skipped (29) + Start at 03:26:44 + Duration 1.24s (transform 644ms, setup 16ms, collect 1.05s, tests 24ms, environment 0ms, prepare 42ms) exit=0 ``` diff --git a/packages/sdk/src/authored-root.ts b/packages/sdk/src/authored-root.ts index 715b798a2..53e4d681c 100644 --- a/packages/sdk/src/authored-root.ts +++ b/packages/sdk/src/authored-root.ts @@ -321,38 +321,42 @@ async function driveRoot( ): Promise { try { const result = await withWorkerLease(peer, dispatch, async rootSignal => { - const callerSignal = options.lifecycle?.signal; - const signal = callerSignal === undefined ? rootSignal : AbortSignal.any([callerSignal, rootSignal]); - if (process.versions['bun'] !== undefined) { - return runAuthoredInNode(metadata, journal.socketPath, dispatch.run_id, { - dataDir: options.dataDir, localAgentStream: options.localAgentStream, - ...(options.agentEnvironment === undefined ? {} : { - agentEnvironment: localAgentCredentialEnvironment(options.agentEnvironment), - }), - ...(options.workerCapacity === undefined ? {} : { workerCapacity: options.workerCapacity }), - ...options.lifecycle, signal, - }); - } - return await executeAuthoredFlow( - loaded.handle, - journal, - metadata.inputPresent ? metadata.input : undefined, - { - getDefinition: loaded.getDefinition, - dataDir: options.dataDir, - flowPath: metadata.flowPath, - localAgentStream: options.localAgentStream, - ...(options.agentEnvironment === undefined ? {} : { agentEnvironment: options.agentEnvironment }), - ...(options.workerCapacity === undefined ? {} : { workerCapacity: options.workerCapacity }), - rootRunId: dispatch.run_id, - extensions: loaded.extensions, - flowGraph: loaded.graph, - ...options.lifecycle, - signal: callerSignal === undefined - ? rootSignal - : AbortSignal.any([callerSignal, rootSignal]), - }, - ); + // A lost root lease must also stop body reads still inside their budget. + const releaseReads = journal.scopeReads?.(rootSignal) ?? (() => {}); + try { + const callerSignal = options.lifecycle?.signal; + const signal = callerSignal === undefined ? rootSignal : AbortSignal.any([callerSignal, rootSignal]); + if (process.versions['bun'] !== undefined) { + return runAuthoredInNode(metadata, journal.socketPath, dispatch.run_id, { + dataDir: options.dataDir, localAgentStream: options.localAgentStream, + ...(options.agentEnvironment === undefined ? {} : { + agentEnvironment: localAgentCredentialEnvironment(options.agentEnvironment), + }), + ...(options.workerCapacity === undefined ? {} : { workerCapacity: options.workerCapacity }), + ...options.lifecycle, signal, + }); + } + return await executeAuthoredFlow( + loaded.handle, + journal, + metadata.inputPresent ? metadata.input : undefined, + { + getDefinition: loaded.getDefinition, + dataDir: options.dataDir, + flowPath: metadata.flowPath, + localAgentStream: options.localAgentStream, + ...(options.agentEnvironment === undefined ? {} : { agentEnvironment: options.agentEnvironment }), + ...(options.workerCapacity === undefined ? {} : { workerCapacity: options.workerCapacity }), + rootRunId: dispatch.run_id, + extensions: loaded.extensions, + flowGraph: loaded.graph, + ...options.lifecycle, + signal: callerSignal === undefined + ? rootSignal + : AbortSignal.any([callerSignal, rootSignal]), + }, + ); + } finally { releaseReads(); } }); await peer.stepComplete( dispatch.run_id, dispatch.step_id, dispatch.attempt, diff --git a/packages/sdk/src/cli/running-step.ts b/packages/sdk/src/cli/running-step.ts index 0ae332e72..924ed2065 100644 --- a/packages/sdk/src/cli/running-step.ts +++ b/packages/sdk/src/cli/running-step.ts @@ -57,8 +57,12 @@ export async function waitForRunningStep(client: JournalClient, runId: string, watch.on('protocol_error', onProtocolError); // A dropped watch loses its completion pushes: a resumable read interruption. const onDisconnected = (error: Error) => { - if (!settled) fail(new AuthoredFlowExecutionError('daemon_unresponsive', - `the completion watch for step "${runningStep.id}" disconnected: ${error.message}`)); + if (settled) return; + const lost = new AuthoredFlowExecutionError('daemon_unresponsive', + `the completion watch for step "${runningStep.id}" disconnected: ${error.message}`); + protocolFailure ??= lost; + fail(lost); + snapshotRead?.abort(lost); }; watch.on('disconnected', onDisconnected); options.signal?.addEventListener('abort', cancel, { once: true }); diff --git a/packages/sdk/src/journal-budgeted-reads.ts b/packages/sdk/src/journal-budgeted-reads.ts index 272d14d62..861610198 100644 --- a/packages/sdk/src/journal-budgeted-reads.ts +++ b/packages/sdk/src/journal-budgeted-reads.ts @@ -15,6 +15,7 @@ export class BudgetedReads { private readonly policy = new JournalReadPolicy(); private reader: JournalClient | undefined; private readerReady: Promise | undefined; + private readonly scoped = new Set(); constructor( private readonly primary: JournalClient, @@ -22,6 +23,12 @@ export class BudgetedReads { private readonly lifecycle?: AbortSignal, ) {} + /** Add an abort source (a root attempt's lease) for as long as the returned release is not called. */ + scope(signal: AbortSignal): () => void { + this.scoped.add(signal); + return () => { this.scoped.delete(signal); }; + } + close(cause?: unknown): void { this.policy.close(); this.reader?.close(cause); @@ -29,8 +36,8 @@ export class BudgetedReads { read(verb: V, params: VerbContract[V]['params'], timeoutMs: number, signal?: AbortSignal): Promise { - const combined = signal === undefined ? this.lifecycle - : this.lifecycle === undefined ? signal : AbortSignal.any([signal, this.lifecycle]); + const sources = [signal, this.lifecycle, ...this.scoped].filter((source): source is AbortSignal => source !== undefined); + const combined = sources.length <= 1 ? sources[0] : AbortSignal.any(sources); return this.policy.read(verb, timeoutMs, this.budgetMs, async (bound, attemptSignal) => { const started = performance.now(); const reader = await this.session().catch(error => { diff --git a/packages/sdk/src/journal-client.ts b/packages/sdk/src/journal-client.ts index 3ed235e67..70ee624a8 100644 --- a/packages/sdk/src/journal-client.ts +++ b/packages/sdk/src/journal-client.ts @@ -227,6 +227,14 @@ export class JournalClient extends EventEmitter { return this.requestOnce(verb, params, timeoutMs, signal); } + /** + * Cancel budgeted reads also when `signal` aborts — a root attempt's lease — + * until the returned release is called. A no-op without a read budget. + */ + scopeReads(signal: AbortSignal): () => void { + return this.budgeted?.scope(signal) ?? (() => {}); + } + /** @internal One unretried request on this session; flow reads go through `request`. */ requestOnce( verb: V, diff --git a/packages/sdk/tests/authored-root.test.ts b/packages/sdk/tests/authored-root.test.ts index 6bb0c741f..af1aca6c0 100644 --- a/packages/sdk/tests/authored-root.test.ts +++ b/packages/sdk/tests/authored-root.test.ts @@ -445,6 +445,19 @@ describe('durable authored root', () => { expect(journal.peer.completions).toEqual([]); }); + it('scopes body reads to the root attempt lease for the life of the attempt', async () => { + const loaded = await fixture(); + const journal = new RootJournal(); + const release = vi.fn(); + const scopeReads = vi.fn((_signal: AbortSignal) => release); + (journal as unknown as { scopeReads: typeof scopeReads }).scopeReads = scopeReads; + await executeDurableAuthoredFlow(loaded, journal as unknown as JournalClient, undefined, + { dataDir: '/unused', admissionKey: 'scoped-reads' }).catch(() => undefined); + expect(scopeReads).toHaveBeenCalledTimes(1); + expect(scopeReads.mock.calls[0]![0]).toBeInstanceOf(AbortSignal); + expect(release).toHaveBeenCalledTimes(1); + }); + it('terminalizes a returned body failure without replaying semantic side effects', async () => { const loaded = await fixture(true); const journal = new RootJournal(); diff --git a/packages/sdk/tests/journal-client-read-timeout.test.ts b/packages/sdk/tests/journal-client-read-timeout.test.ts index ae31fb274..c77247ae7 100644 --- a/packages/sdk/tests/journal-client-read-timeout.test.ts +++ b/packages/sdk/tests/journal-client-read-timeout.test.ts @@ -276,3 +276,18 @@ it('an exhausted post-admission handshake budget is a read interruption, not a b const error = await client.hello('flows-authored-node').catch(caught => caught); expect(isReadInterruptionError(error)).toBe(true); }); + +it('a scoped read signal (the root lease) cancels budgeted reads only while it is attached', async () => { + let entered!: () => void; + let reads = 0; + const started = new Promise(resolve => { entered = resolve; }); + const client = await setup({ 'journal.read': ctx => { reads += 1; if (reads === 1) entered(); else sendResult(ctx, { entries: [] }); } }, 60_000); + const lease = new AbortController(); + const release = client.scopeReads(lease.signal); + const pending = client.journalRead('run', 1); + await started; + lease.abort(new Error('root lease lost')); + await expect(pending).rejects.toThrow('root lease lost'); + release(); + await expect(client.journalRead('run', 1)).resolves.toEqual({ entries: [] }); +}); diff --git a/packages/sdk/tests/running-step-watch.test.ts b/packages/sdk/tests/running-step-watch.test.ts index ecc7a654a..3b533bb6e 100644 --- a/packages/sdk/tests/running-step-watch.test.ts +++ b/packages/sdk/tests/running-step-watch.test.ts @@ -249,3 +249,24 @@ it('a watch connection lost after registration is a read interruption, not a sil expect(isReadInterruption(outcome)).toBe(true); } finally { client.close(); await new Promise(resolve => server.close(() => resolve())); } }); + +it('a watch dropped during an in-flight snapshot is reported at once, not after the read budget', async () => { + const path = sockPath(); + let drop!: () => void; + const server = startLoopback(path, { + hello: sendOk, + 'run.watch': ctx => { sendResult(ctx, {}); drop = () => ctx.socket.destroy(); }, + 'run.get': () => drop(), // the watch drops while the snapshot is in flight + }); + await once(server, 'listening'); + const client = new JournalClient(path, { readBudgetMs: 300_000 }); + try { + await client.connect(); + const outcome = await Promise.race([ + waitForRunningStep(client, 'run', { id: 'step', type: 'agent', leaseDeadlineMs: Date.now() + 30_000 }, {}).then(() => 'returned', error => error), + sleep(4_000).then(() => 'still waiting'), + ]); + expect(outcome).toBeInstanceOf(AuthoredFlowExecutionError); + expect((outcome as AuthoredFlowExecutionError).code).toBe('daemon_unresponsive'); + } finally { client.close(); await new Promise(resolve => server.close(() => resolve())); } +}); From 8c1d19801b697ba0ac22a648bb74f50d4380e9d9 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 03:37:02 -0700 Subject: [PATCH 21/34] fix(sdk): root detection and primary drops keep read interruptions typed - readAuthoredRootMetadata rethrows a read interruption instead of returning 'not authored', so a delayed daemon during resume detection reports a resumable daemon_unresponsive rather than resuming the run declaratively. - An unexpected primary disconnect now ends in-flight budgeted reads with JournalReadInterruptedError (a caller close keeps its plain error), so the authored root parks instead of being terminalized as worker_error. Evidence re-captured at this tree. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: b8262468-ea26-4a6e-bee9-d193832e77dc --- evidence/run-read-timeout/final-focused.log | 114 ++++---- .../mutation-completion-aborts-snapshot.log | 12 +- .../run-read-timeout/mutation-heartbeat.log | 12 +- .../mutation-reader-reconnect.log | 28 +- .../mutation-reader-setup-retry.log | 22 +- evidence/run-read-timeout/mutation-reader.log | 22 +- .../mutation-root-parking.log | 20 +- .../mutation-snapshot-cancel.log | 12 +- .../mutation-watch-cadence.log | 14 +- evidence/run-read-timeout/resume-live.log | 8 +- evidence/run-read-timeout/verification.md | 264 +++++++++--------- packages/sdk/src/authored-root.ts | 7 +- packages/sdk/src/journal-budgeted-reads.ts | 9 +- packages/sdk/src/journal-client.ts | 2 +- packages/sdk/src/journal-read-policy.ts | 2 +- packages/sdk/tests/authored-root.test.ts | 8 + .../tests/journal-client-read-timeout.test.ts | 11 + 17 files changed, 298 insertions(+), 269 deletions(-) diff --git a/evidence/run-read-timeout/final-focused.log b/evidence/run-read-timeout/final-focused.log index 45a13593c..93dc11abb 100644 --- a/evidence/run-read-timeout/final-focused.log +++ b/evidence/run-read-timeout/final-focused.log @@ -2,70 +2,70 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5327ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2039ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2045ms -(node:4039318) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5537ms + ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 462ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2038ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2039ms +(node:4077640) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (29 tests) 399ms - ✓ tests/journal-client.test.ts (17 tests) 70ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 6998ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 450ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1015ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2825ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 388ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 865ms - ✓ tests/authored-flow-slack.test.ts (7 tests) 1252ms - ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 437ms - ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 434ms - ✓ tests/journal-client-read-timeout.test.ts (24 tests) 1877ms - ✓ a recovered read timeout does not become an authored callback failure 363ms - ✓ tests/running-step-watch.test.ts (12 tests) 12287ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms - ✓ cancels promptly while a lease snapshot read is in flight 2006ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2053ms + ✓ tests/authored-root.test.ts (30 tests) 393ms + ✓ tests/journal-client.test.ts (17 tests) 80ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 7062ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 445ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1032ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2778ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 382ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 861ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 1292ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 458ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 451ms + ✓ tests/journal-client-read-timeout.test.ts (25 tests) 1797ms + ✓ tests/running-step-watch.test.ts (12 tests) 12286ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2116ms + ✓ cancels promptly while a lease snapshot read is in flight 2005ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2055ms ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms - ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2003ms + ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2004ms ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2002ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2582ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 692ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 645ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 660ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 585ms - ✓ tests/authored-helpers.test.ts (6 tests) 1962ms - ✓ runs every available provider through the real kernel and resumes completed effects without a second write 904ms - ✓ replays after SIGKILL before confirm with the same token and one successful completion 402ms - ✓ replays after SIGKILL before complete with the same token and one successful completion 427ms - ✓ tests/classify-outcome.test.ts (11 tests) 7429ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2015ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3006ms - ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2002ms - ✓ tests/human-live.test.ts (3 tests) 4589ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2702ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1114ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 772ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2730ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 694ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 665ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 693ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 676ms + ✓ tests/authored-helpers.test.ts (6 tests) 2139ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 1026ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 448ms + ✓ replays after SIGKILL before complete with the same token and one successful completion 425ms + ✓ tests/classify-outcome.test.ts (11 tests) 7423ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2010ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3005ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2000ms + ✓ tests/human-live.test.ts (3 tests) 4749ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2900ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1089ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 758ms ✓ tests/worker-lease.test.ts (7 tests) 19ms - ✓ tests/communication-worker.test.ts (18 tests) 1722ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 24ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 733ms - ✓ tests/memoization.test.ts (58 tests) 67ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2475ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2056ms - ✓ drains read and watch promises before an authored flow completes 418ms - ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms - ✓ tests/journal-client-completion.test.ts (6 tests) 100ms + ✓ tests/communication-worker.test.ts (18 tests) 1716ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 27ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 715ms + ✓ tests/memoization.test.ts (58 tests) 66ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2507ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2057ms + ✓ drains read and watch promises before an authored flow completes 449ms + ✓ tests/worker-lease-sweep.test.ts (4 tests) 9ms + ✓ tests/journal-client-completion.test.ts (6 tests) 106ms ✓ tests/reuse-summary-interruption.test.ts (5 tests) 4ms - ✓ tests/resume-worker-lease.test.ts (3 tests) 8ms - ✓ tests/direct-run-worker-lease.test.ts (3 tests) 9ms - ✓ tests/helper-effect-lease-loss.test.ts (1 test) 229ms - ✓ tests/journal-client-subscriptions.test.ts (1 test) 6ms + ✓ tests/resume-worker-lease.test.ts (3 tests) 9ms + ✓ tests/direct-run-worker-lease.test.ts (3 tests) 10ms + ✓ tests/helper-effect-lease-loss.test.ts (1 test) 232ms + ✓ tests/journal-client-subscriptions.test.ts (1 test) 7ms ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 4ms - ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 47ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms + ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 46ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 17ms Test Files 27 passed (27) - Tests 332 passed (332) - Start at 03:26:49 - Duration 60.48s (transform 1.20s, setup 151ms, collect 6.79s, tests 50.24s, environment 4ms, prepare 970ms) + Tests 334 passed (334) + Start at 03:35:58 + Duration 61.82s (transform 1.34s, setup 155ms, collect 7.29s, tests 50.98s, environment 4ms, prepare 998ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log index e3ff1b5b6..0e2710cba 100644 --- a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log +++ b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log @@ -34,8 +34,8 @@ Received: "still waiting" Test Files 1 failed (1) Tests 1 failed | 11 skipped (12) - Start at 03:26:34 - Duration 3.49s (transform 188ms, setup 16ms, collect 318ms, tests 3.01s, environment 0ms, prepare 41ms) + Start at 03:35:43 + Duration 3.49s (transform 186ms, setup 16ms, collect 316ms, tests 3.01s, environment 0ms, prepare 43ms) exit=1 @@ -44,12 +44,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2060ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2059ms + ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2059ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2058ms Test Files 1 passed (1) Tests 1 passed | 11 skipped (12) - Start at 03:26:38 - Duration 2.54s (transform 188ms, setup 17ms, collect 314ms, tests 2.06s, environment 0ms, prepare 42ms) + Start at 03:35:46 + Duration 2.50s (transform 184ms, setup 15ms, collect 309ms, tests 2.06s, environment 0ms, prepare 31ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-heartbeat.log b/evidence/run-read-timeout/mutation-heartbeat.log index 8e35ca69e..e46ec887d 100644 --- a/evidence/run-read-timeout/mutation-heartbeat.log +++ b/evidence/run-read-timeout/mutation-heartbeat.log @@ -13,8 +13,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 19ms - × a heartbeat timeout is lease loss rather than a worker body failure 19ms + ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 20ms + × a heartbeat timeout is lease loss rather than a worker body failure 20ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -33,8 +33,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 03:26:41 - Duration 455ms (transform 165ms, setup 16ms, collect 289ms, tests 19ms, environment 0ms, prepare 41ms) + Start at 03:35:49 + Duration 461ms (transform 170ms, setup 16ms, collect 277ms, tests 20ms, environment 0ms, prepare 41ms) exit=1 @@ -47,7 +47,7 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb Test Files 1 passed (1) Tests 1 passed (1) - Start at 03:26:42 - Duration 445ms (transform 163ms, setup 16ms, collect 275ms, tests 16ms, environment 0ms, prepare 40ms) + Start at 03:35:50 + Duration 447ms (transform 170ms, setup 17ms, collect 286ms, tests 16ms, environment 0ms, prepare 44ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-reconnect.log b/evidence/run-read-timeout/mutation-reader-reconnect.log index 8a0792eb2..5f6ee12f1 100644 --- a/evidence/run-read-timeout/mutation-reader-reconnect.log +++ b/evidence/run-read-timeout/mutation-reader-reconnect.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: efc2e4cdfdfba782ea0732b5573f4190ae02c486b7de4731a33f280f7bce6fc1 +Original SHA256: 30380437df429b288157a3a895ddb2cede8fe8853889b4a0cb9a3c24a4097814 Replaced: this.drop(reader); @@ -11,14 +11,14 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (24 tests | 1 failed | 23 skipped) 1009ms - × reconnects the reader after it disconnects within the read budget 1008ms - → journal client: run.get read session was interrupted after 80 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ tests/journal-client-read-timeout.test.ts (25 tests | 1 failed | 24 skipped) 1010ms + × reconnects the reader after it disconnects within the read budget 1009ms + → journal client: run.get read session was interrupted after 77 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 80 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 77 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:71:9 69| let last: unknown; 70| const exhausted = () => last instanceof JournalReadInterruptedError @@ -31,7 +31,7 @@ JournalReadInterruptedError: journal client: run.get read session was interrupte Caused by: Error: journal client: not connected (run.get): journal client: connection closed ❯ src/journal-client.ts:252:16 ❯ JournalClient.requestOnce src/journal-client.ts:245:12 - ❯ src/journal-budgeted-reads.ts:54:29 + ❯ src/journal-budgeted-reads.ts:59:29 ❯ src/journal-read-policy.ts:94:18 Caused by: Error: journal client: connection closed @@ -40,22 +40,22 @@ Caused by: Error: journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 23 skipped (24) - Start at 03:26:18 - Duration 1.52s (transform 182ms, setup 18ms, collect 325ms, tests 1.01s, environment 0ms, prepare 46ms) + Tests 1 failed | 24 skipped (25) + Start at 03:35:26 + Duration 1.45s (transform 167ms, setup 15ms, collect 285ms, tests 1.01s, environment 0ms, prepare 41ms) exit=1 -RESTORED SHA256: efc2e4cdfdfba782ea0732b5573f4190ae02c486b7de4731a33f280f7bce6fc1 +RESTORED SHA256: 30380437df429b288157a3a895ddb2cede8fe8853889b4a0cb9a3c24a4097814 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'reconnects the reader after it disconnects' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (24 tests | 23 skipped) 86ms + ✓ tests/journal-client-read-timeout.test.ts (25 tests | 24 skipped) 81ms Test Files 1 passed (1) - Tests 1 passed | 23 skipped (24) - Start at 03:26:20 - Duration 517ms (transform 160ms, setup 17ms, collect 283ms, tests 86ms, environment 0ms, prepare 41ms) + Tests 1 passed | 24 skipped (25) + Start at 03:35:28 + Duration 521ms (transform 159ms, setup 16ms, collect 280ms, tests 81ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-setup-retry.log b/evidence/run-read-timeout/mutation-reader-setup-retry.log index 6bc70f966..015850151 100644 --- a/evidence/run-read-timeout/mutation-reader-setup-retry.log +++ b/evidence/run-read-timeout/mutation-reader-setup-retry.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: efc2e4cdfdfba782ea0732b5573f4190ae02c486b7de4731a33f280f7bce6fc1 +Original SHA256: 30380437df429b288157a3a895ddb2cede8fe8853889b4a0cb9a3c24a4097814 Replaced: if (this.reader === reader) { this.reader = undefined; @@ -16,8 +16,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (24 tests | 1 failed | 23 skipped) 76ms - × retries a reader setup that timed out instead of reading on the primary for good 76ms + ❯ tests/journal-client-read-timeout.test.ts (25 tests | 1 failed | 24 skipped) 78ms + × retries a reader setup that timed out instead of reading on the primary for good 77ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -42,22 +42,22 @@ AssertionError: expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 23 skipped (24) - Start at 03:26:21 - Duration 518ms (transform 160ms, setup 16ms, collect 281ms, tests 76ms, environment 0ms, prepare 40ms) + Tests 1 failed | 24 skipped (25) + Start at 03:35:29 + Duration 514ms (transform 165ms, setup 17ms, collect 289ms, tests 78ms, environment 0ms, prepare 40ms) exit=1 -RESTORED SHA256: efc2e4cdfdfba782ea0732b5573f4190ae02c486b7de4731a33f280f7bce6fc1 +RESTORED SHA256: 30380437df429b288157a3a895ddb2cede8fe8853889b4a0cb9a3c24a4097814 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'retries a reader setup that timed out' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (24 tests | 23 skipped) 70ms + ✓ tests/journal-client-read-timeout.test.ts (25 tests | 24 skipped) 73ms Test Files 1 passed (1) - Tests 1 passed | 23 skipped (24) - Start at 03:26:22 - Duration 518ms (transform 167ms, setup 16ms, collect 293ms, tests 70ms, environment 0ms, prepare 40ms) + Tests 1 passed | 24 skipped (25) + Start at 03:35:30 + Duration 518ms (transform 165ms, setup 18ms, collect 287ms, tests 73ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader.log b/evidence/run-read-timeout/mutation-reader.log index bd205012a..37d5b5f82 100644 --- a/evidence/run-read-timeout/mutation-reader.log +++ b/evidence/run-read-timeout/mutation-reader.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: efc2e4cdfdfba782ea0732b5573f4190ae02c486b7de4731a33f280f7bce6fc1 +Original SHA256: 30380437df429b288157a3a895ddb2cede8fe8853889b4a0cb9a3c24a4097814 Replaced: return await reader.requestOnce(verb, params, remaining, attemptSignal); With: @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (24 tests | 1 failed | 23 skipped) 116ms - × serves a bounded read while an unbounded command is in flight 116ms + ❯ tests/journal-client-read-timeout.test.ts (25 tests | 1 failed | 24 skipped) 113ms + × serves a bounded read while an unbounded command is in flight 112ms → expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -36,22 +36,22 @@ AssertionError: expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 23 skipped (24) - Start at 03:26:16 - Duration 626ms (transform 199ms, setup 14ms, collect 339ms, tests 116ms, environment 0ms, prepare 43ms) + Tests 1 failed | 24 skipped (25) + Start at 03:35:24 + Duration 577ms (transform 172ms, setup 15ms, collect 303ms, tests 113ms, environment 0ms, prepare 40ms) exit=1 -RESTORED SHA256: efc2e4cdfdfba782ea0732b5573f4190ae02c486b7de4731a33f280f7bce6fc1 +RESTORED SHA256: 30380437df429b288157a3a895ddb2cede8fe8853889b4a0cb9a3c24a4097814 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'serves a bounded read' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (24 tests | 23 skipped) 106ms + ✓ tests/journal-client-read-timeout.test.ts (25 tests | 24 skipped) 106ms Test Files 1 passed (1) - Tests 1 passed | 23 skipped (24) - Start at 03:26:17 - Duration 639ms (transform 188ms, setup 21ms, collect 312ms, tests 106ms, environment 0ms, prepare 65ms) + Tests 1 passed | 24 skipped (25) + Start at 03:35:25 + Duration 549ms (transform 162ms, setup 16ms, collect 287ms, tests 106ms, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-root-parking.log b/evidence/run-read-timeout/mutation-root-parking.log index a604e58ff..290e246bb 100644 --- a/evidence/run-read-timeout/mutation-root-parking.log +++ b/evidence/run-read-timeout/mutation-root-parking.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/authored-root.ts -Original SHA256: 79f49acf9e87f35a2ff6a3b757c2d193e4e821eadfd28f96181034f550b01dfa +Original SHA256: 8ab20a03c6def62800d2ef8a49932ec616a3b1c1fcd48c4aff9fa80437bcb35c Replaced: if (isReadInterruptionError(error) || (error instanceof AuthoredFlowExecutionError && error.code === 'daemon_unresponsive')) { @@ -17,7 +17,7 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (29 tests | 3 failed | 26 skipped) 32ms + ❯ tests/authored-root.test.ts (30 tests | 3 failed | 27 skipped) 32ms × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 22ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) @@ -103,22 +103,22 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[3/3]⎯ Test Files 1 failed (1) - Tests 3 failed | 26 skipped (29) - Start at 03:26:42 - Duration 1.29s (transform 668ms, setup 17ms, collect 1.09s, tests 32ms, environment 0ms, prepare 42ms) + Tests 3 failed | 27 skipped (30) + Start at 03:35:51 + Duration 1.29s (transform 661ms, setup 15ms, collect 1.10s, tests 32ms, environment 0ms, prepare 40ms) exit=1 -RESTORED SHA256: 79f49acf9e87f35a2ff6a3b757c2d193e4e821eadfd28f96181034f550b01dfa +RESTORED SHA256: 8ab20a03c6def62800d2ef8a49932ec616a3b1c1fcd48c4aff9fa80437bcb35c $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the root resumable' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (29 tests | 26 skipped) 24ms + ✓ tests/authored-root.test.ts (30 tests | 27 skipped) 23ms Test Files 1 passed (1) - Tests 3 passed | 26 skipped (29) - Start at 03:26:44 - Duration 1.24s (transform 644ms, setup 16ms, collect 1.05s, tests 24ms, environment 0ms, prepare 42ms) + Tests 3 passed | 27 skipped (30) + Start at 03:35:53 + Duration 1.27s (transform 658ms, setup 17ms, collect 1.10s, tests 23ms, environment 0ms, prepare 31ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-snapshot-cancel.log b/evidence/run-read-timeout/mutation-snapshot-cancel.log index 71aef5ff8..2e96db0fb 100644 --- a/evidence/run-read-timeout/mutation-snapshot-cancel.log +++ b/evidence/run-read-timeout/mutation-snapshot-cancel.log @@ -31,8 +31,8 @@ AssertionError: expected 'still waiting' to be an instance of Error Test Files 1 failed (1) Tests 1 failed | 11 skipped (12) - Start at 03:26:28 - Duration 2.70s (transform 192ms, setup 17ms, collect 324ms, tests 2.22s, environment 0ms, prepare 40ms) + Start at 03:35:36 + Duration 2.71s (transform 195ms, setup 16ms, collect 331ms, tests 2.22s, environment 0ms, prepare 41ms) exit=1 @@ -41,12 +41,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2011ms - ✓ cancels promptly while a lease snapshot read is in flight 2010ms + ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2010ms + ✓ cancels promptly while a lease snapshot read is in flight 2009ms Test Files 1 passed (1) Tests 1 passed | 11 skipped (12) - Start at 03:26:31 - Duration 2.49s (transform 184ms, setup 17ms, collect 305ms, tests 2.01s, environment 0ms, prepare 40ms) + Start at 03:35:40 + Duration 2.55s (transform 232ms, setup 22ms, collect 353ms, tests 2.01s, environment 0ms, prepare 57ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-watch-cadence.log b/evidence/run-read-timeout/mutation-watch-cadence.log index 7a82e5803..b532d4076 100644 --- a/evidence/run-read-timeout/mutation-watch-cadence.log +++ b/evidence/run-read-timeout/mutation-watch-cadence.log @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2111ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2111ms + ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2110ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2110ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -30,8 +30,8 @@ AssertionError: expected 41 to be less than or equal to 1 Test Files 1 failed (1) Tests 1 failed | 11 skipped (12) - Start at 03:26:23 - Duration 2.58s (transform 185ms, setup 15ms, collect 316ms, tests 2.11s, environment 0ms, prepare 42ms) + Start at 03:35:31 + Duration 2.56s (transform 179ms, setup 16ms, collect 300ms, tests 2.11s, environment 0ms, prepare 36ms) exit=1 @@ -41,11 +41,11 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2118ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms Test Files 1 passed (1) Tests 1 passed | 11 skipped (12) - Start at 03:26:25 - Duration 2.61s (transform 190ms, setup 17ms, collect 324ms, tests 2.12s, environment 0ms, prepare 40ms) + Start at 03:35:34 + Duration 2.59s (transform 185ms, setup 16ms, collect 311ms, tests 2.12s, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/resume-live.log b/evidence/run-read-timeout/resume-live.log index 19a7a6043..05aec7ca9 100644 --- a/evidence/run-read-timeout/resume-live.log +++ b/evidence/run-read-timeout/resume-live.log @@ -2,12 +2,12 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 944ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 943ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 888ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 887ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 03:27:50 - Duration 2.22s (transform 636ms, setup 21ms, collect 1.04s, tests 944ms, environment 0ms, prepare 85ms) + Start at 03:37:00 + Duration 2.12s (transform 602ms, setup 27ms, collect 967ms, tests 888ms, environment 0ms, prepare 102ms) exit=0 diff --git a/evidence/run-read-timeout/verification.md b/evidence/run-read-timeout/verification.md index ed37579c1..821a500d6 100644 --- a/evidence/run-read-timeout/verification.md +++ b/evidence/run-read-timeout/verification.md @@ -15,71 +15,71 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5327ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2039ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2045ms -(node:4039318) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5537ms + ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 462ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2038ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2039ms +(node:4077640) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (29 tests) 399ms - ✓ tests/journal-client.test.ts (17 tests) 70ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 6998ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 450ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1015ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2825ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 388ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 865ms - ✓ tests/authored-flow-slack.test.ts (7 tests) 1252ms - ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 437ms - ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 434ms - ✓ tests/journal-client-read-timeout.test.ts (24 tests) 1877ms - ✓ a recovered read timeout does not become an authored callback failure 363ms - ✓ tests/running-step-watch.test.ts (12 tests) 12287ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms - ✓ cancels promptly while a lease snapshot read is in flight 2006ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2053ms + ✓ tests/authored-root.test.ts (30 tests) 393ms + ✓ tests/journal-client.test.ts (17 tests) 80ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 7062ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 445ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1032ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2778ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 382ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 861ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 1292ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 458ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 451ms + ✓ tests/journal-client-read-timeout.test.ts (25 tests) 1797ms + ✓ tests/running-step-watch.test.ts (12 tests) 12286ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2116ms + ✓ cancels promptly while a lease snapshot read is in flight 2005ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2055ms ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms - ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2003ms + ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2004ms ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2002ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2582ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 692ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 645ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 660ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 585ms - ✓ tests/authored-helpers.test.ts (6 tests) 1962ms - ✓ runs every available provider through the real kernel and resumes completed effects without a second write 904ms - ✓ replays after SIGKILL before confirm with the same token and one successful completion 402ms - ✓ replays after SIGKILL before complete with the same token and one successful completion 427ms - ✓ tests/classify-outcome.test.ts (11 tests) 7429ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2015ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3006ms - ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2002ms - ✓ tests/human-live.test.ts (3 tests) 4589ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2702ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1114ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 772ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2730ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 694ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 665ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 693ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 676ms + ✓ tests/authored-helpers.test.ts (6 tests) 2139ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 1026ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 448ms + ✓ replays after SIGKILL before complete with the same token and one successful completion 425ms + ✓ tests/classify-outcome.test.ts (11 tests) 7423ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2010ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3005ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2000ms + ✓ tests/human-live.test.ts (3 tests) 4749ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2900ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1089ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 758ms ✓ tests/worker-lease.test.ts (7 tests) 19ms - ✓ tests/communication-worker.test.ts (18 tests) 1722ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 24ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 733ms - ✓ tests/memoization.test.ts (58 tests) 67ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2475ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2056ms - ✓ drains read and watch promises before an authored flow completes 418ms - ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms - ✓ tests/journal-client-completion.test.ts (6 tests) 100ms + ✓ tests/communication-worker.test.ts (18 tests) 1716ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 27ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 715ms + ✓ tests/memoization.test.ts (58 tests) 66ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2507ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2057ms + ✓ drains read and watch promises before an authored flow completes 449ms + ✓ tests/worker-lease-sweep.test.ts (4 tests) 9ms + ✓ tests/journal-client-completion.test.ts (6 tests) 106ms ✓ tests/reuse-summary-interruption.test.ts (5 tests) 4ms - ✓ tests/resume-worker-lease.test.ts (3 tests) 8ms - ✓ tests/direct-run-worker-lease.test.ts (3 tests) 9ms - ✓ tests/helper-effect-lease-loss.test.ts (1 test) 229ms - ✓ tests/journal-client-subscriptions.test.ts (1 test) 6ms + ✓ tests/resume-worker-lease.test.ts (3 tests) 9ms + ✓ tests/direct-run-worker-lease.test.ts (3 tests) 10ms + ✓ tests/helper-effect-lease-loss.test.ts (1 test) 232ms + ✓ tests/journal-client-subscriptions.test.ts (1 test) 7ms ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 4ms - ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 47ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms + ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 46ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 17ms Test Files 27 passed (27) - Tests 332 passed (332) - Start at 03:26:49 - Duration 60.48s (transform 1.20s, setup 151ms, collect 6.79s, tests 50.24s, environment 4ms, prepare 970ms) + Tests 334 passed (334) + Start at 03:35:58 + Duration 61.82s (transform 1.34s, setup 155ms, collect 7.29s, tests 50.98s, environment 4ms, prepare 998ms) exit=0 ``` @@ -91,13 +91,13 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 944ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 943ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 888ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 887ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 03:27:50 - Duration 2.22s (transform 636ms, setup 21ms, collect 1.04s, tests 944ms, environment 0ms, prepare 85ms) + Start at 03:37:00 + Duration 2.12s (transform 602ms, setup 27ms, collect 967ms, tests 888ms, environment 0ms, prepare 102ms) exit=0 ``` @@ -108,7 +108,7 @@ The mutation sections below were regenerated by `cd packages/sdk && python3 ../. ```text Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: efc2e4cdfdfba782ea0732b5573f4190ae02c486b7de4731a33f280f7bce6fc1 +Original SHA256: 30380437df429b288157a3a895ddb2cede8fe8853889b4a0cb9a3c24a4097814 Replaced: return await reader.requestOnce(verb, params, remaining, attemptSignal); With: @@ -119,8 +119,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (24 tests | 1 failed | 23 skipped) 116ms - × serves a bounded read while an unbounded command is in flight 116ms + ❯ tests/journal-client-read-timeout.test.ts (25 tests | 1 failed | 24 skipped) 113ms + × serves a bounded read while an unbounded command is in flight 112ms → expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -145,23 +145,23 @@ AssertionError: expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 23 skipped (24) - Start at 03:26:16 - Duration 626ms (transform 199ms, setup 14ms, collect 339ms, tests 116ms, environment 0ms, prepare 43ms) + Tests 1 failed | 24 skipped (25) + Start at 03:35:24 + Duration 577ms (transform 172ms, setup 15ms, collect 303ms, tests 113ms, environment 0ms, prepare 40ms) exit=1 -RESTORED SHA256: efc2e4cdfdfba782ea0732b5573f4190ae02c486b7de4731a33f280f7bce6fc1 +RESTORED SHA256: 30380437df429b288157a3a895ddb2cede8fe8853889b4a0cb9a3c24a4097814 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'serves a bounded read' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (24 tests | 23 skipped) 106ms + ✓ tests/journal-client-read-timeout.test.ts (25 tests | 24 skipped) 106ms Test Files 1 passed (1) - Tests 1 passed | 23 skipped (24) - Start at 03:26:17 - Duration 639ms (transform 188ms, setup 21ms, collect 312ms, tests 106ms, environment 0ms, prepare 65ms) + Tests 1 passed | 24 skipped (25) + Start at 03:35:25 + Duration 549ms (transform 162ms, setup 16ms, collect 287ms, tests 106ms, environment 0ms, prepare 40ms) exit=0 ``` @@ -170,7 +170,7 @@ exit=0 ```text Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: efc2e4cdfdfba782ea0732b5573f4190ae02c486b7de4731a33f280f7bce6fc1 +Original SHA256: 30380437df429b288157a3a895ddb2cede8fe8853889b4a0cb9a3c24a4097814 Replaced: this.drop(reader); @@ -182,14 +182,14 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (24 tests | 1 failed | 23 skipped) 1009ms - × reconnects the reader after it disconnects within the read budget 1008ms - → journal client: run.get read session was interrupted after 80 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ tests/journal-client-read-timeout.test.ts (25 tests | 1 failed | 24 skipped) 1010ms + × reconnects the reader after it disconnects within the read budget 1009ms + → journal client: run.get read session was interrupted after 77 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 80 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 77 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:71:9 69| let last: unknown; 70| const exhausted = () => last instanceof JournalReadInterruptedError @@ -202,7 +202,7 @@ JournalReadInterruptedError: journal client: run.get read session was interrupte Caused by: Error: journal client: not connected (run.get): journal client: connection closed ❯ src/journal-client.ts:252:16 ❯ JournalClient.requestOnce src/journal-client.ts:245:12 - ❯ src/journal-budgeted-reads.ts:54:29 + ❯ src/journal-budgeted-reads.ts:59:29 ❯ src/journal-read-policy.ts:94:18 Caused by: Error: journal client: connection closed @@ -211,23 +211,23 @@ Caused by: Error: journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 23 skipped (24) - Start at 03:26:18 - Duration 1.52s (transform 182ms, setup 18ms, collect 325ms, tests 1.01s, environment 0ms, prepare 46ms) + Tests 1 failed | 24 skipped (25) + Start at 03:35:26 + Duration 1.45s (transform 167ms, setup 15ms, collect 285ms, tests 1.01s, environment 0ms, prepare 41ms) exit=1 -RESTORED SHA256: efc2e4cdfdfba782ea0732b5573f4190ae02c486b7de4731a33f280f7bce6fc1 +RESTORED SHA256: 30380437df429b288157a3a895ddb2cede8fe8853889b4a0cb9a3c24a4097814 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'reconnects the reader after it disconnects' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (24 tests | 23 skipped) 86ms + ✓ tests/journal-client-read-timeout.test.ts (25 tests | 24 skipped) 81ms Test Files 1 passed (1) - Tests 1 passed | 23 skipped (24) - Start at 03:26:20 - Duration 517ms (transform 160ms, setup 17ms, collect 283ms, tests 86ms, environment 0ms, prepare 41ms) + Tests 1 passed | 24 skipped (25) + Start at 03:35:28 + Duration 521ms (transform 159ms, setup 16ms, collect 280ms, tests 81ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -236,7 +236,7 @@ exit=0 ```text Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: efc2e4cdfdfba782ea0732b5573f4190ae02c486b7de4731a33f280f7bce6fc1 +Original SHA256: 30380437df429b288157a3a895ddb2cede8fe8853889b4a0cb9a3c24a4097814 Replaced: if (this.reader === reader) { this.reader = undefined; @@ -253,8 +253,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (24 tests | 1 failed | 23 skipped) 76ms - × retries a reader setup that timed out instead of reading on the primary for good 76ms + ❯ tests/journal-client-read-timeout.test.ts (25 tests | 1 failed | 24 skipped) 78ms + × retries a reader setup that timed out instead of reading on the primary for good 77ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -279,23 +279,23 @@ AssertionError: expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 23 skipped (24) - Start at 03:26:21 - Duration 518ms (transform 160ms, setup 16ms, collect 281ms, tests 76ms, environment 0ms, prepare 40ms) + Tests 1 failed | 24 skipped (25) + Start at 03:35:29 + Duration 514ms (transform 165ms, setup 17ms, collect 289ms, tests 78ms, environment 0ms, prepare 40ms) exit=1 -RESTORED SHA256: efc2e4cdfdfba782ea0732b5573f4190ae02c486b7de4731a33f280f7bce6fc1 +RESTORED SHA256: 30380437df429b288157a3a895ddb2cede8fe8853889b4a0cb9a3c24a4097814 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'retries a reader setup that timed out' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (24 tests | 23 skipped) 70ms + ✓ tests/journal-client-read-timeout.test.ts (25 tests | 24 skipped) 73ms Test Files 1 passed (1) - Tests 1 passed | 23 skipped (24) - Start at 03:26:22 - Duration 518ms (transform 167ms, setup 16ms, collect 293ms, tests 70ms, environment 0ms, prepare 40ms) + Tests 1 passed | 24 skipped (25) + Start at 03:35:30 + Duration 518ms (transform 165ms, setup 18ms, collect 287ms, tests 73ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -315,8 +315,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2111ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2111ms + ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2110ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2110ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -335,8 +335,8 @@ AssertionError: expected 41 to be less than or equal to 1 Test Files 1 failed (1) Tests 1 failed | 11 skipped (12) - Start at 03:26:23 - Duration 2.58s (transform 185ms, setup 15ms, collect 316ms, tests 2.11s, environment 0ms, prepare 42ms) + Start at 03:35:31 + Duration 2.56s (transform 179ms, setup 16ms, collect 300ms, tests 2.11s, environment 0ms, prepare 36ms) exit=1 @@ -346,12 +346,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2118ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms Test Files 1 passed (1) Tests 1 passed | 11 skipped (12) - Start at 03:26:25 - Duration 2.61s (transform 190ms, setup 17ms, collect 324ms, tests 2.12s, environment 0ms, prepare 40ms) + Start at 03:35:34 + Duration 2.59s (transform 185ms, setup 16ms, collect 311ms, tests 2.12s, environment 0ms, prepare 41ms) exit=0 ``` @@ -392,8 +392,8 @@ AssertionError: expected 'still waiting' to be an instance of Error Test Files 1 failed (1) Tests 1 failed | 11 skipped (12) - Start at 03:26:28 - Duration 2.70s (transform 192ms, setup 17ms, collect 324ms, tests 2.22s, environment 0ms, prepare 40ms) + Start at 03:35:36 + Duration 2.71s (transform 195ms, setup 16ms, collect 331ms, tests 2.22s, environment 0ms, prepare 41ms) exit=1 @@ -402,13 +402,13 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2011ms - ✓ cancels promptly while a lease snapshot read is in flight 2010ms + ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2010ms + ✓ cancels promptly while a lease snapshot read is in flight 2009ms Test Files 1 passed (1) Tests 1 passed | 11 skipped (12) - Start at 03:26:31 - Duration 2.49s (transform 184ms, setup 17ms, collect 305ms, tests 2.01s, environment 0ms, prepare 40ms) + Start at 03:35:40 + Duration 2.55s (transform 232ms, setup 22ms, collect 353ms, tests 2.01s, environment 0ms, prepare 57ms) exit=0 ``` @@ -452,8 +452,8 @@ Received: "still waiting" Test Files 1 failed (1) Tests 1 failed | 11 skipped (12) - Start at 03:26:34 - Duration 3.49s (transform 188ms, setup 16ms, collect 318ms, tests 3.01s, environment 0ms, prepare 41ms) + Start at 03:35:43 + Duration 3.49s (transform 186ms, setup 16ms, collect 316ms, tests 3.01s, environment 0ms, prepare 43ms) exit=1 @@ -462,13 +462,13 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2060ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2059ms + ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2059ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2058ms Test Files 1 passed (1) Tests 1 passed | 11 skipped (12) - Start at 03:26:38 - Duration 2.54s (transform 188ms, setup 17ms, collect 314ms, tests 2.06s, environment 0ms, prepare 42ms) + Start at 03:35:46 + Duration 2.50s (transform 184ms, setup 15ms, collect 309ms, tests 2.06s, environment 0ms, prepare 31ms) exit=0 ``` @@ -491,8 +491,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 19ms - × a heartbeat timeout is lease loss rather than a worker body failure 19ms + ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 20ms + × a heartbeat timeout is lease loss rather than a worker body failure 20ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -511,8 +511,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 03:26:41 - Duration 455ms (transform 165ms, setup 16ms, collect 289ms, tests 19ms, environment 0ms, prepare 41ms) + Start at 03:35:49 + Duration 461ms (transform 170ms, setup 16ms, collect 277ms, tests 20ms, environment 0ms, prepare 41ms) exit=1 @@ -525,8 +525,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb Test Files 1 passed (1) Tests 1 passed (1) - Start at 03:26:42 - Duration 445ms (transform 163ms, setup 16ms, collect 275ms, tests 16ms, environment 0ms, prepare 40ms) + Start at 03:35:50 + Duration 447ms (transform 170ms, setup 17ms, collect 286ms, tests 16ms, environment 0ms, prepare 44ms) exit=0 ``` @@ -535,7 +535,7 @@ exit=0 ```text Source: packages/sdk/src/authored-root.ts -Original SHA256: 79f49acf9e87f35a2ff6a3b757c2d193e4e821eadfd28f96181034f550b01dfa +Original SHA256: 8ab20a03c6def62800d2ef8a49932ec616a3b1c1fcd48c4aff9fa80437bcb35c Replaced: if (isReadInterruptionError(error) || (error instanceof AuthoredFlowExecutionError && error.code === 'daemon_unresponsive')) { @@ -553,7 +553,7 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (29 tests | 3 failed | 26 skipped) 32ms + ❯ tests/authored-root.test.ts (30 tests | 3 failed | 27 skipped) 32ms × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 22ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) @@ -639,23 +639,23 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[3/3]⎯ Test Files 1 failed (1) - Tests 3 failed | 26 skipped (29) - Start at 03:26:42 - Duration 1.29s (transform 668ms, setup 17ms, collect 1.09s, tests 32ms, environment 0ms, prepare 42ms) + Tests 3 failed | 27 skipped (30) + Start at 03:35:51 + Duration 1.29s (transform 661ms, setup 15ms, collect 1.10s, tests 32ms, environment 0ms, prepare 40ms) exit=1 -RESTORED SHA256: 79f49acf9e87f35a2ff6a3b757c2d193e4e821eadfd28f96181034f550b01dfa +RESTORED SHA256: 8ab20a03c6def62800d2ef8a49932ec616a3b1c1fcd48c4aff9fa80437bcb35c $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the root resumable' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (29 tests | 26 skipped) 24ms + ✓ tests/authored-root.test.ts (30 tests | 27 skipped) 23ms Test Files 1 passed (1) - Tests 3 passed | 26 skipped (29) - Start at 03:26:44 - Duration 1.24s (transform 644ms, setup 16ms, collect 1.05s, tests 24ms, environment 0ms, prepare 42ms) + Tests 3 passed | 27 skipped (30) + Start at 03:35:53 + Duration 1.27s (transform 658ms, setup 17ms, collect 1.10s, tests 23ms, environment 0ms, prepare 31ms) exit=0 ``` diff --git a/packages/sdk/src/authored-root.ts b/packages/sdk/src/authored-root.ts index 53e4d681c..9621b36d4 100644 --- a/packages/sdk/src/authored-root.ts +++ b/packages/sdk/src/authored-root.ts @@ -200,7 +200,12 @@ export async function readAuthoredRootMetadata( journal: JournalClient, runId: string, ): Promise { - const step = await rootKernelStep(journal, runId).catch(() => undefined); + // Not finding the root step means "not authored"; an unanswered read means + // nothing yet, and guessing declarative would resume the wrong way. + const step = await rootKernelStep(journal, runId).catch(error => { + if (isReadInterruptionError(error)) throw error; + return undefined; + }); if (step === undefined || step.id !== 'authored-root' || !hasRootStream(step)) return undefined; if (typeof step.instruction !== 'string') { throw new Error('authored root journal has malformed authority metadata'); diff --git a/packages/sdk/src/journal-budgeted-reads.ts b/packages/sdk/src/journal-budgeted-reads.ts index 861610198..441e37198 100644 --- a/packages/sdk/src/journal-budgeted-reads.ts +++ b/packages/sdk/src/journal-budgeted-reads.ts @@ -29,8 +29,13 @@ export class BudgetedReads { return () => { this.scoped.delete(signal); }; } - close(cause?: unknown): void { - this.policy.close(); + /** + * Stop all reads. An unexpected primary drop (`interrupted`) ends in-flight + * reads as a read interruption, so the run stays resumable; a caller close + * keeps its plain error. + */ + close(cause?: unknown, interrupted = false): void { + this.policy.close(interrupted ? new JournalReadInterruptedError('journal', 1, 0, undefined, { cause }) : undefined); this.reader?.close(cause); } diff --git a/packages/sdk/src/journal-client.ts b/packages/sdk/src/journal-client.ts index 70ee624a8..c01c3a9f9 100644 --- a/packages/sdk/src/journal-client.ts +++ b/packages/sdk/src/journal-client.ts @@ -140,7 +140,7 @@ export class JournalClient extends EventEmitter { socket.on('data', (chunk) => this.onData(chunk)); socket.on('close', () => { const closed = new Error('journal client: connection closed'); - this.budgeted?.close(closed); + this.budgeted?.close(closed, this.socket === socket); this.disconnectCause ??= closed; this.failAll(closed); // Only a drop the caller did not ask for; close() clears this.socket first. diff --git a/packages/sdk/src/journal-read-policy.ts b/packages/sdk/src/journal-read-policy.ts index 9c17f8156..6538ec2f7 100644 --- a/packages/sdk/src/journal-read-policy.ts +++ b/packages/sdk/src/journal-read-policy.ts @@ -59,7 +59,7 @@ export class JournalReadPolicy { setMaxListeners(0, this.closed.signal); } - close(): void { this.closed.abort(new Error('journal client: closed by caller')); } + close(reason: unknown = new Error('journal client: closed by caller')): void { this.closed.abort(reason); } async read(verb: string, timeoutMs: number, budgetMs: number, request: (timeoutMs: number, signal: AbortSignal) => Promise, caller?: AbortSignal): Promise { diff --git a/packages/sdk/tests/authored-root.test.ts b/packages/sdk/tests/authored-root.test.ts index af1aca6c0..4c1348aab 100644 --- a/packages/sdk/tests/authored-root.test.ts +++ b/packages/sdk/tests/authored-root.test.ts @@ -458,6 +458,14 @@ describe('durable authored root', () => { expect(release).toHaveBeenCalledTimes(1); }); + it('propagates a read interruption while detecting an authored root instead of guessing declarative', async () => { + const { readAuthoredRootMetadata } = await import('../src/authored-root.js'); + const interrupted = { journalRead: async () => { throw new JournalRequestTimeoutError('journal.read', 10, 3, 300_000, 300_000); } }; + await expect(readAuthoredRootMetadata(interrupted as unknown as JournalClient, 'run')).rejects.toBeInstanceOf(JournalRequestTimeoutError); + const plain = { journalRead: async () => ({ entries: [] }) }; + await expect(readAuthoredRootMetadata(plain as unknown as JournalClient, 'run')).resolves.toBeUndefined(); + }); + it('terminalizes a returned body failure without replaying semantic side effects', async () => { const loaded = await fixture(true); const journal = new RootJournal(); diff --git a/packages/sdk/tests/journal-client-read-timeout.test.ts b/packages/sdk/tests/journal-client-read-timeout.test.ts index c77247ae7..34b949f57 100644 --- a/packages/sdk/tests/journal-client-read-timeout.test.ts +++ b/packages/sdk/tests/journal-client-read-timeout.test.ts @@ -291,3 +291,14 @@ it('a scoped read signal (the root lease) cancels budgeted reads only while it i release(); await expect(client.journalRead('run', 1)).resolves.toEqual({ entries: [] }); }); + +it('an unexpected primary disconnect aborts in-flight reads with a typed read interruption', async () => { + let primary: { destroy(): void } | undefined; + const client = await setup({ + 'stream.append': ctx => { primary = ctx.socket; sendResult(ctx, { seq: 1 }); }, + 'run.get': () => { primary?.destroy(); }, // the reader is healthy; the primary drops + }, 60_000); + await client.streamAppend('run', 'stream', {}); + const error = await client.runGet('run').catch(caught => caught); + expect(isReadInterruptionError(error)).toBe(true); +}); From e4c232af8c9361211f1e9980a888fa58ed01789c Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 03:47:23 -0700 Subject: [PATCH 22/34] fix(sdk): hold the lease read scope for the whole Bun attempt; post-drop reads stay typed - driveRoot returned runAuthoredInNode's promise without awaiting it inside the try, so on Bun the root-lease read scope was released at once; it is now awaited and covers the child's whole run. - Allowlisted reads always take the read policy when one exists, so a read started after the primary dropped gets the typed interruption recorded at the drop instead of an untyped 'not connected'. Evidence re-captured at this tree. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: b8262468-ea26-4a6e-bee9-d193832e77dc --- evidence/run-read-timeout/final-focused.log | 115 +++---- .../mutation-completion-aborts-snapshot.log | 16 +- .../run-read-timeout/mutation-heartbeat.log | 14 +- .../mutation-reader-reconnect.log | 24 +- .../mutation-reader-setup-retry.log | 18 +- evidence/run-read-timeout/mutation-reader.log | 16 +- .../mutation-root-parking.log | 60 ++-- .../mutation-snapshot-cancel.log | 16 +- .../mutation-watch-cadence.log | 12 +- evidence/run-read-timeout/resume-live.log | 8 +- evidence/run-read-timeout/verification.md | 299 +++++++++--------- packages/sdk/src/authored-root.ts | 3 +- packages/sdk/src/journal-client.ts | 4 +- packages/sdk/tests/authored-root.test.ts | 31 ++ .../tests/journal-client-read-timeout.test.ts | 14 + 15 files changed, 352 insertions(+), 298 deletions(-) diff --git a/evidence/run-read-timeout/final-focused.log b/evidence/run-read-timeout/final-focused.log index 93dc11abb..cdfaaa658 100644 --- a/evidence/run-read-timeout/final-focused.log +++ b/evidence/run-read-timeout/final-focused.log @@ -2,70 +2,73 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5537ms - ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 462ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2038ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2039ms -(node:4077640) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5663ms + ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 385ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2045ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2044ms +(node:4127341) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (30 tests) 393ms - ✓ tests/journal-client.test.ts (17 tests) 80ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 7062ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 445ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1032ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2778ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 382ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 861ms - ✓ tests/authored-flow-slack.test.ts (7 tests) 1292ms - ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 458ms - ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 451ms - ✓ tests/journal-client-read-timeout.test.ts (25 tests) 1797ms - ✓ tests/running-step-watch.test.ts (12 tests) 12286ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2116ms - ✓ cancels promptly while a lease snapshot read is in flight 2005ms + ✓ tests/authored-root.test.ts (31 tests) 484ms + ✓ tests/journal-client.test.ts (17 tests) 84ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 8035ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 515ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1257ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2970ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 447ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 1002ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 1354ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 487ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 486ms + ✓ tests/journal-client-read-timeout.test.ts (26 tests) 1842ms + ✓ a recovered read timeout does not become an authored callback failure 312ms + ✓ tests/running-step-watch.test.ts (12 tests) 12293ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms + ✓ cancels promptly while a lease snapshot read is in flight 2006ms ✓ a completion push ends the wait without waiting out an in-flight snapshot 2055ms ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2004ms ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2002ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2730ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 694ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 665ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 693ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 676ms - ✓ tests/authored-helpers.test.ts (6 tests) 2139ms - ✓ runs every available provider through the real kernel and resumes completed effects without a second write 1026ms - ✓ replays after SIGKILL before confirm with the same token and one successful completion 448ms - ✓ replays after SIGKILL before complete with the same token and one successful completion 425ms - ✓ tests/classify-outcome.test.ts (11 tests) 7423ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 3273ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 798ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 858ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 814ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 802ms + ✓ tests/authored-helpers.test.ts (6 tests) 3019ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 1375ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 672ms + ✓ replays after SIGKILL before complete with the same token and one successful completion 690ms + ✓ tests/classify-outcome.test.ts (11 tests) 7427ms ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2010ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3005ms - ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2000ms - ✓ tests/human-live.test.ts (3 tests) 4749ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2900ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1089ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 758ms - ✓ tests/worker-lease.test.ts (7 tests) 19ms - ✓ tests/communication-worker.test.ts (18 tests) 1716ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 27ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 715ms - ✓ tests/memoization.test.ts (58 tests) 66ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2507ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2057ms - ✓ drains read and watch promises before an authored flow completes 449ms - ✓ tests/worker-lease-sweep.test.ts (4 tests) 9ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3006ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms + ✓ tests/human-live.test.ts (3 tests) 5285ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 3027ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1356ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 902ms + ✓ tests/worker-lease.test.ts (7 tests) 28ms + ✓ tests/communication-worker.test.ts (18 tests) 1721ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 39ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 925ms + ✓ reports journal success after completion rejects with lease_conflict 327ms + ✓ reports journal success when a renewal rejects after completion landed 344ms + ✓ tests/memoization.test.ts (58 tests) 77ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2616ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2069ms + ✓ drains read and watch promises before an authored flow completes 547ms + ✓ tests/worker-lease-sweep.test.ts (4 tests) 6ms ✓ tests/journal-client-completion.test.ts (6 tests) 106ms - ✓ tests/reuse-summary-interruption.test.ts (5 tests) 4ms - ✓ tests/resume-worker-lease.test.ts (3 tests) 9ms - ✓ tests/direct-run-worker-lease.test.ts (3 tests) 10ms - ✓ tests/helper-effect-lease-loss.test.ts (1 test) 232ms - ✓ tests/journal-client-subscriptions.test.ts (1 test) 7ms - ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 4ms - ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 46ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 17ms + ✓ tests/reuse-summary-interruption.test.ts (5 tests) 3ms + ✓ tests/resume-worker-lease.test.ts (3 tests) 8ms + ✓ tests/direct-run-worker-lease.test.ts (3 tests) 11ms + ✓ tests/helper-effect-lease-loss.test.ts (1 test) 264ms + ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms + ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 5ms + ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 58ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms Test Files 27 passed (27) - Tests 334 passed (334) - Start at 03:35:58 - Duration 61.82s (transform 1.34s, setup 155ms, collect 7.29s, tests 50.98s, environment 4ms, prepare 998ms) + Tests 336 passed (336) + Start at 03:46:13 + Duration 67.22s (transform 1.52s, setup 196ms, collect 8.43s, tests 54.65s, environment 4ms, prepare 1.14s) exit=0 diff --git a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log index 0e2710cba..8f2228712 100644 --- a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log +++ b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 3012ms - × a completion push ends the wait without waiting out an in-flight snapshot 3011ms + ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 3013ms + × a completion push ends the wait without waiting out an in-flight snapshot 3012ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -34,8 +34,8 @@ Received: "still waiting" Test Files 1 failed (1) Tests 1 failed | 11 skipped (12) - Start at 03:35:43 - Duration 3.49s (transform 186ms, setup 16ms, collect 316ms, tests 3.01s, environment 0ms, prepare 43ms) + Start at 03:45:56 + Duration 3.55s (transform 211ms, setup 19ms, collect 356ms, tests 3.01s, environment 0ms, prepare 45ms) exit=1 @@ -44,12 +44,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2059ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2058ms + ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2063ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2062ms Test Files 1 passed (1) Tests 1 passed | 11 skipped (12) - Start at 03:35:46 - Duration 2.50s (transform 184ms, setup 15ms, collect 309ms, tests 2.06s, environment 0ms, prepare 31ms) + Start at 03:46:00 + Duration 2.57s (transform 210ms, setup 19ms, collect 366ms, tests 2.06s, environment 0ms, prepare 31ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-heartbeat.log b/evidence/run-read-timeout/mutation-heartbeat.log index e46ec887d..362c2525f 100644 --- a/evidence/run-read-timeout/mutation-heartbeat.log +++ b/evidence/run-read-timeout/mutation-heartbeat.log @@ -13,8 +13,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 20ms - × a heartbeat timeout is lease loss rather than a worker body failure 20ms + ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 22ms + × a heartbeat timeout is lease loss rather than a worker body failure 22ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -33,8 +33,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 03:35:49 - Duration 461ms (transform 170ms, setup 16ms, collect 277ms, tests 20ms, environment 0ms, prepare 41ms) + Start at 03:46:03 + Duration 535ms (transform 192ms, setup 19ms, collect 333ms, tests 22ms, environment 0ms, prepare 51ms) exit=1 @@ -43,11 +43,11 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 18ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 03:35:50 - Duration 447ms (transform 170ms, setup 17ms, collect 286ms, tests 16ms, environment 0ms, prepare 44ms) + Start at 03:46:04 + Duration 563ms (transform 197ms, setup 20ms, collect 350ms, tests 18ms, environment 0ms, prepare 51ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-reconnect.log b/evidence/run-read-timeout/mutation-reader-reconnect.log index 5f6ee12f1..3c7383fdb 100644 --- a/evidence/run-read-timeout/mutation-reader-reconnect.log +++ b/evidence/run-read-timeout/mutation-reader-reconnect.log @@ -11,14 +11,14 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (25 tests | 1 failed | 24 skipped) 1010ms + ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 1010ms × reconnects the reader after it disconnects within the read budget 1009ms - → journal client: run.get read session was interrupted after 77 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + → journal client: run.get read session was interrupted after 76 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 77 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 76 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:71:9 69| let last: unknown; 70| const exhausted = () => last instanceof JournalReadInterruptedError @@ -29,8 +29,8 @@ JournalReadInterruptedError: journal client: run.get read session was interrupte ❯ Timeout. src/journal-read-policy.ts:82:39 Caused by: Error: journal client: not connected (run.get): journal client: connection closed - ❯ src/journal-client.ts:252:16 - ❯ JournalClient.requestOnce src/journal-client.ts:245:12 + ❯ src/journal-client.ts:254:16 + ❯ JournalClient.requestOnce src/journal-client.ts:247:12 ❯ src/journal-budgeted-reads.ts:59:29 ❯ src/journal-read-policy.ts:94:18 @@ -40,9 +40,9 @@ Caused by: Error: journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 24 skipped (25) - Start at 03:35:26 - Duration 1.45s (transform 167ms, setup 15ms, collect 285ms, tests 1.01s, environment 0ms, prepare 41ms) + Tests 1 failed | 25 skipped (26) + Start at 03:45:40 + Duration 1.47s (transform 169ms, setup 17ms, collect 298ms, tests 1.01s, environment 0ms, prepare 41ms) exit=1 @@ -51,11 +51,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (25 tests | 24 skipped) 81ms + ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 86ms Test Files 1 passed (1) - Tests 1 passed | 24 skipped (25) - Start at 03:35:28 - Duration 521ms (transform 159ms, setup 16ms, collect 280ms, tests 81ms, environment 0ms, prepare 41ms) + Tests 1 passed | 25 skipped (26) + Start at 03:45:41 + Duration 534ms (transform 164ms, setup 16ms, collect 288ms, tests 86ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-setup-retry.log b/evidence/run-read-timeout/mutation-reader-setup-retry.log index 015850151..83b2058fc 100644 --- a/evidence/run-read-timeout/mutation-reader-setup-retry.log +++ b/evidence/run-read-timeout/mutation-reader-setup-retry.log @@ -16,8 +16,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (25 tests | 1 failed | 24 skipped) 78ms - × retries a reader setup that timed out instead of reading on the primary for good 77ms + ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 81ms + × retries a reader setup that timed out instead of reading on the primary for good 80ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -42,9 +42,9 @@ AssertionError: expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 24 skipped (25) - Start at 03:35:29 - Duration 514ms (transform 165ms, setup 17ms, collect 289ms, tests 78ms, environment 0ms, prepare 40ms) + Tests 1 failed | 25 skipped (26) + Start at 03:45:42 + Duration 535ms (transform 167ms, setup 16ms, collect 291ms, tests 81ms, environment 0ms, prepare 40ms) exit=1 @@ -53,11 +53,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (25 tests | 24 skipped) 73ms + ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 76ms Test Files 1 passed (1) - Tests 1 passed | 24 skipped (25) - Start at 03:35:30 - Duration 518ms (transform 165ms, setup 18ms, collect 287ms, tests 73ms, environment 0ms, prepare 41ms) + Tests 1 passed | 25 skipped (26) + Start at 03:45:43 + Duration 528ms (transform 171ms, setup 16ms, collect 292ms, tests 76ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader.log b/evidence/run-read-timeout/mutation-reader.log index 37d5b5f82..d1e5ce4cb 100644 --- a/evidence/run-read-timeout/mutation-reader.log +++ b/evidence/run-read-timeout/mutation-reader.log @@ -10,7 +10,7 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (25 tests | 1 failed | 24 skipped) 113ms + ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 113ms × serves a bounded read while an unbounded command is in flight 112ms → expected true to be false // Object.is equality @@ -36,9 +36,9 @@ AssertionError: expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 24 skipped (25) - Start at 03:35:24 - Duration 577ms (transform 172ms, setup 15ms, collect 303ms, tests 113ms, environment 0ms, prepare 40ms) + Tests 1 failed | 25 skipped (26) + Start at 03:45:38 + Duration 568ms (transform 165ms, setup 16ms, collect 289ms, tests 113ms, environment 0ms, prepare 41ms) exit=1 @@ -47,11 +47,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (25 tests | 24 skipped) 106ms + ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 107ms Test Files 1 passed (1) - Tests 1 passed | 24 skipped (25) - Start at 03:35:25 - Duration 549ms (transform 162ms, setup 16ms, collect 287ms, tests 106ms, environment 0ms, prepare 40ms) + Tests 1 passed | 25 skipped (26) + Start at 03:45:39 + Duration 550ms (transform 166ms, setup 17ms, collect 292ms, tests 107ms, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-root-parking.log b/evidence/run-read-timeout/mutation-root-parking.log index 290e246bb..a892a0e37 100644 --- a/evidence/run-read-timeout/mutation-root-parking.log +++ b/evidence/run-read-timeout/mutation-root-parking.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/authored-root.ts -Original SHA256: 8ab20a03c6def62800d2ef8a49932ec616a3b1c1fcd48c4aff9fa80437bcb35c +Original SHA256: 4dc10394dadeada752ce12bd86fbbdfb1c231d92902a9aa912ce8ba7fb57c914 Replaced: if (isReadInterruptionError(error) || (error instanceof AuthoredFlowExecutionError && error.code === 'daemon_unresponsive')) { @@ -17,14 +17,14 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (30 tests | 3 failed | 27 skipped) 32ms - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 22ms + ❯ tests/authored-root.test.ts (31 tests | 3 failed | 28 skipped) 26ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 18ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms → expected AuthoredFlowExecutionError: daemon_unresp… { …(7) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 4ms + × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 3ms → expected JournalReadInterruptedError: journal clie… { …(4) } to match object { code: 'daemon_unresponsive', …(2) } (5 matching properties omitted from actual) @@ -44,13 +44,13 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } - } + [JournalRequestTimeoutError: journal client: run.get timed out after 10ms] - ❯ tests/authored-root.test.ts:415:5 - 413| }); - 414| const journal = new RootJournal(); - 415| await expect(executeDurableAuthoredFlow(loaded, journal as unknown… + ❯ tests/authored-root.test.ts:424:5 + 422| }); + 423| const journal = new RootJournal(); + 424| await expect(executeDurableAuthoredFlow(loaded, journal as unknown… | ^ - 416| { dataDir: '/unused', admissionKey: 'read-timeout' })).rejects.t… - 417| code: 'daemon_unresponsive', rootRunId: 'root-run', message: e… + 425| { dataDir: '/unused', admissionKey: 'read-timeout' })).rejects.t… + 426| code: 'daemon_unresponsive', rootRunId: 'root-run', message: e… ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/3]⎯ @@ -68,13 +68,13 @@ AssertionError: expected AuthoredFlowExecutionError: daemon_unresp… { …(7) } "rootRunId": "root-run", } - ❯ tests/authored-root.test.ts:415:5 - 413| }); - 414| const journal = new RootJournal(); - 415| await expect(executeDurableAuthoredFlow(loaded, journal as unknown… + ❯ tests/authored-root.test.ts:424:5 + 422| }); + 423| const journal = new RootJournal(); + 424| await expect(executeDurableAuthoredFlow(loaded, journal as unknown… | ^ - 416| { dataDir: '/unused', admissionKey: 'read-timeout' })).rejects.t… - 417| code: 'daemon_unresponsive', rootRunId: 'root-run', message: e… + 425| { dataDir: '/unused', admissionKey: 'read-timeout' })).rejects.t… + 426| code: 'daemon_unresponsive', rootRunId: 'root-run', message: e… ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[2/3]⎯ @@ -92,33 +92,33 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } - } + [JournalReadInterruptedError: journal client: run.get read session was interrupted after 2 attempts in 40ms (read budget 300000ms): journal client: connection closed] - ❯ tests/authored-root.test.ts:428:5 - 426| const loaded = await fixture(false, 0, async () => { throw interru… - 427| const journal = new RootJournal(); - 428| await expect(executeDurableAuthoredFlow(loaded, journal as unknown… + ❯ tests/authored-root.test.ts:437:5 + 435| const loaded = await fixture(false, 0, async () => { throw interru… + 436| const journal = new RootJournal(); + 437| await expect(executeDurableAuthoredFlow(loaded, journal as unknown… | ^ - 429| { dataDir: '/unused', admissionKey: 'read-disconnect' })).reject… - 430| code: 'daemon_unresponsive', rootRunId: 'root-run', message: e… + 438| { dataDir: '/unused', admissionKey: 'read-disconnect' })).reject… + 439| code: 'daemon_unresponsive', rootRunId: 'root-run', message: e… ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[3/3]⎯ Test Files 1 failed (1) - Tests 3 failed | 27 skipped (30) - Start at 03:35:51 - Duration 1.29s (transform 661ms, setup 15ms, collect 1.10s, tests 32ms, environment 0ms, prepare 40ms) + Tests 3 failed | 28 skipped (31) + Start at 03:46:05 + Duration 1.55s (transform 830ms, setup 14ms, collect 1.38s, tests 26ms, environment 0ms, prepare 33ms) exit=1 -RESTORED SHA256: 8ab20a03c6def62800d2ef8a49932ec616a3b1c1fcd48c4aff9fa80437bcb35c +RESTORED SHA256: 4dc10394dadeada752ce12bd86fbbdfb1c231d92902a9aa912ce8ba7fb57c914 $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the root resumable' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (30 tests | 27 skipped) 23ms + ✓ tests/authored-root.test.ts (31 tests | 28 skipped) 21ms Test Files 1 passed (1) - Tests 3 passed | 27 skipped (30) - Start at 03:35:53 - Duration 1.27s (transform 658ms, setup 17ms, collect 1.10s, tests 23ms, environment 0ms, prepare 31ms) + Tests 3 passed | 28 skipped (31) + Start at 03:46:07 + Duration 1.49s (transform 779ms, setup 17ms, collect 1.27s, tests 21ms, environment 0ms, prepare 54ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-snapshot-cancel.log b/evidence/run-read-timeout/mutation-snapshot-cancel.log index 2e96db0fb..479bd7a3d 100644 --- a/evidence/run-read-timeout/mutation-snapshot-cancel.log +++ b/evidence/run-read-timeout/mutation-snapshot-cancel.log @@ -11,8 +11,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2216ms - × cancels promptly while a lease snapshot read is in flight 2215ms + ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2217ms + × cancels promptly while a lease snapshot read is in flight 2216ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -31,8 +31,8 @@ AssertionError: expected 'still waiting' to be an instance of Error Test Files 1 failed (1) Tests 1 failed | 11 skipped (12) - Start at 03:35:36 - Duration 2.71s (transform 195ms, setup 16ms, collect 331ms, tests 2.22s, environment 0ms, prepare 41ms) + Start at 03:45:50 + Duration 2.77s (transform 216ms, setup 21ms, collect 368ms, tests 2.22s, environment 0ms, prepare 51ms) exit=1 @@ -41,12 +41,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2010ms - ✓ cancels promptly while a lease snapshot read is in flight 2009ms + ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2008ms + ✓ cancels promptly while a lease snapshot read is in flight 2007ms Test Files 1 passed (1) Tests 1 passed | 11 skipped (12) - Start at 03:35:40 - Duration 2.55s (transform 232ms, setup 22ms, collect 353ms, tests 2.01s, environment 0ms, prepare 57ms) + Start at 03:45:53 + Duration 2.53s (transform 207ms, setup 18ms, collect 346ms, tests 2.01s, environment 0ms, prepare 49ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-watch-cadence.log b/evidence/run-read-timeout/mutation-watch-cadence.log index b532d4076..38c7a9cb3 100644 --- a/evidence/run-read-timeout/mutation-watch-cadence.log +++ b/evidence/run-read-timeout/mutation-watch-cadence.log @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2110ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2110ms + ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2112ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2111ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -30,8 +30,8 @@ AssertionError: expected 41 to be less than or equal to 1 Test Files 1 failed (1) Tests 1 failed | 11 skipped (12) - Start at 03:35:31 - Duration 2.56s (transform 179ms, setup 16ms, collect 300ms, tests 2.11s, environment 0ms, prepare 36ms) + Start at 03:45:44 + Duration 2.61s (transform 197ms, setup 18ms, collect 337ms, tests 2.11s, environment 0ms, prepare 41ms) exit=1 @@ -45,7 +45,7 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu Test Files 1 passed (1) Tests 1 passed | 11 skipped (12) - Start at 03:35:34 - Duration 2.59s (transform 185ms, setup 16ms, collect 311ms, tests 2.12s, environment 0ms, prepare 41ms) + Start at 03:45:47 + Duration 2.60s (transform 191ms, setup 13ms, collect 326ms, tests 2.12s, environment 0ms, prepare 42ms) exit=0 diff --git a/evidence/run-read-timeout/resume-live.log b/evidence/run-read-timeout/resume-live.log index 05aec7ca9..21a3d5fe2 100644 --- a/evidence/run-read-timeout/resume-live.log +++ b/evidence/run-read-timeout/resume-live.log @@ -2,12 +2,12 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 888ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 887ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 935ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 934ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 03:37:00 - Duration 2.12s (transform 602ms, setup 27ms, collect 967ms, tests 888ms, environment 0ms, prepare 102ms) + Start at 03:47:21 + Duration 2.27s (transform 627ms, setup 19ms, collect 1.03s, tests 935ms, environment 0ms, prepare 78ms) exit=0 diff --git a/evidence/run-read-timeout/verification.md b/evidence/run-read-timeout/verification.md index 821a500d6..e54641c15 100644 --- a/evidence/run-read-timeout/verification.md +++ b/evidence/run-read-timeout/verification.md @@ -15,71 +15,74 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5537ms - ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 462ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2038ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2039ms -(node:4077640) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5663ms + ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 385ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2045ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2044ms +(node:4127341) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (30 tests) 393ms - ✓ tests/journal-client.test.ts (17 tests) 80ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 7062ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 445ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1032ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2778ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 382ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 861ms - ✓ tests/authored-flow-slack.test.ts (7 tests) 1292ms - ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 458ms - ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 451ms - ✓ tests/journal-client-read-timeout.test.ts (25 tests) 1797ms - ✓ tests/running-step-watch.test.ts (12 tests) 12286ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2116ms - ✓ cancels promptly while a lease snapshot read is in flight 2005ms + ✓ tests/authored-root.test.ts (31 tests) 484ms + ✓ tests/journal-client.test.ts (17 tests) 84ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 8035ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 515ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1257ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2970ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 447ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 1002ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 1354ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 487ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 486ms + ✓ tests/journal-client-read-timeout.test.ts (26 tests) 1842ms + ✓ a recovered read timeout does not become an authored callback failure 312ms + ✓ tests/running-step-watch.test.ts (12 tests) 12293ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms + ✓ cancels promptly while a lease snapshot read is in flight 2006ms ✓ a completion push ends the wait without waiting out an in-flight snapshot 2055ms ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2004ms ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2002ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2730ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 694ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 665ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 693ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 676ms - ✓ tests/authored-helpers.test.ts (6 tests) 2139ms - ✓ runs every available provider through the real kernel and resumes completed effects without a second write 1026ms - ✓ replays after SIGKILL before confirm with the same token and one successful completion 448ms - ✓ replays after SIGKILL before complete with the same token and one successful completion 425ms - ✓ tests/classify-outcome.test.ts (11 tests) 7423ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 3273ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 798ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 858ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 814ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 802ms + ✓ tests/authored-helpers.test.ts (6 tests) 3019ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 1375ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 672ms + ✓ replays after SIGKILL before complete with the same token and one successful completion 690ms + ✓ tests/classify-outcome.test.ts (11 tests) 7427ms ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2010ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3005ms - ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2000ms - ✓ tests/human-live.test.ts (3 tests) 4749ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2900ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1089ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 758ms - ✓ tests/worker-lease.test.ts (7 tests) 19ms - ✓ tests/communication-worker.test.ts (18 tests) 1716ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 27ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 715ms - ✓ tests/memoization.test.ts (58 tests) 66ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2507ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2057ms - ✓ drains read and watch promises before an authored flow completes 449ms - ✓ tests/worker-lease-sweep.test.ts (4 tests) 9ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3006ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms + ✓ tests/human-live.test.ts (3 tests) 5285ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 3027ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1356ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 902ms + ✓ tests/worker-lease.test.ts (7 tests) 28ms + ✓ tests/communication-worker.test.ts (18 tests) 1721ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 39ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 925ms + ✓ reports journal success after completion rejects with lease_conflict 327ms + ✓ reports journal success when a renewal rejects after completion landed 344ms + ✓ tests/memoization.test.ts (58 tests) 77ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2616ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2069ms + ✓ drains read and watch promises before an authored flow completes 547ms + ✓ tests/worker-lease-sweep.test.ts (4 tests) 6ms ✓ tests/journal-client-completion.test.ts (6 tests) 106ms - ✓ tests/reuse-summary-interruption.test.ts (5 tests) 4ms - ✓ tests/resume-worker-lease.test.ts (3 tests) 9ms - ✓ tests/direct-run-worker-lease.test.ts (3 tests) 10ms - ✓ tests/helper-effect-lease-loss.test.ts (1 test) 232ms - ✓ tests/journal-client-subscriptions.test.ts (1 test) 7ms - ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 4ms - ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 46ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 17ms + ✓ tests/reuse-summary-interruption.test.ts (5 tests) 3ms + ✓ tests/resume-worker-lease.test.ts (3 tests) 8ms + ✓ tests/direct-run-worker-lease.test.ts (3 tests) 11ms + ✓ tests/helper-effect-lease-loss.test.ts (1 test) 264ms + ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms + ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 5ms + ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 58ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms Test Files 27 passed (27) - Tests 334 passed (334) - Start at 03:35:58 - Duration 61.82s (transform 1.34s, setup 155ms, collect 7.29s, tests 50.98s, environment 4ms, prepare 998ms) + Tests 336 passed (336) + Start at 03:46:13 + Duration 67.22s (transform 1.52s, setup 196ms, collect 8.43s, tests 54.65s, environment 4ms, prepare 1.14s) exit=0 ``` @@ -91,13 +94,13 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 888ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 887ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 935ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 934ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 03:37:00 - Duration 2.12s (transform 602ms, setup 27ms, collect 967ms, tests 888ms, environment 0ms, prepare 102ms) + Start at 03:47:21 + Duration 2.27s (transform 627ms, setup 19ms, collect 1.03s, tests 935ms, environment 0ms, prepare 78ms) exit=0 ``` @@ -119,7 +122,7 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (25 tests | 1 failed | 24 skipped) 113ms + ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 113ms × serves a bounded read while an unbounded command is in flight 112ms → expected true to be false // Object.is equality @@ -145,9 +148,9 @@ AssertionError: expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 24 skipped (25) - Start at 03:35:24 - Duration 577ms (transform 172ms, setup 15ms, collect 303ms, tests 113ms, environment 0ms, prepare 40ms) + Tests 1 failed | 25 skipped (26) + Start at 03:45:38 + Duration 568ms (transform 165ms, setup 16ms, collect 289ms, tests 113ms, environment 0ms, prepare 41ms) exit=1 @@ -156,12 +159,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (25 tests | 24 skipped) 106ms + ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 107ms Test Files 1 passed (1) - Tests 1 passed | 24 skipped (25) - Start at 03:35:25 - Duration 549ms (transform 162ms, setup 16ms, collect 287ms, tests 106ms, environment 0ms, prepare 40ms) + Tests 1 passed | 25 skipped (26) + Start at 03:45:39 + Duration 550ms (transform 166ms, setup 17ms, collect 292ms, tests 107ms, environment 0ms, prepare 40ms) exit=0 ``` @@ -182,14 +185,14 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (25 tests | 1 failed | 24 skipped) 1010ms + ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 1010ms × reconnects the reader after it disconnects within the read budget 1009ms - → journal client: run.get read session was interrupted after 77 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + → journal client: run.get read session was interrupted after 76 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 77 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 76 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:71:9 69| let last: unknown; 70| const exhausted = () => last instanceof JournalReadInterruptedError @@ -200,8 +203,8 @@ JournalReadInterruptedError: journal client: run.get read session was interrupte ❯ Timeout. src/journal-read-policy.ts:82:39 Caused by: Error: journal client: not connected (run.get): journal client: connection closed - ❯ src/journal-client.ts:252:16 - ❯ JournalClient.requestOnce src/journal-client.ts:245:12 + ❯ src/journal-client.ts:254:16 + ❯ JournalClient.requestOnce src/journal-client.ts:247:12 ❯ src/journal-budgeted-reads.ts:59:29 ❯ src/journal-read-policy.ts:94:18 @@ -211,9 +214,9 @@ Caused by: Error: journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 24 skipped (25) - Start at 03:35:26 - Duration 1.45s (transform 167ms, setup 15ms, collect 285ms, tests 1.01s, environment 0ms, prepare 41ms) + Tests 1 failed | 25 skipped (26) + Start at 03:45:40 + Duration 1.47s (transform 169ms, setup 17ms, collect 298ms, tests 1.01s, environment 0ms, prepare 41ms) exit=1 @@ -222,12 +225,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (25 tests | 24 skipped) 81ms + ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 86ms Test Files 1 passed (1) - Tests 1 passed | 24 skipped (25) - Start at 03:35:28 - Duration 521ms (transform 159ms, setup 16ms, collect 280ms, tests 81ms, environment 0ms, prepare 41ms) + Tests 1 passed | 25 skipped (26) + Start at 03:45:41 + Duration 534ms (transform 164ms, setup 16ms, collect 288ms, tests 86ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -253,8 +256,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (25 tests | 1 failed | 24 skipped) 78ms - × retries a reader setup that timed out instead of reading on the primary for good 77ms + ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 81ms + × retries a reader setup that timed out instead of reading on the primary for good 80ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -279,9 +282,9 @@ AssertionError: expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 24 skipped (25) - Start at 03:35:29 - Duration 514ms (transform 165ms, setup 17ms, collect 289ms, tests 78ms, environment 0ms, prepare 40ms) + Tests 1 failed | 25 skipped (26) + Start at 03:45:42 + Duration 535ms (transform 167ms, setup 16ms, collect 291ms, tests 81ms, environment 0ms, prepare 40ms) exit=1 @@ -290,12 +293,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (25 tests | 24 skipped) 73ms + ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 76ms Test Files 1 passed (1) - Tests 1 passed | 24 skipped (25) - Start at 03:35:30 - Duration 518ms (transform 165ms, setup 18ms, collect 287ms, tests 73ms, environment 0ms, prepare 41ms) + Tests 1 passed | 25 skipped (26) + Start at 03:45:43 + Duration 528ms (transform 171ms, setup 16ms, collect 292ms, tests 76ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -315,8 +318,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2110ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2110ms + ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2112ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2111ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -335,8 +338,8 @@ AssertionError: expected 41 to be less than or equal to 1 Test Files 1 failed (1) Tests 1 failed | 11 skipped (12) - Start at 03:35:31 - Duration 2.56s (transform 179ms, setup 16ms, collect 300ms, tests 2.11s, environment 0ms, prepare 36ms) + Start at 03:45:44 + Duration 2.61s (transform 197ms, setup 18ms, collect 337ms, tests 2.11s, environment 0ms, prepare 41ms) exit=1 @@ -350,8 +353,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu Test Files 1 passed (1) Tests 1 passed | 11 skipped (12) - Start at 03:35:34 - Duration 2.59s (transform 185ms, setup 16ms, collect 311ms, tests 2.12s, environment 0ms, prepare 41ms) + Start at 03:45:47 + Duration 2.60s (transform 191ms, setup 13ms, collect 326ms, tests 2.12s, environment 0ms, prepare 42ms) exit=0 ``` @@ -372,8 +375,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2216ms - × cancels promptly while a lease snapshot read is in flight 2215ms + ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2217ms + × cancels promptly while a lease snapshot read is in flight 2216ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -392,8 +395,8 @@ AssertionError: expected 'still waiting' to be an instance of Error Test Files 1 failed (1) Tests 1 failed | 11 skipped (12) - Start at 03:35:36 - Duration 2.71s (transform 195ms, setup 16ms, collect 331ms, tests 2.22s, environment 0ms, prepare 41ms) + Start at 03:45:50 + Duration 2.77s (transform 216ms, setup 21ms, collect 368ms, tests 2.22s, environment 0ms, prepare 51ms) exit=1 @@ -402,13 +405,13 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2010ms - ✓ cancels promptly while a lease snapshot read is in flight 2009ms + ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2008ms + ✓ cancels promptly while a lease snapshot read is in flight 2007ms Test Files 1 passed (1) Tests 1 passed | 11 skipped (12) - Start at 03:35:40 - Duration 2.55s (transform 232ms, setup 22ms, collect 353ms, tests 2.01s, environment 0ms, prepare 57ms) + Start at 03:45:53 + Duration 2.53s (transform 207ms, setup 18ms, collect 346ms, tests 2.01s, environment 0ms, prepare 49ms) exit=0 ``` @@ -428,8 +431,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 3012ms - × a completion push ends the wait without waiting out an in-flight snapshot 3011ms + ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 3013ms + × a completion push ends the wait without waiting out an in-flight snapshot 3012ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -452,8 +455,8 @@ Received: "still waiting" Test Files 1 failed (1) Tests 1 failed | 11 skipped (12) - Start at 03:35:43 - Duration 3.49s (transform 186ms, setup 16ms, collect 316ms, tests 3.01s, environment 0ms, prepare 43ms) + Start at 03:45:56 + Duration 3.55s (transform 211ms, setup 19ms, collect 356ms, tests 3.01s, environment 0ms, prepare 45ms) exit=1 @@ -462,13 +465,13 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2059ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2058ms + ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2063ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2062ms Test Files 1 passed (1) Tests 1 passed | 11 skipped (12) - Start at 03:35:46 - Duration 2.50s (transform 184ms, setup 15ms, collect 309ms, tests 2.06s, environment 0ms, prepare 31ms) + Start at 03:46:00 + Duration 2.57s (transform 210ms, setup 19ms, collect 366ms, tests 2.06s, environment 0ms, prepare 31ms) exit=0 ``` @@ -491,8 +494,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 20ms - × a heartbeat timeout is lease loss rather than a worker body failure 20ms + ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 22ms + × a heartbeat timeout is lease loss rather than a worker body failure 22ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -511,8 +514,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 03:35:49 - Duration 461ms (transform 170ms, setup 16ms, collect 277ms, tests 20ms, environment 0ms, prepare 41ms) + Start at 03:46:03 + Duration 535ms (transform 192ms, setup 19ms, collect 333ms, tests 22ms, environment 0ms, prepare 51ms) exit=1 @@ -521,12 +524,12 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 18ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 03:35:50 - Duration 447ms (transform 170ms, setup 17ms, collect 286ms, tests 16ms, environment 0ms, prepare 44ms) + Start at 03:46:04 + Duration 563ms (transform 197ms, setup 20ms, collect 350ms, tests 18ms, environment 0ms, prepare 51ms) exit=0 ``` @@ -535,7 +538,7 @@ exit=0 ```text Source: packages/sdk/src/authored-root.ts -Original SHA256: 8ab20a03c6def62800d2ef8a49932ec616a3b1c1fcd48c4aff9fa80437bcb35c +Original SHA256: 4dc10394dadeada752ce12bd86fbbdfb1c231d92902a9aa912ce8ba7fb57c914 Replaced: if (isReadInterruptionError(error) || (error instanceof AuthoredFlowExecutionError && error.code === 'daemon_unresponsive')) { @@ -553,14 +556,14 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (30 tests | 3 failed | 27 skipped) 32ms - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 22ms + ❯ tests/authored-root.test.ts (31 tests | 3 failed | 28 skipped) 26ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 18ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms → expected AuthoredFlowExecutionError: daemon_unresp… { …(7) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 4ms + × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 3ms → expected JournalReadInterruptedError: journal clie… { …(4) } to match object { code: 'daemon_unresponsive', …(2) } (5 matching properties omitted from actual) @@ -580,13 +583,13 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } - } + [JournalRequestTimeoutError: journal client: run.get timed out after 10ms] - ❯ tests/authored-root.test.ts:415:5 - 413| }); - 414| const journal = new RootJournal(); - 415| await expect(executeDurableAuthoredFlow(loaded, journal as unknown… + ❯ tests/authored-root.test.ts:424:5 + 422| }); + 423| const journal = new RootJournal(); + 424| await expect(executeDurableAuthoredFlow(loaded, journal as unknown… | ^ - 416| { dataDir: '/unused', admissionKey: 'read-timeout' })).rejects.t… - 417| code: 'daemon_unresponsive', rootRunId: 'root-run', message: e… + 425| { dataDir: '/unused', admissionKey: 'read-timeout' })).rejects.t… + 426| code: 'daemon_unresponsive', rootRunId: 'root-run', message: e… ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/3]⎯ @@ -604,13 +607,13 @@ AssertionError: expected AuthoredFlowExecutionError: daemon_unresp… { …(7) } "rootRunId": "root-run", } - ❯ tests/authored-root.test.ts:415:5 - 413| }); - 414| const journal = new RootJournal(); - 415| await expect(executeDurableAuthoredFlow(loaded, journal as unknown… + ❯ tests/authored-root.test.ts:424:5 + 422| }); + 423| const journal = new RootJournal(); + 424| await expect(executeDurableAuthoredFlow(loaded, journal as unknown… | ^ - 416| { dataDir: '/unused', admissionKey: 'read-timeout' })).rejects.t… - 417| code: 'daemon_unresponsive', rootRunId: 'root-run', message: e… + 425| { dataDir: '/unused', admissionKey: 'read-timeout' })).rejects.t… + 426| code: 'daemon_unresponsive', rootRunId: 'root-run', message: e… ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[2/3]⎯ @@ -628,34 +631,34 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } - } + [JournalReadInterruptedError: journal client: run.get read session was interrupted after 2 attempts in 40ms (read budget 300000ms): journal client: connection closed] - ❯ tests/authored-root.test.ts:428:5 - 426| const loaded = await fixture(false, 0, async () => { throw interru… - 427| const journal = new RootJournal(); - 428| await expect(executeDurableAuthoredFlow(loaded, journal as unknown… + ❯ tests/authored-root.test.ts:437:5 + 435| const loaded = await fixture(false, 0, async () => { throw interru… + 436| const journal = new RootJournal(); + 437| await expect(executeDurableAuthoredFlow(loaded, journal as unknown… | ^ - 429| { dataDir: '/unused', admissionKey: 'read-disconnect' })).reject… - 430| code: 'daemon_unresponsive', rootRunId: 'root-run', message: e… + 438| { dataDir: '/unused', admissionKey: 'read-disconnect' })).reject… + 439| code: 'daemon_unresponsive', rootRunId: 'root-run', message: e… ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[3/3]⎯ Test Files 1 failed (1) - Tests 3 failed | 27 skipped (30) - Start at 03:35:51 - Duration 1.29s (transform 661ms, setup 15ms, collect 1.10s, tests 32ms, environment 0ms, prepare 40ms) + Tests 3 failed | 28 skipped (31) + Start at 03:46:05 + Duration 1.55s (transform 830ms, setup 14ms, collect 1.38s, tests 26ms, environment 0ms, prepare 33ms) exit=1 -RESTORED SHA256: 8ab20a03c6def62800d2ef8a49932ec616a3b1c1fcd48c4aff9fa80437bcb35c +RESTORED SHA256: 4dc10394dadeada752ce12bd86fbbdfb1c231d92902a9aa912ce8ba7fb57c914 $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the root resumable' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (30 tests | 27 skipped) 23ms + ✓ tests/authored-root.test.ts (31 tests | 28 skipped) 21ms Test Files 1 passed (1) - Tests 3 passed | 27 skipped (30) - Start at 03:35:53 - Duration 1.27s (transform 658ms, setup 17ms, collect 1.10s, tests 23ms, environment 0ms, prepare 31ms) + Tests 3 passed | 28 skipped (31) + Start at 03:46:07 + Duration 1.49s (transform 779ms, setup 17ms, collect 1.27s, tests 21ms, environment 0ms, prepare 54ms) exit=0 ``` diff --git a/packages/sdk/src/authored-root.ts b/packages/sdk/src/authored-root.ts index 9621b36d4..c4dde2778 100644 --- a/packages/sdk/src/authored-root.ts +++ b/packages/sdk/src/authored-root.ts @@ -332,7 +332,8 @@ async function driveRoot( const callerSignal = options.lifecycle?.signal; const signal = callerSignal === undefined ? rootSignal : AbortSignal.any([callerSignal, rootSignal]); if (process.versions['bun'] !== undefined) { - return runAuthoredInNode(metadata, journal.socketPath, dispatch.run_id, { + // Awaited, so the read scope below covers the child's whole run. + return await runAuthoredInNode(metadata, journal.socketPath, dispatch.run_id, { dataDir: options.dataDir, localAgentStream: options.localAgentStream, ...(options.agentEnvironment === undefined ? {} : { agentEnvironment: localAgentCredentialEnvironment(options.agentEnvironment), diff --git a/packages/sdk/src/journal-client.ts b/packages/sdk/src/journal-client.ts index c01c3a9f9..c4be6095e 100644 --- a/packages/sdk/src/journal-client.ts +++ b/packages/sdk/src/journal-client.ts @@ -221,7 +221,9 @@ export class JournalClient extends EventEmitter { private request(verb: V, params: VerbContract[V]['params'], timeoutMs: number | null = this.requestTimeoutMs, signal?: AbortSignal): Promise { - if (this.budgeted !== undefined && this.socket && !this.socket.destroyed && timeoutMs !== null && READ_ONLY_VERBS.has(verb)) { + // Every allowlisted read takes the policy, even after the primary dropped: its + // closed state then answers with the typed interruption recorded at the drop. + if (this.budgeted !== undefined && timeoutMs !== null && READ_ONLY_VERBS.has(verb)) { return this.budgeted.read(verb, params, timeoutMs, signal); } return this.requestOnce(verb, params, timeoutMs, signal); diff --git a/packages/sdk/tests/authored-root.test.ts b/packages/sdk/tests/authored-root.test.ts index 4c1348aab..258b92a71 100644 --- a/packages/sdk/tests/authored-root.test.ts +++ b/packages/sdk/tests/authored-root.test.ts @@ -17,6 +17,15 @@ import type { RunOutcome, StepDispatchEvent } from '../src/protocol.js'; import { AuthoredFlowExecutionError, AuthoredHumanParked } from '../src/authored-flow-error.js'; import { JournalReadInterruptedError } from '../src/journal-read-policy.js'; +const nodeRunner = vi.hoisted(() => ({ override: undefined as undefined | ((...args: unknown[]) => Promise) })); +vi.mock('../src/authored-node-runner.js', async importOriginal => { + const actual = await importOriginal(); + return { ...actual, + assertAuthoredRuntimeAvailable: () => { if (nodeRunner.override === undefined) actual.assertAuthoredRuntimeAvailable(); }, + runAuthoredInNode: (...args: Parameters) => + nodeRunner.override !== undefined ? nodeRunner.override(...args) : actual.runAuthoredInNode(...args) }; +}); + vi.mock('../src/authored-flow-loader.js', async importOriginal => ({ ...await importOriginal(), loadAuthoredFlow: vi.fn(), @@ -466,6 +475,28 @@ describe('durable authored root', () => { await expect(readAuthoredRootMetadata(plain as unknown as JournalClient, 'run')).resolves.toBeUndefined(); }); + it('keeps the lease read scope for the whole Bun-run attempt', async () => { + const loaded = await fixture(); + const journal = new RootJournal(); + const release = vi.fn(); + (journal as unknown as { scopeReads: (signal: AbortSignal) => () => void }).scopeReads = () => release; + let finish!: (value: unknown) => void; + nodeRunner.override = () => new Promise(resolve => { finish = resolve; }); + Object.defineProperty(process.versions, 'bun', { value: '1.4.0', configurable: true }); + try { + const execution = executeDurableAuthoredFlow(loaded, journal as unknown as JournalClient, undefined, + { dataDir: '/unused', admissionKey: 'bun-scope' }).catch(() => undefined); + await vi.waitFor(() => expect(finish).toBeTypeOf('function')); + expect(release).not.toHaveBeenCalled(); // the child is still running + finish({ name: 'f', completionReason: 'success', journalSteps: [] }); + await execution; + expect(release).toHaveBeenCalledTimes(1); + } finally { + delete (process.versions as Record)['bun']; + nodeRunner.override = undefined; + } + }); + it('terminalizes a returned body failure without replaying semantic side effects', async () => { const loaded = await fixture(true); const journal = new RootJournal(); diff --git a/packages/sdk/tests/journal-client-read-timeout.test.ts b/packages/sdk/tests/journal-client-read-timeout.test.ts index 34b949f57..cd595ad75 100644 --- a/packages/sdk/tests/journal-client-read-timeout.test.ts +++ b/packages/sdk/tests/journal-client-read-timeout.test.ts @@ -302,3 +302,17 @@ it('an unexpected primary disconnect aborts in-flight reads with a typed read in const error = await client.runGet('run').catch(caught => caught); expect(isReadInterruptionError(error)).toBe(true); }); + +it('a read started after the primary dropped is still a typed read interruption', async () => { + let primary: { destroy(): void } | undefined; + const client = await setup({ + 'stream.append': ctx => { primary = ctx.socket; sendResult(ctx, { seq: 1 }); }, + 'run.get': ctx => sendResult(ctx, { status: 'running' }), + }, 60_000); + await client.streamAppend('run', 'stream', {}); + const dropped = new Promise(resolve => client.once('disconnected', () => resolve())); + primary?.destroy(); + await dropped; + const error = await client.runGet('run').catch(caught => caught); + expect(isReadInterruptionError(error)).toBe(true); +}); From 6ccad704898c7c7e73f8d2c22ee3383317026a71 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 03:58:34 -0700 Subject: [PATCH 23/34] refactor(sdk): split the journal connection from the client; keep completed outcomes on readback timeouts - journal-connection.ts (222 lines) now owns framing, the socket lifecycle, request correlation and the protocol error types; JournalClient (421 lines, down from 618; main 537) keeps options, budgeted-read integration and the typed verbs, hooking connection drops through connectionClosed. Error types are re-exported from journal-client, so imports are unchanged. - When run.start/run.resume report a completed authored root but reading back its stored result is interrupted, the root raises result_unreadable carrying the known completion reason, and flows run/resume report status completed with a result_unreadable warning instead of a resumable daemon_unresponsive. Evidence re-captured at this tree (8/8 mutations; focused 28 files 343/343; resume-live 1/1). Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: b8262468-ea26-4a6e-bee9-d193832e77dc --- evidence/run-read-timeout/final-focused.log | 127 ++++---- .../mutation-completion-aborts-snapshot.log | 14 +- .../run-read-timeout/mutation-heartbeat.log | 12 +- .../mutation-reader-reconnect.log | 30 +- .../mutation-reader-setup-retry.log | 18 +- evidence/run-read-timeout/mutation-reader.log | 18 +- .../mutation-root-parking.log | 26 +- .../mutation-snapshot-cancel.log | 16 +- .../mutation-watch-cadence.log | 16 +- evidence/run-read-timeout/resume-live.log | 8 +- evidence/run-read-timeout/verification.md | 287 +++++++++--------- packages/sdk/src/authored-flow-error.ts | 2 + packages/sdk/src/authored-root.ts | 20 +- packages/sdk/src/cli/direct-run.ts | 5 +- packages/sdk/src/cli/journal-timeout.ts | 13 + packages/sdk/src/cli/run.ts | 5 +- packages/sdk/src/failure-kinds.ts | 1 + packages/sdk/src/journal-budgeted-reads.ts | 3 +- packages/sdk/src/journal-client.ts | 223 +------------- packages/sdk/src/journal-connection.ts | 222 ++++++++++++++ packages/sdk/tests/authored-root.test.ts | 11 + .../sdk/tests/run-daemon-unresponsive.test.ts | 12 + 22 files changed, 585 insertions(+), 504 deletions(-) create mode 100644 packages/sdk/src/journal-connection.ts diff --git a/evidence/run-read-timeout/final-focused.log b/evidence/run-read-timeout/final-focused.log index cdfaaa658..e49b3bddb 100644 --- a/evidence/run-read-timeout/final-focused.log +++ b/evidence/run-read-timeout/final-focused.log @@ -1,74 +1,71 @@ -$ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/kernel-target/debug/relayflowd npx vitest run tests/journal-client-read-timeout.test.ts tests/journal-client.test.ts tests/journal-client-completion.test.ts tests/journal-client-subscriptions.test.ts tests/running-step-watch.test.ts tests/heartbeat-timeout.test.ts tests/run-daemon-unresponsive.test.ts tests/authored-root.test.ts tests/classify-outcome.test.ts tests/cli.test.ts tests/direct-run-worker-lease.test.ts tests/resume-worker-lease.test.ts tests/worker-lease.test.ts tests/worker-lease-lost.test.ts tests/worker-lease-lost-live.test.ts tests/worker-lease-sweep.test.ts tests/run-read-load-live.test.ts tests/flow-executor-chain.test.ts tests/agent-transcript-live.test.ts tests/human-live.test.ts tests/reuse-summary-interruption.test.ts tests/memoization.test.ts tests/authored-verifier-read-budget.test.ts tests/communication-worker.test.ts tests/helper-effect-lease-loss.test.ts tests/authored-helpers.test.ts tests/authored-flow-slack.test.ts --maxWorkers=1 --minWorkers=1 +$ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/kernel-target/debug/relayflowd npx vitest run tests/journal-client-read-timeout.test.ts tests/journal-client.test.ts tests/journal-client-completion.test.ts tests/journal-client-subscriptions.test.ts tests/running-step-watch.test.ts tests/heartbeat-timeout.test.ts tests/run-daemon-unresponsive.test.ts tests/authored-root.test.ts tests/classify-outcome.test.ts tests/cli.test.ts tests/direct-run-worker-lease.test.ts tests/resume-worker-lease.test.ts tests/worker-lease.test.ts tests/worker-lease-lost.test.ts tests/worker-lease-lost-live.test.ts tests/worker-lease-sweep.test.ts tests/run-read-load-live.test.ts tests/flow-executor-chain.test.ts tests/agent-transcript-live.test.ts tests/human-live.test.ts tests/reuse-summary-interruption.test.ts tests/memoization.test.ts tests/authored-verifier-read-budget.test.ts tests/communication-worker.test.ts tests/helper-effect-lease-loss.test.ts tests/authored-helpers.test.ts tests/authored-flow-slack.test.ts tests/effect-channel.test.ts --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5663ms - ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 385ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2045ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2044ms -(node:4127341) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5256ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2040ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2040ms +(node:4176986) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (31 tests) 484ms - ✓ tests/journal-client.test.ts (17 tests) 84ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 8035ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 515ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1257ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2970ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 447ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 1002ms - ✓ tests/authored-flow-slack.test.ts (7 tests) 1354ms - ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 487ms - ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 486ms - ✓ tests/journal-client-read-timeout.test.ts (26 tests) 1842ms - ✓ a recovered read timeout does not become an authored callback failure 312ms - ✓ tests/running-step-watch.test.ts (12 tests) 12293ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms - ✓ cancels promptly while a lease snapshot read is in flight 2006ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2055ms - ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms - ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2004ms + ✓ tests/authored-root.test.ts (32 tests) 452ms + ✓ tests/journal-client.test.ts (17 tests) 80ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 7535ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 416ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1066ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2855ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 454ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 1043ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 1290ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 436ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 467ms + ✓ tests/journal-client-read-timeout.test.ts (26 tests) 1719ms + ✓ tests/running-step-watch.test.ts (12 tests) 12280ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2114ms + ✓ cancels promptly while a lease snapshot read is in flight 2004ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2054ms + ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2003ms + ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2002ms ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2002ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 3273ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 798ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 858ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 814ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 802ms - ✓ tests/authored-helpers.test.ts (6 tests) 3019ms - ✓ runs every available provider through the real kernel and resumes completed effects without a second write 1375ms - ✓ replays after SIGKILL before confirm with the same token and one successful completion 672ms - ✓ replays after SIGKILL before complete with the same token and one successful completion 690ms - ✓ tests/classify-outcome.test.ts (11 tests) 7427ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2010ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3006ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2469ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 660ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 515ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 643ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 650ms + ✓ tests/authored-helpers.test.ts (6 tests) 1938ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 898ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 399ms + ✓ replays after SIGKILL before complete with the same token and one successful completion 416ms + ✓ tests/classify-outcome.test.ts (11 tests) 7424ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2012ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3004ms ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms - ✓ tests/human-live.test.ts (3 tests) 5285ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 3027ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1356ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 902ms - ✓ tests/worker-lease.test.ts (7 tests) 28ms - ✓ tests/communication-worker.test.ts (18 tests) 1721ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 39ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 925ms - ✓ reports journal success after completion rejects with lease_conflict 327ms - ✓ reports journal success when a renewal rejects after completion landed 344ms - ✓ tests/memoization.test.ts (58 tests) 77ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2616ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2069ms - ✓ drains read and watch promises before an authored flow completes 547ms - ✓ tests/worker-lease-sweep.test.ts (4 tests) 6ms - ✓ tests/journal-client-completion.test.ts (6 tests) 106ms - ✓ tests/reuse-summary-interruption.test.ts (5 tests) 3ms - ✓ tests/resume-worker-lease.test.ts (3 tests) 8ms - ✓ tests/direct-run-worker-lease.test.ts (3 tests) 11ms - ✓ tests/helper-effect-lease-loss.test.ts (1 test) 264ms - ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms - ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 5ms - ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 58ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms + ✓ tests/human-live.test.ts (3 tests) 4505ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2666ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1067ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 771ms + ✓ tests/worker-lease.test.ts (7 tests) 20ms + ✓ tests/communication-worker.test.ts (18 tests) 1713ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 26ms + ✓ tests/effect-channel.test.ts (5 tests) 285ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 739ms + ✓ tests/memoization.test.ts (58 tests) 63ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2408ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2055ms + ✓ drains read and watch promises before an authored flow completes 353ms + ✓ tests/worker-lease-sweep.test.ts (4 tests) 9ms + ✓ tests/journal-client-completion.test.ts (6 tests) 105ms + ✓ tests/reuse-summary-interruption.test.ts (5 tests) 4ms + ✓ tests/run-daemon-unresponsive.test.ts (4 tests) 5ms + ✓ tests/resume-worker-lease.test.ts (3 tests) 9ms + ✓ tests/direct-run-worker-lease.test.ts (3 tests) 10ms + ✓ tests/helper-effect-lease-loss.test.ts (1 test) 229ms + ✓ tests/journal-client-subscriptions.test.ts (1 test) 9ms + ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 35ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 17ms - Test Files 27 passed (27) - Tests 336 passed (336) - Start at 03:46:13 - Duration 67.22s (transform 1.52s, setup 196ms, collect 8.43s, tests 54.65s, environment 4ms, prepare 1.14s) + Test Files 28 passed (28) + Tests 343 passed (343) + Start at 03:57:17 + Duration 61.09s (transform 1.11s, setup 156ms, collect 6.89s, tests 50.63s, environment 4ms, prepare 965ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log index 8f2228712..65ce7c201 100644 --- a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log +++ b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 3013ms - × a completion push ends the wait without waiting out an in-flight snapshot 3012ms + ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 3011ms + × a completion push ends the wait without waiting out an in-flight snapshot 3010ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -34,8 +34,8 @@ Received: "still waiting" Test Files 1 failed (1) Tests 1 failed | 11 skipped (12) - Start at 03:45:56 - Duration 3.55s (transform 211ms, setup 19ms, collect 356ms, tests 3.01s, environment 0ms, prepare 45ms) + Start at 03:57:00 + Duration 3.49s (transform 192ms, setup 16ms, collect 323ms, tests 3.01s, environment 0ms, prepare 40ms) exit=1 @@ -44,12 +44,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2063ms + ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2062ms ✓ a completion push ends the wait without waiting out an in-flight snapshot 2062ms Test Files 1 passed (1) Tests 1 passed | 11 skipped (12) - Start at 03:46:00 - Duration 2.57s (transform 210ms, setup 19ms, collect 366ms, tests 2.06s, environment 0ms, prepare 31ms) + Start at 03:57:04 + Duration 2.53s (transform 194ms, setup 16ms, collect 312ms, tests 2.06s, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-heartbeat.log b/evidence/run-read-timeout/mutation-heartbeat.log index 362c2525f..a14ffa07a 100644 --- a/evidence/run-read-timeout/mutation-heartbeat.log +++ b/evidence/run-read-timeout/mutation-heartbeat.log @@ -13,7 +13,7 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 22ms + ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 24ms × a heartbeat timeout is lease loss rather than a worker body failure 22ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError @@ -33,8 +33,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 03:46:03 - Duration 535ms (transform 192ms, setup 19ms, collect 333ms, tests 22ms, environment 0ms, prepare 51ms) + Start at 03:57:07 + Duration 673ms (transform 211ms, setup 23ms, collect 413ms, tests 24ms, environment 0ms, prepare 63ms) exit=1 @@ -43,11 +43,11 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/heartbeat-timeout.test.ts (1 test) 18ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 19ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 03:46:04 - Duration 563ms (transform 197ms, setup 20ms, collect 350ms, tests 18ms, environment 0ms, prepare 51ms) + Start at 03:57:08 + Duration 644ms (transform 211ms, setup 25ms, collect 388ms, tests 19ms, environment 0ms, prepare 63ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-reconnect.log b/evidence/run-read-timeout/mutation-reader-reconnect.log index 3c7383fdb..41513fbd7 100644 --- a/evidence/run-read-timeout/mutation-reader-reconnect.log +++ b/evidence/run-read-timeout/mutation-reader-reconnect.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: 30380437df429b288157a3a895ddb2cede8fe8853889b4a0cb9a3c24a4097814 +Original SHA256: 3c10585faa843ba188e6d87cc39f0b4e8a6b95a165d9caba104919c1c35598ec Replaced: this.drop(reader); @@ -11,14 +11,14 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 1010ms - × reconnects the reader after it disconnects within the read budget 1009ms - → journal client: run.get read session was interrupted after 76 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 1009ms + × reconnects the reader after it disconnects within the read budget 1008ms + → journal client: run.get read session was interrupted after 80 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 76 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 80 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:71:9 69| let last: unknown; 70| const exhausted = () => last instanceof JournalReadInterruptedError @@ -29,33 +29,33 @@ JournalReadInterruptedError: journal client: run.get read session was interrupte ❯ Timeout. src/journal-read-policy.ts:82:39 Caused by: Error: journal client: not connected (run.get): journal client: connection closed - ❯ src/journal-client.ts:254:16 - ❯ JournalClient.requestOnce src/journal-client.ts:247:12 - ❯ src/journal-budgeted-reads.ts:59:29 + ❯ src/journal-connection.ts:183:16 + ❯ JournalClient.requestOnce src/journal-connection.ts:176:12 + ❯ src/journal-budgeted-reads.ts:60:29 ❯ src/journal-read-policy.ts:94:18 Caused by: Error: journal client: connection closed - ❯ Socket. src/journal-client.ts:142:26 + ❯ Socket. src/journal-connection.ts:89:26 ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) Tests 1 failed | 25 skipped (26) - Start at 03:45:40 - Duration 1.47s (transform 169ms, setup 17ms, collect 298ms, tests 1.01s, environment 0ms, prepare 41ms) + Start at 03:56:44 + Duration 1.47s (transform 172ms, setup 17ms, collect 304ms, tests 1.01s, environment 0ms, prepare 45ms) exit=1 -RESTORED SHA256: 30380437df429b288157a3a895ddb2cede8fe8853889b4a0cb9a3c24a4097814 +RESTORED SHA256: 3c10585faa843ba188e6d87cc39f0b4e8a6b95a165d9caba104919c1c35598ec $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'reconnects the reader after it disconnects' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 86ms + ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 81ms Test Files 1 passed (1) Tests 1 passed | 25 skipped (26) - Start at 03:45:41 - Duration 534ms (transform 164ms, setup 16ms, collect 288ms, tests 86ms, environment 0ms, prepare 41ms) + Start at 03:56:46 + Duration 554ms (transform 172ms, setup 19ms, collect 301ms, tests 81ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-setup-retry.log b/evidence/run-read-timeout/mutation-reader-setup-retry.log index 83b2058fc..436f936e6 100644 --- a/evidence/run-read-timeout/mutation-reader-setup-retry.log +++ b/evidence/run-read-timeout/mutation-reader-setup-retry.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: 30380437df429b288157a3a895ddb2cede8fe8853889b4a0cb9a3c24a4097814 +Original SHA256: 3c10585faa843ba188e6d87cc39f0b4e8a6b95a165d9caba104919c1c35598ec Replaced: if (this.reader === reader) { this.reader = undefined; @@ -16,8 +16,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 81ms - × retries a reader setup that timed out instead of reading on the primary for good 80ms + ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 79ms + × retries a reader setup that timed out instead of reading on the primary for good 78ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -43,21 +43,21 @@ AssertionError: expected 1 to be 2 // Object.is equality Test Files 1 failed (1) Tests 1 failed | 25 skipped (26) - Start at 03:45:42 - Duration 535ms (transform 167ms, setup 16ms, collect 291ms, tests 81ms, environment 0ms, prepare 40ms) + Start at 03:56:47 + Duration 535ms (transform 167ms, setup 17ms, collect 291ms, tests 79ms, environment 0ms, prepare 42ms) exit=1 -RESTORED SHA256: 30380437df429b288157a3a895ddb2cede8fe8853889b4a0cb9a3c24a4097814 +RESTORED SHA256: 3c10585faa843ba188e6d87cc39f0b4e8a6b95a165d9caba104919c1c35598ec $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'retries a reader setup that timed out' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 76ms + ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 67ms Test Files 1 passed (1) Tests 1 passed | 25 skipped (26) - Start at 03:45:43 - Duration 528ms (transform 171ms, setup 16ms, collect 292ms, tests 76ms, environment 0ms, prepare 41ms) + Start at 03:56:48 + Duration 509ms (transform 167ms, setup 15ms, collect 283ms, tests 67ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader.log b/evidence/run-read-timeout/mutation-reader.log index d1e5ce4cb..b269f2f12 100644 --- a/evidence/run-read-timeout/mutation-reader.log +++ b/evidence/run-read-timeout/mutation-reader.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: 30380437df429b288157a3a895ddb2cede8fe8853889b4a0cb9a3c24a4097814 +Original SHA256: 3c10585faa843ba188e6d87cc39f0b4e8a6b95a165d9caba104919c1c35598ec Replaced: return await reader.requestOnce(verb, params, remaining, attemptSignal); With: @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 113ms - × serves a bounded read while an unbounded command is in flight 112ms + ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 109ms + × serves a bounded read while an unbounded command is in flight 109ms → expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -37,21 +37,21 @@ AssertionError: expected true to be false // Object.is equality Test Files 1 failed (1) Tests 1 failed | 25 skipped (26) - Start at 03:45:38 - Duration 568ms (transform 165ms, setup 16ms, collect 289ms, tests 113ms, environment 0ms, prepare 41ms) + Start at 03:56:42 + Duration 561ms (transform 170ms, setup 17ms, collect 292ms, tests 109ms, environment 0ms, prepare 41ms) exit=1 -RESTORED SHA256: 30380437df429b288157a3a895ddb2cede8fe8853889b4a0cb9a3c24a4097814 +RESTORED SHA256: 3c10585faa843ba188e6d87cc39f0b4e8a6b95a165d9caba104919c1c35598ec $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'serves a bounded read' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 107ms + ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 104ms Test Files 1 passed (1) Tests 1 passed | 25 skipped (26) - Start at 03:45:39 - Duration 550ms (transform 166ms, setup 17ms, collect 292ms, tests 107ms, environment 0ms, prepare 40ms) + Start at 03:56:43 + Duration 568ms (transform 175ms, setup 17ms, collect 297ms, tests 104ms, environment 0ms, prepare 42ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-root-parking.log b/evidence/run-read-timeout/mutation-root-parking.log index a892a0e37..fd3558536 100644 --- a/evidence/run-read-timeout/mutation-root-parking.log +++ b/evidence/run-read-timeout/mutation-root-parking.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/authored-root.ts -Original SHA256: 4dc10394dadeada752ce12bd86fbbdfb1c231d92902a9aa912ce8ba7fb57c914 +Original SHA256: 3800932c9ef7768b4bf0a01f08885856e5ccbd3d173cd63574fd599a4fc33230 Replaced: if (isReadInterruptionError(error) || (error instanceof AuthoredFlowExecutionError && error.code === 'daemon_unresponsive')) { @@ -17,14 +17,14 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (31 tests | 3 failed | 28 skipped) 26ms - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 18ms + ❯ tests/authored-root.test.ts (32 tests | 3 failed | 29 skipped) 48ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 33ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 7ms → expected AuthoredFlowExecutionError: daemon_unresp… { …(7) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 3ms + × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 6ms → expected JournalReadInterruptedError: journal clie… { …(4) } to match object { code: 'daemon_unresponsive', …(2) } (5 matching properties omitted from actual) @@ -103,22 +103,22 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[3/3]⎯ Test Files 1 failed (1) - Tests 3 failed | 28 skipped (31) - Start at 03:46:05 - Duration 1.55s (transform 830ms, setup 14ms, collect 1.38s, tests 26ms, environment 0ms, prepare 33ms) + Tests 3 failed | 29 skipped (32) + Start at 03:57:09 + Duration 1.71s (transform 791ms, setup 23ms, collect 1.42s, tests 48ms, environment 0ms, prepare 62ms) exit=1 -RESTORED SHA256: 4dc10394dadeada752ce12bd86fbbdfb1c231d92902a9aa912ce8ba7fb57c914 +RESTORED SHA256: 3800932c9ef7768b4bf0a01f08885856e5ccbd3d173cd63574fd599a4fc33230 $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the root resumable' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (31 tests | 28 skipped) 21ms + ✓ tests/authored-root.test.ts (32 tests | 29 skipped) 35ms Test Files 1 passed (1) - Tests 3 passed | 28 skipped (31) - Start at 03:46:07 - Duration 1.49s (transform 779ms, setup 17ms, collect 1.27s, tests 21ms, environment 0ms, prepare 54ms) + Tests 3 passed | 29 skipped (32) + Start at 03:57:12 + Duration 1.76s (transform 820ms, setup 23ms, collect 1.50s, tests 35ms, environment 0ms, prepare 49ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-snapshot-cancel.log b/evidence/run-read-timeout/mutation-snapshot-cancel.log index 479bd7a3d..a36588881 100644 --- a/evidence/run-read-timeout/mutation-snapshot-cancel.log +++ b/evidence/run-read-timeout/mutation-snapshot-cancel.log @@ -11,8 +11,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2217ms - × cancels promptly while a lease snapshot read is in flight 2216ms + ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2213ms + × cancels promptly while a lease snapshot read is in flight 2212ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -31,8 +31,8 @@ AssertionError: expected 'still waiting' to be an instance of Error Test Files 1 failed (1) Tests 1 failed | 11 skipped (12) - Start at 03:45:50 - Duration 2.77s (transform 216ms, setup 21ms, collect 368ms, tests 2.22s, environment 0ms, prepare 51ms) + Start at 03:56:54 + Duration 2.69s (transform 197ms, setup 18ms, collect 323ms, tests 2.21s, environment 0ms, prepare 40ms) exit=1 @@ -41,12 +41,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2008ms - ✓ cancels promptly while a lease snapshot read is in flight 2007ms + ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2011ms + ✓ cancels promptly while a lease snapshot read is in flight 2011ms Test Files 1 passed (1) Tests 1 passed | 11 skipped (12) - Start at 03:45:53 - Duration 2.53s (transform 207ms, setup 18ms, collect 346ms, tests 2.01s, environment 0ms, prepare 49ms) + Start at 03:56:57 + Duration 2.47s (transform 190ms, setup 17ms, collect 310ms, tests 2.01s, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-watch-cadence.log b/evidence/run-read-timeout/mutation-watch-cadence.log index 38c7a9cb3..adfb19369 100644 --- a/evidence/run-read-timeout/mutation-watch-cadence.log +++ b/evidence/run-read-timeout/mutation-watch-cadence.log @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2112ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2111ms + ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2110ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2109ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -30,8 +30,8 @@ AssertionError: expected 41 to be less than or equal to 1 Test Files 1 failed (1) Tests 1 failed | 11 skipped (12) - Start at 03:45:44 - Duration 2.61s (transform 197ms, setup 18ms, collect 337ms, tests 2.11s, environment 0ms, prepare 41ms) + Start at 03:56:48 + Duration 2.59s (transform 188ms, setup 18ms, collect 320ms, tests 2.11s, environment 0ms, prepare 40ms) exit=1 @@ -40,12 +40,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2118ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms + ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2119ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms Test Files 1 passed (1) Tests 1 passed | 11 skipped (12) - Start at 03:45:47 - Duration 2.60s (transform 191ms, setup 13ms, collect 326ms, tests 2.12s, environment 0ms, prepare 42ms) + Start at 03:56:51 + Duration 2.60s (transform 190ms, setup 16ms, collect 324ms, tests 2.12s, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/resume-live.log b/evidence/run-read-timeout/resume-live.log index 21a3d5fe2..159311474 100644 --- a/evidence/run-read-timeout/resume-live.log +++ b/evidence/run-read-timeout/resume-live.log @@ -2,12 +2,12 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 935ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 934ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 875ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 874ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 03:47:21 - Duration 2.27s (transform 627ms, setup 19ms, collect 1.03s, tests 935ms, environment 0ms, prepare 78ms) + Start at 03:58:18 + Duration 2.04s (transform 609ms, setup 27ms, collect 967ms, tests 875ms, environment 0ms, prepare 63ms) exit=0 diff --git a/evidence/run-read-timeout/verification.md b/evidence/run-read-timeout/verification.md index e54641c15..557661d46 100644 --- a/evidence/run-read-timeout/verification.md +++ b/evidence/run-read-timeout/verification.md @@ -11,78 +11,75 @@ exit=0 ## final-focused.log ```text -$ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/kernel-target/debug/relayflowd npx vitest run tests/journal-client-read-timeout.test.ts tests/journal-client.test.ts tests/journal-client-completion.test.ts tests/journal-client-subscriptions.test.ts tests/running-step-watch.test.ts tests/heartbeat-timeout.test.ts tests/run-daemon-unresponsive.test.ts tests/authored-root.test.ts tests/classify-outcome.test.ts tests/cli.test.ts tests/direct-run-worker-lease.test.ts tests/resume-worker-lease.test.ts tests/worker-lease.test.ts tests/worker-lease-lost.test.ts tests/worker-lease-lost-live.test.ts tests/worker-lease-sweep.test.ts tests/run-read-load-live.test.ts tests/flow-executor-chain.test.ts tests/agent-transcript-live.test.ts tests/human-live.test.ts tests/reuse-summary-interruption.test.ts tests/memoization.test.ts tests/authored-verifier-read-budget.test.ts tests/communication-worker.test.ts tests/helper-effect-lease-loss.test.ts tests/authored-helpers.test.ts tests/authored-flow-slack.test.ts --maxWorkers=1 --minWorkers=1 +$ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/kernel-target/debug/relayflowd npx vitest run tests/journal-client-read-timeout.test.ts tests/journal-client.test.ts tests/journal-client-completion.test.ts tests/journal-client-subscriptions.test.ts tests/running-step-watch.test.ts tests/heartbeat-timeout.test.ts tests/run-daemon-unresponsive.test.ts tests/authored-root.test.ts tests/classify-outcome.test.ts tests/cli.test.ts tests/direct-run-worker-lease.test.ts tests/resume-worker-lease.test.ts tests/worker-lease.test.ts tests/worker-lease-lost.test.ts tests/worker-lease-lost-live.test.ts tests/worker-lease-sweep.test.ts tests/run-read-load-live.test.ts tests/flow-executor-chain.test.ts tests/agent-transcript-live.test.ts tests/human-live.test.ts tests/reuse-summary-interruption.test.ts tests/memoization.test.ts tests/authored-verifier-read-budget.test.ts tests/communication-worker.test.ts tests/helper-effect-lease-loss.test.ts tests/authored-helpers.test.ts tests/authored-flow-slack.test.ts tests/effect-channel.test.ts --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5663ms - ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 385ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2045ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2044ms -(node:4127341) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5256ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2040ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2040ms +(node:4176986) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (31 tests) 484ms - ✓ tests/journal-client.test.ts (17 tests) 84ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 8035ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 515ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1257ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2970ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 447ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 1002ms - ✓ tests/authored-flow-slack.test.ts (7 tests) 1354ms - ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 487ms - ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 486ms - ✓ tests/journal-client-read-timeout.test.ts (26 tests) 1842ms - ✓ a recovered read timeout does not become an authored callback failure 312ms - ✓ tests/running-step-watch.test.ts (12 tests) 12293ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms - ✓ cancels promptly while a lease snapshot read is in flight 2006ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2055ms - ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms - ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2004ms + ✓ tests/authored-root.test.ts (32 tests) 452ms + ✓ tests/journal-client.test.ts (17 tests) 80ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 7535ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 416ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1066ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2855ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 454ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 1043ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 1290ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 436ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 467ms + ✓ tests/journal-client-read-timeout.test.ts (26 tests) 1719ms + ✓ tests/running-step-watch.test.ts (12 tests) 12280ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2114ms + ✓ cancels promptly while a lease snapshot read is in flight 2004ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2054ms + ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2003ms + ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2002ms ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2002ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 3273ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 798ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 858ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 814ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 802ms - ✓ tests/authored-helpers.test.ts (6 tests) 3019ms - ✓ runs every available provider through the real kernel and resumes completed effects without a second write 1375ms - ✓ replays after SIGKILL before confirm with the same token and one successful completion 672ms - ✓ replays after SIGKILL before complete with the same token and one successful completion 690ms - ✓ tests/classify-outcome.test.ts (11 tests) 7427ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2010ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3006ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2469ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 660ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 515ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 643ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 650ms + ✓ tests/authored-helpers.test.ts (6 tests) 1938ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 898ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 399ms + ✓ replays after SIGKILL before complete with the same token and one successful completion 416ms + ✓ tests/classify-outcome.test.ts (11 tests) 7424ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2012ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3004ms ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms - ✓ tests/human-live.test.ts (3 tests) 5285ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 3027ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1356ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 902ms - ✓ tests/worker-lease.test.ts (7 tests) 28ms - ✓ tests/communication-worker.test.ts (18 tests) 1721ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 39ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 925ms - ✓ reports journal success after completion rejects with lease_conflict 327ms - ✓ reports journal success when a renewal rejects after completion landed 344ms - ✓ tests/memoization.test.ts (58 tests) 77ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2616ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2069ms - ✓ drains read and watch promises before an authored flow completes 547ms - ✓ tests/worker-lease-sweep.test.ts (4 tests) 6ms - ✓ tests/journal-client-completion.test.ts (6 tests) 106ms - ✓ tests/reuse-summary-interruption.test.ts (5 tests) 3ms - ✓ tests/resume-worker-lease.test.ts (3 tests) 8ms - ✓ tests/direct-run-worker-lease.test.ts (3 tests) 11ms - ✓ tests/helper-effect-lease-loss.test.ts (1 test) 264ms - ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms - ✓ tests/run-daemon-unresponsive.test.ts (3 tests) 5ms - ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 58ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms - - Test Files 27 passed (27) - Tests 336 passed (336) - Start at 03:46:13 - Duration 67.22s (transform 1.52s, setup 196ms, collect 8.43s, tests 54.65s, environment 4ms, prepare 1.14s) + ✓ tests/human-live.test.ts (3 tests) 4505ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2666ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1067ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 771ms + ✓ tests/worker-lease.test.ts (7 tests) 20ms + ✓ tests/communication-worker.test.ts (18 tests) 1713ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 26ms + ✓ tests/effect-channel.test.ts (5 tests) 285ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 739ms + ✓ tests/memoization.test.ts (58 tests) 63ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2408ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2055ms + ✓ drains read and watch promises before an authored flow completes 353ms + ✓ tests/worker-lease-sweep.test.ts (4 tests) 9ms + ✓ tests/journal-client-completion.test.ts (6 tests) 105ms + ✓ tests/reuse-summary-interruption.test.ts (5 tests) 4ms + ✓ tests/run-daemon-unresponsive.test.ts (4 tests) 5ms + ✓ tests/resume-worker-lease.test.ts (3 tests) 9ms + ✓ tests/direct-run-worker-lease.test.ts (3 tests) 10ms + ✓ tests/helper-effect-lease-loss.test.ts (1 test) 229ms + ✓ tests/journal-client-subscriptions.test.ts (1 test) 9ms + ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 35ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 17ms + + Test Files 28 passed (28) + Tests 343 passed (343) + Start at 03:57:17 + Duration 61.09s (transform 1.11s, setup 156ms, collect 6.89s, tests 50.63s, environment 4ms, prepare 965ms) exit=0 ``` @@ -94,13 +91,13 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 935ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 934ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 875ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 874ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 03:47:21 - Duration 2.27s (transform 627ms, setup 19ms, collect 1.03s, tests 935ms, environment 0ms, prepare 78ms) + Start at 03:58:18 + Duration 2.04s (transform 609ms, setup 27ms, collect 967ms, tests 875ms, environment 0ms, prepare 63ms) exit=0 ``` @@ -111,7 +108,7 @@ The mutation sections below were regenerated by `cd packages/sdk && python3 ../. ```text Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: 30380437df429b288157a3a895ddb2cede8fe8853889b4a0cb9a3c24a4097814 +Original SHA256: 3c10585faa843ba188e6d87cc39f0b4e8a6b95a165d9caba104919c1c35598ec Replaced: return await reader.requestOnce(verb, params, remaining, attemptSignal); With: @@ -122,8 +119,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 113ms - × serves a bounded read while an unbounded command is in flight 112ms + ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 109ms + × serves a bounded read while an unbounded command is in flight 109ms → expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -149,22 +146,22 @@ AssertionError: expected true to be false // Object.is equality Test Files 1 failed (1) Tests 1 failed | 25 skipped (26) - Start at 03:45:38 - Duration 568ms (transform 165ms, setup 16ms, collect 289ms, tests 113ms, environment 0ms, prepare 41ms) + Start at 03:56:42 + Duration 561ms (transform 170ms, setup 17ms, collect 292ms, tests 109ms, environment 0ms, prepare 41ms) exit=1 -RESTORED SHA256: 30380437df429b288157a3a895ddb2cede8fe8853889b4a0cb9a3c24a4097814 +RESTORED SHA256: 3c10585faa843ba188e6d87cc39f0b4e8a6b95a165d9caba104919c1c35598ec $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'serves a bounded read' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 107ms + ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 104ms Test Files 1 passed (1) Tests 1 passed | 25 skipped (26) - Start at 03:45:39 - Duration 550ms (transform 166ms, setup 17ms, collect 292ms, tests 107ms, environment 0ms, prepare 40ms) + Start at 03:56:43 + Duration 568ms (transform 175ms, setup 17ms, collect 297ms, tests 104ms, environment 0ms, prepare 42ms) exit=0 ``` @@ -173,7 +170,7 @@ exit=0 ```text Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: 30380437df429b288157a3a895ddb2cede8fe8853889b4a0cb9a3c24a4097814 +Original SHA256: 3c10585faa843ba188e6d87cc39f0b4e8a6b95a165d9caba104919c1c35598ec Replaced: this.drop(reader); @@ -185,14 +182,14 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 1010ms - × reconnects the reader after it disconnects within the read budget 1009ms - → journal client: run.get read session was interrupted after 76 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 1009ms + × reconnects the reader after it disconnects within the read budget 1008ms + → journal client: run.get read session was interrupted after 80 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 76 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 80 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:71:9 69| let last: unknown; 70| const exhausted = () => last instanceof JournalReadInterruptedError @@ -203,34 +200,34 @@ JournalReadInterruptedError: journal client: run.get read session was interrupte ❯ Timeout. src/journal-read-policy.ts:82:39 Caused by: Error: journal client: not connected (run.get): journal client: connection closed - ❯ src/journal-client.ts:254:16 - ❯ JournalClient.requestOnce src/journal-client.ts:247:12 - ❯ src/journal-budgeted-reads.ts:59:29 + ❯ src/journal-connection.ts:183:16 + ❯ JournalClient.requestOnce src/journal-connection.ts:176:12 + ❯ src/journal-budgeted-reads.ts:60:29 ❯ src/journal-read-policy.ts:94:18 Caused by: Error: journal client: connection closed - ❯ Socket. src/journal-client.ts:142:26 + ❯ Socket. src/journal-connection.ts:89:26 ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) Tests 1 failed | 25 skipped (26) - Start at 03:45:40 - Duration 1.47s (transform 169ms, setup 17ms, collect 298ms, tests 1.01s, environment 0ms, prepare 41ms) + Start at 03:56:44 + Duration 1.47s (transform 172ms, setup 17ms, collect 304ms, tests 1.01s, environment 0ms, prepare 45ms) exit=1 -RESTORED SHA256: 30380437df429b288157a3a895ddb2cede8fe8853889b4a0cb9a3c24a4097814 +RESTORED SHA256: 3c10585faa843ba188e6d87cc39f0b4e8a6b95a165d9caba104919c1c35598ec $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'reconnects the reader after it disconnects' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 86ms + ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 81ms Test Files 1 passed (1) Tests 1 passed | 25 skipped (26) - Start at 03:45:41 - Duration 534ms (transform 164ms, setup 16ms, collect 288ms, tests 86ms, environment 0ms, prepare 41ms) + Start at 03:56:46 + Duration 554ms (transform 172ms, setup 19ms, collect 301ms, tests 81ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -239,7 +236,7 @@ exit=0 ```text Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: 30380437df429b288157a3a895ddb2cede8fe8853889b4a0cb9a3c24a4097814 +Original SHA256: 3c10585faa843ba188e6d87cc39f0b4e8a6b95a165d9caba104919c1c35598ec Replaced: if (this.reader === reader) { this.reader = undefined; @@ -256,8 +253,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 81ms - × retries a reader setup that timed out instead of reading on the primary for good 80ms + ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 79ms + × retries a reader setup that timed out instead of reading on the primary for good 78ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -283,22 +280,22 @@ AssertionError: expected 1 to be 2 // Object.is equality Test Files 1 failed (1) Tests 1 failed | 25 skipped (26) - Start at 03:45:42 - Duration 535ms (transform 167ms, setup 16ms, collect 291ms, tests 81ms, environment 0ms, prepare 40ms) + Start at 03:56:47 + Duration 535ms (transform 167ms, setup 17ms, collect 291ms, tests 79ms, environment 0ms, prepare 42ms) exit=1 -RESTORED SHA256: 30380437df429b288157a3a895ddb2cede8fe8853889b4a0cb9a3c24a4097814 +RESTORED SHA256: 3c10585faa843ba188e6d87cc39f0b4e8a6b95a165d9caba104919c1c35598ec $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'retries a reader setup that timed out' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 76ms + ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 67ms Test Files 1 passed (1) Tests 1 passed | 25 skipped (26) - Start at 03:45:43 - Duration 528ms (transform 171ms, setup 16ms, collect 292ms, tests 76ms, environment 0ms, prepare 41ms) + Start at 03:56:48 + Duration 509ms (transform 167ms, setup 15ms, collect 283ms, tests 67ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -318,8 +315,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2112ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2111ms + ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2110ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2109ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -338,8 +335,8 @@ AssertionError: expected 41 to be less than or equal to 1 Test Files 1 failed (1) Tests 1 failed | 11 skipped (12) - Start at 03:45:44 - Duration 2.61s (transform 197ms, setup 18ms, collect 337ms, tests 2.11s, environment 0ms, prepare 41ms) + Start at 03:56:48 + Duration 2.59s (transform 188ms, setup 18ms, collect 320ms, tests 2.11s, environment 0ms, prepare 40ms) exit=1 @@ -348,13 +345,13 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2118ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms + ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2119ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms Test Files 1 passed (1) Tests 1 passed | 11 skipped (12) - Start at 03:45:47 - Duration 2.60s (transform 191ms, setup 13ms, collect 326ms, tests 2.12s, environment 0ms, prepare 42ms) + Start at 03:56:51 + Duration 2.60s (transform 190ms, setup 16ms, collect 324ms, tests 2.12s, environment 0ms, prepare 41ms) exit=0 ``` @@ -375,8 +372,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2217ms - × cancels promptly while a lease snapshot read is in flight 2216ms + ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2213ms + × cancels promptly while a lease snapshot read is in flight 2212ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -395,8 +392,8 @@ AssertionError: expected 'still waiting' to be an instance of Error Test Files 1 failed (1) Tests 1 failed | 11 skipped (12) - Start at 03:45:50 - Duration 2.77s (transform 216ms, setup 21ms, collect 368ms, tests 2.22s, environment 0ms, prepare 51ms) + Start at 03:56:54 + Duration 2.69s (transform 197ms, setup 18ms, collect 323ms, tests 2.21s, environment 0ms, prepare 40ms) exit=1 @@ -405,13 +402,13 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2008ms - ✓ cancels promptly while a lease snapshot read is in flight 2007ms + ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2011ms + ✓ cancels promptly while a lease snapshot read is in flight 2011ms Test Files 1 passed (1) Tests 1 passed | 11 skipped (12) - Start at 03:45:53 - Duration 2.53s (transform 207ms, setup 18ms, collect 346ms, tests 2.01s, environment 0ms, prepare 49ms) + Start at 03:56:57 + Duration 2.47s (transform 190ms, setup 17ms, collect 310ms, tests 2.01s, environment 0ms, prepare 40ms) exit=0 ``` @@ -431,8 +428,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 3013ms - × a completion push ends the wait without waiting out an in-flight snapshot 3012ms + ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 3011ms + × a completion push ends the wait without waiting out an in-flight snapshot 3010ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -455,8 +452,8 @@ Received: "still waiting" Test Files 1 failed (1) Tests 1 failed | 11 skipped (12) - Start at 03:45:56 - Duration 3.55s (transform 211ms, setup 19ms, collect 356ms, tests 3.01s, environment 0ms, prepare 45ms) + Start at 03:57:00 + Duration 3.49s (transform 192ms, setup 16ms, collect 323ms, tests 3.01s, environment 0ms, prepare 40ms) exit=1 @@ -465,13 +462,13 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2063ms + ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2062ms ✓ a completion push ends the wait without waiting out an in-flight snapshot 2062ms Test Files 1 passed (1) Tests 1 passed | 11 skipped (12) - Start at 03:46:00 - Duration 2.57s (transform 210ms, setup 19ms, collect 366ms, tests 2.06s, environment 0ms, prepare 31ms) + Start at 03:57:04 + Duration 2.53s (transform 194ms, setup 16ms, collect 312ms, tests 2.06s, environment 0ms, prepare 40ms) exit=0 ``` @@ -494,7 +491,7 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 22ms + ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 24ms × a heartbeat timeout is lease loss rather than a worker body failure 22ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError @@ -514,8 +511,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 03:46:03 - Duration 535ms (transform 192ms, setup 19ms, collect 333ms, tests 22ms, environment 0ms, prepare 51ms) + Start at 03:57:07 + Duration 673ms (transform 211ms, setup 23ms, collect 413ms, tests 24ms, environment 0ms, prepare 63ms) exit=1 @@ -524,12 +521,12 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/heartbeat-timeout.test.ts (1 test) 18ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 19ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 03:46:04 - Duration 563ms (transform 197ms, setup 20ms, collect 350ms, tests 18ms, environment 0ms, prepare 51ms) + Start at 03:57:08 + Duration 644ms (transform 211ms, setup 25ms, collect 388ms, tests 19ms, environment 0ms, prepare 63ms) exit=0 ``` @@ -538,7 +535,7 @@ exit=0 ```text Source: packages/sdk/src/authored-root.ts -Original SHA256: 4dc10394dadeada752ce12bd86fbbdfb1c231d92902a9aa912ce8ba7fb57c914 +Original SHA256: 3800932c9ef7768b4bf0a01f08885856e5ccbd3d173cd63574fd599a4fc33230 Replaced: if (isReadInterruptionError(error) || (error instanceof AuthoredFlowExecutionError && error.code === 'daemon_unresponsive')) { @@ -556,14 +553,14 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (31 tests | 3 failed | 28 skipped) 26ms - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 18ms + ❯ tests/authored-root.test.ts (32 tests | 3 failed | 29 skipped) 48ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 33ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 7ms → expected AuthoredFlowExecutionError: daemon_unresp… { …(7) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 3ms + × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 6ms → expected JournalReadInterruptedError: journal clie… { …(4) } to match object { code: 'daemon_unresponsive', …(2) } (5 matching properties omitted from actual) @@ -642,23 +639,23 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[3/3]⎯ Test Files 1 failed (1) - Tests 3 failed | 28 skipped (31) - Start at 03:46:05 - Duration 1.55s (transform 830ms, setup 14ms, collect 1.38s, tests 26ms, environment 0ms, prepare 33ms) + Tests 3 failed | 29 skipped (32) + Start at 03:57:09 + Duration 1.71s (transform 791ms, setup 23ms, collect 1.42s, tests 48ms, environment 0ms, prepare 62ms) exit=1 -RESTORED SHA256: 4dc10394dadeada752ce12bd86fbbdfb1c231d92902a9aa912ce8ba7fb57c914 +RESTORED SHA256: 3800932c9ef7768b4bf0a01f08885856e5ccbd3d173cd63574fd599a4fc33230 $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the root resumable' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (31 tests | 28 skipped) 21ms + ✓ tests/authored-root.test.ts (32 tests | 29 skipped) 35ms Test Files 1 passed (1) - Tests 3 passed | 28 skipped (31) - Start at 03:46:07 - Duration 1.49s (transform 779ms, setup 17ms, collect 1.27s, tests 21ms, environment 0ms, prepare 54ms) + Tests 3 passed | 29 skipped (32) + Start at 03:57:12 + Duration 1.76s (transform 820ms, setup 23ms, collect 1.50s, tests 35ms, environment 0ms, prepare 49ms) exit=0 ``` diff --git a/packages/sdk/src/authored-flow-error.ts b/packages/sdk/src/authored-flow-error.ts index e2ec5828a..307343791 100644 --- a/packages/sdk/src/authored-flow-error.ts +++ b/packages/sdk/src/authored-flow-error.ts @@ -36,6 +36,8 @@ export type AuthoredFlowExecutionErrorCode = * journaled and `flows resume ` continues it. */ | 'daemon_unresponsive' + /** The root run completed; only reading back its stored result failed. */ + | 'result_unreadable' | 'root_lease_lost' | 'unsupported_completion' | 'unsupported_gate' diff --git a/packages/sdk/src/authored-root.ts b/packages/sdk/src/authored-root.ts index c4dde2778..fa548c248 100644 --- a/packages/sdk/src/authored-root.ts +++ b/packages/sdk/src/authored-root.ts @@ -120,7 +120,7 @@ export async function executeDurableAuthoredFlow( options.onAdmitted?.(outcome.run_id); if (outcome.status === 'completed') { dispatchWait.cancel(); - return await completedRootResult(journal, outcome.run_id); + return await completedResult(journal, outcome, outcome.run_id); } assertRootCanDispatch(outcome); options.lifecycle?.onRunStarted?.({ runId: outcome.run_id, flow: definition.name }); @@ -169,7 +169,7 @@ export async function resumeDurableAuthoredFlow( const outcome = await journal.runResume(rootRunId); if (outcome.status === 'completed') { dispatchWait.cancel(); - return await completedRootResult(journal, rootRunId); + return await completedResult(journal, outcome, rootRunId); } assertRootCanDispatch(outcome); await assertNoOpenHumanWait(journal, outcome); @@ -568,6 +568,22 @@ function nextRootDispatch(peer: JournalClient, timeoutMs = 30_000): { return { promise, cancel }; } +/** + * The stored result of a root the daemon already reports completed. If reading + * it back is interrupted, the outcome is still known: say so, rather than + * recasting a finished run as resumable. + */ +function completedResult(journal: JournalClient, outcome: RunOutcome, rootRunId: string): Promise { + return completedRootResult(journal, rootRunId).catch(error => { + if (!isReadInterruptionError(error)) throw error; + const unread = new AuthoredFlowExecutionError('result_unreadable', + `run ${rootRunId} completed (${outcome.completion_reason ?? 'no completion reason'}), but its stored result could not be read: ${error.message}`, + outcome.completion_reason ?? undefined, rootRunId); + unread.rootRunId = rootRunId; + throw unread; + }); +} + async function completedRootResult( journal: JournalClient, rootRunId: string, diff --git a/packages/sdk/src/cli/direct-run.ts b/packages/sdk/src/cli/direct-run.ts index 248a54581..9bf868107 100644 --- a/packages/sdk/src/cli/direct-run.ts +++ b/packages/sdk/src/cli/direct-run.ts @@ -1,4 +1,4 @@ -import { daemonUnresponsiveReport, isReadInterruption } from './journal-timeout.js'; +import { completedResultUnreadableReport, daemonUnresponsiveReport, isReadInterruption } from './journal-timeout.js'; import { FLOW_READ_BUDGET_MS } from '../journal-read-policy.js'; import { onWorkerFailure } from '../worker-lease.js'; import { McpStepError } from '../authored-mcp.js'; @@ -215,6 +215,9 @@ export async function runDirectFlow( if (error instanceof AuthoredFlowExecutionError && (error.code === 'step_failed' || error.code === 'gate_failed')) { return authoredStepFailure('run', base, socketPath, error); } + if (error instanceof AuthoredFlowExecutionError && error.code === 'result_unreadable') { + return completedResultUnreadableReport('run', base, socketPath, error); + } if (isReadInterruption(error)) return daemonUnresponsiveReport('run', base, socketPath, error, base.rootRunId, options, dataDir); if (error instanceof AuthoredFlowExecutionError && error.code === 'root_lease_lost') { return rootLeaseLostReport('run', base, socketPath, error); diff --git a/packages/sdk/src/cli/journal-timeout.ts b/packages/sdk/src/cli/journal-timeout.ts index 1afbb6959..ef7ef44a8 100644 --- a/packages/sdk/src/cli/journal-timeout.ts +++ b/packages/sdk/src/cli/journal-timeout.ts @@ -10,6 +10,19 @@ export function isReadInterruption(error: unknown): boolean { || (error instanceof AuthoredFlowExecutionError && error.code === 'daemon_unresponsive'); } +/** A completed run whose stored result could not be read back: report the known outcome. */ +export function completedResultUnreadableReport(command: RunCommand, base: CheckReport | RunReport, + socketPath: string, error: AuthoredFlowExecutionError): RunExecution { + const runId = error.rootRunId ?? error.runId; + const ok = error.completionReason === 'success'; + return { exitCode: ok ? 0 : 1, report: { + ...base, command, ok, socketPath, status: 'completed', + ...(runId === undefined ? {} : { runId, rootRunId: runId }), + ...(error.completionReason === undefined ? {} : { completionReason: error.completionReason }), + diagnostics: [...base.diagnostics, { severity: 'warning', kind: 'result_unreadable', message: error.message }], + } as RunExecution['report'] }; +} + export async function daemonUnresponsiveReport(command: RunCommand, base: CheckReport | RunReport, socketPath: string, error: unknown, fallbackRunId?: string, options: RunLifecycleOptions = {}, dataDir?: string): Promise { diff --git a/packages/sdk/src/cli/run.ts b/packages/sdk/src/cli/run.ts index 2746e4d1f..e5e24c4b4 100644 --- a/packages/sdk/src/cli/run.ts +++ b/packages/sdk/src/cli/run.ts @@ -1,6 +1,6 @@ import { waitForRunningStep, type RunningStep } from './running-step.js'; export { LEASE_SWEEP_GRACE_MS } from './running-step.js'; -import { daemonUnresponsiveReport, isReadInterruption } from './journal-timeout.js'; +import { completedResultUnreadableReport, daemonUnresponsiveReport, isReadInterruption } from './journal-timeout.js'; import { FLOW_READ_BUDGET_MS } from '../journal-read-policy.js'; import { onWorkerFailure } from '../worker-lease.js'; import { communicationInstruction } from '../communication/spec.js'; @@ -414,6 +414,9 @@ export async function resumeFlow( if (error instanceof AuthoredFlowExecutionError && (error.code === 'step_failed' || error.code === 'gate_failed')) { return authoredStepFailure('resume', base, socketPath, error, runId); } + if (error instanceof AuthoredFlowExecutionError && error.code === 'result_unreadable') { + return completedResultUnreadableReport('resume', base, socketPath, error); + } if (isReadInterruption(error)) return daemonUnresponsiveReport('resume', base, socketPath, error, runId, options, dataDir); if (error instanceof AuthoredFlowExecutionError && error.code === 'root_lease_lost') { return rootLeaseLostReport('resume', base, socketPath, error, runId); diff --git a/packages/sdk/src/failure-kinds.ts b/packages/sdk/src/failure-kinds.ts index 054d47fa8..f962137ff 100644 --- a/packages/sdk/src/failure-kinds.ts +++ b/packages/sdk/src/failure-kinds.ts @@ -162,6 +162,7 @@ export const RUN_FAILURE_KINDS = [ export const RUN_WARNING_KINDS = [ 'connection_file_stale', 'reuse_summary_unavailable', + 'result_unreadable', ] as const; /** diff --git a/packages/sdk/src/journal-budgeted-reads.ts b/packages/sdk/src/journal-budgeted-reads.ts index 441e37198..6d6bbe442 100644 --- a/packages/sdk/src/journal-budgeted-reads.ts +++ b/packages/sdk/src/journal-budgeted-reads.ts @@ -1,4 +1,5 @@ -import { JournalFrameError, JournalProtocolError, type JournalClient } from './journal-client.js'; +import { JournalFrameError, JournalProtocolError } from './journal-connection.js'; +import type { JournalClient } from './journal-client.js'; import { JournalReadInterruptedError, JournalReadPolicy, JournalRequestTimeoutError } from './journal-read-policy.js'; import type { VerbContract } from './protocol.js'; diff --git a/packages/sdk/src/journal-client.ts b/packages/sdk/src/journal-client.ts index c4be6095e..ed1312399 100644 --- a/packages/sdk/src/journal-client.ts +++ b/packages/sdk/src/journal-client.ts @@ -7,24 +7,21 @@ // AGENTS.md rule 4). The kernel binary (`kernel/relayflowd serve`) speaks // this transport, including out-of-band leases, watches, events, and durable // stream plumbing. The client's framing and failure behavior is covered by a -// loopback double in tests. +// loopback double in tests. Framing and the socket lifecycle live in +// journal-connection.ts; budgeted reads in journal-budgeted-reads.ts. -import { JournalRequestTimeoutError, READ_ONLY_VERBS } from './journal-read-policy.js'; +import { READ_ONLY_VERBS } from './journal-read-policy.js'; +import { JournalConnection, JournalProtocolError } from './journal-connection.js'; +export { JournalFrameError, JournalProtocolError } from './journal-connection.js'; import { BudgetedReads } from './journal-budgeted-reads.js'; export { JournalReadInterruptedError, JournalRequestTimeoutError } from './journal-read-policy.js'; -import { EventEmitter } from 'node:events'; export { walkJournal, JournalReadError, type JournalEvent, type JournalReadFailure } from './journal-reader.js'; -import { randomUUID } from 'node:crypto'; -import { createConnection, type Socket } from 'node:net'; import type { VerbContract, EventEmitParams, EventSubmitParams, ReportedCost } from './protocol.js'; import { PROTOCOL_VERSION, type CompletionReason, type EffectRef, type Pins, - type Request, - type Response, - type ServerEvent, type StepUsage, } from './protocol.js'; import type { KernelRunSpec, StepType } from './spec.js'; @@ -55,56 +52,21 @@ export interface JournalClientOptions { connectTimeoutMs?: number; } -interface Pending { - resolve: (value: unknown) => void; - reject: (err: Error) => void; - timer?: ReturnType; -} - -/** A structured rejection returned by relayflowd over the journal protocol. */ -export class JournalProtocolError extends Error { - readonly code: string; - verb?: string; - - constructor(code: string, message: string) { - super(`${code}: ${message}`); - this.name = 'JournalProtocolError'; - this.code = code; - } -} - -/** The daemon sent a frame this client cannot parse: a protocol violation, never retried. */ -export class JournalFrameError extends Error { - constructor(message = 'journal client: malformed frame from server') { - super(message); - this.name = 'JournalFrameError'; - } -} - -export class JournalClient extends EventEmitter { +export class JournalClient extends JournalConnection { private readonly budgeted: BudgetedReads | undefined; private readonly budgetHandshake: boolean; /** Additive request capabilities the daemon advertised at `hello`; none until then. */ private features: ReadonlySet = new Set(); - private socket: Socket | null = null; - private connectingSocket: Socket | undefined; - /** Why the connection ended, so a later "not connected" names its cause rather than hiding it. */ - private disconnectCause: Error | undefined; - private buffer = ''; - private readonly pending = new Map(); - private readonly requestTimeoutMs: number; - private readonly connectTimeoutMs: number; - - constructor( - readonly socketPath: string, - options: JournalClientOptions = {}, - ) { - super(); + + constructor(socketPath: string, options: JournalClientOptions = {}) { + super(socketPath, options.requestTimeoutMs ?? 30_000, options.connectTimeoutMs ?? 2_000); this.budgetHandshake = options.budgetHandshake === true; this.budgeted = options.readBudgetMs === undefined ? undefined : new BudgetedReads(this, options.readBudgetMs, options.readSignal); - this.requestTimeoutMs = options.requestTimeoutMs ?? 30_000; - this.connectTimeoutMs = options.connectTimeoutMs ?? 2_000; + } + + protected override connectionClosed(cause: unknown, unexpected: boolean): void { + this.budgeted?.close(cause, unexpected); } /** Independent session for an SDK helper worker; preserves the caller's registration. */ @@ -114,111 +76,6 @@ export class JournalClient extends EventEmitter { }); } - /** Open the unix socket connection. Rejects on connect failure (fail-closed). */ - connect(): Promise { - return new Promise((resolve, reject) => { - if (this.socket) return resolve(); - const socket = createConnection({ path: this.socketPath }); - this.connectingSocket = socket; - const timer = setTimeout(() => { - socket.removeAllListeners(); - socket.destroy(); - this.failAll(new Error(`journal client: connect timed out after ${this.connectTimeoutMs}ms`)); - reject(new Error(`journal client: connect timed out after ${this.connectTimeoutMs}ms`)); - }, this.connectTimeoutMs); - const onError = (err: Error): void => { - clearTimeout(timer); - socket.removeAllListeners(); - this.failAll(err); - reject(new Error(`journal client: connect failed: ${err.message}`)); - }; - socket.once('error', onError); - socket.once('connect', () => { - clearTimeout(timer); - socket.removeListener('error', onError); - socket.on('error', (err) => { this.disconnectCause ??= err; this.failAll(err); }); - socket.on('data', (chunk) => this.onData(chunk)); - socket.on('close', () => { - const closed = new Error('journal client: connection closed'); - this.budgeted?.close(closed, this.socket === socket); - this.disconnectCause ??= closed; - this.failAll(closed); - // Only a drop the caller did not ask for; close() clears this.socket first. - // Sessions with nothing pending (a registered watch) must hear it too. - if (this.socket === socket) this.emit('disconnected', closed); - }); - this.disconnectCause = undefined; - this.connectingSocket = undefined; - this.socket = socket; - resolve(); - }); - }); - } - - /** - * Close the connection and reject any pending requests. A `cause` (for - * example the worker failure that forced the close) is carried into every - * later "not connected" rejection. - */ - close(cause?: unknown): void { - const closed = cause === undefined - ? new Error('journal client: closed by caller') - : new Error(`journal client: closed after ${cause instanceof Error ? cause.message : String(cause)}`, { cause }); - this.budgeted?.close(cause); - this.connectingSocket?.destroy(closed); - this.disconnectCause ??= closed; - this.failAll(closed); - this.socket?.destroy(); - this.socket = null; - this.buffer = ''; - } - - private onData(chunk: Buffer): void { - this.buffer += chunk.toString('utf8'); - let nl: number; - while ((nl = this.buffer.indexOf('\n')) !== -1) { - const line = this.buffer.slice(0, nl); - this.buffer = this.buffer.slice(nl + 1); - if (line.length > 0) this.onLine(line); - } - } - - private onLine(line: string): void { - let msg: Response | ServerEvent; - try { - msg = JSON.parse(line) as Response | ServerEvent; - } catch { - // A malformed frame is a protocol violation; fail closed. - const error = new JournalFrameError(); - this.failAll(error); - // Sessions with nothing pending (a registered watch) must hear it too. - this.emit('protocol_error', error); - return; - } - - if (typeof (msg as Response).id === 'string' && 'ok' in (msg as Response)) { - const res = msg as Response; - const pending = this.pending.get(res.id); - if (!pending) return; // reply for an already-timed-out request - this.pending.delete(res.id); - if (pending.timer !== undefined) clearTimeout(pending.timer); - if (res.ok) pending.resolve(res.result); - else pending.reject(new JournalProtocolError(res.error.code, res.error.message)); - } else { - const ev = msg as ServerEvent; - this.emit(ev.event, ev.data); - this.emit('event', ev); - } - } - - private failAll(err: Error): void { - for (const [, p] of this.pending) { - if (p.timer !== undefined) clearTimeout(p.timer); - p.reject(err); - } - this.pending.clear(); - } - private request(verb: V, params: VerbContract[V]['params'], timeoutMs: number | null = this.requestTimeoutMs, signal?: AbortSignal): Promise { // Every allowlisted read takes the policy, even after the primary dropped: its @@ -237,60 +94,6 @@ export class JournalClient extends EventEmitter { return this.budgeted?.scope(signal) ?? (() => {}); } - /** @internal One unretried request on this session; flow reads go through `request`. */ - requestOnce( - verb: V, - params: VerbContract[V]['params'], - timeoutMs: number | null = this.requestTimeoutMs, - signal?: AbortSignal, - ): Promise { - return new Promise((resolve, reject) => { - if (signal?.aborted) { - reject(signal.reason); - return; - } - if (!this.socket || this.socket.destroyed) { - const cause = this.disconnectCause; - reject(new Error( - `journal client: not connected (${verb})${cause === undefined ? '' : `: ${cause.message}`}`, - cause === undefined ? undefined : { cause }, - )); - return; - } - const id = randomUUID(); - const started = performance.now(); - const frame: Request = { id, verb: verb as string, params }; - // Cancellation forgets the request; a late reply is ignored like a timed-out one. - const onAbort = () => { - const p = this.pending.get(id); - if (p === undefined) return; - if (p.timer !== undefined) clearTimeout(p.timer); - this.pending.delete(id); - p.reject(signal!.reason); - }; - const settle = (fn: (arg: A) => void) => (arg: A) => { - signal?.removeEventListener('abort', onAbort); - fn(arg); - }; - const timer = timeoutMs === null ? undefined : setTimeout(() => { - this.pending.delete(id); - settle(reject)(new JournalRequestTimeoutError(verb, timeoutMs, 1, performance.now() - started)); - }, timeoutMs); - this.pending.set(id, { resolve: settle(resolve as (v: unknown) => void), reject: settle(reject), timer }); - signal?.addEventListener('abort', onAbort, { once: true }); - this.socket.write(JSON.stringify(frame) + '\n', (err) => { - if (err) { - const p = this.pending.get(id); - if (p) { - if (p.timer !== undefined) clearTimeout(p.timer); - this.pending.delete(id); - p.reject(new Error(`journal client: ${verb} write failed: ${err.message}`)); - } - } - }); - }); - } - // --- Typed verb methods (gate 1 minimal set, kernel DESIGN.md §5) -------- /** Handshake; version mismatch is a hard error. Records the daemon's additive features. */ diff --git a/packages/sdk/src/journal-connection.ts b/packages/sdk/src/journal-connection.ts new file mode 100644 index 000000000..b2cd78ab2 --- /dev/null +++ b/packages/sdk/src/journal-connection.ts @@ -0,0 +1,222 @@ +// Journal-protocol v0 client (kernel DESIGN.md §5). +// +// A real client implementation of the wire protocol — newline-delimited JSON +// over a unix socket. It correlates requests by `id`, demultiplexes +// server-pushed events, and is fail-closed: a connection drop or write error +// rejects every pending request (a journal write that fails fails the step; +// AGENTS.md rule 4). The kernel binary (`kernel/relayflowd serve`) speaks +// this transport, including out-of-band leases, watches, events, and durable +// stream plumbing. The client's framing and failure behavior is covered by a +// loopback double in tests. + +import { EventEmitter } from 'node:events'; +import { randomUUID } from 'node:crypto'; +import { createConnection, type Socket } from 'node:net'; +import { JournalRequestTimeoutError } from './journal-read-policy.js'; +import type { Request, Response, ServerEvent, VerbContract } from './protocol.js'; + +interface Pending { + resolve: (value: unknown) => void; + reject: (err: Error) => void; + timer?: ReturnType; +} + +/** A structured rejection returned by relayflowd over the journal protocol. */ +export class JournalProtocolError extends Error { + readonly code: string; + verb?: string; + + constructor(code: string, message: string) { + super(`${code}: ${message}`); + this.name = 'JournalProtocolError'; + this.code = code; + } +} + +/** The daemon sent a frame this client cannot parse: a protocol violation, never retried. */ +export class JournalFrameError extends Error { + constructor(message = 'journal client: malformed frame from server') { + super(message); + this.name = 'JournalFrameError'; + } +} + +/** Framing, socket lifecycle and request correlation; the typed verbs live on JournalClient. */ +export class JournalConnection extends EventEmitter { + protected socket: Socket | null = null; + private connectingSocket: Socket | undefined; + /** Why the connection ended, so a later "not connected" names its cause rather than hiding it. */ + protected disconnectCause: Error | undefined; + private buffer = ''; + private readonly pending = new Map(); + + constructor( + readonly socketPath: string, + protected readonly requestTimeoutMs: number, + protected readonly connectTimeoutMs: number, + ) { + super(); + } + + /** Hook for sessions layered on this one: `unexpected` is a drop the caller did not ask for. */ + protected connectionClosed(_cause: unknown, _unexpected: boolean): void {} + + /** Open the unix socket connection. Rejects on connect failure (fail-closed). */ + connect(): Promise { + return new Promise((resolve, reject) => { + if (this.socket) return resolve(); + const socket = createConnection({ path: this.socketPath }); + this.connectingSocket = socket; + const timer = setTimeout(() => { + socket.removeAllListeners(); + socket.destroy(); + this.failAll(new Error(`journal client: connect timed out after ${this.connectTimeoutMs}ms`)); + reject(new Error(`journal client: connect timed out after ${this.connectTimeoutMs}ms`)); + }, this.connectTimeoutMs); + const onError = (err: Error): void => { + clearTimeout(timer); + socket.removeAllListeners(); + this.failAll(err); + reject(new Error(`journal client: connect failed: ${err.message}`)); + }; + socket.once('error', onError); + socket.once('connect', () => { + clearTimeout(timer); + socket.removeListener('error', onError); + socket.on('error', (err) => { this.disconnectCause ??= err; this.failAll(err); }); + socket.on('data', (chunk) => this.onData(chunk)); + socket.on('close', () => { + const closed = new Error('journal client: connection closed'); + this.connectionClosed(closed, this.socket === socket); + this.disconnectCause ??= closed; + this.failAll(closed); + // Only a drop the caller did not ask for; close() clears this.socket first. + // Sessions with nothing pending (a registered watch) must hear it too. + if (this.socket === socket) this.emit('disconnected', closed); + }); + this.disconnectCause = undefined; + this.connectingSocket = undefined; + this.socket = socket; + resolve(); + }); + }); + } + + /** + * Close the connection and reject any pending requests. A `cause` (for + * example the worker failure that forced the close) is carried into every + * later "not connected" rejection. + */ + close(cause?: unknown): void { + const closed = cause === undefined + ? new Error('journal client: closed by caller') + : new Error(`journal client: closed after ${cause instanceof Error ? cause.message : String(cause)}`, { cause }); + this.connectionClosed(cause, false); + this.connectingSocket?.destroy(closed); + this.disconnectCause ??= closed; + this.failAll(closed); + this.socket?.destroy(); + this.socket = null; + this.buffer = ''; + } + + private onData(chunk: Buffer): void { + this.buffer += chunk.toString('utf8'); + let nl: number; + while ((nl = this.buffer.indexOf('\n')) !== -1) { + const line = this.buffer.slice(0, nl); + this.buffer = this.buffer.slice(nl + 1); + if (line.length > 0) this.onLine(line); + } + } + + private onLine(line: string): void { + let msg: Response | ServerEvent; + try { + msg = JSON.parse(line) as Response | ServerEvent; + } catch { + // A malformed frame is a protocol violation; fail closed. + const error = new JournalFrameError(); + this.failAll(error); + // Sessions with nothing pending (a registered watch) must hear it too. + this.emit('protocol_error', error); + return; + } + + if (typeof (msg as Response).id === 'string' && 'ok' in (msg as Response)) { + const res = msg as Response; + const pending = this.pending.get(res.id); + if (!pending) return; // reply for an already-timed-out request + this.pending.delete(res.id); + if (pending.timer !== undefined) clearTimeout(pending.timer); + if (res.ok) pending.resolve(res.result); + else pending.reject(new JournalProtocolError(res.error.code, res.error.message)); + } else { + const ev = msg as ServerEvent; + this.emit(ev.event, ev.data); + this.emit('event', ev); + } + } + + private failAll(err: Error): void { + for (const [, p] of this.pending) { + if (p.timer !== undefined) clearTimeout(p.timer); + p.reject(err); + } + this.pending.clear(); + } + + /** @internal One unretried request on this session; flow reads go through `request`. */ + requestOnce( + verb: V, + params: VerbContract[V]['params'], + timeoutMs: number | null = this.requestTimeoutMs, + signal?: AbortSignal, + ): Promise { + return new Promise((resolve, reject) => { + if (signal?.aborted) { + reject(signal.reason); + return; + } + if (!this.socket || this.socket.destroyed) { + const cause = this.disconnectCause; + reject(new Error( + `journal client: not connected (${verb})${cause === undefined ? '' : `: ${cause.message}`}`, + cause === undefined ? undefined : { cause }, + )); + return; + } + const id = randomUUID(); + const started = performance.now(); + const frame: Request = { id, verb: verb as string, params }; + // Cancellation forgets the request; a late reply is ignored like a timed-out one. + const onAbort = () => { + const p = this.pending.get(id); + if (p === undefined) return; + if (p.timer !== undefined) clearTimeout(p.timer); + this.pending.delete(id); + p.reject(signal!.reason); + }; + const settle = (fn: (arg: A) => void) => (arg: A) => { + signal?.removeEventListener('abort', onAbort); + fn(arg); + }; + const timer = timeoutMs === null ? undefined : setTimeout(() => { + this.pending.delete(id); + settle(reject)(new JournalRequestTimeoutError(verb, timeoutMs, 1, performance.now() - started)); + }, timeoutMs); + this.pending.set(id, { resolve: settle(resolve as (v: unknown) => void), reject: settle(reject), timer }); + signal?.addEventListener('abort', onAbort, { once: true }); + this.socket.write(JSON.stringify(frame) + '\n', (err) => { + if (err) { + const p = this.pending.get(id); + if (p) { + if (p.timer !== undefined) clearTimeout(p.timer); + this.pending.delete(id); + p.reject(new Error(`journal client: ${verb} write failed: ${err.message}`)); + } + } + }); + }); + } +} diff --git a/packages/sdk/tests/authored-root.test.ts b/packages/sdk/tests/authored-root.test.ts index 258b92a71..e1d94b53f 100644 --- a/packages/sdk/tests/authored-root.test.ts +++ b/packages/sdk/tests/authored-root.test.ts @@ -497,6 +497,17 @@ describe('durable authored root', () => { } }); + it('keeps a completed root outcome when reading back its result is interrupted', async () => { + const loaded = await fixture(); + const journal = new RootJournal(); + (journal as unknown as { startStatus: RunOutcome }).startStatus = outcome('root-run', 'completed', 'success'); + journal.journalRead = async () => { throw new JournalRequestTimeoutError('journal.read', 10, 3, 300_000, 300_000); }; + await expect(executeDurableAuthoredFlow(loaded, journal as unknown as JournalClient, undefined, + { dataDir: '/unused', admissionKey: 'completed-unread' })).rejects.toMatchObject({ + code: 'result_unreadable', completionReason: 'success', rootRunId: 'root-run', + }); + }); + it('terminalizes a returned body failure without replaying semantic side effects', async () => { const loaded = await fixture(true); const journal = new RootJournal(); diff --git a/packages/sdk/tests/run-daemon-unresponsive.test.ts b/packages/sdk/tests/run-daemon-unresponsive.test.ts index 3ba6e0d66..fd3142b51 100644 --- a/packages/sdk/tests/run-daemon-unresponsive.test.ts +++ b/packages/sdk/tests/run-daemon-unresponsive.test.ts @@ -34,3 +34,15 @@ it('reports an interrupted read session as resumable rather than a failed run', diagnostics: [{ kind: 'daemon_unresponsive', message: expect.stringContaining('flows resume saved-run') }], } }); }); + +it('reports a completed run whose result could not be read as completed, not resumable', async () => { + const { completedResultUnreadableReport } = await import('../src/cli/journal-timeout.js'); + const { AuthoredFlowExecutionError } = await import('../src/authored-flow-error.js'); + const error = new AuthoredFlowExecutionError('result_unreadable', 'run r completed (success), but its stored result could not be read', 'success', 'r'); + error.rootRunId = 'r'; + const execution = completedResultUnreadableReport('resume', { command: 'resume', ok: true, diagnostics: [] } as never, '/socket', error); + expect(execution.exitCode).toBe(0); + expect(execution.report).toMatchObject({ ok: true, status: 'completed', completionReason: 'success', runId: 'r', + diagnostics: [{ severity: 'warning', kind: 'result_unreadable' }] }); + expect(JSON.stringify(execution.report)).not.toContain('flows resume'); +}); From 711ae22381f404497b654d0f611c843cd2911b5f Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 04:06:37 -0700 Subject: [PATCH 24/34] fix(sdk): an unread authored result is an unconfirmed verdict, not success MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The kernel completes the authored-root step with success whatever the body declared (step_failed, needs_human, ...); the authored verdict is only in the stored output. result_unreadable no longer carries the kernel's reason, and the report says the run is finished with an unconfirmed verdict — ok false, exit 1, no completionReason — and how to read it with flows status. Evidence re-captured at this tree. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: b8262468-ea26-4a6e-bee9-d193832e77dc --- evidence/run-read-timeout/final-focused.log | 114 ++++---- .../mutation-completion-aborts-snapshot.log | 16 +- .../run-read-timeout/mutation-heartbeat.log | 14 +- .../mutation-reader-reconnect.log | 18 +- .../mutation-reader-setup-retry.log | 14 +- evidence/run-read-timeout/mutation-reader.log | 14 +- .../mutation-root-parking.log | 22 +- .../mutation-snapshot-cancel.log | 14 +- .../mutation-watch-cadence.log | 16 +- evidence/run-read-timeout/resume-live.log | 8 +- evidence/run-read-timeout/verification.md | 250 +++++++++--------- packages/sdk/src/authored-root.ts | 6 +- packages/sdk/src/cli/journal-timeout.ts | 16 +- packages/sdk/tests/authored-root.test.ts | 2 +- .../sdk/tests/run-daemon-unresponsive.test.ts | 11 +- 15 files changed, 271 insertions(+), 264 deletions(-) diff --git a/evidence/run-read-timeout/final-focused.log b/evidence/run-read-timeout/final-focused.log index e49b3bddb..95dd769b2 100644 --- a/evidence/run-read-timeout/final-focused.log +++ b/evidence/run-read-timeout/final-focused.log @@ -2,70 +2,70 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5256ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2040ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2040ms -(node:4176986) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5264ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2038ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2049ms +(node:50017) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (32 tests) 452ms - ✓ tests/journal-client.test.ts (17 tests) 80ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 7535ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 416ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1066ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2855ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 454ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 1043ms - ✓ tests/authored-flow-slack.test.ts (7 tests) 1290ms - ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 436ms - ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 467ms - ✓ tests/journal-client-read-timeout.test.ts (26 tests) 1719ms - ✓ tests/running-step-watch.test.ts (12 tests) 12280ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2114ms - ✓ cancels promptly while a lease snapshot read is in flight 2004ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2054ms - ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2003ms - ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2002ms - ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2002ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2469ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 660ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 515ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 643ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 650ms - ✓ tests/authored-helpers.test.ts (6 tests) 1938ms - ✓ runs every available provider through the real kernel and resumes completed effects without a second write 898ms - ✓ replays after SIGKILL before confirm with the same token and one successful completion 399ms + ✓ tests/authored-root.test.ts (32 tests) 443ms + ✓ tests/journal-client.test.ts (17 tests) 69ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 6851ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 468ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 979ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2721ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 355ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 830ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 1150ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 410ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 392ms + ✓ tests/journal-client-read-timeout.test.ts (26 tests) 1788ms + ✓ tests/running-step-watch.test.ts (12 tests) 12294ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2119ms + ✓ cancels promptly while a lease snapshot read is in flight 2005ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2055ms + ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms + ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2005ms + ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2003ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2550ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 656ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 603ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 675ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 615ms + ✓ tests/authored-helpers.test.ts (6 tests) 1946ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 884ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 419ms ✓ replays after SIGKILL before complete with the same token and one successful completion 416ms - ✓ tests/classify-outcome.test.ts (11 tests) 7424ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2012ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3004ms + ✓ tests/classify-outcome.test.ts (11 tests) 7429ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2013ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3007ms ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms - ✓ tests/human-live.test.ts (3 tests) 4505ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2666ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1067ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 771ms - ✓ tests/worker-lease.test.ts (7 tests) 20ms - ✓ tests/communication-worker.test.ts (18 tests) 1713ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 26ms - ✓ tests/effect-channel.test.ts (5 tests) 285ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 739ms - ✓ tests/memoization.test.ts (58 tests) 63ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2408ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2055ms - ✓ drains read and watch promises before an authored flow completes 353ms - ✓ tests/worker-lease-sweep.test.ts (4 tests) 9ms - ✓ tests/journal-client-completion.test.ts (6 tests) 105ms - ✓ tests/reuse-summary-interruption.test.ts (5 tests) 4ms + ✓ tests/human-live.test.ts (3 tests) 4728ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2841ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1080ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 806ms + ✓ tests/worker-lease.test.ts (7 tests) 21ms + ✓ tests/communication-worker.test.ts (18 tests) 1714ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 30ms + ✓ tests/effect-channel.test.ts (5 tests) 316ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 710ms + ✓ tests/memoization.test.ts (58 tests) 65ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2456ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2064ms + ✓ drains read and watch promises before an authored flow completes 391ms + ✓ tests/worker-lease-sweep.test.ts (4 tests) 6ms + ✓ tests/journal-client-completion.test.ts (6 tests) 101ms ✓ tests/run-daemon-unresponsive.test.ts (4 tests) 5ms - ✓ tests/resume-worker-lease.test.ts (3 tests) 9ms - ✓ tests/direct-run-worker-lease.test.ts (3 tests) 10ms - ✓ tests/helper-effect-lease-loss.test.ts (1 test) 229ms + ✓ tests/reuse-summary-interruption.test.ts (5 tests) 4ms + ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms + ✓ tests/direct-run-worker-lease.test.ts (3 tests) 9ms + ✓ tests/helper-effect-lease-loss.test.ts (1 test) 234ms ✓ tests/journal-client-subscriptions.test.ts (1 test) 9ms - ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 35ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 17ms + ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 47ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 14ms Test Files 28 passed (28) Tests 343 passed (343) - Start at 03:57:17 - Duration 61.09s (transform 1.11s, setup 156ms, collect 6.89s, tests 50.63s, environment 4ms, prepare 965ms) + Start at 04:05:34 + Duration 60.68s (transform 1.14s, setup 164ms, collect 6.84s, tests 50.26s, environment 4ms, prepare 1.02s) exit=0 diff --git a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log index 65ce7c201..91c803704 100644 --- a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log +++ b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 3011ms - × a completion push ends the wait without waiting out an in-flight snapshot 3010ms + ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 3012ms + × a completion push ends the wait without waiting out an in-flight snapshot 3011ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -34,8 +34,8 @@ Received: "still waiting" Test Files 1 failed (1) Tests 1 failed | 11 skipped (12) - Start at 03:57:00 - Duration 3.49s (transform 192ms, setup 16ms, collect 323ms, tests 3.01s, environment 0ms, prepare 40ms) + Start at 04:05:19 + Duration 3.50s (transform 194ms, setup 16ms, collect 328ms, tests 3.01s, environment 0ms, prepare 41ms) exit=1 @@ -44,12 +44,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2062ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2062ms + ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2061ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2060ms Test Files 1 passed (1) Tests 1 passed | 11 skipped (12) - Start at 03:57:04 - Duration 2.53s (transform 194ms, setup 16ms, collect 312ms, tests 2.06s, environment 0ms, prepare 40ms) + Start at 04:05:22 + Duration 2.53s (transform 185ms, setup 17ms, collect 309ms, tests 2.06s, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-heartbeat.log b/evidence/run-read-timeout/mutation-heartbeat.log index a14ffa07a..dc48d593d 100644 --- a/evidence/run-read-timeout/mutation-heartbeat.log +++ b/evidence/run-read-timeout/mutation-heartbeat.log @@ -13,8 +13,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 24ms - × a heartbeat timeout is lease loss rather than a worker body failure 22ms + ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 19ms + × a heartbeat timeout is lease loss rather than a worker body failure 19ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -33,8 +33,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 03:57:07 - Duration 673ms (transform 211ms, setup 23ms, collect 413ms, tests 24ms, environment 0ms, prepare 63ms) + Start at 04:05:25 + Duration 455ms (transform 168ms, setup 18ms, collect 284ms, tests 19ms, environment 0ms, prepare 41ms) exit=1 @@ -43,11 +43,11 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/heartbeat-timeout.test.ts (1 test) 19ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 03:57:08 - Duration 644ms (transform 211ms, setup 25ms, collect 388ms, tests 19ms, environment 0ms, prepare 63ms) + Start at 04:05:26 + Duration 459ms (transform 169ms, setup 16ms, collect 290ms, tests 15ms, environment 0ms, prepare 44ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-reconnect.log b/evidence/run-read-timeout/mutation-reader-reconnect.log index 41513fbd7..5c9abc20e 100644 --- a/evidence/run-read-timeout/mutation-reader-reconnect.log +++ b/evidence/run-read-timeout/mutation-reader-reconnect.log @@ -11,14 +11,14 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 1009ms - × reconnects the reader after it disconnects within the read budget 1008ms - → journal client: run.get read session was interrupted after 80 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 1010ms + × reconnects the reader after it disconnects within the read budget 1009ms + → journal client: run.get read session was interrupted after 78 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 80 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 78 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:71:9 69| let last: unknown; 70| const exhausted = () => last instanceof JournalReadInterruptedError @@ -41,8 +41,8 @@ Caused by: Error: journal client: connection closed Test Files 1 failed (1) Tests 1 failed | 25 skipped (26) - Start at 03:56:44 - Duration 1.47s (transform 172ms, setup 17ms, collect 304ms, tests 1.01s, environment 0ms, prepare 45ms) + Start at 04:05:02 + Duration 1.47s (transform 167ms, setup 16ms, collect 293ms, tests 1.01s, environment 0ms, prepare 45ms) exit=1 @@ -51,11 +51,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 81ms + ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 87ms Test Files 1 passed (1) Tests 1 passed | 25 skipped (26) - Start at 03:56:46 - Duration 554ms (transform 172ms, setup 19ms, collect 301ms, tests 81ms, environment 0ms, prepare 41ms) + Start at 04:05:04 + Duration 526ms (transform 165ms, setup 17ms, collect 283ms, tests 87ms, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-setup-retry.log b/evidence/run-read-timeout/mutation-reader-setup-retry.log index 436f936e6..4dd0af898 100644 --- a/evidence/run-read-timeout/mutation-reader-setup-retry.log +++ b/evidence/run-read-timeout/mutation-reader-setup-retry.log @@ -16,8 +16,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 79ms - × retries a reader setup that timed out instead of reading on the primary for good 78ms + ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 71ms + × retries a reader setup that timed out instead of reading on the primary for good 70ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -43,8 +43,8 @@ AssertionError: expected 1 to be 2 // Object.is equality Test Files 1 failed (1) Tests 1 failed | 25 skipped (26) - Start at 03:56:47 - Duration 535ms (transform 167ms, setup 17ms, collect 291ms, tests 79ms, environment 0ms, prepare 42ms) + Start at 04:05:05 + Duration 501ms (transform 163ms, setup 17ms, collect 279ms, tests 71ms, environment 0ms, prepare 40ms) exit=1 @@ -53,11 +53,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 67ms + ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 69ms Test Files 1 passed (1) Tests 1 passed | 25 skipped (26) - Start at 03:56:48 - Duration 509ms (transform 167ms, setup 15ms, collect 283ms, tests 67ms, environment 0ms, prepare 41ms) + Start at 04:05:06 + Duration 523ms (transform 169ms, setup 16ms, collect 292ms, tests 69ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader.log b/evidence/run-read-timeout/mutation-reader.log index b269f2f12..086862663 100644 --- a/evidence/run-read-timeout/mutation-reader.log +++ b/evidence/run-read-timeout/mutation-reader.log @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 109ms - × serves a bounded read while an unbounded command is in flight 109ms + ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 113ms + × serves a bounded read while an unbounded command is in flight 112ms → expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -37,8 +37,8 @@ AssertionError: expected true to be false // Object.is equality Test Files 1 failed (1) Tests 1 failed | 25 skipped (26) - Start at 03:56:42 - Duration 561ms (transform 170ms, setup 17ms, collect 292ms, tests 109ms, environment 0ms, prepare 41ms) + Start at 04:05:00 + Duration 553ms (transform 167ms, setup 16ms, collect 286ms, tests 113ms, environment 0ms, prepare 42ms) exit=1 @@ -47,11 +47,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 104ms + ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 106ms Test Files 1 passed (1) Tests 1 passed | 25 skipped (26) - Start at 03:56:43 - Duration 568ms (transform 175ms, setup 17ms, collect 297ms, tests 104ms, environment 0ms, prepare 42ms) + Start at 04:05:01 + Duration 553ms (transform 165ms, setup 16ms, collect 284ms, tests 106ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-root-parking.log b/evidence/run-read-timeout/mutation-root-parking.log index fd3558536..a9613e789 100644 --- a/evidence/run-read-timeout/mutation-root-parking.log +++ b/evidence/run-read-timeout/mutation-root-parking.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/authored-root.ts -Original SHA256: 3800932c9ef7768b4bf0a01f08885856e5ccbd3d173cd63574fd599a4fc33230 +Original SHA256: 2186915328814fe8fb40c1da6692cd7ea674942d8d68cca939f1f2b83fdc422c Replaced: if (isReadInterruptionError(error) || (error instanceof AuthoredFlowExecutionError && error.code === 'daemon_unresponsive')) { @@ -17,14 +17,14 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (32 tests | 3 failed | 29 skipped) 48ms - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 33ms + ❯ tests/authored-root.test.ts (32 tests | 3 failed | 29 skipped) 29ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 21ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 7ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms → expected AuthoredFlowExecutionError: daemon_unresp… { …(7) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 6ms + × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 3ms → expected JournalReadInterruptedError: journal clie… { …(4) } to match object { code: 'daemon_unresponsive', …(2) } (5 matching properties omitted from actual) @@ -104,21 +104,21 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } Test Files 1 failed (1) Tests 3 failed | 29 skipped (32) - Start at 03:57:09 - Duration 1.71s (transform 791ms, setup 23ms, collect 1.42s, tests 48ms, environment 0ms, prepare 62ms) + Start at 04:05:27 + Duration 1.28s (transform 669ms, setup 16ms, collect 1.10s, tests 29ms, environment 0ms, prepare 40ms) exit=1 -RESTORED SHA256: 3800932c9ef7768b4bf0a01f08885856e5ccbd3d173cd63574fd599a4fc33230 +RESTORED SHA256: 2186915328814fe8fb40c1da6692cd7ea674942d8d68cca939f1f2b83fdc422c $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the root resumable' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (32 tests | 29 skipped) 35ms + ✓ tests/authored-root.test.ts (32 tests | 29 skipped) 24ms Test Files 1 passed (1) Tests 3 passed | 29 skipped (32) - Start at 03:57:12 - Duration 1.76s (transform 820ms, setup 23ms, collect 1.50s, tests 35ms, environment 0ms, prepare 49ms) + Start at 04:05:29 + Duration 1.29s (transform 666ms, setup 16ms, collect 1.11s, tests 24ms, environment 0ms, prepare 30ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-snapshot-cancel.log b/evidence/run-read-timeout/mutation-snapshot-cancel.log index a36588881..67ef3f567 100644 --- a/evidence/run-read-timeout/mutation-snapshot-cancel.log +++ b/evidence/run-read-timeout/mutation-snapshot-cancel.log @@ -11,8 +11,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2213ms - × cancels promptly while a lease snapshot read is in flight 2212ms + ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2216ms + × cancels promptly while a lease snapshot read is in flight 2216ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -31,8 +31,8 @@ AssertionError: expected 'still waiting' to be an instance of Error Test Files 1 failed (1) Tests 1 failed | 11 skipped (12) - Start at 03:56:54 - Duration 2.69s (transform 197ms, setup 18ms, collect 323ms, tests 2.21s, environment 0ms, prepare 40ms) + Start at 04:05:13 + Duration 2.69s (transform 192ms, setup 13ms, collect 331ms, tests 2.22s, environment 0ms, prepare 41ms) exit=1 @@ -41,12 +41,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2011ms + ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2012ms ✓ cancels promptly while a lease snapshot read is in flight 2011ms Test Files 1 passed (1) Tests 1 passed | 11 skipped (12) - Start at 03:56:57 - Duration 2.47s (transform 190ms, setup 17ms, collect 310ms, tests 2.01s, environment 0ms, prepare 40ms) + Start at 04:05:16 + Duration 2.51s (transform 197ms, setup 17ms, collect 332ms, tests 2.01s, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-watch-cadence.log b/evidence/run-read-timeout/mutation-watch-cadence.log index adfb19369..c62fa5253 100644 --- a/evidence/run-read-timeout/mutation-watch-cadence.log +++ b/evidence/run-read-timeout/mutation-watch-cadence.log @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2110ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2109ms + ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2109ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2108ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -30,8 +30,8 @@ AssertionError: expected 41 to be less than or equal to 1 Test Files 1 failed (1) Tests 1 failed | 11 skipped (12) - Start at 03:56:48 - Duration 2.59s (transform 188ms, setup 18ms, collect 320ms, tests 2.11s, environment 0ms, prepare 40ms) + Start at 04:05:07 + Duration 2.57s (transform 198ms, setup 15ms, collect 327ms, tests 2.11s, environment 0ms, prepare 31ms) exit=1 @@ -40,12 +40,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2119ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms + ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2118ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms Test Files 1 passed (1) Tests 1 passed | 11 skipped (12) - Start at 03:56:51 - Duration 2.60s (transform 190ms, setup 16ms, collect 324ms, tests 2.12s, environment 0ms, prepare 41ms) + Start at 04:05:10 + Duration 2.59s (transform 194ms, setup 17ms, collect 325ms, tests 2.12s, environment 0ms, prepare 34ms) exit=0 diff --git a/evidence/run-read-timeout/resume-live.log b/evidence/run-read-timeout/resume-live.log index 159311474..40885b754 100644 --- a/evidence/run-read-timeout/resume-live.log +++ b/evidence/run-read-timeout/resume-live.log @@ -2,12 +2,12 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 875ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 874ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 925ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 924ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 03:58:18 - Duration 2.04s (transform 609ms, setup 27ms, collect 967ms, tests 875ms, environment 0ms, prepare 63ms) + Start at 04:06:35 + Duration 2.20s (transform 612ms, setup 17ms, collect 1.03s, tests 925ms, environment 0ms, prepare 71ms) exit=0 diff --git a/evidence/run-read-timeout/verification.md b/evidence/run-read-timeout/verification.md index 557661d46..2a4971f87 100644 --- a/evidence/run-read-timeout/verification.md +++ b/evidence/run-read-timeout/verification.md @@ -15,71 +15,71 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5256ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2040ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2040ms -(node:4176986) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5264ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2038ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2049ms +(node:50017) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (32 tests) 452ms - ✓ tests/journal-client.test.ts (17 tests) 80ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 7535ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 416ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1066ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2855ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 454ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 1043ms - ✓ tests/authored-flow-slack.test.ts (7 tests) 1290ms - ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 436ms - ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 467ms - ✓ tests/journal-client-read-timeout.test.ts (26 tests) 1719ms - ✓ tests/running-step-watch.test.ts (12 tests) 12280ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2114ms - ✓ cancels promptly while a lease snapshot read is in flight 2004ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2054ms - ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2003ms - ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2002ms - ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2002ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2469ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 660ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 515ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 643ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 650ms - ✓ tests/authored-helpers.test.ts (6 tests) 1938ms - ✓ runs every available provider through the real kernel and resumes completed effects without a second write 898ms - ✓ replays after SIGKILL before confirm with the same token and one successful completion 399ms + ✓ tests/authored-root.test.ts (32 tests) 443ms + ✓ tests/journal-client.test.ts (17 tests) 69ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 6851ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 468ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 979ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2721ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 355ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 830ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 1150ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 410ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 392ms + ✓ tests/journal-client-read-timeout.test.ts (26 tests) 1788ms + ✓ tests/running-step-watch.test.ts (12 tests) 12294ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2119ms + ✓ cancels promptly while a lease snapshot read is in flight 2005ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2055ms + ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms + ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2005ms + ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2003ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2550ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 656ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 603ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 675ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 615ms + ✓ tests/authored-helpers.test.ts (6 tests) 1946ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 884ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 419ms ✓ replays after SIGKILL before complete with the same token and one successful completion 416ms - ✓ tests/classify-outcome.test.ts (11 tests) 7424ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2012ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3004ms + ✓ tests/classify-outcome.test.ts (11 tests) 7429ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2013ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3007ms ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms - ✓ tests/human-live.test.ts (3 tests) 4505ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2666ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1067ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 771ms - ✓ tests/worker-lease.test.ts (7 tests) 20ms - ✓ tests/communication-worker.test.ts (18 tests) 1713ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 26ms - ✓ tests/effect-channel.test.ts (5 tests) 285ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 739ms - ✓ tests/memoization.test.ts (58 tests) 63ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2408ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2055ms - ✓ drains read and watch promises before an authored flow completes 353ms - ✓ tests/worker-lease-sweep.test.ts (4 tests) 9ms - ✓ tests/journal-client-completion.test.ts (6 tests) 105ms - ✓ tests/reuse-summary-interruption.test.ts (5 tests) 4ms + ✓ tests/human-live.test.ts (3 tests) 4728ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2841ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1080ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 806ms + ✓ tests/worker-lease.test.ts (7 tests) 21ms + ✓ tests/communication-worker.test.ts (18 tests) 1714ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 30ms + ✓ tests/effect-channel.test.ts (5 tests) 316ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 710ms + ✓ tests/memoization.test.ts (58 tests) 65ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2456ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2064ms + ✓ drains read and watch promises before an authored flow completes 391ms + ✓ tests/worker-lease-sweep.test.ts (4 tests) 6ms + ✓ tests/journal-client-completion.test.ts (6 tests) 101ms ✓ tests/run-daemon-unresponsive.test.ts (4 tests) 5ms - ✓ tests/resume-worker-lease.test.ts (3 tests) 9ms - ✓ tests/direct-run-worker-lease.test.ts (3 tests) 10ms - ✓ tests/helper-effect-lease-loss.test.ts (1 test) 229ms + ✓ tests/reuse-summary-interruption.test.ts (5 tests) 4ms + ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms + ✓ tests/direct-run-worker-lease.test.ts (3 tests) 9ms + ✓ tests/helper-effect-lease-loss.test.ts (1 test) 234ms ✓ tests/journal-client-subscriptions.test.ts (1 test) 9ms - ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 35ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 17ms + ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 47ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 14ms Test Files 28 passed (28) Tests 343 passed (343) - Start at 03:57:17 - Duration 61.09s (transform 1.11s, setup 156ms, collect 6.89s, tests 50.63s, environment 4ms, prepare 965ms) + Start at 04:05:34 + Duration 60.68s (transform 1.14s, setup 164ms, collect 6.84s, tests 50.26s, environment 4ms, prepare 1.02s) exit=0 ``` @@ -91,13 +91,13 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 875ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 874ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 925ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 924ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 03:58:18 - Duration 2.04s (transform 609ms, setup 27ms, collect 967ms, tests 875ms, environment 0ms, prepare 63ms) + Start at 04:06:35 + Duration 2.20s (transform 612ms, setup 17ms, collect 1.03s, tests 925ms, environment 0ms, prepare 71ms) exit=0 ``` @@ -119,8 +119,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 109ms - × serves a bounded read while an unbounded command is in flight 109ms + ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 113ms + × serves a bounded read while an unbounded command is in flight 112ms → expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -146,8 +146,8 @@ AssertionError: expected true to be false // Object.is equality Test Files 1 failed (1) Tests 1 failed | 25 skipped (26) - Start at 03:56:42 - Duration 561ms (transform 170ms, setup 17ms, collect 292ms, tests 109ms, environment 0ms, prepare 41ms) + Start at 04:05:00 + Duration 553ms (transform 167ms, setup 16ms, collect 286ms, tests 113ms, environment 0ms, prepare 42ms) exit=1 @@ -156,12 +156,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 104ms + ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 106ms Test Files 1 passed (1) Tests 1 passed | 25 skipped (26) - Start at 03:56:43 - Duration 568ms (transform 175ms, setup 17ms, collect 297ms, tests 104ms, environment 0ms, prepare 42ms) + Start at 04:05:01 + Duration 553ms (transform 165ms, setup 16ms, collect 284ms, tests 106ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -182,14 +182,14 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 1009ms - × reconnects the reader after it disconnects within the read budget 1008ms - → journal client: run.get read session was interrupted after 80 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 1010ms + × reconnects the reader after it disconnects within the read budget 1009ms + → journal client: run.get read session was interrupted after 78 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 80 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 78 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:71:9 69| let last: unknown; 70| const exhausted = () => last instanceof JournalReadInterruptedError @@ -212,8 +212,8 @@ Caused by: Error: journal client: connection closed Test Files 1 failed (1) Tests 1 failed | 25 skipped (26) - Start at 03:56:44 - Duration 1.47s (transform 172ms, setup 17ms, collect 304ms, tests 1.01s, environment 0ms, prepare 45ms) + Start at 04:05:02 + Duration 1.47s (transform 167ms, setup 16ms, collect 293ms, tests 1.01s, environment 0ms, prepare 45ms) exit=1 @@ -222,12 +222,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 81ms + ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 87ms Test Files 1 passed (1) Tests 1 passed | 25 skipped (26) - Start at 03:56:46 - Duration 554ms (transform 172ms, setup 19ms, collect 301ms, tests 81ms, environment 0ms, prepare 41ms) + Start at 04:05:04 + Duration 526ms (transform 165ms, setup 17ms, collect 283ms, tests 87ms, environment 0ms, prepare 40ms) exit=0 ``` @@ -253,8 +253,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 79ms - × retries a reader setup that timed out instead of reading on the primary for good 78ms + ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 71ms + × retries a reader setup that timed out instead of reading on the primary for good 70ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -280,8 +280,8 @@ AssertionError: expected 1 to be 2 // Object.is equality Test Files 1 failed (1) Tests 1 failed | 25 skipped (26) - Start at 03:56:47 - Duration 535ms (transform 167ms, setup 17ms, collect 291ms, tests 79ms, environment 0ms, prepare 42ms) + Start at 04:05:05 + Duration 501ms (transform 163ms, setup 17ms, collect 279ms, tests 71ms, environment 0ms, prepare 40ms) exit=1 @@ -290,12 +290,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 67ms + ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 69ms Test Files 1 passed (1) Tests 1 passed | 25 skipped (26) - Start at 03:56:48 - Duration 509ms (transform 167ms, setup 15ms, collect 283ms, tests 67ms, environment 0ms, prepare 41ms) + Start at 04:05:06 + Duration 523ms (transform 169ms, setup 16ms, collect 292ms, tests 69ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -315,8 +315,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2110ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2109ms + ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2109ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2108ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -335,8 +335,8 @@ AssertionError: expected 41 to be less than or equal to 1 Test Files 1 failed (1) Tests 1 failed | 11 skipped (12) - Start at 03:56:48 - Duration 2.59s (transform 188ms, setup 18ms, collect 320ms, tests 2.11s, environment 0ms, prepare 40ms) + Start at 04:05:07 + Duration 2.57s (transform 198ms, setup 15ms, collect 327ms, tests 2.11s, environment 0ms, prepare 31ms) exit=1 @@ -345,13 +345,13 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2119ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms + ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2118ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms Test Files 1 passed (1) Tests 1 passed | 11 skipped (12) - Start at 03:56:51 - Duration 2.60s (transform 190ms, setup 16ms, collect 324ms, tests 2.12s, environment 0ms, prepare 41ms) + Start at 04:05:10 + Duration 2.59s (transform 194ms, setup 17ms, collect 325ms, tests 2.12s, environment 0ms, prepare 34ms) exit=0 ``` @@ -372,8 +372,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2213ms - × cancels promptly while a lease snapshot read is in flight 2212ms + ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2216ms + × cancels promptly while a lease snapshot read is in flight 2216ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -392,8 +392,8 @@ AssertionError: expected 'still waiting' to be an instance of Error Test Files 1 failed (1) Tests 1 failed | 11 skipped (12) - Start at 03:56:54 - Duration 2.69s (transform 197ms, setup 18ms, collect 323ms, tests 2.21s, environment 0ms, prepare 40ms) + Start at 04:05:13 + Duration 2.69s (transform 192ms, setup 13ms, collect 331ms, tests 2.22s, environment 0ms, prepare 41ms) exit=1 @@ -402,13 +402,13 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2011ms + ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2012ms ✓ cancels promptly while a lease snapshot read is in flight 2011ms Test Files 1 passed (1) Tests 1 passed | 11 skipped (12) - Start at 03:56:57 - Duration 2.47s (transform 190ms, setup 17ms, collect 310ms, tests 2.01s, environment 0ms, prepare 40ms) + Start at 04:05:16 + Duration 2.51s (transform 197ms, setup 17ms, collect 332ms, tests 2.01s, environment 0ms, prepare 41ms) exit=0 ``` @@ -428,8 +428,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 3011ms - × a completion push ends the wait without waiting out an in-flight snapshot 3010ms + ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 3012ms + × a completion push ends the wait without waiting out an in-flight snapshot 3011ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -452,8 +452,8 @@ Received: "still waiting" Test Files 1 failed (1) Tests 1 failed | 11 skipped (12) - Start at 03:57:00 - Duration 3.49s (transform 192ms, setup 16ms, collect 323ms, tests 3.01s, environment 0ms, prepare 40ms) + Start at 04:05:19 + Duration 3.50s (transform 194ms, setup 16ms, collect 328ms, tests 3.01s, environment 0ms, prepare 41ms) exit=1 @@ -462,13 +462,13 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2062ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2062ms + ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2061ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2060ms Test Files 1 passed (1) Tests 1 passed | 11 skipped (12) - Start at 03:57:04 - Duration 2.53s (transform 194ms, setup 16ms, collect 312ms, tests 2.06s, environment 0ms, prepare 40ms) + Start at 04:05:22 + Duration 2.53s (transform 185ms, setup 17ms, collect 309ms, tests 2.06s, environment 0ms, prepare 41ms) exit=0 ``` @@ -491,8 +491,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 24ms - × a heartbeat timeout is lease loss rather than a worker body failure 22ms + ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 19ms + × a heartbeat timeout is lease loss rather than a worker body failure 19ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -511,8 +511,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 03:57:07 - Duration 673ms (transform 211ms, setup 23ms, collect 413ms, tests 24ms, environment 0ms, prepare 63ms) + Start at 04:05:25 + Duration 455ms (transform 168ms, setup 18ms, collect 284ms, tests 19ms, environment 0ms, prepare 41ms) exit=1 @@ -521,12 +521,12 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/heartbeat-timeout.test.ts (1 test) 19ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 03:57:08 - Duration 644ms (transform 211ms, setup 25ms, collect 388ms, tests 19ms, environment 0ms, prepare 63ms) + Start at 04:05:26 + Duration 459ms (transform 169ms, setup 16ms, collect 290ms, tests 15ms, environment 0ms, prepare 44ms) exit=0 ``` @@ -535,7 +535,7 @@ exit=0 ```text Source: packages/sdk/src/authored-root.ts -Original SHA256: 3800932c9ef7768b4bf0a01f08885856e5ccbd3d173cd63574fd599a4fc33230 +Original SHA256: 2186915328814fe8fb40c1da6692cd7ea674942d8d68cca939f1f2b83fdc422c Replaced: if (isReadInterruptionError(error) || (error instanceof AuthoredFlowExecutionError && error.code === 'daemon_unresponsive')) { @@ -553,14 +553,14 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (32 tests | 3 failed | 29 skipped) 48ms - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 33ms + ❯ tests/authored-root.test.ts (32 tests | 3 failed | 29 skipped) 29ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 21ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 7ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms → expected AuthoredFlowExecutionError: daemon_unresp… { …(7) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 6ms + × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 3ms → expected JournalReadInterruptedError: journal clie… { …(4) } to match object { code: 'daemon_unresponsive', …(2) } (5 matching properties omitted from actual) @@ -640,22 +640,22 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } Test Files 1 failed (1) Tests 3 failed | 29 skipped (32) - Start at 03:57:09 - Duration 1.71s (transform 791ms, setup 23ms, collect 1.42s, tests 48ms, environment 0ms, prepare 62ms) + Start at 04:05:27 + Duration 1.28s (transform 669ms, setup 16ms, collect 1.10s, tests 29ms, environment 0ms, prepare 40ms) exit=1 -RESTORED SHA256: 3800932c9ef7768b4bf0a01f08885856e5ccbd3d173cd63574fd599a4fc33230 +RESTORED SHA256: 2186915328814fe8fb40c1da6692cd7ea674942d8d68cca939f1f2b83fdc422c $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the root resumable' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (32 tests | 29 skipped) 35ms + ✓ tests/authored-root.test.ts (32 tests | 29 skipped) 24ms Test Files 1 passed (1) Tests 3 passed | 29 skipped (32) - Start at 03:57:12 - Duration 1.76s (transform 820ms, setup 23ms, collect 1.50s, tests 35ms, environment 0ms, prepare 49ms) + Start at 04:05:29 + Duration 1.29s (transform 666ms, setup 16ms, collect 1.11s, tests 24ms, environment 0ms, prepare 30ms) exit=0 ``` diff --git a/packages/sdk/src/authored-root.ts b/packages/sdk/src/authored-root.ts index fa548c248..e30ae630c 100644 --- a/packages/sdk/src/authored-root.ts +++ b/packages/sdk/src/authored-root.ts @@ -576,9 +576,11 @@ function nextRootDispatch(peer: JournalClient, timeoutMs = 30_000): { function completedResult(journal: JournalClient, outcome: RunOutcome, rootRunId: string): Promise { return completedRootResult(journal, rootRunId).catch(error => { if (!isReadInterruptionError(error)) throw error; + // The kernel completes the root step with success whatever the body + // declared; the authored verdict lives only in the unread output. const unread = new AuthoredFlowExecutionError('result_unreadable', - `run ${rootRunId} completed (${outcome.completion_reason ?? 'no completion reason'}), but its stored result could not be read: ${error.message}`, - outcome.completion_reason ?? undefined, rootRunId); + `run ${rootRunId} completed, but its stored result (the flow's verdict) could not be read: ${error.message}`, + undefined, rootRunId); unread.rootRunId = rootRunId; throw unread; }); diff --git a/packages/sdk/src/cli/journal-timeout.ts b/packages/sdk/src/cli/journal-timeout.ts index ef7ef44a8..a6a20c6d0 100644 --- a/packages/sdk/src/cli/journal-timeout.ts +++ b/packages/sdk/src/cli/journal-timeout.ts @@ -10,16 +10,20 @@ export function isReadInterruption(error: unknown): boolean { || (error instanceof AuthoredFlowExecutionError && error.code === 'daemon_unresponsive'); } -/** A completed run whose stored result could not be read back: report the known outcome. */ +/** + * A completed run whose stored result could not be read back. The run is + * finished — nothing to resume — but its authored verdict is unknown, so the + * report claims neither success nor a reason, and says how to read it later. + */ export function completedResultUnreadableReport(command: RunCommand, base: CheckReport | RunReport, socketPath: string, error: AuthoredFlowExecutionError): RunExecution { const runId = error.rootRunId ?? error.runId; - const ok = error.completionReason === 'success'; - return { exitCode: ok ? 0 : 1, report: { - ...base, command, ok, socketPath, status: 'completed', + return { exitCode: 1, report: { + ...base, command, ok: false, socketPath, status: 'completed', ...(runId === undefined ? {} : { runId, rootRunId: runId }), - ...(error.completionReason === undefined ? {} : { completionReason: error.completionReason }), - diagnostics: [...base.diagnostics, { severity: 'warning', kind: 'result_unreadable', message: error.message }], + diagnostics: [...base.diagnostics, { severity: 'warning', kind: 'result_unreadable', + message: `${error.message}. The run is finished; its verdict is unconfirmed` + + (runId === undefined ? '.' : ` — read it with \`flows status ${runId}\` once relayflowd answers.`) }], } as RunExecution['report'] }; } diff --git a/packages/sdk/tests/authored-root.test.ts b/packages/sdk/tests/authored-root.test.ts index e1d94b53f..dc553647f 100644 --- a/packages/sdk/tests/authored-root.test.ts +++ b/packages/sdk/tests/authored-root.test.ts @@ -504,7 +504,7 @@ describe('durable authored root', () => { journal.journalRead = async () => { throw new JournalRequestTimeoutError('journal.read', 10, 3, 300_000, 300_000); }; await expect(executeDurableAuthoredFlow(loaded, journal as unknown as JournalClient, undefined, { dataDir: '/unused', admissionKey: 'completed-unread' })).rejects.toMatchObject({ - code: 'result_unreadable', completionReason: 'success', rootRunId: 'root-run', + code: 'result_unreadable', completionReason: undefined, rootRunId: 'root-run', }); }); diff --git a/packages/sdk/tests/run-daemon-unresponsive.test.ts b/packages/sdk/tests/run-daemon-unresponsive.test.ts index fd3142b51..3afde1c50 100644 --- a/packages/sdk/tests/run-daemon-unresponsive.test.ts +++ b/packages/sdk/tests/run-daemon-unresponsive.test.ts @@ -35,14 +35,15 @@ it('reports an interrupted read session as resumable rather than a failed run', } }); }); -it('reports a completed run whose result could not be read as completed, not resumable', async () => { +it('reports a completed run with an unread result as finished but with an unconfirmed verdict', async () => { const { completedResultUnreadableReport } = await import('../src/cli/journal-timeout.js'); const { AuthoredFlowExecutionError } = await import('../src/authored-flow-error.js'); - const error = new AuthoredFlowExecutionError('result_unreadable', 'run r completed (success), but its stored result could not be read', 'success', 'r'); + const error = new AuthoredFlowExecutionError('result_unreadable', 'run r completed, but its stored result (the flow\'s verdict) could not be read', undefined, 'r'); error.rootRunId = 'r'; const execution = completedResultUnreadableReport('resume', { command: 'resume', ok: true, diagnostics: [] } as never, '/socket', error); - expect(execution.exitCode).toBe(0); - expect(execution.report).toMatchObject({ ok: true, status: 'completed', completionReason: 'success', runId: 'r', - diagnostics: [{ severity: 'warning', kind: 'result_unreadable' }] }); + expect(execution.exitCode).toBe(1); + expect(execution.report).toMatchObject({ ok: false, status: 'completed', runId: 'r', + diagnostics: [{ severity: 'warning', kind: 'result_unreadable', message: expect.stringContaining('flows status r') }] }); + expect(execution.report).not.toHaveProperty('completionReason'); expect(JSON.stringify(execution.report)).not.toContain('flows resume'); }); From c7af06006224cf88d6a9e6bce6d5d2676335e0c7 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 04:17:21 -0700 Subject: [PATCH 25/34] fix(sdk): only definite daemon answers decide authored-root detection and readback A canceled budgeted read rejects with the signal's reason, which is not a read interruption, so both catches misread it: - readAuthoredRootMetadata treats a run as not authored only on a definite answer (a protocol refusal, or a journal with no root step); anything else, cancellation included, propagates instead of resuming declaratively. - A completed root's result readback is result_unreadable unless the daemon definitively answered (a refusal, a malformed frame, or no stored result), so a canceled readback keeps the completed state. Evidence re-captured at this tree. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: b8262468-ea26-4a6e-bee9-d193832e77dc --- evidence/run-read-timeout/final-focused.log | 114 ++++---- .../mutation-completion-aborts-snapshot.log | 16 +- .../run-read-timeout/mutation-heartbeat.log | 12 +- .../mutation-reader-reconnect.log | 18 +- .../mutation-reader-setup-retry.log | 14 +- evidence/run-read-timeout/mutation-reader.log | 12 +- .../mutation-root-parking.log | 26 +- .../mutation-snapshot-cancel.log | 16 +- .../mutation-watch-cadence.log | 16 +- evidence/run-read-timeout/resume-live.log | 8 +- evidence/run-read-timeout/verification.md | 252 +++++++++--------- packages/sdk/src/authored-root.ts | 28 +- packages/sdk/tests/authored-root.test.ts | 17 ++ 13 files changed, 285 insertions(+), 264 deletions(-) diff --git a/evidence/run-read-timeout/final-focused.log b/evidence/run-read-timeout/final-focused.log index 95dd769b2..330c588d8 100644 --- a/evidence/run-read-timeout/final-focused.log +++ b/evidence/run-read-timeout/final-focused.log @@ -2,70 +2,70 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5264ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2038ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2049ms -(node:50017) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5245ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2037ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2037ms +(node:96527) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (32 tests) 443ms - ✓ tests/journal-client.test.ts (17 tests) 69ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 6851ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 468ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 979ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2721ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 355ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 830ms - ✓ tests/authored-flow-slack.test.ts (7 tests) 1150ms - ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 410ms - ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 392ms - ✓ tests/journal-client-read-timeout.test.ts (26 tests) 1788ms - ✓ tests/running-step-watch.test.ts (12 tests) 12294ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2119ms - ✓ cancels promptly while a lease snapshot read is in flight 2005ms + ✓ tests/authored-root.test.ts (34 tests) 454ms + ✓ tests/journal-client.test.ts (17 tests) 80ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 6900ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 459ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1044ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2719ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 351ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 833ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 1141ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 421ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 382ms + ✓ tests/journal-client-read-timeout.test.ts (26 tests) 1796ms + ✓ tests/running-step-watch.test.ts (12 tests) 12284ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2116ms + ✓ cancels promptly while a lease snapshot read is in flight 2006ms ✓ a completion push ends the wait without waiting out an in-flight snapshot 2055ms - ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms - ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2005ms - ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2003ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2550ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 656ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 603ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 675ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 615ms - ✓ tests/authored-helpers.test.ts (6 tests) 1946ms - ✓ runs every available provider through the real kernel and resumes completed effects without a second write 884ms - ✓ replays after SIGKILL before confirm with the same token and one successful completion 419ms - ✓ replays after SIGKILL before complete with the same token and one successful completion 416ms + ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2003ms + ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2002ms + ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2002ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2582ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 646ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 624ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 647ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 664ms + ✓ tests/authored-helpers.test.ts (6 tests) 1908ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 890ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 388ms + ✓ replays after SIGKILL before complete with the same token and one successful completion 410ms ✓ tests/classify-outcome.test.ts (11 tests) 7429ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2013ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3007ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2012ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3008ms ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms - ✓ tests/human-live.test.ts (3 tests) 4728ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2841ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1080ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 806ms - ✓ tests/worker-lease.test.ts (7 tests) 21ms - ✓ tests/communication-worker.test.ts (18 tests) 1714ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 30ms - ✓ tests/effect-channel.test.ts (5 tests) 316ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 710ms - ✓ tests/memoization.test.ts (58 tests) 65ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2456ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2064ms - ✓ drains read and watch promises before an authored flow completes 391ms - ✓ tests/worker-lease-sweep.test.ts (4 tests) 6ms - ✓ tests/journal-client-completion.test.ts (6 tests) 101ms + ✓ tests/human-live.test.ts (3 tests) 4766ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2877ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1190ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 699ms + ✓ tests/worker-lease.test.ts (7 tests) 15ms + ✓ tests/communication-worker.test.ts (18 tests) 1711ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 25ms + ✓ tests/effect-channel.test.ts (5 tests) 282ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 693ms + ✓ tests/memoization.test.ts (58 tests) 59ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2494ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2058ms + ✓ drains read and watch promises before an authored flow completes 435ms + ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms + ✓ tests/journal-client-completion.test.ts (6 tests) 105ms ✓ tests/run-daemon-unresponsive.test.ts (4 tests) 5ms ✓ tests/reuse-summary-interruption.test.ts (5 tests) 4ms - ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms - ✓ tests/direct-run-worker-lease.test.ts (3 tests) 9ms - ✓ tests/helper-effect-lease-loss.test.ts (1 test) 234ms - ✓ tests/journal-client-subscriptions.test.ts (1 test) 9ms - ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 47ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 14ms + ✓ tests/resume-worker-lease.test.ts (3 tests) 8ms + ✓ tests/direct-run-worker-lease.test.ts (3 tests) 7ms + ✓ tests/helper-effect-lease-loss.test.ts (1 test) 243ms + ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms + ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 39ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 18ms Test Files 28 passed (28) - Tests 343 passed (343) - Start at 04:05:34 - Duration 60.68s (transform 1.14s, setup 164ms, collect 6.84s, tests 50.26s, environment 4ms, prepare 1.02s) + Tests 345 passed (345) + Start at 04:16:18 + Duration 60.34s (transform 1.11s, setup 158ms, collect 6.53s, tests 50.31s, environment 4ms, prepare 989ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log index 91c803704..548193106 100644 --- a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log +++ b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 3012ms - × a completion push ends the wait without waiting out an in-flight snapshot 3011ms + ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 3013ms + × a completion push ends the wait without waiting out an in-flight snapshot 3012ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -34,8 +34,8 @@ Received: "still waiting" Test Files 1 failed (1) Tests 1 failed | 11 skipped (12) - Start at 04:05:19 - Duration 3.50s (transform 194ms, setup 16ms, collect 328ms, tests 3.01s, environment 0ms, prepare 41ms) + Start at 04:16:03 + Duration 3.50s (transform 193ms, setup 16ms, collect 322ms, tests 3.01s, environment 0ms, prepare 44ms) exit=1 @@ -44,12 +44,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2061ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2060ms + ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2063ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2062ms Test Files 1 passed (1) Tests 1 passed | 11 skipped (12) - Start at 04:05:22 - Duration 2.53s (transform 185ms, setup 17ms, collect 309ms, tests 2.06s, environment 0ms, prepare 41ms) + Start at 04:16:07 + Duration 2.53s (transform 189ms, setup 15ms, collect 322ms, tests 2.06s, environment 0ms, prepare 37ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-heartbeat.log b/evidence/run-read-timeout/mutation-heartbeat.log index dc48d593d..18c8a04fd 100644 --- a/evidence/run-read-timeout/mutation-heartbeat.log +++ b/evidence/run-read-timeout/mutation-heartbeat.log @@ -13,8 +13,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 19ms - × a heartbeat timeout is lease loss rather than a worker body failure 19ms + ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 18ms + × a heartbeat timeout is lease loss rather than a worker body failure 18ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -33,8 +33,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 04:05:25 - Duration 455ms (transform 168ms, setup 18ms, collect 284ms, tests 19ms, environment 0ms, prepare 41ms) + Start at 04:16:10 + Duration 456ms (transform 164ms, setup 16ms, collect 275ms, tests 18ms, environment 0ms, prepare 41ms) exit=1 @@ -47,7 +47,7 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb Test Files 1 passed (1) Tests 1 passed (1) - Start at 04:05:26 - Duration 459ms (transform 169ms, setup 16ms, collect 290ms, tests 15ms, environment 0ms, prepare 44ms) + Start at 04:16:11 + Duration 469ms (transform 167ms, setup 16ms, collect 292ms, tests 15ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-reconnect.log b/evidence/run-read-timeout/mutation-reader-reconnect.log index 5c9abc20e..8e126df8c 100644 --- a/evidence/run-read-timeout/mutation-reader-reconnect.log +++ b/evidence/run-read-timeout/mutation-reader-reconnect.log @@ -11,14 +11,14 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 1010ms - × reconnects the reader after it disconnects within the read budget 1009ms - → journal client: run.get read session was interrupted after 78 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 1008ms + × reconnects the reader after it disconnects within the read budget 1007ms + → journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 78 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:71:9 69| let last: unknown; 70| const exhausted = () => last instanceof JournalReadInterruptedError @@ -41,8 +41,8 @@ Caused by: Error: journal client: connection closed Test Files 1 failed (1) Tests 1 failed | 25 skipped (26) - Start at 04:05:02 - Duration 1.47s (transform 167ms, setup 16ms, collect 293ms, tests 1.01s, environment 0ms, prepare 45ms) + Start at 04:15:47 + Duration 1.41s (transform 161ms, setup 16ms, collect 267ms, tests 1.01s, environment 0ms, prepare 29ms) exit=1 @@ -51,11 +51,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 87ms + ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 81ms Test Files 1 passed (1) Tests 1 passed | 25 skipped (26) - Start at 04:05:04 - Duration 526ms (transform 165ms, setup 17ms, collect 283ms, tests 87ms, environment 0ms, prepare 40ms) + Start at 04:15:49 + Duration 520ms (transform 165ms, setup 14ms, collect 292ms, tests 81ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-setup-retry.log b/evidence/run-read-timeout/mutation-reader-setup-retry.log index 4dd0af898..115bbb043 100644 --- a/evidence/run-read-timeout/mutation-reader-setup-retry.log +++ b/evidence/run-read-timeout/mutation-reader-setup-retry.log @@ -16,8 +16,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 71ms - × retries a reader setup that timed out instead of reading on the primary for good 70ms + ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 80ms + × retries a reader setup that timed out instead of reading on the primary for good 79ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -43,8 +43,8 @@ AssertionError: expected 1 to be 2 // Object.is equality Test Files 1 failed (1) Tests 1 failed | 25 skipped (26) - Start at 04:05:05 - Duration 501ms (transform 163ms, setup 17ms, collect 279ms, tests 71ms, environment 0ms, prepare 40ms) + Start at 04:15:50 + Duration 524ms (transform 168ms, setup 17ms, collect 293ms, tests 80ms, environment 0ms, prepare 42ms) exit=1 @@ -53,11 +53,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 69ms + ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 74ms Test Files 1 passed (1) Tests 1 passed | 25 skipped (26) - Start at 04:05:06 - Duration 523ms (transform 169ms, setup 16ms, collect 292ms, tests 69ms, environment 0ms, prepare 41ms) + Start at 04:15:51 + Duration 508ms (transform 164ms, setup 16ms, collect 287ms, tests 74ms, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader.log b/evidence/run-read-timeout/mutation-reader.log index 086862663..5c6cf836b 100644 --- a/evidence/run-read-timeout/mutation-reader.log +++ b/evidence/run-read-timeout/mutation-reader.log @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 113ms - × serves a bounded read while an unbounded command is in flight 112ms + ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 112ms + × serves a bounded read while an unbounded command is in flight 111ms → expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -37,8 +37,8 @@ AssertionError: expected true to be false // Object.is equality Test Files 1 failed (1) Tests 1 failed | 25 skipped (26) - Start at 04:05:00 - Duration 553ms (transform 167ms, setup 16ms, collect 286ms, tests 113ms, environment 0ms, prepare 42ms) + Start at 04:15:45 + Duration 535ms (transform 170ms, setup 16ms, collect 283ms, tests 112ms, environment 0ms, prepare 29ms) exit=1 @@ -51,7 +51,7 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t Test Files 1 passed (1) Tests 1 passed | 25 skipped (26) - Start at 04:05:01 - Duration 553ms (transform 165ms, setup 16ms, collect 284ms, tests 106ms, environment 0ms, prepare 41ms) + Start at 04:15:46 + Duration 543ms (transform 161ms, setup 16ms, collect 282ms, tests 106ms, environment 0ms, prepare 43ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-root-parking.log b/evidence/run-read-timeout/mutation-root-parking.log index a9613e789..569dd142d 100644 --- a/evidence/run-read-timeout/mutation-root-parking.log +++ b/evidence/run-read-timeout/mutation-root-parking.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/authored-root.ts -Original SHA256: 2186915328814fe8fb40c1da6692cd7ea674942d8d68cca939f1f2b83fdc422c +Original SHA256: 9adbef378df4b191653880ac6bd08cb526ff96b9b7a29c36d54a0df3301acb0b Replaced: if (isReadInterruptionError(error) || (error instanceof AuthoredFlowExecutionError && error.code === 'daemon_unresponsive')) { @@ -17,14 +17,14 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (32 tests | 3 failed | 29 skipped) 29ms - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 21ms + ❯ tests/authored-root.test.ts (34 tests | 3 failed | 31 skipped) 21ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 15ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 3ms → expected AuthoredFlowExecutionError: daemon_unresp… { …(7) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 3ms + × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 2ms → expected JournalReadInterruptedError: journal clie… { …(4) } to match object { code: 'daemon_unresponsive', …(2) } (5 matching properties omitted from actual) @@ -103,22 +103,22 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[3/3]⎯ Test Files 1 failed (1) - Tests 3 failed | 29 skipped (32) - Start at 04:05:27 - Duration 1.28s (transform 669ms, setup 16ms, collect 1.10s, tests 29ms, environment 0ms, prepare 40ms) + Tests 3 failed | 31 skipped (34) + Start at 04:16:12 + Duration 1.25s (transform 656ms, setup 18ms, collect 1.06s, tests 21ms, environment 0ms, prepare 42ms) exit=1 -RESTORED SHA256: 2186915328814fe8fb40c1da6692cd7ea674942d8d68cca939f1f2b83fdc422c +RESTORED SHA256: 9adbef378df4b191653880ac6bd08cb526ff96b9b7a29c36d54a0df3301acb0b $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the root resumable' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (32 tests | 29 skipped) 24ms + ✓ tests/authored-root.test.ts (34 tests | 31 skipped) 22ms Test Files 1 passed (1) - Tests 3 passed | 29 skipped (32) - Start at 04:05:29 - Duration 1.29s (transform 666ms, setup 16ms, collect 1.11s, tests 24ms, environment 0ms, prepare 30ms) + Tests 3 passed | 31 skipped (34) + Start at 04:16:13 + Duration 1.28s (transform 673ms, setup 15ms, collect 1.10s, tests 22ms, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-snapshot-cancel.log b/evidence/run-read-timeout/mutation-snapshot-cancel.log index 67ef3f567..e9bcf3a9b 100644 --- a/evidence/run-read-timeout/mutation-snapshot-cancel.log +++ b/evidence/run-read-timeout/mutation-snapshot-cancel.log @@ -11,8 +11,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2216ms - × cancels promptly while a lease snapshot read is in flight 2216ms + ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2215ms + × cancels promptly while a lease snapshot read is in flight 2215ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -31,8 +31,8 @@ AssertionError: expected 'still waiting' to be an instance of Error Test Files 1 failed (1) Tests 1 failed | 11 skipped (12) - Start at 04:05:13 - Duration 2.69s (transform 192ms, setup 13ms, collect 331ms, tests 2.22s, environment 0ms, prepare 41ms) + Start at 04:15:57 + Duration 2.71s (transform 196ms, setup 16ms, collect 331ms, tests 2.22s, environment 0ms, prepare 40ms) exit=1 @@ -41,12 +41,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2012ms - ✓ cancels promptly while a lease snapshot read is in flight 2011ms + ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2010ms + ✓ cancels promptly while a lease snapshot read is in flight 2010ms Test Files 1 passed (1) Tests 1 passed | 11 skipped (12) - Start at 04:05:16 - Duration 2.51s (transform 197ms, setup 17ms, collect 332ms, tests 2.01s, environment 0ms, prepare 41ms) + Start at 04:16:00 + Duration 2.45s (transform 190ms, setup 15ms, collect 309ms, tests 2.01s, environment 0ms, prepare 31ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-watch-cadence.log b/evidence/run-read-timeout/mutation-watch-cadence.log index c62fa5253..00820fea7 100644 --- a/evidence/run-read-timeout/mutation-watch-cadence.log +++ b/evidence/run-read-timeout/mutation-watch-cadence.log @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2109ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2108ms + ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2110ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2109ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -30,8 +30,8 @@ AssertionError: expected 41 to be less than or equal to 1 Test Files 1 failed (1) Tests 1 failed | 11 skipped (12) - Start at 04:05:07 - Duration 2.57s (transform 198ms, setup 15ms, collect 327ms, tests 2.11s, environment 0ms, prepare 31ms) + Start at 04:15:52 + Duration 2.58s (transform 194ms, setup 17ms, collect 319ms, tests 2.11s, environment 0ms, prepare 40ms) exit=1 @@ -40,12 +40,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2118ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms + ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2119ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms Test Files 1 passed (1) Tests 1 passed | 11 skipped (12) - Start at 04:05:10 - Duration 2.59s (transform 194ms, setup 17ms, collect 325ms, tests 2.12s, environment 0ms, prepare 34ms) + Start at 04:15:54 + Duration 2.60s (transform 190ms, setup 16ms, collect 318ms, tests 2.12s, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/resume-live.log b/evidence/run-read-timeout/resume-live.log index 40885b754..33da6d8f8 100644 --- a/evidence/run-read-timeout/resume-live.log +++ b/evidence/run-read-timeout/resume-live.log @@ -2,12 +2,12 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 925ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 924ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 933ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 933ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 04:06:35 - Duration 2.20s (transform 612ms, setup 17ms, collect 1.03s, tests 925ms, environment 0ms, prepare 71ms) + Start at 04:17:19 + Duration 2.34s (transform 678ms, setup 22ms, collect 1.14s, tests 933ms, environment 0ms, prepare 85ms) exit=0 diff --git a/evidence/run-read-timeout/verification.md b/evidence/run-read-timeout/verification.md index 2a4971f87..f0bace86f 100644 --- a/evidence/run-read-timeout/verification.md +++ b/evidence/run-read-timeout/verification.md @@ -15,71 +15,71 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5264ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2038ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2049ms -(node:50017) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5245ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2037ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2037ms +(node:96527) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (32 tests) 443ms - ✓ tests/journal-client.test.ts (17 tests) 69ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 6851ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 468ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 979ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2721ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 355ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 830ms - ✓ tests/authored-flow-slack.test.ts (7 tests) 1150ms - ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 410ms - ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 392ms - ✓ tests/journal-client-read-timeout.test.ts (26 tests) 1788ms - ✓ tests/running-step-watch.test.ts (12 tests) 12294ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2119ms - ✓ cancels promptly while a lease snapshot read is in flight 2005ms + ✓ tests/authored-root.test.ts (34 tests) 454ms + ✓ tests/journal-client.test.ts (17 tests) 80ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 6900ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 459ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1044ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2719ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 351ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 833ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 1141ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 421ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 382ms + ✓ tests/journal-client-read-timeout.test.ts (26 tests) 1796ms + ✓ tests/running-step-watch.test.ts (12 tests) 12284ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2116ms + ✓ cancels promptly while a lease snapshot read is in flight 2006ms ✓ a completion push ends the wait without waiting out an in-flight snapshot 2055ms - ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms - ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2005ms - ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2003ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2550ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 656ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 603ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 675ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 615ms - ✓ tests/authored-helpers.test.ts (6 tests) 1946ms - ✓ runs every available provider through the real kernel and resumes completed effects without a second write 884ms - ✓ replays after SIGKILL before confirm with the same token and one successful completion 419ms - ✓ replays after SIGKILL before complete with the same token and one successful completion 416ms + ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2003ms + ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2002ms + ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2002ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2582ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 646ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 624ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 647ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 664ms + ✓ tests/authored-helpers.test.ts (6 tests) 1908ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 890ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 388ms + ✓ replays after SIGKILL before complete with the same token and one successful completion 410ms ✓ tests/classify-outcome.test.ts (11 tests) 7429ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2013ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3007ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2012ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3008ms ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms - ✓ tests/human-live.test.ts (3 tests) 4728ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2841ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1080ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 806ms - ✓ tests/worker-lease.test.ts (7 tests) 21ms - ✓ tests/communication-worker.test.ts (18 tests) 1714ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 30ms - ✓ tests/effect-channel.test.ts (5 tests) 316ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 710ms - ✓ tests/memoization.test.ts (58 tests) 65ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2456ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2064ms - ✓ drains read and watch promises before an authored flow completes 391ms - ✓ tests/worker-lease-sweep.test.ts (4 tests) 6ms - ✓ tests/journal-client-completion.test.ts (6 tests) 101ms + ✓ tests/human-live.test.ts (3 tests) 4766ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2877ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1190ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 699ms + ✓ tests/worker-lease.test.ts (7 tests) 15ms + ✓ tests/communication-worker.test.ts (18 tests) 1711ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 25ms + ✓ tests/effect-channel.test.ts (5 tests) 282ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 693ms + ✓ tests/memoization.test.ts (58 tests) 59ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2494ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2058ms + ✓ drains read and watch promises before an authored flow completes 435ms + ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms + ✓ tests/journal-client-completion.test.ts (6 tests) 105ms ✓ tests/run-daemon-unresponsive.test.ts (4 tests) 5ms ✓ tests/reuse-summary-interruption.test.ts (5 tests) 4ms - ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms - ✓ tests/direct-run-worker-lease.test.ts (3 tests) 9ms - ✓ tests/helper-effect-lease-loss.test.ts (1 test) 234ms - ✓ tests/journal-client-subscriptions.test.ts (1 test) 9ms - ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 47ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 14ms + ✓ tests/resume-worker-lease.test.ts (3 tests) 8ms + ✓ tests/direct-run-worker-lease.test.ts (3 tests) 7ms + ✓ tests/helper-effect-lease-loss.test.ts (1 test) 243ms + ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms + ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 39ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 18ms Test Files 28 passed (28) - Tests 343 passed (343) - Start at 04:05:34 - Duration 60.68s (transform 1.14s, setup 164ms, collect 6.84s, tests 50.26s, environment 4ms, prepare 1.02s) + Tests 345 passed (345) + Start at 04:16:18 + Duration 60.34s (transform 1.11s, setup 158ms, collect 6.53s, tests 50.31s, environment 4ms, prepare 989ms) exit=0 ``` @@ -91,13 +91,13 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 925ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 924ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 933ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 933ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 04:06:35 - Duration 2.20s (transform 612ms, setup 17ms, collect 1.03s, tests 925ms, environment 0ms, prepare 71ms) + Start at 04:17:19 + Duration 2.34s (transform 678ms, setup 22ms, collect 1.14s, tests 933ms, environment 0ms, prepare 85ms) exit=0 ``` @@ -119,8 +119,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 113ms - × serves a bounded read while an unbounded command is in flight 112ms + ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 112ms + × serves a bounded read while an unbounded command is in flight 111ms → expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -146,8 +146,8 @@ AssertionError: expected true to be false // Object.is equality Test Files 1 failed (1) Tests 1 failed | 25 skipped (26) - Start at 04:05:00 - Duration 553ms (transform 167ms, setup 16ms, collect 286ms, tests 113ms, environment 0ms, prepare 42ms) + Start at 04:15:45 + Duration 535ms (transform 170ms, setup 16ms, collect 283ms, tests 112ms, environment 0ms, prepare 29ms) exit=1 @@ -160,8 +160,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t Test Files 1 passed (1) Tests 1 passed | 25 skipped (26) - Start at 04:05:01 - Duration 553ms (transform 165ms, setup 16ms, collect 284ms, tests 106ms, environment 0ms, prepare 41ms) + Start at 04:15:46 + Duration 543ms (transform 161ms, setup 16ms, collect 282ms, tests 106ms, environment 0ms, prepare 43ms) exit=0 ``` @@ -182,14 +182,14 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 1010ms - × reconnects the reader after it disconnects within the read budget 1009ms - → journal client: run.get read session was interrupted after 78 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 1008ms + × reconnects the reader after it disconnects within the read budget 1007ms + → journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 78 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:71:9 69| let last: unknown; 70| const exhausted = () => last instanceof JournalReadInterruptedError @@ -212,8 +212,8 @@ Caused by: Error: journal client: connection closed Test Files 1 failed (1) Tests 1 failed | 25 skipped (26) - Start at 04:05:02 - Duration 1.47s (transform 167ms, setup 16ms, collect 293ms, tests 1.01s, environment 0ms, prepare 45ms) + Start at 04:15:47 + Duration 1.41s (transform 161ms, setup 16ms, collect 267ms, tests 1.01s, environment 0ms, prepare 29ms) exit=1 @@ -222,12 +222,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 87ms + ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 81ms Test Files 1 passed (1) Tests 1 passed | 25 skipped (26) - Start at 04:05:04 - Duration 526ms (transform 165ms, setup 17ms, collect 283ms, tests 87ms, environment 0ms, prepare 40ms) + Start at 04:15:49 + Duration 520ms (transform 165ms, setup 14ms, collect 292ms, tests 81ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -253,8 +253,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 71ms - × retries a reader setup that timed out instead of reading on the primary for good 70ms + ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 80ms + × retries a reader setup that timed out instead of reading on the primary for good 79ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -280,8 +280,8 @@ AssertionError: expected 1 to be 2 // Object.is equality Test Files 1 failed (1) Tests 1 failed | 25 skipped (26) - Start at 04:05:05 - Duration 501ms (transform 163ms, setup 17ms, collect 279ms, tests 71ms, environment 0ms, prepare 40ms) + Start at 04:15:50 + Duration 524ms (transform 168ms, setup 17ms, collect 293ms, tests 80ms, environment 0ms, prepare 42ms) exit=1 @@ -290,12 +290,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 69ms + ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 74ms Test Files 1 passed (1) Tests 1 passed | 25 skipped (26) - Start at 04:05:06 - Duration 523ms (transform 169ms, setup 16ms, collect 292ms, tests 69ms, environment 0ms, prepare 41ms) + Start at 04:15:51 + Duration 508ms (transform 164ms, setup 16ms, collect 287ms, tests 74ms, environment 0ms, prepare 40ms) exit=0 ``` @@ -315,8 +315,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2109ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2108ms + ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2110ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2109ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -335,8 +335,8 @@ AssertionError: expected 41 to be less than or equal to 1 Test Files 1 failed (1) Tests 1 failed | 11 skipped (12) - Start at 04:05:07 - Duration 2.57s (transform 198ms, setup 15ms, collect 327ms, tests 2.11s, environment 0ms, prepare 31ms) + Start at 04:15:52 + Duration 2.58s (transform 194ms, setup 17ms, collect 319ms, tests 2.11s, environment 0ms, prepare 40ms) exit=1 @@ -345,13 +345,13 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2118ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms + ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2119ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms Test Files 1 passed (1) Tests 1 passed | 11 skipped (12) - Start at 04:05:10 - Duration 2.59s (transform 194ms, setup 17ms, collect 325ms, tests 2.12s, environment 0ms, prepare 34ms) + Start at 04:15:54 + Duration 2.60s (transform 190ms, setup 16ms, collect 318ms, tests 2.12s, environment 0ms, prepare 41ms) exit=0 ``` @@ -372,8 +372,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2216ms - × cancels promptly while a lease snapshot read is in flight 2216ms + ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2215ms + × cancels promptly while a lease snapshot read is in flight 2215ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -392,8 +392,8 @@ AssertionError: expected 'still waiting' to be an instance of Error Test Files 1 failed (1) Tests 1 failed | 11 skipped (12) - Start at 04:05:13 - Duration 2.69s (transform 192ms, setup 13ms, collect 331ms, tests 2.22s, environment 0ms, prepare 41ms) + Start at 04:15:57 + Duration 2.71s (transform 196ms, setup 16ms, collect 331ms, tests 2.22s, environment 0ms, prepare 40ms) exit=1 @@ -402,13 +402,13 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2012ms - ✓ cancels promptly while a lease snapshot read is in flight 2011ms + ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2010ms + ✓ cancels promptly while a lease snapshot read is in flight 2010ms Test Files 1 passed (1) Tests 1 passed | 11 skipped (12) - Start at 04:05:16 - Duration 2.51s (transform 197ms, setup 17ms, collect 332ms, tests 2.01s, environment 0ms, prepare 41ms) + Start at 04:16:00 + Duration 2.45s (transform 190ms, setup 15ms, collect 309ms, tests 2.01s, environment 0ms, prepare 31ms) exit=0 ``` @@ -428,8 +428,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 3012ms - × a completion push ends the wait without waiting out an in-flight snapshot 3011ms + ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 3013ms + × a completion push ends the wait without waiting out an in-flight snapshot 3012ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -452,8 +452,8 @@ Received: "still waiting" Test Files 1 failed (1) Tests 1 failed | 11 skipped (12) - Start at 04:05:19 - Duration 3.50s (transform 194ms, setup 16ms, collect 328ms, tests 3.01s, environment 0ms, prepare 41ms) + Start at 04:16:03 + Duration 3.50s (transform 193ms, setup 16ms, collect 322ms, tests 3.01s, environment 0ms, prepare 44ms) exit=1 @@ -462,13 +462,13 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2061ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2060ms + ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2063ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2062ms Test Files 1 passed (1) Tests 1 passed | 11 skipped (12) - Start at 04:05:22 - Duration 2.53s (transform 185ms, setup 17ms, collect 309ms, tests 2.06s, environment 0ms, prepare 41ms) + Start at 04:16:07 + Duration 2.53s (transform 189ms, setup 15ms, collect 322ms, tests 2.06s, environment 0ms, prepare 37ms) exit=0 ``` @@ -491,8 +491,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 19ms - × a heartbeat timeout is lease loss rather than a worker body failure 19ms + ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 18ms + × a heartbeat timeout is lease loss rather than a worker body failure 18ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -511,8 +511,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 04:05:25 - Duration 455ms (transform 168ms, setup 18ms, collect 284ms, tests 19ms, environment 0ms, prepare 41ms) + Start at 04:16:10 + Duration 456ms (transform 164ms, setup 16ms, collect 275ms, tests 18ms, environment 0ms, prepare 41ms) exit=1 @@ -525,8 +525,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb Test Files 1 passed (1) Tests 1 passed (1) - Start at 04:05:26 - Duration 459ms (transform 169ms, setup 16ms, collect 290ms, tests 15ms, environment 0ms, prepare 44ms) + Start at 04:16:11 + Duration 469ms (transform 167ms, setup 16ms, collect 292ms, tests 15ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -535,7 +535,7 @@ exit=0 ```text Source: packages/sdk/src/authored-root.ts -Original SHA256: 2186915328814fe8fb40c1da6692cd7ea674942d8d68cca939f1f2b83fdc422c +Original SHA256: 9adbef378df4b191653880ac6bd08cb526ff96b9b7a29c36d54a0df3301acb0b Replaced: if (isReadInterruptionError(error) || (error instanceof AuthoredFlowExecutionError && error.code === 'daemon_unresponsive')) { @@ -553,14 +553,14 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (32 tests | 3 failed | 29 skipped) 29ms - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 21ms + ❯ tests/authored-root.test.ts (34 tests | 3 failed | 31 skipped) 21ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 15ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 3ms → expected AuthoredFlowExecutionError: daemon_unresp… { …(7) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 3ms + × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 2ms → expected JournalReadInterruptedError: journal clie… { …(4) } to match object { code: 'daemon_unresponsive', …(2) } (5 matching properties omitted from actual) @@ -639,23 +639,23 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[3/3]⎯ Test Files 1 failed (1) - Tests 3 failed | 29 skipped (32) - Start at 04:05:27 - Duration 1.28s (transform 669ms, setup 16ms, collect 1.10s, tests 29ms, environment 0ms, prepare 40ms) + Tests 3 failed | 31 skipped (34) + Start at 04:16:12 + Duration 1.25s (transform 656ms, setup 18ms, collect 1.06s, tests 21ms, environment 0ms, prepare 42ms) exit=1 -RESTORED SHA256: 2186915328814fe8fb40c1da6692cd7ea674942d8d68cca939f1f2b83fdc422c +RESTORED SHA256: 9adbef378df4b191653880ac6bd08cb526ff96b9b7a29c36d54a0df3301acb0b $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the root resumable' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (32 tests | 29 skipped) 24ms + ✓ tests/authored-root.test.ts (34 tests | 31 skipped) 22ms Test Files 1 passed (1) - Tests 3 passed | 29 skipped (32) - Start at 04:05:29 - Duration 1.29s (transform 666ms, setup 16ms, collect 1.11s, tests 24ms, environment 0ms, prepare 30ms) + Tests 3 passed | 31 skipped (34) + Start at 04:16:13 + Duration 1.28s (transform 673ms, setup 15ms, collect 1.10s, tests 22ms, environment 0ms, prepare 40ms) exit=0 ``` diff --git a/packages/sdk/src/authored-root.ts b/packages/sdk/src/authored-root.ts index e30ae630c..e6e2e4d30 100644 --- a/packages/sdk/src/authored-root.ts +++ b/packages/sdk/src/authored-root.ts @@ -13,7 +13,7 @@ import { type LoadedAuthoredFlow, type SurfaceModuleAuthority, } from './authored-flow-loader.js'; -import { JournalClient, JournalProtocolError } from './journal-client.js'; +import { JournalClient, JournalFrameError, JournalProtocolError } from './journal-client.js'; import type { RunOutcome, StepDispatchEvent } from './protocol.js'; import { SPEC_SCHEMA_VERSION } from './spec.js'; import type { RunLifecycleOptions } from './cli/run.js'; @@ -200,11 +200,12 @@ export async function readAuthoredRootMetadata( journal: JournalClient, runId: string, ): Promise { - // Not finding the root step means "not authored"; an unanswered read means - // nothing yet, and guessing declarative would resume the wrong way. + // Only a definite answer means "not authored": the daemon refused the read, or + // the journal has no root step. An unanswered or canceled read means nothing + // yet, and guessing declarative would resume the run the wrong way. const step = await rootKernelStep(journal, runId).catch(error => { - if (isReadInterruptionError(error)) throw error; - return undefined; + if (error instanceof JournalProtocolError || error === NO_ROOT_STEP) return undefined; + throw error; }); if (step === undefined || step.id !== 'authored-root' || !hasRootStream(step)) return undefined; if (typeof step.instruction !== 'string') { @@ -475,14 +476,14 @@ function rootSpec(metadata: AuthoredRootMetadata, stream: string) { })); } +const NO_ROOT_STEP = new Error('authored root journal has no root step'); + async function rootKernelStep(journal: JournalClient, runId: string): Promise> { const entries = (await journal.journalRead(runId, 1)).entries as Array>; const spawned = entries.find(entry => entry.entry_type === 'run.spawned'); const payload = spawned?.payload as { spec?: { steps?: unknown[] } } | undefined; const step = payload?.spec?.steps?.[0]; - if (typeof step !== 'object' || step === null || Array.isArray(step)) { - throw new Error('authored root journal has no root step'); - } + if (typeof step !== 'object' || step === null || Array.isArray(step)) throw NO_ROOT_STEP; return step as Record; } @@ -575,7 +576,10 @@ function nextRootDispatch(peer: JournalClient, timeoutMs = 30_000): { */ function completedResult(journal: JournalClient, outcome: RunOutcome, rootRunId: string): Promise { return completedRootResult(journal, rootRunId).catch(error => { - if (!isReadInterruptionError(error)) throw error; + // A definite answer stands (a refusal, a bad frame, no stored result); + // anything else — interrupted or canceled — left the verdict unread. + if (error instanceof JournalProtocolError || error instanceof JournalFrameError + || error === NO_DURABLE_RESULT) throw error; // The kernel completes the root step with success whatever the body // declared; the authored verdict lives only in the unread output. const unread = new AuthoredFlowExecutionError('result_unreadable', @@ -586,6 +590,8 @@ function completedResult(journal: JournalClient, outcome: RunOutcome, rootRunId: }); } +const NO_DURABLE_RESULT = new Error('completed authored root has no durable result'); + async function completedRootResult( journal: JournalClient, rootRunId: string, @@ -595,9 +601,7 @@ async function completedRootResult( && entry.step_id === 'authored-root' && (entry.payload as { completionReason?: unknown } | undefined)?.completionReason === 'success'); const output = (completed?.payload as { output?: unknown } | undefined)?.output; - if (!isCompletedRootOutput(output)) { - throw new Error('completed authored root has no durable result'); - } + if (!isCompletedRootOutput(output)) throw NO_DURABLE_RESULT; // The stored detail, never a freshly computed one: redaction reads the // CURRENT environment, so recomputing here would let a completed run report // something its journal does not hold. diff --git a/packages/sdk/tests/authored-root.test.ts b/packages/sdk/tests/authored-root.test.ts index dc553647f..3b85d17f6 100644 --- a/packages/sdk/tests/authored-root.test.ts +++ b/packages/sdk/tests/authored-root.test.ts @@ -508,6 +508,23 @@ describe('durable authored root', () => { }); }); + it('propagates a cancellation while detecting an authored root, but trusts a definite answer', async () => { + const { readAuthoredRootMetadata } = await import('../src/authored-root.js'); + const canceled = { journalRead: async () => { throw new DOMException('This operation was aborted', 'AbortError'); } }; + await expect(readAuthoredRootMetadata(canceled as unknown as JournalClient, 'run')).rejects.toMatchObject({ name: 'AbortError' }); + const absent = { journalRead: async () => { throw new JournalProtocolError('run_not_found', 'no such run'); } }; + await expect(readAuthoredRootMetadata(absent as unknown as JournalClient, 'run')).resolves.toBeUndefined(); + }); + + it('keeps a completed root outcome when reading back its result is canceled', async () => { + const loaded = await fixture(); + const journal = new RootJournal(); + (journal as unknown as { startStatus: RunOutcome }).startStatus = outcome('root-run', 'completed', 'success'); + journal.journalRead = async () => { throw new DOMException('This operation was aborted', 'AbortError'); }; + await expect(executeDurableAuthoredFlow(loaded, journal as unknown as JournalClient, undefined, + { dataDir: '/unused', admissionKey: 'completed-canceled' })).rejects.toMatchObject({ code: 'result_unreadable', rootRunId: 'root-run' }); + }); + it('terminalizes a returned body failure without replaying semantic side effects', async () => { const loaded = await fixture(true); const journal = new RootJournal(); From 6c85fbfd6792d2f2e71c60711994137b63ff4652 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 04:27:16 -0700 Subject: [PATCH 26/34] fix(sdk): canceling a read abandons a reader setup still in progress A read canceled while its dedicated reader was still connecting or awaiting hello left the serialized job waiting out the setup timeout (up to 30 s), so an immediate retry or root re-drive queued behind abandoned work. The setup is now raced against the attempt's signal: an abort closes the half-open reader and frees the queue at once. Evidence re-captured at this tree. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: b8262468-ea26-4a6e-bee9-d193832e77dc --- evidence/run-read-timeout/final-focused.log | 111 ++++---- .../mutation-completion-aborts-snapshot.log | 12 +- .../run-read-timeout/mutation-heartbeat.log | 12 +- .../mutation-reader-reconnect.log | 22 +- .../mutation-reader-setup-retry.log | 22 +- evidence/run-read-timeout/mutation-reader.log | 22 +- .../mutation-root-parking.log | 18 +- .../mutation-snapshot-cancel.log | 16 +- .../mutation-watch-cadence.log | 16 +- evidence/run-read-timeout/resume-live.log | 8 +- evidence/run-read-timeout/verification.md | 259 +++++++++--------- packages/sdk/src/journal-budgeted-reads.ts | 20 +- .../tests/journal-client-read-timeout.test.ts | 24 ++ 13 files changed, 305 insertions(+), 257 deletions(-) diff --git a/evidence/run-read-timeout/final-focused.log b/evidence/run-read-timeout/final-focused.log index 330c588d8..0101e5626 100644 --- a/evidence/run-read-timeout/final-focused.log +++ b/evidence/run-read-timeout/final-focused.log @@ -2,70 +2,73 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5245ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2037ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2037ms -(node:96527) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5364ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2044ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2038ms +(node:150258) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (34 tests) 454ms - ✓ tests/journal-client.test.ts (17 tests) 80ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 6900ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 459ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1044ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2719ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 351ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 833ms - ✓ tests/authored-flow-slack.test.ts (7 tests) 1141ms - ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 421ms - ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 382ms - ✓ tests/journal-client-read-timeout.test.ts (26 tests) 1796ms - ✓ tests/running-step-watch.test.ts (12 tests) 12284ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2116ms - ✓ cancels promptly while a lease snapshot read is in flight 2006ms + ✓ tests/authored-root.test.ts (34 tests) 466ms + ✓ tests/journal-client.test.ts (17 tests) 82ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 7848ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 524ms + ✓ flow executor LLM and output-binding chain > fails invalid LLM output before the next step: not JSON 311ms + ✓ flow executor LLM and output-binding chain > fails invalid LLM output before the next step: {"message":7} 314ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1231ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2925ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 379ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 935ms + ✓ tests/journal-client-read-timeout.test.ts (27 tests) 1834ms + ✓ a recovered read timeout does not become an authored callback failure 317ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 1214ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 413ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 431ms + ✓ tests/running-step-watch.test.ts (12 tests) 12287ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms + ✓ cancels promptly while a lease snapshot read is in flight 2005ms ✓ a completion push ends the wait without waiting out an in-flight snapshot 2055ms - ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2003ms - ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2002ms - ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2002ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2582ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 646ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 624ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 647ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 664ms - ✓ tests/authored-helpers.test.ts (6 tests) 1908ms - ✓ runs every available provider through the real kernel and resumes completed effects without a second write 890ms - ✓ replays after SIGKILL before confirm with the same token and one successful completion 388ms - ✓ replays after SIGKILL before complete with the same token and one successful completion 410ms + ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms + ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2003ms + ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2003ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2599ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 670ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 650ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 691ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 587ms + ✓ tests/authored-helpers.test.ts (6 tests) 1775ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 910ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 345ms + ✓ replays after SIGKILL before complete with the same token and one successful completion 331ms ✓ tests/classify-outcome.test.ts (11 tests) 7429ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2012ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3008ms - ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms - ✓ tests/human-live.test.ts (3 tests) 4766ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2877ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1190ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 699ms - ✓ tests/worker-lease.test.ts (7 tests) 15ms - ✓ tests/communication-worker.test.ts (18 tests) 1711ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 25ms - ✓ tests/effect-channel.test.ts (5 tests) 282ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 693ms - ✓ tests/memoization.test.ts (58 tests) 59ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2494ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2058ms - ✓ drains read and watch promises before an authored flow completes 435ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2013ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3007ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2002ms + ✓ tests/human-live.test.ts (3 tests) 4597ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2634ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1189ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 772ms + ✓ tests/worker-lease.test.ts (7 tests) 12ms + ✓ tests/communication-worker.test.ts (18 tests) 1708ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 21ms + ✓ tests/effect-channel.test.ts (5 tests) 253ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 700ms + ✓ tests/memoization.test.ts (58 tests) 68ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2499ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2059ms + ✓ drains read and watch promises before an authored flow completes 439ms ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms ✓ tests/journal-client-completion.test.ts (6 tests) 105ms ✓ tests/run-daemon-unresponsive.test.ts (4 tests) 5ms ✓ tests/reuse-summary-interruption.test.ts (5 tests) 4ms ✓ tests/resume-worker-lease.test.ts (3 tests) 8ms - ✓ tests/direct-run-worker-lease.test.ts (3 tests) 7ms - ✓ tests/helper-effect-lease-loss.test.ts (1 test) 243ms + ✓ tests/direct-run-worker-lease.test.ts (3 tests) 10ms + ✓ tests/helper-effect-lease-loss.test.ts (1 test) 228ms ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms - ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 39ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 18ms + ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 48ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms Test Files 28 passed (28) - Tests 345 passed (345) - Start at 04:16:18 - Duration 60.34s (transform 1.11s, setup 158ms, collect 6.53s, tests 50.31s, environment 4ms, prepare 989ms) + Tests 346 passed (346) + Start at 04:26:11 + Duration 61.79s (transform 1.15s, setup 162ms, collect 6.95s, tests 51.20s, environment 4ms, prepare 1.05s) exit=0 diff --git a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log index 548193106..a9bf5e2d3 100644 --- a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log +++ b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log @@ -34,8 +34,8 @@ Received: "still waiting" Test Files 1 failed (1) Tests 1 failed | 11 skipped (12) - Start at 04:16:03 - Duration 3.50s (transform 193ms, setup 16ms, collect 322ms, tests 3.01s, environment 0ms, prepare 44ms) + Start at 04:25:55 + Duration 4.12s (transform 456ms, setup 59ms, collect 722ms, tests 3.01s, environment 0ms, prepare 131ms) exit=1 @@ -44,12 +44,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2063ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2062ms + ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2062ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2061ms Test Files 1 passed (1) Tests 1 passed | 11 skipped (12) - Start at 04:16:07 - Duration 2.53s (transform 189ms, setup 15ms, collect 322ms, tests 2.06s, environment 0ms, prepare 37ms) + Start at 04:26:00 + Duration 2.62s (transform 216ms, setup 20ms, collect 365ms, tests 2.06s, environment 0ms, prepare 49ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-heartbeat.log b/evidence/run-read-timeout/mutation-heartbeat.log index 18c8a04fd..1cfa2a6cc 100644 --- a/evidence/run-read-timeout/mutation-heartbeat.log +++ b/evidence/run-read-timeout/mutation-heartbeat.log @@ -14,7 +14,7 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 18ms - × a heartbeat timeout is lease loss rather than a worker body failure 18ms + × a heartbeat timeout is lease loss rather than a worker body failure 17ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -33,8 +33,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 04:16:10 - Duration 456ms (transform 164ms, setup 16ms, collect 275ms, tests 18ms, environment 0ms, prepare 41ms) + Start at 04:26:03 + Duration 485ms (transform 173ms, setup 19ms, collect 295ms, tests 18ms, environment 0ms, prepare 45ms) exit=1 @@ -43,11 +43,11 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 18ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 04:16:11 - Duration 469ms (transform 167ms, setup 16ms, collect 292ms, tests 15ms, environment 0ms, prepare 41ms) + Start at 04:26:04 + Duration 500ms (transform 177ms, setup 18ms, collect 315ms, tests 18ms, environment 0ms, prepare 43ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-reconnect.log b/evidence/run-read-timeout/mutation-reader-reconnect.log index 8e126df8c..73476c8fc 100644 --- a/evidence/run-read-timeout/mutation-reader-reconnect.log +++ b/evidence/run-read-timeout/mutation-reader-reconnect.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: 3c10585faa843ba188e6d87cc39f0b4e8a6b95a165d9caba104919c1c35598ec +Original SHA256: 668d4d45dd732da1c653985dda4c2c3ddb115dab7462c0c1187c3e820f779b6a Replaced: this.drop(reader); @@ -11,8 +11,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 1008ms - × reconnects the reader after it disconnects within the read budget 1007ms + ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 1012ms + × reconnects the reader after it disconnects within the read budget 1011ms → journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -40,22 +40,22 @@ Caused by: Error: journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 25 skipped (26) - Start at 04:15:47 - Duration 1.41s (transform 161ms, setup 16ms, collect 267ms, tests 1.01s, environment 0ms, prepare 29ms) + Tests 1 failed | 26 skipped (27) + Start at 04:25:37 + Duration 1.61s (transform 207ms, setup 25ms, collect 367ms, tests 1.01s, environment 0ms, prepare 59ms) exit=1 -RESTORED SHA256: 3c10585faa843ba188e6d87cc39f0b4e8a6b95a165d9caba104919c1c35598ec +RESTORED SHA256: 668d4d45dd732da1c653985dda4c2c3ddb115dab7462c0c1187c3e820f779b6a $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'reconnects the reader after it disconnects' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 81ms + ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 97ms Test Files 1 passed (1) - Tests 1 passed | 25 skipped (26) - Start at 04:15:49 - Duration 520ms (transform 165ms, setup 14ms, collect 292ms, tests 81ms, environment 0ms, prepare 41ms) + Tests 1 passed | 26 skipped (27) + Start at 04:25:39 + Duration 647ms (transform 202ms, setup 15ms, collect 372ms, tests 97ms, environment 0ms, prepare 48ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-setup-retry.log b/evidence/run-read-timeout/mutation-reader-setup-retry.log index 115bbb043..34d53d189 100644 --- a/evidence/run-read-timeout/mutation-reader-setup-retry.log +++ b/evidence/run-read-timeout/mutation-reader-setup-retry.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: 3c10585faa843ba188e6d87cc39f0b4e8a6b95a165d9caba104919c1c35598ec +Original SHA256: 668d4d45dd732da1c653985dda4c2c3ddb115dab7462c0c1187c3e820f779b6a Replaced: if (this.reader === reader) { this.reader = undefined; @@ -16,8 +16,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 80ms - × retries a reader setup that timed out instead of reading on the primary for good 79ms + ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 82ms + × retries a reader setup that timed out instead of reading on the primary for good 81ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -42,22 +42,22 @@ AssertionError: expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 25 skipped (26) - Start at 04:15:50 - Duration 524ms (transform 168ms, setup 17ms, collect 293ms, tests 80ms, environment 0ms, prepare 42ms) + Tests 1 failed | 26 skipped (27) + Start at 04:25:40 + Duration 649ms (transform 210ms, setup 17ms, collect 359ms, tests 82ms, environment 0ms, prepare 60ms) exit=1 -RESTORED SHA256: 3c10585faa843ba188e6d87cc39f0b4e8a6b95a165d9caba104919c1c35598ec +RESTORED SHA256: 668d4d45dd732da1c653985dda4c2c3ddb115dab7462c0c1187c3e820f779b6a $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'retries a reader setup that timed out' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 74ms + ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 74ms Test Files 1 passed (1) - Tests 1 passed | 25 skipped (26) - Start at 04:15:51 - Duration 508ms (transform 164ms, setup 16ms, collect 287ms, tests 74ms, environment 0ms, prepare 40ms) + Tests 1 passed | 26 skipped (27) + Start at 04:25:41 + Duration 617ms (transform 199ms, setup 19ms, collect 348ms, tests 74ms, environment 0ms, prepare 45ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader.log b/evidence/run-read-timeout/mutation-reader.log index 5c6cf836b..58639eba4 100644 --- a/evidence/run-read-timeout/mutation-reader.log +++ b/evidence/run-read-timeout/mutation-reader.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: 3c10585faa843ba188e6d87cc39f0b4e8a6b95a165d9caba104919c1c35598ec +Original SHA256: 668d4d45dd732da1c653985dda4c2c3ddb115dab7462c0c1187c3e820f779b6a Replaced: return await reader.requestOnce(verb, params, remaining, attemptSignal); With: @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 112ms - × serves a bounded read while an unbounded command is in flight 111ms + ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 117ms + × serves a bounded read while an unbounded command is in flight 116ms → expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -36,22 +36,22 @@ AssertionError: expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 25 skipped (26) - Start at 04:15:45 - Duration 535ms (transform 170ms, setup 16ms, collect 283ms, tests 112ms, environment 0ms, prepare 29ms) + Tests 1 failed | 26 skipped (27) + Start at 04:25:35 + Duration 662ms (transform 208ms, setup 18ms, collect 379ms, tests 117ms, environment 0ms, prepare 39ms) exit=1 -RESTORED SHA256: 3c10585faa843ba188e6d87cc39f0b4e8a6b95a165d9caba104919c1c35598ec +RESTORED SHA256: 668d4d45dd732da1c653985dda4c2c3ddb115dab7462c0c1187c3e820f779b6a $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'serves a bounded read' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 106ms + ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 106ms Test Files 1 passed (1) - Tests 1 passed | 25 skipped (26) - Start at 04:15:46 - Duration 543ms (transform 161ms, setup 16ms, collect 282ms, tests 106ms, environment 0ms, prepare 43ms) + Tests 1 passed | 26 skipped (27) + Start at 04:25:36 + Duration 617ms (transform 181ms, setup 21ms, collect 314ms, tests 106ms, environment 0ms, prepare 54ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-root-parking.log b/evidence/run-read-timeout/mutation-root-parking.log index 569dd142d..b409de7fc 100644 --- a/evidence/run-read-timeout/mutation-root-parking.log +++ b/evidence/run-read-timeout/mutation-root-parking.log @@ -17,14 +17,14 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (34 tests | 3 failed | 31 skipped) 21ms - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 15ms + ❯ tests/authored-root.test.ts (34 tests | 3 failed | 31 skipped) 32ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 23ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 3ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms → expected AuthoredFlowExecutionError: daemon_unresp… { …(7) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 2ms + × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 4ms → expected JournalReadInterruptedError: journal clie… { …(4) } to match object { code: 'daemon_unresponsive', …(2) } (5 matching properties omitted from actual) @@ -104,8 +104,8 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } Test Files 1 failed (1) Tests 3 failed | 31 skipped (34) - Start at 04:16:12 - Duration 1.25s (transform 656ms, setup 18ms, collect 1.06s, tests 21ms, environment 0ms, prepare 42ms) + Start at 04:26:04 + Duration 1.29s (transform 655ms, setup 16ms, collect 1.09s, tests 32ms, environment 0ms, prepare 41ms) exit=1 @@ -114,11 +114,11 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (34 tests | 31 skipped) 22ms + ✓ tests/authored-root.test.ts (34 tests | 31 skipped) 25ms Test Files 1 passed (1) Tests 3 passed | 31 skipped (34) - Start at 04:16:13 - Duration 1.28s (transform 673ms, setup 15ms, collect 1.10s, tests 22ms, environment 0ms, prepare 40ms) + Start at 04:26:06 + Duration 1.28s (transform 654ms, setup 16ms, collect 1.09s, tests 25ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-snapshot-cancel.log b/evidence/run-read-timeout/mutation-snapshot-cancel.log index e9bcf3a9b..33686434d 100644 --- a/evidence/run-read-timeout/mutation-snapshot-cancel.log +++ b/evidence/run-read-timeout/mutation-snapshot-cancel.log @@ -11,8 +11,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2215ms - × cancels promptly while a lease snapshot read is in flight 2215ms + ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2213ms + × cancels promptly while a lease snapshot read is in flight 2212ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -31,8 +31,8 @@ AssertionError: expected 'still waiting' to be an instance of Error Test Files 1 failed (1) Tests 1 failed | 11 skipped (12) - Start at 04:15:57 - Duration 2.71s (transform 196ms, setup 16ms, collect 331ms, tests 2.22s, environment 0ms, prepare 40ms) + Start at 04:25:49 + Duration 2.69s (transform 192ms, setup 18ms, collect 318ms, tests 2.21s, environment 0ms, prepare 43ms) exit=1 @@ -41,12 +41,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2010ms - ✓ cancels promptly while a lease snapshot read is in flight 2010ms + ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2027ms + ✓ cancels promptly while a lease snapshot read is in flight 2025ms Test Files 1 passed (1) Tests 1 passed | 11 skipped (12) - Start at 04:16:00 - Duration 2.45s (transform 190ms, setup 15ms, collect 309ms, tests 2.01s, environment 0ms, prepare 31ms) + Start at 04:25:52 + Duration 2.67s (transform 245ms, setup 38ms, collect 374ms, tests 2.03s, environment 0ms, prepare 87ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-watch-cadence.log b/evidence/run-read-timeout/mutation-watch-cadence.log index 00820fea7..19b01a1d4 100644 --- a/evidence/run-read-timeout/mutation-watch-cadence.log +++ b/evidence/run-read-timeout/mutation-watch-cadence.log @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2110ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2109ms + ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2111ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2110ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -30,8 +30,8 @@ AssertionError: expected 41 to be less than or equal to 1 Test Files 1 failed (1) Tests 1 failed | 11 skipped (12) - Start at 04:15:52 - Duration 2.58s (transform 194ms, setup 17ms, collect 319ms, tests 2.11s, environment 0ms, prepare 40ms) + Start at 04:25:42 + Duration 2.68s (transform 222ms, setup 18ms, collect 385ms, tests 2.11s, environment 0ms, prepare 45ms) exit=1 @@ -40,12 +40,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2119ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms + ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2118ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms Test Files 1 passed (1) Tests 1 passed | 11 skipped (12) - Start at 04:15:54 - Duration 2.60s (transform 190ms, setup 16ms, collect 318ms, tests 2.12s, environment 0ms, prepare 41ms) + Start at 04:25:46 + Duration 2.58s (transform 191ms, setup 15ms, collect 319ms, tests 2.12s, environment 0ms, prepare 30ms) exit=0 diff --git a/evidence/run-read-timeout/resume-live.log b/evidence/run-read-timeout/resume-live.log index 33da6d8f8..f3681f7df 100644 --- a/evidence/run-read-timeout/resume-live.log +++ b/evidence/run-read-timeout/resume-live.log @@ -2,12 +2,12 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 933ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 933ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 902ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 901ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 04:17:19 - Duration 2.34s (transform 678ms, setup 22ms, collect 1.14s, tests 933ms, environment 0ms, prepare 85ms) + Start at 04:27:13 + Duration 2.17s (transform 618ms, setup 15ms, collect 1.02s, tests 902ms, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/verification.md b/evidence/run-read-timeout/verification.md index f0bace86f..318f263b5 100644 --- a/evidence/run-read-timeout/verification.md +++ b/evidence/run-read-timeout/verification.md @@ -15,71 +15,74 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5245ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2037ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2037ms -(node:96527) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5364ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2044ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2038ms +(node:150258) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (34 tests) 454ms - ✓ tests/journal-client.test.ts (17 tests) 80ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 6900ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 459ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1044ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2719ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 351ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 833ms - ✓ tests/authored-flow-slack.test.ts (7 tests) 1141ms - ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 421ms - ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 382ms - ✓ tests/journal-client-read-timeout.test.ts (26 tests) 1796ms - ✓ tests/running-step-watch.test.ts (12 tests) 12284ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2116ms - ✓ cancels promptly while a lease snapshot read is in flight 2006ms + ✓ tests/authored-root.test.ts (34 tests) 466ms + ✓ tests/journal-client.test.ts (17 tests) 82ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 7848ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 524ms + ✓ flow executor LLM and output-binding chain > fails invalid LLM output before the next step: not JSON 311ms + ✓ flow executor LLM and output-binding chain > fails invalid LLM output before the next step: {"message":7} 314ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1231ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2925ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 379ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 935ms + ✓ tests/journal-client-read-timeout.test.ts (27 tests) 1834ms + ✓ a recovered read timeout does not become an authored callback failure 317ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 1214ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 413ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 431ms + ✓ tests/running-step-watch.test.ts (12 tests) 12287ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms + ✓ cancels promptly while a lease snapshot read is in flight 2005ms ✓ a completion push ends the wait without waiting out an in-flight snapshot 2055ms - ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2003ms - ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2002ms - ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2002ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2582ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 646ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 624ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 647ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 664ms - ✓ tests/authored-helpers.test.ts (6 tests) 1908ms - ✓ runs every available provider through the real kernel and resumes completed effects without a second write 890ms - ✓ replays after SIGKILL before confirm with the same token and one successful completion 388ms - ✓ replays after SIGKILL before complete with the same token and one successful completion 410ms + ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms + ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2003ms + ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2003ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2599ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 670ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 650ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 691ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 587ms + ✓ tests/authored-helpers.test.ts (6 tests) 1775ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 910ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 345ms + ✓ replays after SIGKILL before complete with the same token and one successful completion 331ms ✓ tests/classify-outcome.test.ts (11 tests) 7429ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2012ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3008ms - ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms - ✓ tests/human-live.test.ts (3 tests) 4766ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2877ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1190ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 699ms - ✓ tests/worker-lease.test.ts (7 tests) 15ms - ✓ tests/communication-worker.test.ts (18 tests) 1711ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 25ms - ✓ tests/effect-channel.test.ts (5 tests) 282ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 693ms - ✓ tests/memoization.test.ts (58 tests) 59ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2494ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2058ms - ✓ drains read and watch promises before an authored flow completes 435ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2013ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3007ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2002ms + ✓ tests/human-live.test.ts (3 tests) 4597ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2634ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1189ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 772ms + ✓ tests/worker-lease.test.ts (7 tests) 12ms + ✓ tests/communication-worker.test.ts (18 tests) 1708ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 21ms + ✓ tests/effect-channel.test.ts (5 tests) 253ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 700ms + ✓ tests/memoization.test.ts (58 tests) 68ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2499ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2059ms + ✓ drains read and watch promises before an authored flow completes 439ms ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms ✓ tests/journal-client-completion.test.ts (6 tests) 105ms ✓ tests/run-daemon-unresponsive.test.ts (4 tests) 5ms ✓ tests/reuse-summary-interruption.test.ts (5 tests) 4ms ✓ tests/resume-worker-lease.test.ts (3 tests) 8ms - ✓ tests/direct-run-worker-lease.test.ts (3 tests) 7ms - ✓ tests/helper-effect-lease-loss.test.ts (1 test) 243ms + ✓ tests/direct-run-worker-lease.test.ts (3 tests) 10ms + ✓ tests/helper-effect-lease-loss.test.ts (1 test) 228ms ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms - ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 39ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 18ms + ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 48ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms Test Files 28 passed (28) - Tests 345 passed (345) - Start at 04:16:18 - Duration 60.34s (transform 1.11s, setup 158ms, collect 6.53s, tests 50.31s, environment 4ms, prepare 989ms) + Tests 346 passed (346) + Start at 04:26:11 + Duration 61.79s (transform 1.15s, setup 162ms, collect 6.95s, tests 51.20s, environment 4ms, prepare 1.05s) exit=0 ``` @@ -91,13 +94,13 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 933ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 933ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 902ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 901ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 04:17:19 - Duration 2.34s (transform 678ms, setup 22ms, collect 1.14s, tests 933ms, environment 0ms, prepare 85ms) + Start at 04:27:13 + Duration 2.17s (transform 618ms, setup 15ms, collect 1.02s, tests 902ms, environment 0ms, prepare 40ms) exit=0 ``` @@ -108,7 +111,7 @@ The mutation sections below were regenerated by `cd packages/sdk && python3 ../. ```text Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: 3c10585faa843ba188e6d87cc39f0b4e8a6b95a165d9caba104919c1c35598ec +Original SHA256: 668d4d45dd732da1c653985dda4c2c3ddb115dab7462c0c1187c3e820f779b6a Replaced: return await reader.requestOnce(verb, params, remaining, attemptSignal); With: @@ -119,8 +122,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 112ms - × serves a bounded read while an unbounded command is in flight 111ms + ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 117ms + × serves a bounded read while an unbounded command is in flight 116ms → expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -145,23 +148,23 @@ AssertionError: expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 25 skipped (26) - Start at 04:15:45 - Duration 535ms (transform 170ms, setup 16ms, collect 283ms, tests 112ms, environment 0ms, prepare 29ms) + Tests 1 failed | 26 skipped (27) + Start at 04:25:35 + Duration 662ms (transform 208ms, setup 18ms, collect 379ms, tests 117ms, environment 0ms, prepare 39ms) exit=1 -RESTORED SHA256: 3c10585faa843ba188e6d87cc39f0b4e8a6b95a165d9caba104919c1c35598ec +RESTORED SHA256: 668d4d45dd732da1c653985dda4c2c3ddb115dab7462c0c1187c3e820f779b6a $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'serves a bounded read' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 106ms + ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 106ms Test Files 1 passed (1) - Tests 1 passed | 25 skipped (26) - Start at 04:15:46 - Duration 543ms (transform 161ms, setup 16ms, collect 282ms, tests 106ms, environment 0ms, prepare 43ms) + Tests 1 passed | 26 skipped (27) + Start at 04:25:36 + Duration 617ms (transform 181ms, setup 21ms, collect 314ms, tests 106ms, environment 0ms, prepare 54ms) exit=0 ``` @@ -170,7 +173,7 @@ exit=0 ```text Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: 3c10585faa843ba188e6d87cc39f0b4e8a6b95a165d9caba104919c1c35598ec +Original SHA256: 668d4d45dd732da1c653985dda4c2c3ddb115dab7462c0c1187c3e820f779b6a Replaced: this.drop(reader); @@ -182,8 +185,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 1008ms - × reconnects the reader after it disconnects within the read budget 1007ms + ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 1012ms + × reconnects the reader after it disconnects within the read budget 1011ms → journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -211,23 +214,23 @@ Caused by: Error: journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 25 skipped (26) - Start at 04:15:47 - Duration 1.41s (transform 161ms, setup 16ms, collect 267ms, tests 1.01s, environment 0ms, prepare 29ms) + Tests 1 failed | 26 skipped (27) + Start at 04:25:37 + Duration 1.61s (transform 207ms, setup 25ms, collect 367ms, tests 1.01s, environment 0ms, prepare 59ms) exit=1 -RESTORED SHA256: 3c10585faa843ba188e6d87cc39f0b4e8a6b95a165d9caba104919c1c35598ec +RESTORED SHA256: 668d4d45dd732da1c653985dda4c2c3ddb115dab7462c0c1187c3e820f779b6a $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'reconnects the reader after it disconnects' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 81ms + ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 97ms Test Files 1 passed (1) - Tests 1 passed | 25 skipped (26) - Start at 04:15:49 - Duration 520ms (transform 165ms, setup 14ms, collect 292ms, tests 81ms, environment 0ms, prepare 41ms) + Tests 1 passed | 26 skipped (27) + Start at 04:25:39 + Duration 647ms (transform 202ms, setup 15ms, collect 372ms, tests 97ms, environment 0ms, prepare 48ms) exit=0 ``` @@ -236,7 +239,7 @@ exit=0 ```text Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: 3c10585faa843ba188e6d87cc39f0b4e8a6b95a165d9caba104919c1c35598ec +Original SHA256: 668d4d45dd732da1c653985dda4c2c3ddb115dab7462c0c1187c3e820f779b6a Replaced: if (this.reader === reader) { this.reader = undefined; @@ -253,8 +256,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (26 tests | 1 failed | 25 skipped) 80ms - × retries a reader setup that timed out instead of reading on the primary for good 79ms + ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 82ms + × retries a reader setup that timed out instead of reading on the primary for good 81ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -279,23 +282,23 @@ AssertionError: expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 25 skipped (26) - Start at 04:15:50 - Duration 524ms (transform 168ms, setup 17ms, collect 293ms, tests 80ms, environment 0ms, prepare 42ms) + Tests 1 failed | 26 skipped (27) + Start at 04:25:40 + Duration 649ms (transform 210ms, setup 17ms, collect 359ms, tests 82ms, environment 0ms, prepare 60ms) exit=1 -RESTORED SHA256: 3c10585faa843ba188e6d87cc39f0b4e8a6b95a165d9caba104919c1c35598ec +RESTORED SHA256: 668d4d45dd732da1c653985dda4c2c3ddb115dab7462c0c1187c3e820f779b6a $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'retries a reader setup that timed out' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (26 tests | 25 skipped) 74ms + ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 74ms Test Files 1 passed (1) - Tests 1 passed | 25 skipped (26) - Start at 04:15:51 - Duration 508ms (transform 164ms, setup 16ms, collect 287ms, tests 74ms, environment 0ms, prepare 40ms) + Tests 1 passed | 26 skipped (27) + Start at 04:25:41 + Duration 617ms (transform 199ms, setup 19ms, collect 348ms, tests 74ms, environment 0ms, prepare 45ms) exit=0 ``` @@ -315,8 +318,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2110ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2109ms + ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2111ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2110ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -335,8 +338,8 @@ AssertionError: expected 41 to be less than or equal to 1 Test Files 1 failed (1) Tests 1 failed | 11 skipped (12) - Start at 04:15:52 - Duration 2.58s (transform 194ms, setup 17ms, collect 319ms, tests 2.11s, environment 0ms, prepare 40ms) + Start at 04:25:42 + Duration 2.68s (transform 222ms, setup 18ms, collect 385ms, tests 2.11s, environment 0ms, prepare 45ms) exit=1 @@ -345,13 +348,13 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2119ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms + ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2118ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms Test Files 1 passed (1) Tests 1 passed | 11 skipped (12) - Start at 04:15:54 - Duration 2.60s (transform 190ms, setup 16ms, collect 318ms, tests 2.12s, environment 0ms, prepare 41ms) + Start at 04:25:46 + Duration 2.58s (transform 191ms, setup 15ms, collect 319ms, tests 2.12s, environment 0ms, prepare 30ms) exit=0 ``` @@ -372,8 +375,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2215ms - × cancels promptly while a lease snapshot read is in flight 2215ms + ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2213ms + × cancels promptly while a lease snapshot read is in flight 2212ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -392,8 +395,8 @@ AssertionError: expected 'still waiting' to be an instance of Error Test Files 1 failed (1) Tests 1 failed | 11 skipped (12) - Start at 04:15:57 - Duration 2.71s (transform 196ms, setup 16ms, collect 331ms, tests 2.22s, environment 0ms, prepare 40ms) + Start at 04:25:49 + Duration 2.69s (transform 192ms, setup 18ms, collect 318ms, tests 2.21s, environment 0ms, prepare 43ms) exit=1 @@ -402,13 +405,13 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2010ms - ✓ cancels promptly while a lease snapshot read is in flight 2010ms + ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2027ms + ✓ cancels promptly while a lease snapshot read is in flight 2025ms Test Files 1 passed (1) Tests 1 passed | 11 skipped (12) - Start at 04:16:00 - Duration 2.45s (transform 190ms, setup 15ms, collect 309ms, tests 2.01s, environment 0ms, prepare 31ms) + Start at 04:25:52 + Duration 2.67s (transform 245ms, setup 38ms, collect 374ms, tests 2.03s, environment 0ms, prepare 87ms) exit=0 ``` @@ -452,8 +455,8 @@ Received: "still waiting" Test Files 1 failed (1) Tests 1 failed | 11 skipped (12) - Start at 04:16:03 - Duration 3.50s (transform 193ms, setup 16ms, collect 322ms, tests 3.01s, environment 0ms, prepare 44ms) + Start at 04:25:55 + Duration 4.12s (transform 456ms, setup 59ms, collect 722ms, tests 3.01s, environment 0ms, prepare 131ms) exit=1 @@ -462,13 +465,13 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2063ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2062ms + ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2062ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2061ms Test Files 1 passed (1) Tests 1 passed | 11 skipped (12) - Start at 04:16:07 - Duration 2.53s (transform 189ms, setup 15ms, collect 322ms, tests 2.06s, environment 0ms, prepare 37ms) + Start at 04:26:00 + Duration 2.62s (transform 216ms, setup 20ms, collect 365ms, tests 2.06s, environment 0ms, prepare 49ms) exit=0 ``` @@ -492,7 +495,7 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 18ms - × a heartbeat timeout is lease loss rather than a worker body failure 18ms + × a heartbeat timeout is lease loss rather than a worker body failure 17ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -511,8 +514,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 04:16:10 - Duration 456ms (transform 164ms, setup 16ms, collect 275ms, tests 18ms, environment 0ms, prepare 41ms) + Start at 04:26:03 + Duration 485ms (transform 173ms, setup 19ms, collect 295ms, tests 18ms, environment 0ms, prepare 45ms) exit=1 @@ -521,12 +524,12 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 18ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 04:16:11 - Duration 469ms (transform 167ms, setup 16ms, collect 292ms, tests 15ms, environment 0ms, prepare 41ms) + Start at 04:26:04 + Duration 500ms (transform 177ms, setup 18ms, collect 315ms, tests 18ms, environment 0ms, prepare 43ms) exit=0 ``` @@ -553,14 +556,14 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (34 tests | 3 failed | 31 skipped) 21ms - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 15ms + ❯ tests/authored-root.test.ts (34 tests | 3 failed | 31 skipped) 32ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 23ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 3ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms → expected AuthoredFlowExecutionError: daemon_unresp… { …(7) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) - × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 2ms + × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 4ms → expected JournalReadInterruptedError: journal clie… { …(4) } to match object { code: 'daemon_unresponsive', …(2) } (5 matching properties omitted from actual) @@ -640,8 +643,8 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } Test Files 1 failed (1) Tests 3 failed | 31 skipped (34) - Start at 04:16:12 - Duration 1.25s (transform 656ms, setup 18ms, collect 1.06s, tests 21ms, environment 0ms, prepare 42ms) + Start at 04:26:04 + Duration 1.29s (transform 655ms, setup 16ms, collect 1.09s, tests 32ms, environment 0ms, prepare 41ms) exit=1 @@ -650,12 +653,12 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (34 tests | 31 skipped) 22ms + ✓ tests/authored-root.test.ts (34 tests | 31 skipped) 25ms Test Files 1 passed (1) Tests 3 passed | 31 skipped (34) - Start at 04:16:13 - Duration 1.28s (transform 673ms, setup 15ms, collect 1.10s, tests 22ms, environment 0ms, prepare 40ms) + Start at 04:26:06 + Duration 1.28s (transform 654ms, setup 16ms, collect 1.09s, tests 25ms, environment 0ms, prepare 41ms) exit=0 ``` diff --git a/packages/sdk/src/journal-budgeted-reads.ts b/packages/sdk/src/journal-budgeted-reads.ts index 6d6bbe442..46ac0dde9 100644 --- a/packages/sdk/src/journal-budgeted-reads.ts +++ b/packages/sdk/src/journal-budgeted-reads.ts @@ -46,7 +46,7 @@ export class BudgetedReads { const combined = sources.length <= 1 ? sources[0] : AbortSignal.any(sources); return this.policy.read(verb, timeoutMs, this.budgetMs, async (bound, attemptSignal) => { const started = performance.now(); - const reader = await this.session().catch(error => { + const reader = await this.untilAborted(this.session(), attemptSignal).catch(error => { attemptSignal.throwIfAborted(); // A malformed handshake frame is a protocol violation: fail closed. if (error instanceof JournalFrameError) throw error; @@ -95,6 +95,24 @@ export class BudgetedReads { return this.readerReady; } + /** + * Settle with the session, or at once with the attempt's abort — abandoning a + * setup still in progress, so the serialized queue never waits on it. + */ + private untilAborted(setup: Promise, signal: AbortSignal): Promise { + return new Promise((resolve, reject) => { + const onAbort = () => { + const pending = this.reader; + if (pending !== undefined) this.drop(pending); + reject(signal.reason); + }; + if (signal.aborted) { onAbort(); return; } + signal.addEventListener('abort', onAbort, { once: true }); + setup.then(value => { signal.removeEventListener('abort', onAbort); resolve(value); }, + error => { signal.removeEventListener('abort', onAbort); reject(error); }); + }); + } + /** Forget a reader whose transport failed so the next attempt reconnects one. */ private drop(reader: JournalClient): void { if (this.reader !== reader) return; diff --git a/packages/sdk/tests/journal-client-read-timeout.test.ts b/packages/sdk/tests/journal-client-read-timeout.test.ts index cd595ad75..c667343da 100644 --- a/packages/sdk/tests/journal-client-read-timeout.test.ts +++ b/packages/sdk/tests/journal-client-read-timeout.test.ts @@ -316,3 +316,27 @@ it('a read started after the primary dropped is still a typed read interruption' const error = await client.runGet('run').catch(caught => caught); expect(isReadInterruptionError(error)).toBe(true); }); + +it('canceling a read during reader setup frees the queue for the next read at once', async () => { + let hellos = 0; + let firstHello!: () => void; + const helloSeen = new Promise(resolve => { firstHello = resolve; }); + const path = sockPath(); + const server = startLoopback(path, { + hello: ctx => { if (++hellos === 1) firstHello(); else sendOk(ctx); }, // the first reader hello hangs + 'run.get': ctx => sendResult(ctx, { status: 'running' }), + }, { serialize: true }); + servers.push(server); + await once(server, 'listening'); + const client = new JournalClient(path, { requestTimeoutMs: 20_000, readBudgetMs: 60_000 }); + clients.push(client); + await client.connect(); + const controller = new AbortController(); + const first = client.runGet('run', { signal: controller.signal }); + await helloSeen; + controller.abort(new Error('re-drive')); + await expect(first).rejects.toThrow('re-drive'); + const begun = performance.now(); + await expect(client.runGet('run')).resolves.toMatchObject({ status: 'running' }); + expect(performance.now() - begun).toBeLessThan(2_000); +}); From 4e50eea7e92ee9a68a135e66fed91870c7429772 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 04:37:50 -0700 Subject: [PATCH 27/34] fix(sdk): a completion replayed before a lost watch registration stands run.watch writes the replay before its response; if the socket dropped after a replayed step.completed but before the response, the wait still reported a resumable daemon_unresponsive. A completion already folded from the replay now ends the wait successfully. Evidence re-captured at this tree. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: b8262468-ea26-4a6e-bee9-d193832e77dc --- evidence/run-read-timeout/final-focused.log | 104 ++++--- .../mutation-completion-aborts-snapshot.log | 24 +- .../run-read-timeout/mutation-heartbeat.log | 14 +- .../mutation-reader-reconnect.log | 18 +- .../mutation-reader-setup-retry.log | 14 +- evidence/run-read-timeout/mutation-reader.log | 12 +- .../mutation-root-parking.log | 14 +- .../mutation-snapshot-cancel.log | 24 +- .../mutation-watch-cadence.log | 22 +- evidence/run-read-timeout/resume-live.log | 8 +- evidence/run-read-timeout/verification.md | 254 +++++++++--------- packages/sdk/src/cli/running-step.ts | 3 + packages/sdk/tests/running-step-watch.test.ts | 15 ++ 13 files changed, 270 insertions(+), 256 deletions(-) diff --git a/evidence/run-read-timeout/final-focused.log b/evidence/run-read-timeout/final-focused.log index 0101e5626..699c502ad 100644 --- a/evidence/run-read-timeout/final-focused.log +++ b/evidence/run-read-timeout/final-focused.log @@ -2,73 +2,71 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5364ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2044ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2038ms -(node:150258) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5303ms + ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 315ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2036ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2046ms +(node:195226) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (34 tests) 466ms - ✓ tests/journal-client.test.ts (17 tests) 82ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 7848ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 524ms - ✓ flow executor LLM and output-binding chain > fails invalid LLM output before the next step: not JSON 311ms - ✓ flow executor LLM and output-binding chain > fails invalid LLM output before the next step: {"message":7} 314ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1231ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2925ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 379ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 935ms - ✓ tests/journal-client-read-timeout.test.ts (27 tests) 1834ms - ✓ a recovered read timeout does not become an authored callback failure 317ms - ✓ tests/authored-flow-slack.test.ts (7 tests) 1214ms - ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 413ms - ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 431ms - ✓ tests/running-step-watch.test.ts (12 tests) 12287ms + ✓ tests/authored-root.test.ts (34 tests) 449ms + ✓ tests/journal-client.test.ts (17 tests) 79ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 7114ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 492ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1046ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2799ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 388ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 884ms + ✓ tests/journal-client-read-timeout.test.ts (27 tests) 1794ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 1164ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 463ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 335ms + ✓ tests/running-step-watch.test.ts (13 tests) 12302ms ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms - ✓ cancels promptly while a lease snapshot read is in flight 2005ms + ✓ cancels promptly while a lease snapshot read is in flight 2006ms ✓ a completion push ends the wait without waiting out an in-flight snapshot 2055ms - ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms + ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2003ms ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2003ms - ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2003ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2599ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 670ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 650ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 691ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 587ms - ✓ tests/authored-helpers.test.ts (6 tests) 1775ms - ✓ runs every available provider through the real kernel and resumes completed effects without a second write 910ms - ✓ replays after SIGKILL before confirm with the same token and one successful completion 345ms - ✓ replays after SIGKILL before complete with the same token and one successful completion 331ms - ✓ tests/classify-outcome.test.ts (11 tests) 7429ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2013ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3007ms + ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2002ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2534ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 588ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 645ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 646ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 654ms + ✓ tests/authored-helpers.test.ts (6 tests) 1973ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 898ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 440ms + ✓ replays after SIGKILL before complete with the same token and one successful completion 433ms + ✓ tests/classify-outcome.test.ts (11 tests) 7426ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2010ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3005ms ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2002ms - ✓ tests/human-live.test.ts (3 tests) 4597ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2634ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1189ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 772ms + ✓ tests/human-live.test.ts (3 tests) 4919ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2949ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1157ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 812ms ✓ tests/worker-lease.test.ts (7 tests) 12ms - ✓ tests/communication-worker.test.ts (18 tests) 1708ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 21ms - ✓ tests/effect-channel.test.ts (5 tests) 253ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 700ms - ✓ tests/memoization.test.ts (58 tests) 68ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2499ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2059ms - ✓ drains read and watch promises before an authored flow completes 439ms + ✓ tests/communication-worker.test.ts (18 tests) 1716ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 20ms + ✓ tests/effect-channel.test.ts (5 tests) 279ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 699ms + ✓ tests/memoization.test.ts (58 tests) 62ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2485ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2057ms + ✓ drains read and watch promises before an authored flow completes 428ms ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms - ✓ tests/journal-client-completion.test.ts (6 tests) 105ms + ✓ tests/journal-client-completion.test.ts (6 tests) 103ms ✓ tests/run-daemon-unresponsive.test.ts (4 tests) 5ms ✓ tests/reuse-summary-interruption.test.ts (5 tests) 4ms ✓ tests/resume-worker-lease.test.ts (3 tests) 8ms ✓ tests/direct-run-worker-lease.test.ts (3 tests) 10ms - ✓ tests/helper-effect-lease-loss.test.ts (1 test) 228ms + ✓ tests/helper-effect-lease-loss.test.ts (1 test) 186ms ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 48ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms Test Files 28 passed (28) - Tests 346 passed (346) - Start at 04:26:11 - Duration 61.79s (transform 1.15s, setup 162ms, collect 6.95s, tests 51.20s, environment 4ms, prepare 1.05s) + Tests 347 passed (347) + Start at 04:36:46 + Duration 60.99s (transform 1.13s, setup 157ms, collect 6.78s, tests 50.73s, environment 4ms, prepare 992ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log index a9bf5e2d3..c7516b2d4 100644 --- a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log +++ b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/cli/running-step.ts -Original SHA256: 2d95d1b21da7acb5e91e79236cb099604beabaed7e1e0443c96aeccf4c704c3e +Original SHA256: 251ef48cb8616a41911ba353fd9ea7d9aae299049b941c87ef9927e9429cd1d8 Replaced: snapshotRead?.abort(new Error(`step "${runningStep.id}" completed during the lease snapshot`)); With: @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 3013ms - × a completion push ends the wait without waiting out an in-flight snapshot 3012ms + ❯ tests/running-step-watch.test.ts (13 tests | 1 failed | 12 skipped) 3013ms + × a completion push ends the wait without waiting out an in-flight snapshot 3013ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -33,23 +33,23 @@ Received: "still waiting" ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 11 skipped (12) - Start at 04:25:55 - Duration 4.12s (transform 456ms, setup 59ms, collect 722ms, tests 3.01s, environment 0ms, prepare 131ms) + Tests 1 failed | 12 skipped (13) + Start at 04:36:31 + Duration 3.50s (transform 190ms, setup 17ms, collect 324ms, tests 3.01s, environment 0ms, prepare 41ms) exit=1 -RESTORED SHA256: 2d95d1b21da7acb5e91e79236cb099604beabaed7e1e0443c96aeccf4c704c3e +RESTORED SHA256: 251ef48cb8616a41911ba353fd9ea7d9aae299049b941c87ef9927e9429cd1d8 $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a completion push ends the wait' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2062ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2061ms + ✓ tests/running-step-watch.test.ts (13 tests | 12 skipped) 2063ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2062ms Test Files 1 passed (1) - Tests 1 passed | 11 skipped (12) - Start at 04:26:00 - Duration 2.62s (transform 216ms, setup 20ms, collect 365ms, tests 2.06s, environment 0ms, prepare 49ms) + Tests 1 passed | 12 skipped (13) + Start at 04:36:35 + Duration 2.52s (transform 182ms, setup 17ms, collect 300ms, tests 2.06s, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-heartbeat.log b/evidence/run-read-timeout/mutation-heartbeat.log index 1cfa2a6cc..5d45bdc62 100644 --- a/evidence/run-read-timeout/mutation-heartbeat.log +++ b/evidence/run-read-timeout/mutation-heartbeat.log @@ -13,8 +13,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 18ms - × a heartbeat timeout is lease loss rather than a worker body failure 17ms + ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 21ms + × a heartbeat timeout is lease loss rather than a worker body failure 20ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -33,8 +33,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 04:26:03 - Duration 485ms (transform 173ms, setup 19ms, collect 295ms, tests 18ms, environment 0ms, prepare 45ms) + Start at 04:36:38 + Duration 475ms (transform 173ms, setup 21ms, collect 290ms, tests 21ms, environment 0ms, prepare 41ms) exit=1 @@ -43,11 +43,11 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/heartbeat-timeout.test.ts (1 test) 18ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 04:26:04 - Duration 500ms (transform 177ms, setup 18ms, collect 315ms, tests 18ms, environment 0ms, prepare 43ms) + Start at 04:36:39 + Duration 470ms (transform 168ms, setup 18ms, collect 290ms, tests 16ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-reconnect.log b/evidence/run-read-timeout/mutation-reader-reconnect.log index 73476c8fc..4cfadf1e2 100644 --- a/evidence/run-read-timeout/mutation-reader-reconnect.log +++ b/evidence/run-read-timeout/mutation-reader-reconnect.log @@ -11,14 +11,14 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 1012ms - × reconnects the reader after it disconnects within the read budget 1011ms - → journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 1009ms + × reconnects the reader after it disconnects within the read budget 1008ms + → journal client: run.get read session was interrupted after 74 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 74 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:71:9 69| let last: unknown; 70| const exhausted = () => last instanceof JournalReadInterruptedError @@ -41,8 +41,8 @@ Caused by: Error: journal client: connection closed Test Files 1 failed (1) Tests 1 failed | 26 skipped (27) - Start at 04:25:37 - Duration 1.61s (transform 207ms, setup 25ms, collect 367ms, tests 1.01s, environment 0ms, prepare 59ms) + Start at 04:36:15 + Duration 1.48s (transform 172ms, setup 18ms, collect 300ms, tests 1.01s, environment 0ms, prepare 42ms) exit=1 @@ -51,11 +51,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 97ms + ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 85ms Test Files 1 passed (1) Tests 1 passed | 26 skipped (27) - Start at 04:25:39 - Duration 647ms (transform 202ms, setup 15ms, collect 372ms, tests 97ms, environment 0ms, prepare 48ms) + Start at 04:36:17 + Duration 520ms (transform 161ms, setup 17ms, collect 279ms, tests 85ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-setup-retry.log b/evidence/run-read-timeout/mutation-reader-setup-retry.log index 34d53d189..9f8cae962 100644 --- a/evidence/run-read-timeout/mutation-reader-setup-retry.log +++ b/evidence/run-read-timeout/mutation-reader-setup-retry.log @@ -16,8 +16,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 82ms - × retries a reader setup that timed out instead of reading on the primary for good 81ms + ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 71ms + × retries a reader setup that timed out instead of reading on the primary for good 70ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -43,8 +43,8 @@ AssertionError: expected 1 to be 2 // Object.is equality Test Files 1 failed (1) Tests 1 failed | 26 skipped (27) - Start at 04:25:40 - Duration 649ms (transform 210ms, setup 17ms, collect 359ms, tests 82ms, environment 0ms, prepare 60ms) + Start at 04:36:18 + Duration 539ms (transform 179ms, setup 16ms, collect 306ms, tests 71ms, environment 0ms, prepare 41ms) exit=1 @@ -53,11 +53,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 74ms + ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 70ms Test Files 1 passed (1) Tests 1 passed | 26 skipped (27) - Start at 04:25:41 - Duration 617ms (transform 199ms, setup 19ms, collect 348ms, tests 74ms, environment 0ms, prepare 45ms) + Start at 04:36:19 + Duration 518ms (transform 167ms, setup 17ms, collect 287ms, tests 70ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader.log b/evidence/run-read-timeout/mutation-reader.log index 58639eba4..2c038e211 100644 --- a/evidence/run-read-timeout/mutation-reader.log +++ b/evidence/run-read-timeout/mutation-reader.log @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 117ms - × serves a bounded read while an unbounded command is in flight 116ms + ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 113ms + × serves a bounded read while an unbounded command is in flight 112ms → expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -37,8 +37,8 @@ AssertionError: expected true to be false // Object.is equality Test Files 1 failed (1) Tests 1 failed | 26 skipped (27) - Start at 04:25:35 - Duration 662ms (transform 208ms, setup 18ms, collect 379ms, tests 117ms, environment 0ms, prepare 39ms) + Start at 04:36:13 + Duration 576ms (transform 173ms, setup 17ms, collect 299ms, tests 113ms, environment 0ms, prepare 41ms) exit=1 @@ -51,7 +51,7 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t Test Files 1 passed (1) Tests 1 passed | 26 skipped (27) - Start at 04:25:36 - Duration 617ms (transform 181ms, setup 21ms, collect 314ms, tests 106ms, environment 0ms, prepare 54ms) + Start at 04:36:14 + Duration 556ms (transform 168ms, setup 16ms, collect 293ms, tests 106ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-root-parking.log b/evidence/run-read-timeout/mutation-root-parking.log index b409de7fc..2c5260f85 100644 --- a/evidence/run-read-timeout/mutation-root-parking.log +++ b/evidence/run-read-timeout/mutation-root-parking.log @@ -17,8 +17,8 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (34 tests | 3 failed | 31 skipped) 32ms - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 23ms + ❯ tests/authored-root.test.ts (34 tests | 3 failed | 31 skipped) 30ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 22ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms @@ -104,8 +104,8 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } Test Files 1 failed (1) Tests 3 failed | 31 skipped (34) - Start at 04:26:04 - Duration 1.29s (transform 655ms, setup 16ms, collect 1.09s, tests 32ms, environment 0ms, prepare 41ms) + Start at 04:36:40 + Duration 1.29s (transform 657ms, setup 17ms, collect 1.09s, tests 30ms, environment 0ms, prepare 41ms) exit=1 @@ -114,11 +114,11 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (34 tests | 31 skipped) 25ms + ✓ tests/authored-root.test.ts (34 tests | 31 skipped) 23ms Test Files 1 passed (1) Tests 3 passed | 31 skipped (34) - Start at 04:26:06 - Duration 1.28s (transform 654ms, setup 16ms, collect 1.09s, tests 25ms, environment 0ms, prepare 41ms) + Start at 04:36:41 + Duration 1.30s (transform 673ms, setup 16ms, collect 1.11s, tests 23ms, environment 0ms, prepare 42ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-snapshot-cancel.log b/evidence/run-read-timeout/mutation-snapshot-cancel.log index 33686434d..3f1dbb09c 100644 --- a/evidence/run-read-timeout/mutation-snapshot-cancel.log +++ b/evidence/run-read-timeout/mutation-snapshot-cancel.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/cli/running-step.ts -Original SHA256: 2d95d1b21da7acb5e91e79236cb099604beabaed7e1e0443c96aeccf4c704c3e +Original SHA256: 251ef48cb8616a41911ba353fd9ea7d9aae299049b941c87ef9927e9429cd1d8 Replaced: const signal = options.signal === undefined ? read.signal : AbortSignal.any([options.signal, read.signal]); @@ -11,8 +11,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2213ms - × cancels promptly while a lease snapshot read is in flight 2212ms + ❯ tests/running-step-watch.test.ts (13 tests | 1 failed | 12 skipped) 2217ms + × cancels promptly while a lease snapshot read is in flight 2216ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -30,23 +30,23 @@ AssertionError: expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 11 skipped (12) - Start at 04:25:49 - Duration 2.69s (transform 192ms, setup 18ms, collect 318ms, tests 2.21s, environment 0ms, prepare 43ms) + Tests 1 failed | 12 skipped (13) + Start at 04:36:25 + Duration 2.70s (transform 189ms, setup 17ms, collect 322ms, tests 2.22s, environment 0ms, prepare 41ms) exit=1 -RESTORED SHA256: 2d95d1b21da7acb5e91e79236cb099604beabaed7e1e0443c96aeccf4c704c3e +RESTORED SHA256: 251ef48cb8616a41911ba353fd9ea7d9aae299049b941c87ef9927e9429cd1d8 $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels promptly while a lease snapshot read is in flight' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2027ms - ✓ cancels promptly while a lease snapshot read is in flight 2025ms + ✓ tests/running-step-watch.test.ts (13 tests | 12 skipped) 2011ms + ✓ cancels promptly while a lease snapshot read is in flight 2011ms Test Files 1 passed (1) - Tests 1 passed | 11 skipped (12) - Start at 04:25:52 - Duration 2.67s (transform 245ms, setup 38ms, collect 374ms, tests 2.03s, environment 0ms, prepare 87ms) + Tests 1 passed | 12 skipped (13) + Start at 04:36:28 + Duration 2.46s (transform 188ms, setup 16ms, collect 307ms, tests 2.01s, environment 0ms, prepare 33ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-watch-cadence.log b/evidence/run-read-timeout/mutation-watch-cadence.log index 19b01a1d4..14350fe6d 100644 --- a/evidence/run-read-timeout/mutation-watch-cadence.log +++ b/evidence/run-read-timeout/mutation-watch-cadence.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/cli/running-step.ts -Original SHA256: 2d95d1b21da7acb5e91e79236cb099604beabaed7e1e0443c96aeccf4c704c3e +Original SHA256: 251ef48cb8616a41911ba353fd9ea7d9aae299049b941c87ef9927e9429cd1d8 Replaced: const LEASE_POLL_MS = 2_000; With: @@ -10,7 +10,7 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2111ms + ❯ tests/running-step-watch.test.ts (13 tests | 1 failed | 12 skipped) 2111ms × uses pushes for completion with lease-cadence reads and releases its watcher 2110ms → expected 41 to be less than or equal to 1 @@ -29,23 +29,23 @@ AssertionError: expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 11 skipped (12) - Start at 04:25:42 - Duration 2.68s (transform 222ms, setup 18ms, collect 385ms, tests 2.11s, environment 0ms, prepare 45ms) + Tests 1 failed | 12 skipped (13) + Start at 04:36:20 + Duration 2.54s (transform 182ms, setup 13ms, collect 307ms, tests 2.11s, environment 0ms, prepare 29ms) exit=1 -RESTORED SHA256: 2d95d1b21da7acb5e91e79236cb099604beabaed7e1e0443c96aeccf4c704c3e +RESTORED SHA256: 251ef48cb8616a41911ba353fd9ea7d9aae299049b941c87ef9927e9429cd1d8 $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pushes for completion' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2118ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms + ✓ tests/running-step-watch.test.ts (13 tests | 12 skipped) 2116ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2115ms Test Files 1 passed (1) - Tests 1 passed | 11 skipped (12) - Start at 04:25:46 - Duration 2.58s (transform 191ms, setup 15ms, collect 319ms, tests 2.12s, environment 0ms, prepare 30ms) + Tests 1 passed | 12 skipped (13) + Start at 04:36:23 + Duration 2.59s (transform 191ms, setup 17ms, collect 326ms, tests 2.12s, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/resume-live.log b/evidence/run-read-timeout/resume-live.log index f3681f7df..940e0c5ce 100644 --- a/evidence/run-read-timeout/resume-live.log +++ b/evidence/run-read-timeout/resume-live.log @@ -2,12 +2,12 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 902ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 901ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 929ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 928ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 04:27:13 - Duration 2.17s (transform 618ms, setup 15ms, collect 1.02s, tests 902ms, environment 0ms, prepare 40ms) + Start at 04:37:47 + Duration 2.20s (transform 614ms, setup 16ms, collect 1.01s, tests 929ms, environment 0ms, prepare 62ms) exit=0 diff --git a/evidence/run-read-timeout/verification.md b/evidence/run-read-timeout/verification.md index 318f263b5..2062ebe2b 100644 --- a/evidence/run-read-timeout/verification.md +++ b/evidence/run-read-timeout/verification.md @@ -15,74 +15,72 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5364ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2044ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2038ms -(node:150258) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5303ms + ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 315ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2036ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2046ms +(node:195226) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (34 tests) 466ms - ✓ tests/journal-client.test.ts (17 tests) 82ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 7848ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 524ms - ✓ flow executor LLM and output-binding chain > fails invalid LLM output before the next step: not JSON 311ms - ✓ flow executor LLM and output-binding chain > fails invalid LLM output before the next step: {"message":7} 314ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1231ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2925ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 379ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 935ms - ✓ tests/journal-client-read-timeout.test.ts (27 tests) 1834ms - ✓ a recovered read timeout does not become an authored callback failure 317ms - ✓ tests/authored-flow-slack.test.ts (7 tests) 1214ms - ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 413ms - ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 431ms - ✓ tests/running-step-watch.test.ts (12 tests) 12287ms + ✓ tests/authored-root.test.ts (34 tests) 449ms + ✓ tests/journal-client.test.ts (17 tests) 79ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 7114ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 492ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1046ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2799ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 388ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 884ms + ✓ tests/journal-client-read-timeout.test.ts (27 tests) 1794ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 1164ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 463ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 335ms + ✓ tests/running-step-watch.test.ts (13 tests) 12302ms ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms - ✓ cancels promptly while a lease snapshot read is in flight 2005ms + ✓ cancels promptly while a lease snapshot read is in flight 2006ms ✓ a completion push ends the wait without waiting out an in-flight snapshot 2055ms - ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms + ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2003ms ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2003ms - ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2003ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2599ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 670ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 650ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 691ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 587ms - ✓ tests/authored-helpers.test.ts (6 tests) 1775ms - ✓ runs every available provider through the real kernel and resumes completed effects without a second write 910ms - ✓ replays after SIGKILL before confirm with the same token and one successful completion 345ms - ✓ replays after SIGKILL before complete with the same token and one successful completion 331ms - ✓ tests/classify-outcome.test.ts (11 tests) 7429ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2013ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3007ms + ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2002ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2534ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 588ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 645ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 646ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 654ms + ✓ tests/authored-helpers.test.ts (6 tests) 1973ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 898ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 440ms + ✓ replays after SIGKILL before complete with the same token and one successful completion 433ms + ✓ tests/classify-outcome.test.ts (11 tests) 7426ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2010ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3005ms ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2002ms - ✓ tests/human-live.test.ts (3 tests) 4597ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2634ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1189ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 772ms + ✓ tests/human-live.test.ts (3 tests) 4919ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2949ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1157ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 812ms ✓ tests/worker-lease.test.ts (7 tests) 12ms - ✓ tests/communication-worker.test.ts (18 tests) 1708ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 21ms - ✓ tests/effect-channel.test.ts (5 tests) 253ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 700ms - ✓ tests/memoization.test.ts (58 tests) 68ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2499ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2059ms - ✓ drains read and watch promises before an authored flow completes 439ms + ✓ tests/communication-worker.test.ts (18 tests) 1716ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 20ms + ✓ tests/effect-channel.test.ts (5 tests) 279ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 699ms + ✓ tests/memoization.test.ts (58 tests) 62ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2485ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2057ms + ✓ drains read and watch promises before an authored flow completes 428ms ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms - ✓ tests/journal-client-completion.test.ts (6 tests) 105ms + ✓ tests/journal-client-completion.test.ts (6 tests) 103ms ✓ tests/run-daemon-unresponsive.test.ts (4 tests) 5ms ✓ tests/reuse-summary-interruption.test.ts (5 tests) 4ms ✓ tests/resume-worker-lease.test.ts (3 tests) 8ms ✓ tests/direct-run-worker-lease.test.ts (3 tests) 10ms - ✓ tests/helper-effect-lease-loss.test.ts (1 test) 228ms + ✓ tests/helper-effect-lease-loss.test.ts (1 test) 186ms ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 48ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms Test Files 28 passed (28) - Tests 346 passed (346) - Start at 04:26:11 - Duration 61.79s (transform 1.15s, setup 162ms, collect 6.95s, tests 51.20s, environment 4ms, prepare 1.05s) + Tests 347 passed (347) + Start at 04:36:46 + Duration 60.99s (transform 1.13s, setup 157ms, collect 6.78s, tests 50.73s, environment 4ms, prepare 992ms) exit=0 ``` @@ -94,13 +92,13 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 902ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 901ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 929ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 928ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 04:27:13 - Duration 2.17s (transform 618ms, setup 15ms, collect 1.02s, tests 902ms, environment 0ms, prepare 40ms) + Start at 04:37:47 + Duration 2.20s (transform 614ms, setup 16ms, collect 1.01s, tests 929ms, environment 0ms, prepare 62ms) exit=0 ``` @@ -122,8 +120,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 117ms - × serves a bounded read while an unbounded command is in flight 116ms + ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 113ms + × serves a bounded read while an unbounded command is in flight 112ms → expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -149,8 +147,8 @@ AssertionError: expected true to be false // Object.is equality Test Files 1 failed (1) Tests 1 failed | 26 skipped (27) - Start at 04:25:35 - Duration 662ms (transform 208ms, setup 18ms, collect 379ms, tests 117ms, environment 0ms, prepare 39ms) + Start at 04:36:13 + Duration 576ms (transform 173ms, setup 17ms, collect 299ms, tests 113ms, environment 0ms, prepare 41ms) exit=1 @@ -163,8 +161,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t Test Files 1 passed (1) Tests 1 passed | 26 skipped (27) - Start at 04:25:36 - Duration 617ms (transform 181ms, setup 21ms, collect 314ms, tests 106ms, environment 0ms, prepare 54ms) + Start at 04:36:14 + Duration 556ms (transform 168ms, setup 16ms, collect 293ms, tests 106ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -185,14 +183,14 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 1012ms - × reconnects the reader after it disconnects within the read budget 1011ms - → journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 1009ms + × reconnects the reader after it disconnects within the read budget 1008ms + → journal client: run.get read session was interrupted after 74 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 74 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:71:9 69| let last: unknown; 70| const exhausted = () => last instanceof JournalReadInterruptedError @@ -215,8 +213,8 @@ Caused by: Error: journal client: connection closed Test Files 1 failed (1) Tests 1 failed | 26 skipped (27) - Start at 04:25:37 - Duration 1.61s (transform 207ms, setup 25ms, collect 367ms, tests 1.01s, environment 0ms, prepare 59ms) + Start at 04:36:15 + Duration 1.48s (transform 172ms, setup 18ms, collect 300ms, tests 1.01s, environment 0ms, prepare 42ms) exit=1 @@ -225,12 +223,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 97ms + ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 85ms Test Files 1 passed (1) Tests 1 passed | 26 skipped (27) - Start at 04:25:39 - Duration 647ms (transform 202ms, setup 15ms, collect 372ms, tests 97ms, environment 0ms, prepare 48ms) + Start at 04:36:17 + Duration 520ms (transform 161ms, setup 17ms, collect 279ms, tests 85ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -256,8 +254,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 82ms - × retries a reader setup that timed out instead of reading on the primary for good 81ms + ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 71ms + × retries a reader setup that timed out instead of reading on the primary for good 70ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -283,8 +281,8 @@ AssertionError: expected 1 to be 2 // Object.is equality Test Files 1 failed (1) Tests 1 failed | 26 skipped (27) - Start at 04:25:40 - Duration 649ms (transform 210ms, setup 17ms, collect 359ms, tests 82ms, environment 0ms, prepare 60ms) + Start at 04:36:18 + Duration 539ms (transform 179ms, setup 16ms, collect 306ms, tests 71ms, environment 0ms, prepare 41ms) exit=1 @@ -293,12 +291,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 74ms + ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 70ms Test Files 1 passed (1) Tests 1 passed | 26 skipped (27) - Start at 04:25:41 - Duration 617ms (transform 199ms, setup 19ms, collect 348ms, tests 74ms, environment 0ms, prepare 45ms) + Start at 04:36:19 + Duration 518ms (transform 167ms, setup 17ms, collect 287ms, tests 70ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -307,7 +305,7 @@ exit=0 ```text Source: packages/sdk/src/cli/running-step.ts -Original SHA256: 2d95d1b21da7acb5e91e79236cb099604beabaed7e1e0443c96aeccf4c704c3e +Original SHA256: 251ef48cb8616a41911ba353fd9ea7d9aae299049b941c87ef9927e9429cd1d8 Replaced: const LEASE_POLL_MS = 2_000; With: @@ -318,7 +316,7 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2111ms + ❯ tests/running-step-watch.test.ts (13 tests | 1 failed | 12 skipped) 2111ms × uses pushes for completion with lease-cadence reads and releases its watcher 2110ms → expected 41 to be less than or equal to 1 @@ -337,24 +335,24 @@ AssertionError: expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 11 skipped (12) - Start at 04:25:42 - Duration 2.68s (transform 222ms, setup 18ms, collect 385ms, tests 2.11s, environment 0ms, prepare 45ms) + Tests 1 failed | 12 skipped (13) + Start at 04:36:20 + Duration 2.54s (transform 182ms, setup 13ms, collect 307ms, tests 2.11s, environment 0ms, prepare 29ms) exit=1 -RESTORED SHA256: 2d95d1b21da7acb5e91e79236cb099604beabaed7e1e0443c96aeccf4c704c3e +RESTORED SHA256: 251ef48cb8616a41911ba353fd9ea7d9aae299049b941c87ef9927e9429cd1d8 $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pushes for completion' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2118ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms + ✓ tests/running-step-watch.test.ts (13 tests | 12 skipped) 2116ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2115ms Test Files 1 passed (1) - Tests 1 passed | 11 skipped (12) - Start at 04:25:46 - Duration 2.58s (transform 191ms, setup 15ms, collect 319ms, tests 2.12s, environment 0ms, prepare 30ms) + Tests 1 passed | 12 skipped (13) + Start at 04:36:23 + Duration 2.59s (transform 191ms, setup 17ms, collect 326ms, tests 2.12s, environment 0ms, prepare 41ms) exit=0 ``` @@ -363,7 +361,7 @@ exit=0 ```text Source: packages/sdk/src/cli/running-step.ts -Original SHA256: 2d95d1b21da7acb5e91e79236cb099604beabaed7e1e0443c96aeccf4c704c3e +Original SHA256: 251ef48cb8616a41911ba353fd9ea7d9aae299049b941c87ef9927e9429cd1d8 Replaced: const signal = options.signal === undefined ? read.signal : AbortSignal.any([options.signal, read.signal]); @@ -375,8 +373,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 2213ms - × cancels promptly while a lease snapshot read is in flight 2212ms + ❯ tests/running-step-watch.test.ts (13 tests | 1 failed | 12 skipped) 2217ms + × cancels promptly while a lease snapshot read is in flight 2216ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -394,24 +392,24 @@ AssertionError: expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 11 skipped (12) - Start at 04:25:49 - Duration 2.69s (transform 192ms, setup 18ms, collect 318ms, tests 2.21s, environment 0ms, prepare 43ms) + Tests 1 failed | 12 skipped (13) + Start at 04:36:25 + Duration 2.70s (transform 189ms, setup 17ms, collect 322ms, tests 2.22s, environment 0ms, prepare 41ms) exit=1 -RESTORED SHA256: 2d95d1b21da7acb5e91e79236cb099604beabaed7e1e0443c96aeccf4c704c3e +RESTORED SHA256: 251ef48cb8616a41911ba353fd9ea7d9aae299049b941c87ef9927e9429cd1d8 $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels promptly while a lease snapshot read is in flight' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2027ms - ✓ cancels promptly while a lease snapshot read is in flight 2025ms + ✓ tests/running-step-watch.test.ts (13 tests | 12 skipped) 2011ms + ✓ cancels promptly while a lease snapshot read is in flight 2011ms Test Files 1 passed (1) - Tests 1 passed | 11 skipped (12) - Start at 04:25:52 - Duration 2.67s (transform 245ms, setup 38ms, collect 374ms, tests 2.03s, environment 0ms, prepare 87ms) + Tests 1 passed | 12 skipped (13) + Start at 04:36:28 + Duration 2.46s (transform 188ms, setup 16ms, collect 307ms, tests 2.01s, environment 0ms, prepare 33ms) exit=0 ``` @@ -420,7 +418,7 @@ exit=0 ```text Source: packages/sdk/src/cli/running-step.ts -Original SHA256: 2d95d1b21da7acb5e91e79236cb099604beabaed7e1e0443c96aeccf4c704c3e +Original SHA256: 251ef48cb8616a41911ba353fd9ea7d9aae299049b941c87ef9927e9429cd1d8 Replaced: snapshotRead?.abort(new Error(`step "${runningStep.id}" completed during the lease snapshot`)); With: @@ -431,8 +429,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (12 tests | 1 failed | 11 skipped) 3013ms - × a completion push ends the wait without waiting out an in-flight snapshot 3012ms + ❯ tests/running-step-watch.test.ts (13 tests | 1 failed | 12 skipped) 3013ms + × a completion push ends the wait without waiting out an in-flight snapshot 3013ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -454,24 +452,24 @@ Received: "still waiting" ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 11 skipped (12) - Start at 04:25:55 - Duration 4.12s (transform 456ms, setup 59ms, collect 722ms, tests 3.01s, environment 0ms, prepare 131ms) + Tests 1 failed | 12 skipped (13) + Start at 04:36:31 + Duration 3.50s (transform 190ms, setup 17ms, collect 324ms, tests 3.01s, environment 0ms, prepare 41ms) exit=1 -RESTORED SHA256: 2d95d1b21da7acb5e91e79236cb099604beabaed7e1e0443c96aeccf4c704c3e +RESTORED SHA256: 251ef48cb8616a41911ba353fd9ea7d9aae299049b941c87ef9927e9429cd1d8 $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a completion push ends the wait' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (12 tests | 11 skipped) 2062ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2061ms + ✓ tests/running-step-watch.test.ts (13 tests | 12 skipped) 2063ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2062ms Test Files 1 passed (1) - Tests 1 passed | 11 skipped (12) - Start at 04:26:00 - Duration 2.62s (transform 216ms, setup 20ms, collect 365ms, tests 2.06s, environment 0ms, prepare 49ms) + Tests 1 passed | 12 skipped (13) + Start at 04:36:35 + Duration 2.52s (transform 182ms, setup 17ms, collect 300ms, tests 2.06s, environment 0ms, prepare 40ms) exit=0 ``` @@ -494,8 +492,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 18ms - × a heartbeat timeout is lease loss rather than a worker body failure 17ms + ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 21ms + × a heartbeat timeout is lease loss rather than a worker body failure 20ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -514,8 +512,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 04:26:03 - Duration 485ms (transform 173ms, setup 19ms, collect 295ms, tests 18ms, environment 0ms, prepare 45ms) + Start at 04:36:38 + Duration 475ms (transform 173ms, setup 21ms, collect 290ms, tests 21ms, environment 0ms, prepare 41ms) exit=1 @@ -524,12 +522,12 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/heartbeat-timeout.test.ts (1 test) 18ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 04:26:04 - Duration 500ms (transform 177ms, setup 18ms, collect 315ms, tests 18ms, environment 0ms, prepare 43ms) + Start at 04:36:39 + Duration 470ms (transform 168ms, setup 18ms, collect 290ms, tests 16ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -556,8 +554,8 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (34 tests | 3 failed | 31 skipped) 32ms - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 23ms + ❯ tests/authored-root.test.ts (34 tests | 3 failed | 31 skipped) 30ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 22ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms @@ -643,8 +641,8 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } Test Files 1 failed (1) Tests 3 failed | 31 skipped (34) - Start at 04:26:04 - Duration 1.29s (transform 655ms, setup 16ms, collect 1.09s, tests 32ms, environment 0ms, prepare 41ms) + Start at 04:36:40 + Duration 1.29s (transform 657ms, setup 17ms, collect 1.09s, tests 30ms, environment 0ms, prepare 41ms) exit=1 @@ -653,12 +651,12 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (34 tests | 31 skipped) 25ms + ✓ tests/authored-root.test.ts (34 tests | 31 skipped) 23ms Test Files 1 passed (1) Tests 3 passed | 31 skipped (34) - Start at 04:26:06 - Duration 1.28s (transform 654ms, setup 16ms, collect 1.09s, tests 25ms, environment 0ms, prepare 41ms) + Start at 04:36:41 + Duration 1.30s (transform 673ms, setup 16ms, collect 1.11s, tests 23ms, environment 0ms, prepare 42ms) exit=0 ``` diff --git a/packages/sdk/src/cli/running-step.ts b/packages/sdk/src/cli/running-step.ts index 924ed2065..0263b3056 100644 --- a/packages/sdk/src/cli/running-step.ts +++ b/packages/sdk/src/cli/running-step.ts @@ -72,6 +72,9 @@ export async function waitForRunningStep(client: JournalClient, runId: string, // read interruption: the step may still be running and the run resumable. await Promise.race([watch.connect().then(() => watch.hello('flows-step-watch')) .then(() => watch.runWatch(runId)).catch(error => { + // The replay precedes the response: a completion already folded from it + // is authoritative even if registration never answered. + if (settled) return; // Refusals and malformed frames are protocol facts, not lost transport: fail closed. if (error instanceof JournalProtocolError || error instanceof JournalFrameError) throw error; throw new AuthoredFlowExecutionError('daemon_unresponsive', diff --git a/packages/sdk/tests/running-step-watch.test.ts b/packages/sdk/tests/running-step-watch.test.ts index 3b533bb6e..5bb4da74e 100644 --- a/packages/sdk/tests/running-step-watch.test.ts +++ b/packages/sdk/tests/running-step-watch.test.ts @@ -270,3 +270,18 @@ it('a watch dropped during an in-flight snapshot is reported at once, not after expect((outcome as AuthoredFlowExecutionError).code).toBe('daemon_unresponsive'); } finally { client.close(); await new Promise(resolve => server.close(() => resolve())); } }); + +it('honours a replayed completion even if the watch drops before registration answers', async () => { + const path = sockPath(); + const server = startLoopback(path, { hello: sendOk, 'run.watch': (ctx, params) => { + ctx.send({ event: 'entry', data: { run_id: String(params['run_id']), step_id: 'step', entry_type: 'step.completed' } }); + setTimeout(() => ctx.socket.destroy(), 10); // replay delivered, response never sent + } }); + await once(server, 'listening'); + const client = new JournalClient(path); + try { + await client.connect(); + await expect(waitForRunningStep(client, 'run', { id: 'step', type: 'agent', leaseDeadlineMs: Date.now() + 30_000 }, {})) + .resolves.toBeUndefined(); + } finally { client.close(); await new Promise(resolve => server.close(() => resolve())); } +}); From 324aaf3c5856930d863f0dd3a65ee770dfe27149 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 04:49:22 -0700 Subject: [PATCH 28/34] refactor(sdk): authored-root readback policy in its own module; narrower definite answers - authored-root-readback.ts (96 lines) now owns reading a root back from its journal and what a failed read means; authored-root.ts goes from 687 to 611 lines (main: 598). - Root detection propagates daemon-side read failures (journal_write_failed, internal) as well as interruptions and cancellations; only a refusal of the read or a journal without a root step means 'not authored' (resume from a later epoch still works). - A completed root's readback is result_unreadable for any failure but a malformed frame or a journal with no stored result, protocol errors such as journal_write_failed included. - A completion folded from a replay that may have been truncated by a dropped watch is confirmed with one snapshot before the wait returns. Evidence re-captured at this tree. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: b8262468-ea26-4a6e-bee9-d193832e77dc --- evidence/run-read-timeout/final-focused.log | 112 ++++---- .../mutation-completion-aborts-snapshot.log | 24 +- .../run-read-timeout/mutation-heartbeat.log | 12 +- .../mutation-reader-reconnect.log | 14 +- .../mutation-reader-setup-retry.log | 14 +- evidence/run-read-timeout/mutation-reader.log | 10 +- .../mutation-root-parking.log | 22 +- .../mutation-snapshot-cancel.log | 22 +- .../mutation-watch-cadence.log | 22 +- evidence/run-read-timeout/resume-live.log | 8 +- evidence/run-read-timeout/verification.md | 260 +++++++++--------- packages/sdk/src/authored-root-readback.ts | 100 +++++++ packages/sdk/src/authored-root.ts | 82 +----- packages/sdk/src/cli/running-step.ts | 13 +- packages/sdk/tests/authored-root.test.ts | 15 + packages/sdk/tests/running-step-watch.test.ts | 25 +- 16 files changed, 412 insertions(+), 343 deletions(-) create mode 100644 packages/sdk/src/authored-root-readback.ts diff --git a/evidence/run-read-timeout/final-focused.log b/evidence/run-read-timeout/final-focused.log index 699c502ad..2a457140f 100644 --- a/evidence/run-read-timeout/final-focused.log +++ b/evidence/run-read-timeout/final-focused.log @@ -2,71 +2,71 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5303ms - ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 315ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2036ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2046ms -(node:195226) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5339ms + ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 325ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2037ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2048ms +(node:244392) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (34 tests) 449ms - ✓ tests/journal-client.test.ts (17 tests) 79ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 7114ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 492ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1046ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2799ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 388ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 884ms - ✓ tests/journal-client-read-timeout.test.ts (27 tests) 1794ms - ✓ tests/authored-flow-slack.test.ts (7 tests) 1164ms - ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 463ms - ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 335ms - ✓ tests/running-step-watch.test.ts (13 tests) 12302ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms - ✓ cancels promptly while a lease snapshot read is in flight 2006ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2055ms + ✓ tests/authored-root.test.ts (36 tests) 417ms + ✓ tests/journal-client.test.ts (17 tests) 84ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 7509ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 434ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1078ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2951ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 419ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 977ms + ✓ tests/journal-client-read-timeout.test.ts (27 tests) 1805ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 1152ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 419ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 393ms + ✓ tests/running-step-watch.test.ts (14 tests) 12317ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms + ✓ cancels promptly while a lease snapshot read is in flight 2005ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2054ms ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2003ms ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2003ms - ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2002ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2534ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 588ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 645ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 646ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 654ms - ✓ tests/authored-helpers.test.ts (6 tests) 1973ms - ✓ runs every available provider through the real kernel and resumes completed effects without a second write 898ms - ✓ replays after SIGKILL before confirm with the same token and one successful completion 440ms - ✓ replays after SIGKILL before complete with the same token and one successful completion 433ms - ✓ tests/classify-outcome.test.ts (11 tests) 7426ms + ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2001ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2448ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 654ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 590ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 561ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 642ms + ✓ tests/authored-helpers.test.ts (6 tests) 2538ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 1067ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 663ms + ✓ replays after SIGKILL before complete with the same token and one successful completion 594ms + ✓ tests/classify-outcome.test.ts (11 tests) 7423ms ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2010ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3005ms - ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2002ms - ✓ tests/human-live.test.ts (3 tests) 4919ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2949ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1157ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 812ms - ✓ tests/worker-lease.test.ts (7 tests) 12ms - ✓ tests/communication-worker.test.ts (18 tests) 1716ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 20ms - ✓ tests/effect-channel.test.ts (5 tests) 279ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 699ms - ✓ tests/memoization.test.ts (58 tests) 62ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2485ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2057ms - ✓ drains read and watch promises before an authored flow completes 428ms - ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms - ✓ tests/journal-client-completion.test.ts (6 tests) 103ms - ✓ tests/run-daemon-unresponsive.test.ts (4 tests) 5ms - ✓ tests/reuse-summary-interruption.test.ts (5 tests) 4ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3006ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2000ms + ✓ tests/human-live.test.ts (3 tests) 4757ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2824ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1247ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 685ms + ✓ tests/worker-lease.test.ts (7 tests) 8ms + ✓ tests/communication-worker.test.ts (18 tests) 1711ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 15ms + ✓ tests/effect-channel.test.ts (5 tests) 250ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 721ms + ✓ tests/memoization.test.ts (58 tests) 65ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2453ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2059ms + ✓ drains read and watch promises before an authored flow completes 393ms + ✓ tests/worker-lease-sweep.test.ts (4 tests) 9ms + ✓ tests/journal-client-completion.test.ts (6 tests) 105ms + ✓ tests/run-daemon-unresponsive.test.ts (4 tests) 6ms + ✓ tests/reuse-summary-interruption.test.ts (5 tests) 3ms ✓ tests/resume-worker-lease.test.ts (3 tests) 8ms ✓ tests/direct-run-worker-lease.test.ts (3 tests) 10ms - ✓ tests/helper-effect-lease-loss.test.ts (1 test) 186ms + ✓ tests/helper-effect-lease-loss.test.ts (1 test) 225ms ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms - ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 48ms + ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 34ms ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms Test Files 28 passed (28) - Tests 347 passed (347) - Start at 04:36:46 - Duration 60.99s (transform 1.13s, setup 157ms, collect 6.78s, tests 50.73s, environment 4ms, prepare 992ms) + Tests 350 passed (350) + Start at 04:48:18 + Duration 61.95s (transform 1.12s, setup 159ms, collect 6.95s, tests 51.44s, environment 4ms, prepare 994ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log index c7516b2d4..9ec27620c 100644 --- a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log +++ b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/cli/running-step.ts -Original SHA256: 251ef48cb8616a41911ba353fd9ea7d9aae299049b941c87ef9927e9429cd1d8 +Original SHA256: 2a5f6adeca146e6188ac5e6d129db004e71574cf09df397fd4b9127c50e9b5aa Replaced: snapshotRead?.abort(new Error(`step "${runningStep.id}" completed during the lease snapshot`)); With: @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (13 tests | 1 failed | 12 skipped) 3013ms - × a completion push ends the wait without waiting out an in-flight snapshot 3013ms + ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 3013ms + × a completion push ends the wait without waiting out an in-flight snapshot 3012ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -33,23 +33,23 @@ Received: "still waiting" ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 12 skipped (13) - Start at 04:36:31 - Duration 3.50s (transform 190ms, setup 17ms, collect 324ms, tests 3.01s, environment 0ms, prepare 41ms) + Tests 1 failed | 13 skipped (14) + Start at 04:48:03 + Duration 3.50s (transform 189ms, setup 16ms, collect 333ms, tests 3.01s, environment 0ms, prepare 42ms) exit=1 -RESTORED SHA256: 251ef48cb8616a41911ba353fd9ea7d9aae299049b941c87ef9927e9429cd1d8 +RESTORED SHA256: 2a5f6adeca146e6188ac5e6d129db004e71574cf09df397fd4b9127c50e9b5aa $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a completion push ends the wait' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (13 tests | 12 skipped) 2063ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2062ms + ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2058ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2058ms Test Files 1 passed (1) - Tests 1 passed | 12 skipped (13) - Start at 04:36:35 - Duration 2.52s (transform 182ms, setup 17ms, collect 300ms, tests 2.06s, environment 0ms, prepare 40ms) + Tests 1 passed | 13 skipped (14) + Start at 04:48:07 + Duration 2.58s (transform 201ms, setup 18ms, collect 347ms, tests 2.06s, environment 0ms, prepare 47ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-heartbeat.log b/evidence/run-read-timeout/mutation-heartbeat.log index 5d45bdc62..b2c05b961 100644 --- a/evidence/run-read-timeout/mutation-heartbeat.log +++ b/evidence/run-read-timeout/mutation-heartbeat.log @@ -13,8 +13,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 21ms - × a heartbeat timeout is lease loss rather than a worker body failure 20ms + ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 19ms + × a heartbeat timeout is lease loss rather than a worker body failure 18ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -33,8 +33,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 04:36:38 - Duration 475ms (transform 173ms, setup 21ms, collect 290ms, tests 21ms, environment 0ms, prepare 41ms) + Start at 04:48:10 + Duration 464ms (transform 163ms, setup 17ms, collect 280ms, tests 19ms, environment 0ms, prepare 44ms) exit=1 @@ -47,7 +47,7 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb Test Files 1 passed (1) Tests 1 passed (1) - Start at 04:36:39 - Duration 470ms (transform 168ms, setup 18ms, collect 290ms, tests 16ms, environment 0ms, prepare 41ms) + Start at 04:48:11 + Duration 450ms (transform 165ms, setup 16ms, collect 288ms, tests 16ms, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-reconnect.log b/evidence/run-read-timeout/mutation-reader-reconnect.log index 4cfadf1e2..555835c68 100644 --- a/evidence/run-read-timeout/mutation-reader-reconnect.log +++ b/evidence/run-read-timeout/mutation-reader-reconnect.log @@ -13,12 +13,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 1009ms × reconnects the reader after it disconnects within the read budget 1008ms - → journal client: run.get read session was interrupted after 74 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + → journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 74 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:71:9 69| let last: unknown; 70| const exhausted = () => last instanceof JournalReadInterruptedError @@ -41,8 +41,8 @@ Caused by: Error: journal client: connection closed Test Files 1 failed (1) Tests 1 failed | 26 skipped (27) - Start at 04:36:15 - Duration 1.48s (transform 172ms, setup 18ms, collect 300ms, tests 1.01s, environment 0ms, prepare 42ms) + Start at 04:47:47 + Duration 1.47s (transform 179ms, setup 17ms, collect 308ms, tests 1.01s, environment 0ms, prepare 41ms) exit=1 @@ -51,11 +51,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 85ms + ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 73ms Test Files 1 passed (1) Tests 1 passed | 26 skipped (27) - Start at 04:36:17 - Duration 520ms (transform 161ms, setup 17ms, collect 279ms, tests 85ms, environment 0ms, prepare 41ms) + Start at 04:47:49 + Duration 534ms (transform 169ms, setup 16ms, collect 299ms, tests 73ms, environment 0ms, prepare 45ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-setup-retry.log b/evidence/run-read-timeout/mutation-reader-setup-retry.log index 9f8cae962..01d7d977e 100644 --- a/evidence/run-read-timeout/mutation-reader-setup-retry.log +++ b/evidence/run-read-timeout/mutation-reader-setup-retry.log @@ -16,8 +16,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 71ms - × retries a reader setup that timed out instead of reading on the primary for good 70ms + ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 78ms + × retries a reader setup that timed out instead of reading on the primary for good 77ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -43,8 +43,8 @@ AssertionError: expected 1 to be 2 // Object.is equality Test Files 1 failed (1) Tests 1 failed | 26 skipped (27) - Start at 04:36:18 - Duration 539ms (transform 179ms, setup 16ms, collect 306ms, tests 71ms, environment 0ms, prepare 41ms) + Start at 04:47:50 + Duration 526ms (transform 168ms, setup 16ms, collect 299ms, tests 78ms, environment 0ms, prepare 41ms) exit=1 @@ -53,11 +53,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 70ms + ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 74ms Test Files 1 passed (1) Tests 1 passed | 26 skipped (27) - Start at 04:36:19 - Duration 518ms (transform 167ms, setup 17ms, collect 287ms, tests 70ms, environment 0ms, prepare 41ms) + Start at 04:47:50 + Duration 526ms (transform 167ms, setup 16ms, collect 290ms, tests 74ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader.log b/evidence/run-read-timeout/mutation-reader.log index 2c038e211..fc126bd0d 100644 --- a/evidence/run-read-timeout/mutation-reader.log +++ b/evidence/run-read-timeout/mutation-reader.log @@ -37,8 +37,8 @@ AssertionError: expected true to be false // Object.is equality Test Files 1 failed (1) Tests 1 failed | 26 skipped (27) - Start at 04:36:13 - Duration 576ms (transform 173ms, setup 17ms, collect 299ms, tests 113ms, environment 0ms, prepare 41ms) + Start at 04:47:45 + Duration 560ms (transform 167ms, setup 16ms, collect 283ms, tests 113ms, environment 0ms, prepare 41ms) exit=1 @@ -47,11 +47,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 106ms + ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 104ms Test Files 1 passed (1) Tests 1 passed | 26 skipped (27) - Start at 04:36:14 - Duration 556ms (transform 168ms, setup 16ms, collect 293ms, tests 106ms, environment 0ms, prepare 41ms) + Start at 04:47:46 + Duration 538ms (transform 165ms, setup 17ms, collect 273ms, tests 104ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-root-parking.log b/evidence/run-read-timeout/mutation-root-parking.log index 2c5260f85..c95eff99b 100644 --- a/evidence/run-read-timeout/mutation-root-parking.log +++ b/evidence/run-read-timeout/mutation-root-parking.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/authored-root.ts -Original SHA256: 9adbef378df4b191653880ac6bd08cb526ff96b9b7a29c36d54a0df3301acb0b +Original SHA256: 8e3bd4ebe8b470c1486f1ab6adcf2fd81ba19e6a173e2531e7cbc59b16bcba29 Replaced: if (isReadInterruptionError(error) || (error instanceof AuthoredFlowExecutionError && error.code === 'daemon_unresponsive')) { @@ -17,8 +17,8 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (34 tests | 3 failed | 31 skipped) 30ms - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 22ms + ❯ tests/authored-root.test.ts (36 tests | 3 failed | 33 skipped) 32ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 23ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms @@ -103,22 +103,22 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[3/3]⎯ Test Files 1 failed (1) - Tests 3 failed | 31 skipped (34) - Start at 04:36:40 - Duration 1.29s (transform 657ms, setup 17ms, collect 1.09s, tests 30ms, environment 0ms, prepare 41ms) + Tests 3 failed | 33 skipped (36) + Start at 04:48:12 + Duration 1.31s (transform 678ms, setup 17ms, collect 1.12s, tests 32ms, environment 0ms, prepare 44ms) exit=1 -RESTORED SHA256: 9adbef378df4b191653880ac6bd08cb526ff96b9b7a29c36d54a0df3301acb0b +RESTORED SHA256: 8e3bd4ebe8b470c1486f1ab6adcf2fd81ba19e6a173e2531e7cbc59b16bcba29 $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the root resumable' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (34 tests | 31 skipped) 23ms + ✓ tests/authored-root.test.ts (36 tests | 33 skipped) 18ms Test Files 1 passed (1) - Tests 3 passed | 31 skipped (34) - Start at 04:36:41 - Duration 1.30s (transform 673ms, setup 16ms, collect 1.11s, tests 23ms, environment 0ms, prepare 42ms) + Tests 3 passed | 33 skipped (36) + Start at 04:48:13 + Duration 1.29s (transform 675ms, setup 16ms, collect 1.11s, tests 18ms, environment 0ms, prepare 42ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-snapshot-cancel.log b/evidence/run-read-timeout/mutation-snapshot-cancel.log index 3f1dbb09c..ebdde6d03 100644 --- a/evidence/run-read-timeout/mutation-snapshot-cancel.log +++ b/evidence/run-read-timeout/mutation-snapshot-cancel.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/cli/running-step.ts -Original SHA256: 251ef48cb8616a41911ba353fd9ea7d9aae299049b941c87ef9927e9429cd1d8 +Original SHA256: 2a5f6adeca146e6188ac5e6d129db004e71574cf09df397fd4b9127c50e9b5aa Replaced: const signal = options.signal === undefined ? read.signal : AbortSignal.any([options.signal, read.signal]); @@ -11,8 +11,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (13 tests | 1 failed | 12 skipped) 2217ms - × cancels promptly while a lease snapshot read is in flight 2216ms + ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2215ms + × cancels promptly while a lease snapshot read is in flight 2215ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -30,23 +30,23 @@ AssertionError: expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 12 skipped (13) - Start at 04:36:25 - Duration 2.70s (transform 189ms, setup 17ms, collect 322ms, tests 2.22s, environment 0ms, prepare 41ms) + Tests 1 failed | 13 skipped (14) + Start at 04:47:57 + Duration 2.68s (transform 189ms, setup 15ms, collect 317ms, tests 2.22s, environment 0ms, prepare 41ms) exit=1 -RESTORED SHA256: 251ef48cb8616a41911ba353fd9ea7d9aae299049b941c87ef9927e9429cd1d8 +RESTORED SHA256: 2a5f6adeca146e6188ac5e6d129db004e71574cf09df397fd4b9127c50e9b5aa $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels promptly while a lease snapshot read is in flight' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (13 tests | 12 skipped) 2011ms + ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2011ms ✓ cancels promptly while a lease snapshot read is in flight 2011ms Test Files 1 passed (1) - Tests 1 passed | 12 skipped (13) - Start at 04:36:28 - Duration 2.46s (transform 188ms, setup 16ms, collect 307ms, tests 2.01s, environment 0ms, prepare 33ms) + Tests 1 passed | 13 skipped (14) + Start at 04:48:00 + Duration 2.48s (transform 182ms, setup 15ms, collect 317ms, tests 2.01s, environment 0ms, prepare 29ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-watch-cadence.log b/evidence/run-read-timeout/mutation-watch-cadence.log index 14350fe6d..2f33a6c98 100644 --- a/evidence/run-read-timeout/mutation-watch-cadence.log +++ b/evidence/run-read-timeout/mutation-watch-cadence.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/cli/running-step.ts -Original SHA256: 251ef48cb8616a41911ba353fd9ea7d9aae299049b941c87ef9927e9429cd1d8 +Original SHA256: 2a5f6adeca146e6188ac5e6d129db004e71574cf09df397fd4b9127c50e9b5aa Replaced: const LEASE_POLL_MS = 2_000; With: @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (13 tests | 1 failed | 12 skipped) 2111ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2110ms + ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2109ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2108ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -29,23 +29,23 @@ AssertionError: expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 12 skipped (13) - Start at 04:36:20 - Duration 2.54s (transform 182ms, setup 13ms, collect 307ms, tests 2.11s, environment 0ms, prepare 29ms) + Tests 1 failed | 13 skipped (14) + Start at 04:47:51 + Duration 2.57s (transform 182ms, setup 17ms, collect 306ms, tests 2.11s, environment 0ms, prepare 42ms) exit=1 -RESTORED SHA256: 251ef48cb8616a41911ba353fd9ea7d9aae299049b941c87ef9927e9429cd1d8 +RESTORED SHA256: 2a5f6adeca146e6188ac5e6d129db004e71574cf09df397fd4b9127c50e9b5aa $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pushes for completion' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (13 tests | 12 skipped) 2116ms + ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2115ms ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2115ms Test Files 1 passed (1) - Tests 1 passed | 12 skipped (13) - Start at 04:36:23 - Duration 2.59s (transform 191ms, setup 17ms, collect 326ms, tests 2.12s, environment 0ms, prepare 41ms) + Tests 1 passed | 13 skipped (14) + Start at 04:47:54 + Duration 2.59s (transform 185ms, setup 17ms, collect 319ms, tests 2.12s, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/resume-live.log b/evidence/run-read-timeout/resume-live.log index 940e0c5ce..220511f39 100644 --- a/evidence/run-read-timeout/resume-live.log +++ b/evidence/run-read-timeout/resume-live.log @@ -2,12 +2,12 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 929ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 928ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 911ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 911ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 04:37:47 - Duration 2.20s (transform 614ms, setup 16ms, collect 1.01s, tests 929ms, environment 0ms, prepare 62ms) + Start at 04:49:20 + Duration 2.15s (transform 618ms, setup 20ms, collect 1.01s, tests 911ms, environment 0ms, prepare 90ms) exit=0 diff --git a/evidence/run-read-timeout/verification.md b/evidence/run-read-timeout/verification.md index 2062ebe2b..a6b6a4685 100644 --- a/evidence/run-read-timeout/verification.md +++ b/evidence/run-read-timeout/verification.md @@ -15,72 +15,72 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5303ms - ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 315ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2036ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2046ms -(node:195226) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5339ms + ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 325ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2037ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2048ms +(node:244392) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (34 tests) 449ms - ✓ tests/journal-client.test.ts (17 tests) 79ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 7114ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 492ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1046ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2799ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 388ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 884ms - ✓ tests/journal-client-read-timeout.test.ts (27 tests) 1794ms - ✓ tests/authored-flow-slack.test.ts (7 tests) 1164ms - ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 463ms - ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 335ms - ✓ tests/running-step-watch.test.ts (13 tests) 12302ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms - ✓ cancels promptly while a lease snapshot read is in flight 2006ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2055ms + ✓ tests/authored-root.test.ts (36 tests) 417ms + ✓ tests/journal-client.test.ts (17 tests) 84ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 7509ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 434ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1078ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2951ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 419ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 977ms + ✓ tests/journal-client-read-timeout.test.ts (27 tests) 1805ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 1152ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 419ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 393ms + ✓ tests/running-step-watch.test.ts (14 tests) 12317ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms + ✓ cancels promptly while a lease snapshot read is in flight 2005ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2054ms ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2003ms ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2003ms - ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2002ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2534ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 588ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 645ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 646ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 654ms - ✓ tests/authored-helpers.test.ts (6 tests) 1973ms - ✓ runs every available provider through the real kernel and resumes completed effects without a second write 898ms - ✓ replays after SIGKILL before confirm with the same token and one successful completion 440ms - ✓ replays after SIGKILL before complete with the same token and one successful completion 433ms - ✓ tests/classify-outcome.test.ts (11 tests) 7426ms + ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2001ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2448ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 654ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 590ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 561ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 642ms + ✓ tests/authored-helpers.test.ts (6 tests) 2538ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 1067ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 663ms + ✓ replays after SIGKILL before complete with the same token and one successful completion 594ms + ✓ tests/classify-outcome.test.ts (11 tests) 7423ms ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2010ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3005ms - ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2002ms - ✓ tests/human-live.test.ts (3 tests) 4919ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2949ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1157ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 812ms - ✓ tests/worker-lease.test.ts (7 tests) 12ms - ✓ tests/communication-worker.test.ts (18 tests) 1716ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 20ms - ✓ tests/effect-channel.test.ts (5 tests) 279ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 699ms - ✓ tests/memoization.test.ts (58 tests) 62ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2485ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2057ms - ✓ drains read and watch promises before an authored flow completes 428ms - ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms - ✓ tests/journal-client-completion.test.ts (6 tests) 103ms - ✓ tests/run-daemon-unresponsive.test.ts (4 tests) 5ms - ✓ tests/reuse-summary-interruption.test.ts (5 tests) 4ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3006ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2000ms + ✓ tests/human-live.test.ts (3 tests) 4757ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2824ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1247ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 685ms + ✓ tests/worker-lease.test.ts (7 tests) 8ms + ✓ tests/communication-worker.test.ts (18 tests) 1711ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 15ms + ✓ tests/effect-channel.test.ts (5 tests) 250ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 721ms + ✓ tests/memoization.test.ts (58 tests) 65ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2453ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2059ms + ✓ drains read and watch promises before an authored flow completes 393ms + ✓ tests/worker-lease-sweep.test.ts (4 tests) 9ms + ✓ tests/journal-client-completion.test.ts (6 tests) 105ms + ✓ tests/run-daemon-unresponsive.test.ts (4 tests) 6ms + ✓ tests/reuse-summary-interruption.test.ts (5 tests) 3ms ✓ tests/resume-worker-lease.test.ts (3 tests) 8ms ✓ tests/direct-run-worker-lease.test.ts (3 tests) 10ms - ✓ tests/helper-effect-lease-loss.test.ts (1 test) 186ms + ✓ tests/helper-effect-lease-loss.test.ts (1 test) 225ms ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms - ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 48ms + ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 34ms ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms Test Files 28 passed (28) - Tests 347 passed (347) - Start at 04:36:46 - Duration 60.99s (transform 1.13s, setup 157ms, collect 6.78s, tests 50.73s, environment 4ms, prepare 992ms) + Tests 350 passed (350) + Start at 04:48:18 + Duration 61.95s (transform 1.12s, setup 159ms, collect 6.95s, tests 51.44s, environment 4ms, prepare 994ms) exit=0 ``` @@ -92,13 +92,13 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 929ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 928ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 911ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 911ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 04:37:47 - Duration 2.20s (transform 614ms, setup 16ms, collect 1.01s, tests 929ms, environment 0ms, prepare 62ms) + Start at 04:49:20 + Duration 2.15s (transform 618ms, setup 20ms, collect 1.01s, tests 911ms, environment 0ms, prepare 90ms) exit=0 ``` @@ -147,8 +147,8 @@ AssertionError: expected true to be false // Object.is equality Test Files 1 failed (1) Tests 1 failed | 26 skipped (27) - Start at 04:36:13 - Duration 576ms (transform 173ms, setup 17ms, collect 299ms, tests 113ms, environment 0ms, prepare 41ms) + Start at 04:47:45 + Duration 560ms (transform 167ms, setup 16ms, collect 283ms, tests 113ms, environment 0ms, prepare 41ms) exit=1 @@ -157,12 +157,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 106ms + ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 104ms Test Files 1 passed (1) Tests 1 passed | 26 skipped (27) - Start at 04:36:14 - Duration 556ms (transform 168ms, setup 16ms, collect 293ms, tests 106ms, environment 0ms, prepare 41ms) + Start at 04:47:46 + Duration 538ms (transform 165ms, setup 17ms, collect 273ms, tests 104ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -185,12 +185,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 1009ms × reconnects the reader after it disconnects within the read budget 1008ms - → journal client: run.get read session was interrupted after 74 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + → journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 74 attempts in 1001ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:71:9 69| let last: unknown; 70| const exhausted = () => last instanceof JournalReadInterruptedError @@ -213,8 +213,8 @@ Caused by: Error: journal client: connection closed Test Files 1 failed (1) Tests 1 failed | 26 skipped (27) - Start at 04:36:15 - Duration 1.48s (transform 172ms, setup 18ms, collect 300ms, tests 1.01s, environment 0ms, prepare 42ms) + Start at 04:47:47 + Duration 1.47s (transform 179ms, setup 17ms, collect 308ms, tests 1.01s, environment 0ms, prepare 41ms) exit=1 @@ -223,12 +223,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 85ms + ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 73ms Test Files 1 passed (1) Tests 1 passed | 26 skipped (27) - Start at 04:36:17 - Duration 520ms (transform 161ms, setup 17ms, collect 279ms, tests 85ms, environment 0ms, prepare 41ms) + Start at 04:47:49 + Duration 534ms (transform 169ms, setup 16ms, collect 299ms, tests 73ms, environment 0ms, prepare 45ms) exit=0 ``` @@ -254,8 +254,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 71ms - × retries a reader setup that timed out instead of reading on the primary for good 70ms + ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 78ms + × retries a reader setup that timed out instead of reading on the primary for good 77ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -281,8 +281,8 @@ AssertionError: expected 1 to be 2 // Object.is equality Test Files 1 failed (1) Tests 1 failed | 26 skipped (27) - Start at 04:36:18 - Duration 539ms (transform 179ms, setup 16ms, collect 306ms, tests 71ms, environment 0ms, prepare 41ms) + Start at 04:47:50 + Duration 526ms (transform 168ms, setup 16ms, collect 299ms, tests 78ms, environment 0ms, prepare 41ms) exit=1 @@ -291,12 +291,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 70ms + ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 74ms Test Files 1 passed (1) Tests 1 passed | 26 skipped (27) - Start at 04:36:19 - Duration 518ms (transform 167ms, setup 17ms, collect 287ms, tests 70ms, environment 0ms, prepare 41ms) + Start at 04:47:50 + Duration 526ms (transform 167ms, setup 16ms, collect 290ms, tests 74ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -305,7 +305,7 @@ exit=0 ```text Source: packages/sdk/src/cli/running-step.ts -Original SHA256: 251ef48cb8616a41911ba353fd9ea7d9aae299049b941c87ef9927e9429cd1d8 +Original SHA256: 2a5f6adeca146e6188ac5e6d129db004e71574cf09df397fd4b9127c50e9b5aa Replaced: const LEASE_POLL_MS = 2_000; With: @@ -316,8 +316,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (13 tests | 1 failed | 12 skipped) 2111ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2110ms + ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2109ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2108ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -335,24 +335,24 @@ AssertionError: expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 12 skipped (13) - Start at 04:36:20 - Duration 2.54s (transform 182ms, setup 13ms, collect 307ms, tests 2.11s, environment 0ms, prepare 29ms) + Tests 1 failed | 13 skipped (14) + Start at 04:47:51 + Duration 2.57s (transform 182ms, setup 17ms, collect 306ms, tests 2.11s, environment 0ms, prepare 42ms) exit=1 -RESTORED SHA256: 251ef48cb8616a41911ba353fd9ea7d9aae299049b941c87ef9927e9429cd1d8 +RESTORED SHA256: 2a5f6adeca146e6188ac5e6d129db004e71574cf09df397fd4b9127c50e9b5aa $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pushes for completion' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (13 tests | 12 skipped) 2116ms + ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2115ms ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2115ms Test Files 1 passed (1) - Tests 1 passed | 12 skipped (13) - Start at 04:36:23 - Duration 2.59s (transform 191ms, setup 17ms, collect 326ms, tests 2.12s, environment 0ms, prepare 41ms) + Tests 1 passed | 13 skipped (14) + Start at 04:47:54 + Duration 2.59s (transform 185ms, setup 17ms, collect 319ms, tests 2.12s, environment 0ms, prepare 40ms) exit=0 ``` @@ -361,7 +361,7 @@ exit=0 ```text Source: packages/sdk/src/cli/running-step.ts -Original SHA256: 251ef48cb8616a41911ba353fd9ea7d9aae299049b941c87ef9927e9429cd1d8 +Original SHA256: 2a5f6adeca146e6188ac5e6d129db004e71574cf09df397fd4b9127c50e9b5aa Replaced: const signal = options.signal === undefined ? read.signal : AbortSignal.any([options.signal, read.signal]); @@ -373,8 +373,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (13 tests | 1 failed | 12 skipped) 2217ms - × cancels promptly while a lease snapshot read is in flight 2216ms + ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2215ms + × cancels promptly while a lease snapshot read is in flight 2215ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -392,24 +392,24 @@ AssertionError: expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 12 skipped (13) - Start at 04:36:25 - Duration 2.70s (transform 189ms, setup 17ms, collect 322ms, tests 2.22s, environment 0ms, prepare 41ms) + Tests 1 failed | 13 skipped (14) + Start at 04:47:57 + Duration 2.68s (transform 189ms, setup 15ms, collect 317ms, tests 2.22s, environment 0ms, prepare 41ms) exit=1 -RESTORED SHA256: 251ef48cb8616a41911ba353fd9ea7d9aae299049b941c87ef9927e9429cd1d8 +RESTORED SHA256: 2a5f6adeca146e6188ac5e6d129db004e71574cf09df397fd4b9127c50e9b5aa $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels promptly while a lease snapshot read is in flight' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (13 tests | 12 skipped) 2011ms + ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2011ms ✓ cancels promptly while a lease snapshot read is in flight 2011ms Test Files 1 passed (1) - Tests 1 passed | 12 skipped (13) - Start at 04:36:28 - Duration 2.46s (transform 188ms, setup 16ms, collect 307ms, tests 2.01s, environment 0ms, prepare 33ms) + Tests 1 passed | 13 skipped (14) + Start at 04:48:00 + Duration 2.48s (transform 182ms, setup 15ms, collect 317ms, tests 2.01s, environment 0ms, prepare 29ms) exit=0 ``` @@ -418,7 +418,7 @@ exit=0 ```text Source: packages/sdk/src/cli/running-step.ts -Original SHA256: 251ef48cb8616a41911ba353fd9ea7d9aae299049b941c87ef9927e9429cd1d8 +Original SHA256: 2a5f6adeca146e6188ac5e6d129db004e71574cf09df397fd4b9127c50e9b5aa Replaced: snapshotRead?.abort(new Error(`step "${runningStep.id}" completed during the lease snapshot`)); With: @@ -429,8 +429,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (13 tests | 1 failed | 12 skipped) 3013ms - × a completion push ends the wait without waiting out an in-flight snapshot 3013ms + ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 3013ms + × a completion push ends the wait without waiting out an in-flight snapshot 3012ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -452,24 +452,24 @@ Received: "still waiting" ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 12 skipped (13) - Start at 04:36:31 - Duration 3.50s (transform 190ms, setup 17ms, collect 324ms, tests 3.01s, environment 0ms, prepare 41ms) + Tests 1 failed | 13 skipped (14) + Start at 04:48:03 + Duration 3.50s (transform 189ms, setup 16ms, collect 333ms, tests 3.01s, environment 0ms, prepare 42ms) exit=1 -RESTORED SHA256: 251ef48cb8616a41911ba353fd9ea7d9aae299049b941c87ef9927e9429cd1d8 +RESTORED SHA256: 2a5f6adeca146e6188ac5e6d129db004e71574cf09df397fd4b9127c50e9b5aa $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a completion push ends the wait' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (13 tests | 12 skipped) 2063ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2062ms + ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2058ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2058ms Test Files 1 passed (1) - Tests 1 passed | 12 skipped (13) - Start at 04:36:35 - Duration 2.52s (transform 182ms, setup 17ms, collect 300ms, tests 2.06s, environment 0ms, prepare 40ms) + Tests 1 passed | 13 skipped (14) + Start at 04:48:07 + Duration 2.58s (transform 201ms, setup 18ms, collect 347ms, tests 2.06s, environment 0ms, prepare 47ms) exit=0 ``` @@ -492,8 +492,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 21ms - × a heartbeat timeout is lease loss rather than a worker body failure 20ms + ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 19ms + × a heartbeat timeout is lease loss rather than a worker body failure 18ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -512,8 +512,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 04:36:38 - Duration 475ms (transform 173ms, setup 21ms, collect 290ms, tests 21ms, environment 0ms, prepare 41ms) + Start at 04:48:10 + Duration 464ms (transform 163ms, setup 17ms, collect 280ms, tests 19ms, environment 0ms, prepare 44ms) exit=1 @@ -526,8 +526,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb Test Files 1 passed (1) Tests 1 passed (1) - Start at 04:36:39 - Duration 470ms (transform 168ms, setup 18ms, collect 290ms, tests 16ms, environment 0ms, prepare 41ms) + Start at 04:48:11 + Duration 450ms (transform 165ms, setup 16ms, collect 288ms, tests 16ms, environment 0ms, prepare 40ms) exit=0 ``` @@ -536,7 +536,7 @@ exit=0 ```text Source: packages/sdk/src/authored-root.ts -Original SHA256: 9adbef378df4b191653880ac6bd08cb526ff96b9b7a29c36d54a0df3301acb0b +Original SHA256: 8e3bd4ebe8b470c1486f1ab6adcf2fd81ba19e6a173e2531e7cbc59b16bcba29 Replaced: if (isReadInterruptionError(error) || (error instanceof AuthoredFlowExecutionError && error.code === 'daemon_unresponsive')) { @@ -554,8 +554,8 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (34 tests | 3 failed | 31 skipped) 30ms - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 22ms + ❯ tests/authored-root.test.ts (36 tests | 3 failed | 33 skipped) 32ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 23ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms @@ -640,23 +640,23 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[3/3]⎯ Test Files 1 failed (1) - Tests 3 failed | 31 skipped (34) - Start at 04:36:40 - Duration 1.29s (transform 657ms, setup 17ms, collect 1.09s, tests 30ms, environment 0ms, prepare 41ms) + Tests 3 failed | 33 skipped (36) + Start at 04:48:12 + Duration 1.31s (transform 678ms, setup 17ms, collect 1.12s, tests 32ms, environment 0ms, prepare 44ms) exit=1 -RESTORED SHA256: 9adbef378df4b191653880ac6bd08cb526ff96b9b7a29c36d54a0df3301acb0b +RESTORED SHA256: 8e3bd4ebe8b470c1486f1ab6adcf2fd81ba19e6a173e2531e7cbc59b16bcba29 $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the root resumable' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (34 tests | 31 skipped) 23ms + ✓ tests/authored-root.test.ts (36 tests | 33 skipped) 18ms Test Files 1 passed (1) - Tests 3 passed | 31 skipped (34) - Start at 04:36:41 - Duration 1.30s (transform 673ms, setup 16ms, collect 1.11s, tests 23ms, environment 0ms, prepare 42ms) + Tests 3 passed | 33 skipped (36) + Start at 04:48:13 + Duration 1.29s (transform 675ms, setup 16ms, collect 1.11s, tests 18ms, environment 0ms, prepare 42ms) exit=0 ``` diff --git a/packages/sdk/src/authored-root-readback.ts b/packages/sdk/src/authored-root-readback.ts new file mode 100644 index 000000000..e80d25ee8 --- /dev/null +++ b/packages/sdk/src/authored-root-readback.ts @@ -0,0 +1,100 @@ +import { isDurableCompletionDetail, isLoweredCompletion } from './authored-completion.js'; +import type { AuthoredFlowExecutionResult } from './authored-flow-executor.js'; +import { AuthoredFlowExecutionError } from './authored-flow-error.js'; +import { isSurfaceCompletionReason } from './authored-step-output.js'; +import { JournalFrameError, JournalProtocolError, type JournalClient } from './journal-client.js'; +import type { RunOutcome } from './protocol.js'; + +/* + * Reading an authored root back from its journal, and what a failed read + * means. Only a definite answer from the daemon may change how a run is + * treated; an unanswered, canceled or failed read decides nothing. + */ + +const NO_ROOT_STEP = new Error('authored root journal has no root step'); + +export async function rootKernelStep(journal: JournalClient, runId: string): Promise> { + const entries = (await journal.journalRead(runId, 1)).entries as Array>; + const spawned = entries.find(entry => entry.entry_type === 'run.spawned'); + const payload = spawned?.payload as { spec?: { steps?: unknown[] } } | undefined; + const step = payload?.spec?.steps?.[0]; + if (typeof step !== 'object' || step === null || Array.isArray(step)) throw NO_ROOT_STEP; + return step as Record; +} + +/** Codes relayflowd uses when it failed to read its own storage (server/protocol.rs). */ +const DAEMON_READ_FAILURES = new Set(['journal_write_failed', 'internal']); + +/** + * The root step of `runId`, or undefined when the daemon definitively answered + * that it has none: a refusal of the read (run_not_found, an unreadable early + * epoch, ...) or a journal without a root step. A daemon-side read failure, an + * interruption or a cancellation propagates instead, so a delayed, canceled or + * failing daemon never makes resume guess the declarative path. + */ +export async function rootStepIfPresent(journal: JournalClient, runId: string): Promise | undefined> { + return rootKernelStep(journal, runId).catch(error => { + if ((error instanceof JournalProtocolError && !DAEMON_READ_FAILURES.has(error.code)) || error === NO_ROOT_STEP) return undefined; + throw error; + }); +} + +/** + * The stored result of a root the daemon already reports completed. If reading + * it back is interrupted, the outcome is still known: say so, rather than + * recasting a finished run as resumable. + */ +export function completedResult(journal: JournalClient, outcome: RunOutcome, rootRunId: string): Promise { + return completedRootResult(journal, rootRunId).catch(error => { + // A bad frame or a journal with no stored result is a definite fact; any + // other failure — interrupted, canceled, or a daemon-side read error such + // as journal_write_failed — left the verdict unread, not the run unfinished. + if (error instanceof JournalFrameError || error === NO_DURABLE_RESULT) throw error; + // The kernel completes the root step with success whatever the body + // declared; the authored verdict lives only in the unread output. + const unread = new AuthoredFlowExecutionError('result_unreadable', + `run ${rootRunId} completed, but its stored result (the flow's verdict) could not be read: ${error.message}`, + undefined, rootRunId); + unread.rootRunId = rootRunId; + throw unread; + }); +} + +const NO_DURABLE_RESULT = new Error('completed authored root has no durable result'); + +async function completedRootResult( + journal: JournalClient, + rootRunId: string, +): Promise { + const entries = (await journal.journalRead(rootRunId, 1)).entries as Array>; + const completed = entries.slice().reverse().find(entry => entry.entry_type === 'step.completed' + && entry.step_id === 'authored-root' + && (entry.payload as { completionReason?: unknown } | undefined)?.completionReason === 'success'); + const output = (completed?.payload as { output?: unknown } | undefined)?.output; + if (!isCompletedRootOutput(output)) throw NO_DURABLE_RESULT; + // The stored detail, never a freshly computed one: redaction reads the + // CURRENT environment, so recomputing here would let a completed run report + // something its journal does not hold. + return Object.freeze({ + name: output.name, + completionReason: output.completionReason, + ...(output.completionDetail === undefined ? {} : { completionDetail: output.completionDetail }), + journalSteps: Object.freeze(output.journalSteps.map(step => Object.freeze({ ...step }))), + rootRunId, + }); +} + +function isCompletedRootOutput(value: unknown): value is Omit { + if (typeof value !== 'object' || value === null || Array.isArray(value)) return false; + const output = value as Partial; + return typeof output.name === 'string' + && isLoweredCompletion(output.completionReason) + // A malformed detail fails the whole readback rather than being dropped: + // silently discarding it would report the verdict without the evidence + // the journal says it was recorded with. + && (output.completionDetail === undefined || isDurableCompletionDetail(output.completionDetail)) + && Array.isArray(output.journalSteps) + && output.journalSteps.every(step => typeof step === 'object' && step !== null + && typeof step.id === 'string' && typeof step.runId === 'string' + && isSurfaceCompletionReason(step.completionReason)); +} diff --git a/packages/sdk/src/authored-root.ts b/packages/sdk/src/authored-root.ts index e6e2e4d30..ce5a70aca 100644 --- a/packages/sdk/src/authored-root.ts +++ b/packages/sdk/src/authored-root.ts @@ -6,23 +6,22 @@ import { readFile } from 'node:fs/promises'; import { canonicalize } from './canonical.js'; import { compileSpec, toKernelSpec } from './compile.js'; import { executeAuthoredFlow, type AuthoredFlowExecutionResult, type AuthoredFlowSuspendedResult } from './authored-flow-executor.js'; -import { isDurableCompletionDetail, isLoweredCompletion } from './authored-completion.js'; import { AUTHORED_ROOT_KIND } from './authored-verdict.js'; import { loadAuthoredFlow, type LoadedAuthoredFlow, type SurfaceModuleAuthority, } from './authored-flow-loader.js'; -import { JournalClient, JournalFrameError, JournalProtocolError } from './journal-client.js'; +import { JournalClient, JournalProtocolError } from './journal-client.js'; import type { RunOutcome, StepDispatchEvent } from './protocol.js'; import { SPEC_SCHEMA_VERSION } from './spec.js'; import type { RunLifecycleOptions } from './cli/run.js'; import { isLeaseLost, withWorkerLease } from './worker-lease.js'; import { AuthoredFlowExecutionError, AuthoredHumanParked } from './authored-flow-error.js'; import { readOpenHumanWaits, resumeCommand } from './authored-human.js'; -import { isSurfaceCompletionReason } from './authored-step-output.js'; import { readSubscriptionPark } from './authored-subscription-park.js'; import { localAgentCredentialEnvironment } from './local-agent-environment.js'; +import { completedResult, rootKernelStep, rootStepIfPresent } from './authored-root-readback.js'; export type DurableAuthoredFlowResult = | (AuthoredFlowExecutionResult & { readonly rootRunId: string }) @@ -200,13 +199,7 @@ export async function readAuthoredRootMetadata( journal: JournalClient, runId: string, ): Promise { - // Only a definite answer means "not authored": the daemon refused the read, or - // the journal has no root step. An unanswered or canceled read means nothing - // yet, and guessing declarative would resume the run the wrong way. - const step = await rootKernelStep(journal, runId).catch(error => { - if (error instanceof JournalProtocolError || error === NO_ROOT_STEP) return undefined; - throw error; - }); + const step = await rootStepIfPresent(journal, runId); if (step === undefined || step.id !== 'authored-root' || !hasRootStream(step)) return undefined; if (typeof step.instruction !== 'string') { throw new Error('authored root journal has malformed authority metadata'); @@ -476,16 +469,6 @@ function rootSpec(metadata: AuthoredRootMetadata, stream: string) { })); } -const NO_ROOT_STEP = new Error('authored root journal has no root step'); - -async function rootKernelStep(journal: JournalClient, runId: string): Promise> { - const entries = (await journal.journalRead(runId, 1)).entries as Array>; - const spawned = entries.find(entry => entry.entry_type === 'run.spawned'); - const payload = spawned?.payload as { spec?: { steps?: unknown[] } } | undefined; - const step = payload?.spec?.steps?.[0]; - if (typeof step !== 'object' || step === null || Array.isArray(step)) throw NO_ROOT_STEP; - return step as Record; -} function rootStreamFromMetadata(step: Record): string { const surfaces = step.surfaces as { streams?: Array<{ stream?: unknown }> } | undefined; @@ -569,65 +552,6 @@ function nextRootDispatch(peer: JournalClient, timeoutMs = 30_000): { return { promise, cancel }; } -/** - * The stored result of a root the daemon already reports completed. If reading - * it back is interrupted, the outcome is still known: say so, rather than - * recasting a finished run as resumable. - */ -function completedResult(journal: JournalClient, outcome: RunOutcome, rootRunId: string): Promise { - return completedRootResult(journal, rootRunId).catch(error => { - // A definite answer stands (a refusal, a bad frame, no stored result); - // anything else — interrupted or canceled — left the verdict unread. - if (error instanceof JournalProtocolError || error instanceof JournalFrameError - || error === NO_DURABLE_RESULT) throw error; - // The kernel completes the root step with success whatever the body - // declared; the authored verdict lives only in the unread output. - const unread = new AuthoredFlowExecutionError('result_unreadable', - `run ${rootRunId} completed, but its stored result (the flow's verdict) could not be read: ${error.message}`, - undefined, rootRunId); - unread.rootRunId = rootRunId; - throw unread; - }); -} - -const NO_DURABLE_RESULT = new Error('completed authored root has no durable result'); - -async function completedRootResult( - journal: JournalClient, - rootRunId: string, -): Promise { - const entries = (await journal.journalRead(rootRunId, 1)).entries as Array>; - const completed = entries.slice().reverse().find(entry => entry.entry_type === 'step.completed' - && entry.step_id === 'authored-root' - && (entry.payload as { completionReason?: unknown } | undefined)?.completionReason === 'success'); - const output = (completed?.payload as { output?: unknown } | undefined)?.output; - if (!isCompletedRootOutput(output)) throw NO_DURABLE_RESULT; - // The stored detail, never a freshly computed one: redaction reads the - // CURRENT environment, so recomputing here would let a completed run report - // something its journal does not hold. - return Object.freeze({ - name: output.name, - completionReason: output.completionReason, - ...(output.completionDetail === undefined ? {} : { completionDetail: output.completionDetail }), - journalSteps: Object.freeze(output.journalSteps.map(step => Object.freeze({ ...step }))), - rootRunId, - }); -} - -function isCompletedRootOutput(value: unknown): value is Omit { - if (typeof value !== 'object' || value === null || Array.isArray(value)) return false; - const output = value as Partial; - return typeof output.name === 'string' - && isLoweredCompletion(output.completionReason) - // A malformed detail fails the whole readback rather than being dropped: - // silently discarding it would report the verdict without the evidence - // the journal says it was recorded with. - && (output.completionDetail === undefined || isDurableCompletionDetail(output.completionDetail)) - && Array.isArray(output.journalSteps) - && output.journalSteps.every(step => typeof step === 'object' && step !== null - && typeof step.id === 'string' && typeof step.runId === 'string' - && isSurfaceCompletionReason(step.completionReason)); -} function assertRootCanDispatch(outcome: RunOutcome): void { // The kernel reports `parked` after handing an agent lease to its worker; diff --git a/packages/sdk/src/cli/running-step.ts b/packages/sdk/src/cli/running-step.ts index 0263b3056..2b3ca12cf 100644 --- a/packages/sdk/src/cli/running-step.ts +++ b/packages/sdk/src/cli/running-step.ts @@ -70,16 +70,23 @@ export async function waitForRunningStep(client: JournalClient, runId: string, try { // Every transport failure up to and including the watch registration is a // read interruption: the step may still be running and the run resumable. + let unconfirmed: unknown; await Promise.race([watch.connect().then(() => watch.hello('flows-step-watch')) .then(() => watch.runWatch(runId)).catch(error => { - // The replay precedes the response: a completion already folded from it - // is authoritative even if registration never answered. - if (settled) return; + // The replay precedes the response, but a drop may have truncated it + // (an older completion, the retry not yet seen): confirm below. + if (settled) { unconfirmed = error; return; } // Refusals and malformed frames are protocol facts, not lost transport: fail closed. if (error instanceof JournalProtocolError || error instanceof JournalFrameError) throw error; throw new AuthoredFlowExecutionError('daemon_unresponsive', `could not establish the completion watch: ${error instanceof Error ? error.message : String(error)}`); }), completed]); + if (unconfirmed !== undefined) { + const snapshot = await client.runGet(runId, options.signal === undefined ? {} : { signal: options.signal }); + if (snapshot.steps[runningStep.id]?.state !== 'running') return; + throw new AuthoredFlowExecutionError('daemon_unresponsive', + `could not establish the completion watch: ${unconfirmed instanceof Error ? unconfirmed.message : String(unconfirmed)}`); + } ready = true; if (settled) return; while (true) { diff --git a/packages/sdk/tests/authored-root.test.ts b/packages/sdk/tests/authored-root.test.ts index 3b85d17f6..ab7867413 100644 --- a/packages/sdk/tests/authored-root.test.ts +++ b/packages/sdk/tests/authored-root.test.ts @@ -525,6 +525,21 @@ describe('durable authored root', () => { { dataDir: '/unused', admissionKey: 'completed-canceled' })).rejects.toMatchObject({ code: 'result_unreadable', rootRunId: 'root-run' }); }); + it('propagates a daemon-side read failure while detecting an authored root', async () => { + const { readAuthoredRootMetadata } = await import('../src/authored-root.js'); + const storage = { journalRead: async () => { throw new JournalProtocolError('journal_write_failed', 'storage failed'); } }; + await expect(readAuthoredRootMetadata(storage as unknown as JournalClient, 'run')).rejects.toMatchObject({ code: 'journal_write_failed' }); + }); + + it('keeps a completed root outcome when its readback meets a storage protocol error', async () => { + const loaded = await fixture(); + const journal = new RootJournal(); + (journal as unknown as { startStatus: RunOutcome }).startStatus = outcome('root-run', 'completed', 'success'); + journal.journalRead = async () => { throw new JournalProtocolError('journal_write_failed', 'storage failed'); }; + await expect(executeDurableAuthoredFlow(loaded, journal as unknown as JournalClient, undefined, + { dataDir: '/unused', admissionKey: 'completed-storage' })).rejects.toMatchObject({ code: 'result_unreadable', rootRunId: 'root-run' }); + }); + it('terminalizes a returned body failure without replaying semantic side effects', async () => { const loaded = await fixture(true); const journal = new RootJournal(); diff --git a/packages/sdk/tests/running-step-watch.test.ts b/packages/sdk/tests/running-step-watch.test.ts index 5bb4da74e..b161921d9 100644 --- a/packages/sdk/tests/running-step-watch.test.ts +++ b/packages/sdk/tests/running-step-watch.test.ts @@ -276,7 +276,7 @@ it('honours a replayed completion even if the watch drops before registration an const server = startLoopback(path, { hello: sendOk, 'run.watch': (ctx, params) => { ctx.send({ event: 'entry', data: { run_id: String(params['run_id']), step_id: 'step', entry_type: 'step.completed' } }); setTimeout(() => ctx.socket.destroy(), 10); // replay delivered, response never sent - } }); + }, 'run.get': ctx => sendResult(ctx, { steps: { step: { state: 'completed' } } }) }); await once(server, 'listening'); const client = new JournalClient(path); try { @@ -285,3 +285,26 @@ it('honours a replayed completion even if the watch drops before registration an .resolves.toBeUndefined(); } finally { client.close(); await new Promise(resolve => server.close(() => resolve())); } }); + +it('confirms a replayed completion with a snapshot when the replay may be truncated', async () => { + for (const [state, expected] of [['completed', 'returned'], ['running', 'interrupted']] as const) { + const path = sockPath(); + const server = startLoopback(path, { + hello: sendOk, + 'run.watch': (ctx, params) => { + // An older attempt's completion arrives; the drop may hide a later retry. + ctx.send({ event: 'entry', data: { run_id: String(params['run_id']), step_id: 'step', entry_type: 'step.completed' } }); + setTimeout(() => ctx.socket.destroy(), 10); + }, + 'run.get': ctx => sendResult(ctx, { steps: { step: { state, lease_deadline_ms: Date.now() + 30_000 } } }), + }); + await once(server, 'listening'); + const client = new JournalClient(path); + try { + await client.connect(); + const outcome = await waitForRunningStep(client, 'run', { id: 'step', type: 'agent', leaseDeadlineMs: Date.now() + 30_000 }, {}) + .then(() => 'returned', error => (isReadInterruption(error) ? 'interrupted' : `other: ${error.message}`)); + expect(outcome).toBe(expected); + } finally { client.close(); await new Promise(resolve => server.close(() => resolve())); } + } +}); From 46778f74efbd53e7a2bd2122d26ce53b5ec97c1c Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 05:07:37 -0700 Subject: [PATCH 29/34] fix(sdk): no reuse-summary read after failed-step inspection was interrupted A step_failed run whose failed-step inspection exhausted the read budget kept its known failure but left no marker, so the optional reuse summary started a second full budget against the same delayed daemon. The interruption now adds an inspection_interrupted warning, and attachReuseSummary skips its read when that or a daemon_unresponsive/unreachable diagnostic is present. Evidence re-captured at this tree. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: b8262468-ea26-4a6e-bee9-d193832e77dc --- evidence/run-read-timeout/final-focused.log | 114 ++++----- .../mutation-completion-aborts-snapshot.log | 16 +- .../run-read-timeout/mutation-heartbeat.log | 8 +- .../mutation-reader-reconnect.log | 18 +- .../mutation-reader-setup-retry.log | 14 +- evidence/run-read-timeout/mutation-reader.log | 14 +- .../mutation-root-parking.log | 14 +- .../mutation-snapshot-cancel.log | 14 +- .../mutation-watch-cadence.log | 16 +- evidence/run-read-timeout/resume-live.log | 8 +- evidence/run-read-timeout/verification.md | 236 +++++++++--------- packages/sdk/src/cli/reuse.ts | 4 +- packages/sdk/src/cli/run.ts | 7 +- packages/sdk/src/failure-kinds.ts | 1 + .../tests/reuse-summary-interruption.test.ts | 12 + .../sdk/tests/run-daemon-unresponsive.test.ts | 11 + 16 files changed, 268 insertions(+), 239 deletions(-) diff --git a/evidence/run-read-timeout/final-focused.log b/evidence/run-read-timeout/final-focused.log index 2a457140f..5446831ec 100644 --- a/evidence/run-read-timeout/final-focused.log +++ b/evidence/run-read-timeout/final-focused.log @@ -2,71 +2,71 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5339ms - ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 325ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2037ms + ✓ tests/cli.test.ts (71 tests) 5630ms + ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 436ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2047ms ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2048ms -(node:244392) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. +(node:356923) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (36 tests) 417ms - ✓ tests/journal-client.test.ts (17 tests) 84ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 7509ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 434ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1078ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2951ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 419ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 977ms - ✓ tests/journal-client-read-timeout.test.ts (27 tests) 1805ms - ✓ tests/authored-flow-slack.test.ts (7 tests) 1152ms - ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 419ms - ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 393ms - ✓ tests/running-step-watch.test.ts (14 tests) 12317ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms - ✓ cancels promptly while a lease snapshot read is in flight 2005ms + ✓ tests/authored-root.test.ts (36 tests) 461ms + ✓ tests/journal-client.test.ts (17 tests) 83ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 6771ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 486ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 949ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2727ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 308ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 842ms + ✓ tests/journal-client-read-timeout.test.ts (27 tests) 1702ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 1165ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 445ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 359ms + ✓ tests/running-step-watch.test.ts (14 tests) 12322ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2116ms + ✓ cancels promptly while a lease snapshot read is in flight 2006ms ✓ a completion push ends the wait without waiting out an in-flight snapshot 2054ms - ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2003ms + ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2003ms - ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2001ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2448ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 654ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 590ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 561ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 642ms - ✓ tests/authored-helpers.test.ts (6 tests) 2538ms - ✓ runs every available provider through the real kernel and resumes completed effects without a second write 1067ms - ✓ replays after SIGKILL before confirm with the same token and one successful completion 663ms - ✓ replays after SIGKILL before complete with the same token and one successful completion 594ms - ✓ tests/classify-outcome.test.ts (11 tests) 7423ms + ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2002ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2574ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 661ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 589ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 666ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 657ms + ✓ tests/authored-helpers.test.ts (6 tests) 1859ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 878ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 425ms + ✓ replays after SIGKILL before complete with the same token and one successful completion 325ms + ✓ tests/classify-outcome.test.ts (11 tests) 7426ms ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2010ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3006ms - ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2000ms - ✓ tests/human-live.test.ts (3 tests) 4757ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2824ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1247ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 685ms - ✓ tests/worker-lease.test.ts (7 tests) 8ms - ✓ tests/communication-worker.test.ts (18 tests) 1711ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 15ms - ✓ tests/effect-channel.test.ts (5 tests) 250ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 721ms - ✓ tests/memoization.test.ts (58 tests) 65ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2453ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2059ms - ✓ drains read and watch promises before an authored flow completes 393ms - ✓ tests/worker-lease-sweep.test.ts (4 tests) 9ms - ✓ tests/journal-client-completion.test.ts (6 tests) 105ms - ✓ tests/run-daemon-unresponsive.test.ts (4 tests) 6ms - ✓ tests/reuse-summary-interruption.test.ts (5 tests) 3ms - ✓ tests/resume-worker-lease.test.ts (3 tests) 8ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3008ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2002ms + ✓ tests/human-live.test.ts (3 tests) 4300ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2560ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1082ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 658ms + ✓ tests/worker-lease.test.ts (7 tests) 19ms + ✓ tests/communication-worker.test.ts (18 tests) 1719ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 20ms + ✓ tests/effect-channel.test.ts (5 tests) 278ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 619ms + ✓ tests/run-daemon-unresponsive.test.ts (5 tests) 6ms + ✓ tests/memoization.test.ts (58 tests) 64ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2440ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2056ms + ✓ drains read and watch promises before an authored flow completes 383ms + ✓ tests/reuse-summary-interruption.test.ts (6 tests) 4ms + ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms + ✓ tests/journal-client-completion.test.ts (6 tests) 104ms + ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms ✓ tests/direct-run-worker-lease.test.ts (3 tests) 10ms - ✓ tests/helper-effect-lease-loss.test.ts (1 test) 225ms + ✓ tests/helper-effect-lease-loss.test.ts (1 test) 223ms ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms - ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 34ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms + ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 47ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 14ms Test Files 28 passed (28) - Tests 350 passed (350) - Start at 04:48:18 - Duration 61.95s (transform 1.12s, setup 159ms, collect 6.95s, tests 51.44s, environment 4ms, prepare 994ms) + Tests 352 passed (352) + Start at 05:06:34 + Duration 60.19s (transform 1.26s, setup 150ms, collect 6.87s, tests 49.88s, environment 4ms, prepare 969ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log index 9ec27620c..edaff1494 100644 --- a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log +++ b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 3013ms - × a completion push ends the wait without waiting out an in-flight snapshot 3012ms + ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 3010ms + × a completion push ends the wait without waiting out an in-flight snapshot 3010ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -34,8 +34,8 @@ Received: "still waiting" Test Files 1 failed (1) Tests 1 failed | 13 skipped (14) - Start at 04:48:03 - Duration 3.50s (transform 189ms, setup 16ms, collect 333ms, tests 3.01s, environment 0ms, prepare 42ms) + Start at 05:06:19 + Duration 3.49s (transform 192ms, setup 15ms, collect 325ms, tests 3.01s, environment 0ms, prepare 41ms) exit=1 @@ -44,12 +44,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2058ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2058ms + ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2062ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2061ms Test Files 1 passed (1) Tests 1 passed | 13 skipped (14) - Start at 04:48:07 - Duration 2.58s (transform 201ms, setup 18ms, collect 347ms, tests 2.06s, environment 0ms, prepare 47ms) + Start at 05:06:23 + Duration 2.53s (transform 182ms, setup 14ms, collect 311ms, tests 2.06s, environment 0ms, prepare 44ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-heartbeat.log b/evidence/run-read-timeout/mutation-heartbeat.log index b2c05b961..d513e89b3 100644 --- a/evidence/run-read-timeout/mutation-heartbeat.log +++ b/evidence/run-read-timeout/mutation-heartbeat.log @@ -33,8 +33,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 04:48:10 - Duration 464ms (transform 163ms, setup 17ms, collect 280ms, tests 19ms, environment 0ms, prepare 44ms) + Start at 05:06:26 + Duration 472ms (transform 167ms, setup 17ms, collect 290ms, tests 19ms, environment 0ms, prepare 42ms) exit=1 @@ -47,7 +47,7 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb Test Files 1 passed (1) Tests 1 passed (1) - Start at 04:48:11 - Duration 450ms (transform 165ms, setup 16ms, collect 288ms, tests 16ms, environment 0ms, prepare 40ms) + Start at 05:06:27 + Duration 443ms (transform 165ms, setup 17ms, collect 284ms, tests 16ms, environment 0ms, prepare 29ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-reconnect.log b/evidence/run-read-timeout/mutation-reader-reconnect.log index 555835c68..f35b5c230 100644 --- a/evidence/run-read-timeout/mutation-reader-reconnect.log +++ b/evidence/run-read-timeout/mutation-reader-reconnect.log @@ -11,14 +11,14 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 1009ms - × reconnects the reader after it disconnects within the read budget 1008ms - → journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 1006ms + × reconnects the reader after it disconnects within the read budget 1006ms + → journal client: run.get read session was interrupted after 80 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 80 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:71:9 69| let last: unknown; 70| const exhausted = () => last instanceof JournalReadInterruptedError @@ -41,8 +41,8 @@ Caused by: Error: journal client: connection closed Test Files 1 failed (1) Tests 1 failed | 26 skipped (27) - Start at 04:47:47 - Duration 1.47s (transform 179ms, setup 17ms, collect 308ms, tests 1.01s, environment 0ms, prepare 41ms) + Start at 05:06:03 + Duration 1.46s (transform 169ms, setup 17ms, collect 292ms, tests 1.01s, environment 0ms, prepare 41ms) exit=1 @@ -51,11 +51,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 73ms + ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 72ms Test Files 1 passed (1) Tests 1 passed | 26 skipped (27) - Start at 04:47:49 - Duration 534ms (transform 169ms, setup 16ms, collect 299ms, tests 73ms, environment 0ms, prepare 45ms) + Start at 05:06:05 + Duration 515ms (transform 168ms, setup 16ms, collect 294ms, tests 72ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-setup-retry.log b/evidence/run-read-timeout/mutation-reader-setup-retry.log index 01d7d977e..e2ac4c707 100644 --- a/evidence/run-read-timeout/mutation-reader-setup-retry.log +++ b/evidence/run-read-timeout/mutation-reader-setup-retry.log @@ -16,8 +16,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 78ms - × retries a reader setup that timed out instead of reading on the primary for good 77ms + ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 76ms + × retries a reader setup that timed out instead of reading on the primary for good 75ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -43,8 +43,8 @@ AssertionError: expected 1 to be 2 // Object.is equality Test Files 1 failed (1) Tests 1 failed | 26 skipped (27) - Start at 04:47:50 - Duration 526ms (transform 168ms, setup 16ms, collect 299ms, tests 78ms, environment 0ms, prepare 41ms) + Start at 05:06:06 + Duration 540ms (transform 173ms, setup 17ms, collect 298ms, tests 76ms, environment 0ms, prepare 41ms) exit=1 @@ -53,11 +53,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 74ms + ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 75ms Test Files 1 passed (1) Tests 1 passed | 26 skipped (27) - Start at 04:47:50 - Duration 526ms (transform 167ms, setup 16ms, collect 290ms, tests 74ms, environment 0ms, prepare 41ms) + Start at 05:06:06 + Duration 526ms (transform 164ms, setup 17ms, collect 290ms, tests 75ms, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader.log b/evidence/run-read-timeout/mutation-reader.log index fc126bd0d..5c80bf709 100644 --- a/evidence/run-read-timeout/mutation-reader.log +++ b/evidence/run-read-timeout/mutation-reader.log @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 113ms - × serves a bounded read while an unbounded command is in flight 112ms + ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 109ms + × serves a bounded read while an unbounded command is in flight 108ms → expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -37,8 +37,8 @@ AssertionError: expected true to be false // Object.is equality Test Files 1 failed (1) Tests 1 failed | 26 skipped (27) - Start at 04:47:45 - Duration 560ms (transform 167ms, setup 16ms, collect 283ms, tests 113ms, environment 0ms, prepare 41ms) + Start at 05:06:01 + Duration 506ms (transform 154ms, setup 15ms, collect 263ms, tests 109ms, environment 0ms, prepare 28ms) exit=1 @@ -47,11 +47,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 104ms + ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 106ms Test Files 1 passed (1) Tests 1 passed | 26 skipped (27) - Start at 04:47:46 - Duration 538ms (transform 165ms, setup 17ms, collect 273ms, tests 104ms, environment 0ms, prepare 41ms) + Start at 05:06:02 + Duration 543ms (transform 166ms, setup 16ms, collect 288ms, tests 106ms, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-root-parking.log b/evidence/run-read-timeout/mutation-root-parking.log index c95eff99b..f812617ae 100644 --- a/evidence/run-read-timeout/mutation-root-parking.log +++ b/evidence/run-read-timeout/mutation-root-parking.log @@ -17,8 +17,8 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (36 tests | 3 failed | 33 skipped) 32ms - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 23ms + ❯ tests/authored-root.test.ts (36 tests | 3 failed | 33 skipped) 31ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 22ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms @@ -104,8 +104,8 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } Test Files 1 failed (1) Tests 3 failed | 33 skipped (36) - Start at 04:48:12 - Duration 1.31s (transform 678ms, setup 17ms, collect 1.12s, tests 32ms, environment 0ms, prepare 44ms) + Start at 05:06:27 + Duration 1.30s (transform 662ms, setup 17ms, collect 1.11s, tests 31ms, environment 0ms, prepare 41ms) exit=1 @@ -114,11 +114,11 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (36 tests | 33 skipped) 18ms + ✓ tests/authored-root.test.ts (36 tests | 33 skipped) 24ms Test Files 1 passed (1) Tests 3 passed | 33 skipped (36) - Start at 04:48:13 - Duration 1.29s (transform 675ms, setup 16ms, collect 1.11s, tests 18ms, environment 0ms, prepare 42ms) + Start at 05:06:29 + Duration 1.37s (transform 701ms, setup 17ms, collect 1.18s, tests 24ms, environment 0ms, prepare 43ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-snapshot-cancel.log b/evidence/run-read-timeout/mutation-snapshot-cancel.log index ebdde6d03..1757816db 100644 --- a/evidence/run-read-timeout/mutation-snapshot-cancel.log +++ b/evidence/run-read-timeout/mutation-snapshot-cancel.log @@ -11,8 +11,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2215ms - × cancels promptly while a lease snapshot read is in flight 2215ms + ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2217ms + × cancels promptly while a lease snapshot read is in flight 2216ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -31,8 +31,8 @@ AssertionError: expected 'still waiting' to be an instance of Error Test Files 1 failed (1) Tests 1 failed | 13 skipped (14) - Start at 04:47:57 - Duration 2.68s (transform 189ms, setup 15ms, collect 317ms, tests 2.22s, environment 0ms, prepare 41ms) + Start at 05:06:13 + Duration 2.70s (transform 187ms, setup 17ms, collect 320ms, tests 2.22s, environment 0ms, prepare 40ms) exit=1 @@ -41,12 +41,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2011ms + ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2012ms ✓ cancels promptly while a lease snapshot read is in flight 2011ms Test Files 1 passed (1) Tests 1 passed | 13 skipped (14) - Start at 04:48:00 - Duration 2.48s (transform 182ms, setup 15ms, collect 317ms, tests 2.01s, environment 0ms, prepare 29ms) + Start at 05:06:16 + Duration 2.48s (transform 185ms, setup 17ms, collect 317ms, tests 2.01s, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-watch-cadence.log b/evidence/run-read-timeout/mutation-watch-cadence.log index 2f33a6c98..529925293 100644 --- a/evidence/run-read-timeout/mutation-watch-cadence.log +++ b/evidence/run-read-timeout/mutation-watch-cadence.log @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2109ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2108ms + ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2110ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2109ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -30,8 +30,8 @@ AssertionError: expected 41 to be less than or equal to 1 Test Files 1 failed (1) Tests 1 failed | 13 skipped (14) - Start at 04:47:51 - Duration 2.57s (transform 182ms, setup 17ms, collect 306ms, tests 2.11s, environment 0ms, prepare 42ms) + Start at 05:06:07 + Duration 2.60s (transform 192ms, setup 16ms, collect 332ms, tests 2.11s, environment 0ms, prepare 42ms) exit=1 @@ -40,12 +40,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2115ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2115ms + ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2118ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms Test Files 1 passed (1) Tests 1 passed | 13 skipped (14) - Start at 04:47:54 - Duration 2.59s (transform 185ms, setup 17ms, collect 319ms, tests 2.12s, environment 0ms, prepare 40ms) + Start at 05:06:10 + Duration 2.59s (transform 185ms, setup 16ms, collect 309ms, tests 2.12s, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/resume-live.log b/evidence/run-read-timeout/resume-live.log index 220511f39..7ba13d37c 100644 --- a/evidence/run-read-timeout/resume-live.log +++ b/evidence/run-read-timeout/resume-live.log @@ -2,12 +2,12 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 911ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 911ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 943ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 943ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 04:49:20 - Duration 2.15s (transform 618ms, setup 20ms, collect 1.01s, tests 911ms, environment 0ms, prepare 90ms) + Start at 05:07:35 + Duration 2.18s (transform 616ms, setup 19ms, collect 1.00s, tests 943ms, environment 0ms, prepare 58ms) exit=0 diff --git a/evidence/run-read-timeout/verification.md b/evidence/run-read-timeout/verification.md index a6b6a4685..982f1bf06 100644 --- a/evidence/run-read-timeout/verification.md +++ b/evidence/run-read-timeout/verification.md @@ -15,72 +15,72 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5339ms - ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 325ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2037ms + ✓ tests/cli.test.ts (71 tests) 5630ms + ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 436ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2047ms ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2048ms -(node:244392) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. +(node:356923) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (36 tests) 417ms - ✓ tests/journal-client.test.ts (17 tests) 84ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 7509ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 434ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1078ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2951ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 419ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 977ms - ✓ tests/journal-client-read-timeout.test.ts (27 tests) 1805ms - ✓ tests/authored-flow-slack.test.ts (7 tests) 1152ms - ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 419ms - ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 393ms - ✓ tests/running-step-watch.test.ts (14 tests) 12317ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms - ✓ cancels promptly while a lease snapshot read is in flight 2005ms + ✓ tests/authored-root.test.ts (36 tests) 461ms + ✓ tests/journal-client.test.ts (17 tests) 83ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 6771ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 486ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 949ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2727ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 308ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 842ms + ✓ tests/journal-client-read-timeout.test.ts (27 tests) 1702ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 1165ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 445ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 359ms + ✓ tests/running-step-watch.test.ts (14 tests) 12322ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2116ms + ✓ cancels promptly while a lease snapshot read is in flight 2006ms ✓ a completion push ends the wait without waiting out an in-flight snapshot 2054ms - ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2003ms + ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2003ms - ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2001ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2448ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 654ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 590ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 561ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 642ms - ✓ tests/authored-helpers.test.ts (6 tests) 2538ms - ✓ runs every available provider through the real kernel and resumes completed effects without a second write 1067ms - ✓ replays after SIGKILL before confirm with the same token and one successful completion 663ms - ✓ replays after SIGKILL before complete with the same token and one successful completion 594ms - ✓ tests/classify-outcome.test.ts (11 tests) 7423ms + ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2002ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2574ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 661ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 589ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 666ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 657ms + ✓ tests/authored-helpers.test.ts (6 tests) 1859ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 878ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 425ms + ✓ replays after SIGKILL before complete with the same token and one successful completion 325ms + ✓ tests/classify-outcome.test.ts (11 tests) 7426ms ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2010ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3006ms - ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2000ms - ✓ tests/human-live.test.ts (3 tests) 4757ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2824ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1247ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 685ms - ✓ tests/worker-lease.test.ts (7 tests) 8ms - ✓ tests/communication-worker.test.ts (18 tests) 1711ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 15ms - ✓ tests/effect-channel.test.ts (5 tests) 250ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 721ms - ✓ tests/memoization.test.ts (58 tests) 65ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2453ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2059ms - ✓ drains read and watch promises before an authored flow completes 393ms - ✓ tests/worker-lease-sweep.test.ts (4 tests) 9ms - ✓ tests/journal-client-completion.test.ts (6 tests) 105ms - ✓ tests/run-daemon-unresponsive.test.ts (4 tests) 6ms - ✓ tests/reuse-summary-interruption.test.ts (5 tests) 3ms - ✓ tests/resume-worker-lease.test.ts (3 tests) 8ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3008ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2002ms + ✓ tests/human-live.test.ts (3 tests) 4300ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2560ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1082ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 658ms + ✓ tests/worker-lease.test.ts (7 tests) 19ms + ✓ tests/communication-worker.test.ts (18 tests) 1719ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 20ms + ✓ tests/effect-channel.test.ts (5 tests) 278ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 619ms + ✓ tests/run-daemon-unresponsive.test.ts (5 tests) 6ms + ✓ tests/memoization.test.ts (58 tests) 64ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2440ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2056ms + ✓ drains read and watch promises before an authored flow completes 383ms + ✓ tests/reuse-summary-interruption.test.ts (6 tests) 4ms + ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms + ✓ tests/journal-client-completion.test.ts (6 tests) 104ms + ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms ✓ tests/direct-run-worker-lease.test.ts (3 tests) 10ms - ✓ tests/helper-effect-lease-loss.test.ts (1 test) 225ms + ✓ tests/helper-effect-lease-loss.test.ts (1 test) 223ms ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms - ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 34ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms + ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 47ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 14ms Test Files 28 passed (28) - Tests 350 passed (350) - Start at 04:48:18 - Duration 61.95s (transform 1.12s, setup 159ms, collect 6.95s, tests 51.44s, environment 4ms, prepare 994ms) + Tests 352 passed (352) + Start at 05:06:34 + Duration 60.19s (transform 1.26s, setup 150ms, collect 6.87s, tests 49.88s, environment 4ms, prepare 969ms) exit=0 ``` @@ -92,13 +92,13 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 911ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 911ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 943ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 943ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 04:49:20 - Duration 2.15s (transform 618ms, setup 20ms, collect 1.01s, tests 911ms, environment 0ms, prepare 90ms) + Start at 05:07:35 + Duration 2.18s (transform 616ms, setup 19ms, collect 1.00s, tests 943ms, environment 0ms, prepare 58ms) exit=0 ``` @@ -120,8 +120,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 113ms - × serves a bounded read while an unbounded command is in flight 112ms + ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 109ms + × serves a bounded read while an unbounded command is in flight 108ms → expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -147,8 +147,8 @@ AssertionError: expected true to be false // Object.is equality Test Files 1 failed (1) Tests 1 failed | 26 skipped (27) - Start at 04:47:45 - Duration 560ms (transform 167ms, setup 16ms, collect 283ms, tests 113ms, environment 0ms, prepare 41ms) + Start at 05:06:01 + Duration 506ms (transform 154ms, setup 15ms, collect 263ms, tests 109ms, environment 0ms, prepare 28ms) exit=1 @@ -157,12 +157,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 104ms + ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 106ms Test Files 1 passed (1) Tests 1 passed | 26 skipped (27) - Start at 04:47:46 - Duration 538ms (transform 165ms, setup 17ms, collect 273ms, tests 104ms, environment 0ms, prepare 41ms) + Start at 05:06:02 + Duration 543ms (transform 166ms, setup 16ms, collect 288ms, tests 106ms, environment 0ms, prepare 40ms) exit=0 ``` @@ -183,14 +183,14 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 1009ms - × reconnects the reader after it disconnects within the read budget 1008ms - → journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 1006ms + × reconnects the reader after it disconnects within the read budget 1006ms + → journal client: run.get read session was interrupted after 80 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 80 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:71:9 69| let last: unknown; 70| const exhausted = () => last instanceof JournalReadInterruptedError @@ -213,8 +213,8 @@ Caused by: Error: journal client: connection closed Test Files 1 failed (1) Tests 1 failed | 26 skipped (27) - Start at 04:47:47 - Duration 1.47s (transform 179ms, setup 17ms, collect 308ms, tests 1.01s, environment 0ms, prepare 41ms) + Start at 05:06:03 + Duration 1.46s (transform 169ms, setup 17ms, collect 292ms, tests 1.01s, environment 0ms, prepare 41ms) exit=1 @@ -223,12 +223,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 73ms + ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 72ms Test Files 1 passed (1) Tests 1 passed | 26 skipped (27) - Start at 04:47:49 - Duration 534ms (transform 169ms, setup 16ms, collect 299ms, tests 73ms, environment 0ms, prepare 45ms) + Start at 05:06:05 + Duration 515ms (transform 168ms, setup 16ms, collect 294ms, tests 72ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -254,8 +254,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 78ms - × retries a reader setup that timed out instead of reading on the primary for good 77ms + ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 76ms + × retries a reader setup that timed out instead of reading on the primary for good 75ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -281,8 +281,8 @@ AssertionError: expected 1 to be 2 // Object.is equality Test Files 1 failed (1) Tests 1 failed | 26 skipped (27) - Start at 04:47:50 - Duration 526ms (transform 168ms, setup 16ms, collect 299ms, tests 78ms, environment 0ms, prepare 41ms) + Start at 05:06:06 + Duration 540ms (transform 173ms, setup 17ms, collect 298ms, tests 76ms, environment 0ms, prepare 41ms) exit=1 @@ -291,12 +291,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 74ms + ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 75ms Test Files 1 passed (1) Tests 1 passed | 26 skipped (27) - Start at 04:47:50 - Duration 526ms (transform 167ms, setup 16ms, collect 290ms, tests 74ms, environment 0ms, prepare 41ms) + Start at 05:06:06 + Duration 526ms (transform 164ms, setup 17ms, collect 290ms, tests 75ms, environment 0ms, prepare 40ms) exit=0 ``` @@ -316,8 +316,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2109ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2108ms + ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2110ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2109ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -336,8 +336,8 @@ AssertionError: expected 41 to be less than or equal to 1 Test Files 1 failed (1) Tests 1 failed | 13 skipped (14) - Start at 04:47:51 - Duration 2.57s (transform 182ms, setup 17ms, collect 306ms, tests 2.11s, environment 0ms, prepare 42ms) + Start at 05:06:07 + Duration 2.60s (transform 192ms, setup 16ms, collect 332ms, tests 2.11s, environment 0ms, prepare 42ms) exit=1 @@ -346,13 +346,13 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2115ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2115ms + ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2118ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms Test Files 1 passed (1) Tests 1 passed | 13 skipped (14) - Start at 04:47:54 - Duration 2.59s (transform 185ms, setup 17ms, collect 319ms, tests 2.12s, environment 0ms, prepare 40ms) + Start at 05:06:10 + Duration 2.59s (transform 185ms, setup 16ms, collect 309ms, tests 2.12s, environment 0ms, prepare 40ms) exit=0 ``` @@ -373,8 +373,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2215ms - × cancels promptly while a lease snapshot read is in flight 2215ms + ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2217ms + × cancels promptly while a lease snapshot read is in flight 2216ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -393,8 +393,8 @@ AssertionError: expected 'still waiting' to be an instance of Error Test Files 1 failed (1) Tests 1 failed | 13 skipped (14) - Start at 04:47:57 - Duration 2.68s (transform 189ms, setup 15ms, collect 317ms, tests 2.22s, environment 0ms, prepare 41ms) + Start at 05:06:13 + Duration 2.70s (transform 187ms, setup 17ms, collect 320ms, tests 2.22s, environment 0ms, prepare 40ms) exit=1 @@ -403,13 +403,13 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2011ms + ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2012ms ✓ cancels promptly while a lease snapshot read is in flight 2011ms Test Files 1 passed (1) Tests 1 passed | 13 skipped (14) - Start at 04:48:00 - Duration 2.48s (transform 182ms, setup 15ms, collect 317ms, tests 2.01s, environment 0ms, prepare 29ms) + Start at 05:06:16 + Duration 2.48s (transform 185ms, setup 17ms, collect 317ms, tests 2.01s, environment 0ms, prepare 40ms) exit=0 ``` @@ -429,8 +429,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 3013ms - × a completion push ends the wait without waiting out an in-flight snapshot 3012ms + ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 3010ms + × a completion push ends the wait without waiting out an in-flight snapshot 3010ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -453,8 +453,8 @@ Received: "still waiting" Test Files 1 failed (1) Tests 1 failed | 13 skipped (14) - Start at 04:48:03 - Duration 3.50s (transform 189ms, setup 16ms, collect 333ms, tests 3.01s, environment 0ms, prepare 42ms) + Start at 05:06:19 + Duration 3.49s (transform 192ms, setup 15ms, collect 325ms, tests 3.01s, environment 0ms, prepare 41ms) exit=1 @@ -463,13 +463,13 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2058ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2058ms + ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2062ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2061ms Test Files 1 passed (1) Tests 1 passed | 13 skipped (14) - Start at 04:48:07 - Duration 2.58s (transform 201ms, setup 18ms, collect 347ms, tests 2.06s, environment 0ms, prepare 47ms) + Start at 05:06:23 + Duration 2.53s (transform 182ms, setup 14ms, collect 311ms, tests 2.06s, environment 0ms, prepare 44ms) exit=0 ``` @@ -512,8 +512,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 04:48:10 - Duration 464ms (transform 163ms, setup 17ms, collect 280ms, tests 19ms, environment 0ms, prepare 44ms) + Start at 05:06:26 + Duration 472ms (transform 167ms, setup 17ms, collect 290ms, tests 19ms, environment 0ms, prepare 42ms) exit=1 @@ -526,8 +526,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb Test Files 1 passed (1) Tests 1 passed (1) - Start at 04:48:11 - Duration 450ms (transform 165ms, setup 16ms, collect 288ms, tests 16ms, environment 0ms, prepare 40ms) + Start at 05:06:27 + Duration 443ms (transform 165ms, setup 17ms, collect 284ms, tests 16ms, environment 0ms, prepare 29ms) exit=0 ``` @@ -554,8 +554,8 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (36 tests | 3 failed | 33 skipped) 32ms - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 23ms + ❯ tests/authored-root.test.ts (36 tests | 3 failed | 33 skipped) 31ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 22ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms @@ -641,8 +641,8 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } Test Files 1 failed (1) Tests 3 failed | 33 skipped (36) - Start at 04:48:12 - Duration 1.31s (transform 678ms, setup 17ms, collect 1.12s, tests 32ms, environment 0ms, prepare 44ms) + Start at 05:06:27 + Duration 1.30s (transform 662ms, setup 17ms, collect 1.11s, tests 31ms, environment 0ms, prepare 41ms) exit=1 @@ -651,12 +651,12 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (36 tests | 33 skipped) 18ms + ✓ tests/authored-root.test.ts (36 tests | 33 skipped) 24ms Test Files 1 passed (1) Tests 3 passed | 33 skipped (36) - Start at 04:48:13 - Duration 1.29s (transform 675ms, setup 16ms, collect 1.11s, tests 18ms, environment 0ms, prepare 42ms) + Start at 05:06:29 + Duration 1.37s (transform 701ms, setup 17ms, collect 1.18s, tests 24ms, environment 0ms, prepare 43ms) exit=0 ``` diff --git a/packages/sdk/src/cli/reuse.ts b/packages/sdk/src/cli/reuse.ts index 26f012188..0fae54c61 100644 --- a/packages/sdk/src/cli/reuse.ts +++ b/packages/sdk/src/cli/reuse.ts @@ -33,8 +33,8 @@ export async function attachReuseSummary(execution: RunExecution, client: Journa runId: string, fromRunId: string): Promise { // Classification already ended on an unanswered read: the run is resumable and // another read would only spend a fresh budget against the same delayed daemon. - if (execution.report.diagnostics.some(diagnostic => - diagnostic.kind === 'daemon_unresponsive' || diagnostic.kind === 'daemon_unreachable')) return execution; + if (execution.report.diagnostics.some(diagnostic => diagnostic.kind === 'daemon_unresponsive' + || diagnostic.kind === 'daemon_unreachable' || diagnostic.kind === 'inspection_interrupted')) return execution; try { execution.report.reuse = await reuseSummary(client, runId, fromRunId); } catch (error) { diff --git a/packages/sdk/src/cli/run.ts b/packages/sdk/src/cli/run.ts index e5e24c4b4..20733bfe6 100644 --- a/packages/sdk/src/cli/run.ts +++ b/packages/sdk/src/cli/run.ts @@ -952,6 +952,7 @@ async function classifyOutcomeInner( kind: current.completion_reason, message: `Run "${current.run_id}" failed with completionReason: ${current.completion_reason}.`, }; + let interrupted: RunDiagnostic | undefined; if (current.completion_reason === 'step_failed') { let details: StepFailedDetails | undefined; try { @@ -959,6 +960,10 @@ async function classifyOutcomeInner( } catch (error) { // Inspection must not erase the already known run failure. diagnostic.message += ` Could not inspect the failed step: ${errorMessage(error)}`; + // Mark an unanswered read so optional follow-up reads do not spend a + // fresh budget on the same delayed daemon. + if (isReadInterruption(error)) interrupted = { severity: 'warning', kind: 'inspection_interrupted', + message: `The failed step could not be inspected: ${errorMessage(error)}` }; } if (details !== undefined) { Object.assign(diagnostic, details); @@ -975,7 +980,7 @@ async function classifyOutcomeInner( exitCode: 1, report: { ...report, - diagnostics: [...report.diagnostics, diagnostic], + diagnostics: [...report.diagnostics, diagnostic, ...(interrupted === undefined ? [] : [interrupted])], }, }; } diff --git a/packages/sdk/src/failure-kinds.ts b/packages/sdk/src/failure-kinds.ts index f962137ff..5050806e9 100644 --- a/packages/sdk/src/failure-kinds.ts +++ b/packages/sdk/src/failure-kinds.ts @@ -163,6 +163,7 @@ export const RUN_WARNING_KINDS = [ 'connection_file_stale', 'reuse_summary_unavailable', 'result_unreadable', + 'inspection_interrupted', ] as const; /** diff --git a/packages/sdk/tests/reuse-summary-interruption.test.ts b/packages/sdk/tests/reuse-summary-interruption.test.ts index 5c9d2926d..71230bf93 100644 --- a/packages/sdk/tests/reuse-summary-interruption.test.ts +++ b/packages/sdk/tests/reuse-summary-interruption.test.ts @@ -46,3 +46,15 @@ it.each(['daemon_unresponsive', 'daemon_unreachable'])('does not start another r expect(execution).toBe(interrupted); expect(execution.report.reuse).toBeUndefined(); }); + +it('does not start another read after failed-step inspection was interrupted', async () => { + let reads = 0; + const client = { journalRead: async () => { reads += 1; return { entries: [] }; } } as unknown as JournalClient; + const failed: RunExecution = { exitCode: 1, report: { command: 'run', ok: false, status: 'failed', runId: 'run-1', diagnostics: [ + { severity: 'failure', kind: 'step_failed', message: 'Run failed. Could not inspect the failed step: read timed out' }, + { severity: 'warning', kind: 'inspection_interrupted', message: 'The failed step could not be inspected: read timed out' }, + ] } as never }; + const execution = await attachReuseSummary(failed, client, 'run-1', 'prior'); + expect(reads).toBe(0); + expect(execution).toBe(failed); +}); diff --git a/packages/sdk/tests/run-daemon-unresponsive.test.ts b/packages/sdk/tests/run-daemon-unresponsive.test.ts index 3afde1c50..21305b76d 100644 --- a/packages/sdk/tests/run-daemon-unresponsive.test.ts +++ b/packages/sdk/tests/run-daemon-unresponsive.test.ts @@ -47,3 +47,14 @@ it('reports a completed run with an unread result as finished but with an unconf expect(execution.report).not.toHaveProperty('completionReason'); expect(JSON.stringify(execution.report)).not.toContain('flows resume'); }); + +it('marks a step_failed run whose inspection was interrupted', async () => { + vi.mocked(probeSocket).mockResolvedValue({ reachable: true }); + const client = { journalRead: async () => { throw new JournalRequestTimeoutError('journal.read', 10, 3, 300_000, 300_000); }, + runGet: async () => { throw new JournalRequestTimeoutError('run.get', 10, 3, 300_000, 300_000); } }; + const result = await classifyOutcome(client as unknown as JournalClient, 'run', { + run_id: 'failed-run', status: 'failed', completion_reason: 'step_failed', completed_steps: 1, + }, { command: 'run', ok: true, diagnostics: [] } as never, '/socket', {}); + expect(result.report).toMatchObject({ status: 'failed', completionReason: 'step_failed' }); + expect(result.report.diagnostics).toContainEqual(expect.objectContaining({ severity: 'warning', kind: 'inspection_interrupted' })); +}); From c6a1cf9392f4ddd8c6fe94222d38db1ef8501ccf Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 05:19:19 -0700 Subject: [PATCH 30/34] fix(sdk): communication workers keep a retried attempt's history read resumable A retried communicating agent reads its history through a worker client that had no read budget, and a timeout there was completed as worker_error. The worker client now has the flow read budget, completeCommunicationDispatch rethrows read interruptions instead of completing the attempt, and onWorkerFailure hands an interrupted worker attempt back to the kernel (as it does a lost lease) rather than failing the CLI. Evidence re-captured at this tree. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: b8262468-ea26-4a6e-bee9-d193832e77dc --- evidence/run-read-timeout/final-focused.log | 107 ++++---- .../mutation-completion-aborts-snapshot.log | 14 +- .../run-read-timeout/mutation-heartbeat.log | 14 +- .../mutation-reader-reconnect.log | 18 +- .../mutation-reader-setup-retry.log | 14 +- evidence/run-read-timeout/mutation-reader.log | 14 +- .../mutation-root-parking.log | 12 +- .../mutation-snapshot-cancel.log | 16 +- .../mutation-watch-cadence.log | 12 +- evidence/run-read-timeout/resume-live.log | 8 +- evidence/run-read-timeout/verification.md | 229 +++++++++--------- packages/sdk/src/communication/local.ts | 4 +- packages/sdk/src/communication/worker.ts | 6 +- packages/sdk/src/worker-lease.ts | 5 +- .../sdk/tests/communication-worker.test.ts | 15 ++ packages/sdk/tests/worker-lease.test.ts | 12 + 16 files changed, 268 insertions(+), 232 deletions(-) diff --git a/evidence/run-read-timeout/final-focused.log b/evidence/run-read-timeout/final-focused.log index 5446831ec..51bb845b8 100644 --- a/evidence/run-read-timeout/final-focused.log +++ b/evidence/run-read-timeout/final-focused.log @@ -1,72 +1,73 @@ -$ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/kernel-target/debug/relayflowd npx vitest run tests/journal-client-read-timeout.test.ts tests/journal-client.test.ts tests/journal-client-completion.test.ts tests/journal-client-subscriptions.test.ts tests/running-step-watch.test.ts tests/heartbeat-timeout.test.ts tests/run-daemon-unresponsive.test.ts tests/authored-root.test.ts tests/classify-outcome.test.ts tests/cli.test.ts tests/direct-run-worker-lease.test.ts tests/resume-worker-lease.test.ts tests/worker-lease.test.ts tests/worker-lease-lost.test.ts tests/worker-lease-lost-live.test.ts tests/worker-lease-sweep.test.ts tests/run-read-load-live.test.ts tests/flow-executor-chain.test.ts tests/agent-transcript-live.test.ts tests/human-live.test.ts tests/reuse-summary-interruption.test.ts tests/memoization.test.ts tests/authored-verifier-read-budget.test.ts tests/communication-worker.test.ts tests/helper-effect-lease-loss.test.ts tests/authored-helpers.test.ts tests/authored-flow-slack.test.ts tests/effect-channel.test.ts --maxWorkers=1 --minWorkers=1 +$ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/kernel-target/debug/relayflowd npx vitest run tests/journal-client-read-timeout.test.ts tests/journal-client.test.ts tests/journal-client-completion.test.ts tests/journal-client-subscriptions.test.ts tests/running-step-watch.test.ts tests/heartbeat-timeout.test.ts tests/run-daemon-unresponsive.test.ts tests/authored-root.test.ts tests/classify-outcome.test.ts tests/cli.test.ts tests/direct-run-worker-lease.test.ts tests/resume-worker-lease.test.ts tests/worker-lease.test.ts tests/worker-lease-lost.test.ts tests/worker-lease-lost-live.test.ts tests/worker-lease-sweep.test.ts tests/run-read-load-live.test.ts tests/flow-executor-chain.test.ts tests/agent-transcript-live.test.ts tests/human-live.test.ts tests/reuse-summary-interruption.test.ts tests/memoization.test.ts tests/authored-verifier-read-budget.test.ts tests/communication-worker.test.ts tests/communication.test.ts tests/helper-effect-lease-loss.test.ts tests/authored-helpers.test.ts tests/authored-flow-slack.test.ts tests/effect-channel.test.ts --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5630ms - ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 436ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2047ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2048ms -(node:356923) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5331ms + ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 319ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2039ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2049ms +(node:414643) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (36 tests) 461ms - ✓ tests/journal-client.test.ts (17 tests) 83ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 6771ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 486ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 949ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2727ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 308ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 842ms - ✓ tests/journal-client-read-timeout.test.ts (27 tests) 1702ms - ✓ tests/authored-flow-slack.test.ts (7 tests) 1165ms - ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 445ms - ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 359ms - ✓ tests/running-step-watch.test.ts (14 tests) 12322ms + ✓ tests/authored-root.test.ts (36 tests) 456ms + ✓ tests/journal-client.test.ts (17 tests) 79ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 6978ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 464ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1062ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2739ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 351ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 793ms + ✓ tests/journal-client-read-timeout.test.ts (27 tests) 1801ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 1149ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 405ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 402ms + ✓ tests/running-step-watch.test.ts (14 tests) 12320ms ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2116ms ✓ cancels promptly while a lease snapshot read is in flight 2006ms ✓ a completion push ends the wait without waiting out an in-flight snapshot 2054ms ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms - ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2003ms - ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2002ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2574ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 661ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 589ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 666ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 657ms - ✓ tests/authored-helpers.test.ts (6 tests) 1859ms - ✓ runs every available provider through the real kernel and resumes completed effects without a second write 878ms - ✓ replays after SIGKILL before confirm with the same token and one successful completion 425ms - ✓ replays after SIGKILL before complete with the same token and one successful completion 325ms - ✓ tests/classify-outcome.test.ts (11 tests) 7426ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2010ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3008ms + ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2004ms + ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2001ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2517ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 655ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 529ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 681ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 652ms + ✓ tests/authored-helpers.test.ts (6 tests) 1855ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 801ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 411ms + ✓ replays after SIGKILL before complete with the same token and one successful completion 413ms + ✓ tests/classify-outcome.test.ts (11 tests) 7424ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2007ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3006ms ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2002ms - ✓ tests/human-live.test.ts (3 tests) 4300ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2560ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1082ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 658ms - ✓ tests/worker-lease.test.ts (7 tests) 19ms - ✓ tests/communication-worker.test.ts (18 tests) 1719ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 20ms - ✓ tests/effect-channel.test.ts (5 tests) 278ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 619ms + ✓ tests/human-live.test.ts (3 tests) 4576ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2824ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1017ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 734ms + ✓ tests/communication-worker.test.ts (19 tests) 1713ms + ✓ tests/worker-lease.test.ts (8 tests) 12ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 27ms + ✓ tests/communication.test.ts (10 tests) 14ms + ✓ tests/effect-channel.test.ts (5 tests) 277ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 707ms ✓ tests/run-daemon-unresponsive.test.ts (5 tests) 6ms ✓ tests/memoization.test.ts (58 tests) 64ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2440ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2056ms - ✓ drains read and watch promises before an authored flow completes 383ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2488ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2058ms + ✓ drains read and watch promises before an authored flow completes 429ms ✓ tests/reuse-summary-interruption.test.ts (6 tests) 4ms ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms - ✓ tests/journal-client-completion.test.ts (6 tests) 104ms + ✓ tests/journal-client-completion.test.ts (6 tests) 105ms ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms - ✓ tests/direct-run-worker-lease.test.ts (3 tests) 10ms - ✓ tests/helper-effect-lease-loss.test.ts (1 test) 223ms + ✓ tests/direct-run-worker-lease.test.ts (3 tests) 13ms + ✓ tests/helper-effect-lease-loss.test.ts (1 test) 232ms ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 47ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 14ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 17ms - Test Files 28 passed (28) - Tests 352 passed (352) - Start at 05:06:34 - Duration 60.19s (transform 1.26s, setup 150ms, collect 6.87s, tests 49.88s, environment 4ms, prepare 969ms) + Test Files 29 passed (29) + Tests 364 passed (364) + Start at 05:18:16 + Duration 60.69s (transform 1.13s, setup 158ms, collect 6.83s, tests 50.24s, environment 4ms, prepare 1.03s) exit=0 diff --git a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log index edaff1494..b2ce240bc 100644 --- a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log +++ b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log @@ -11,7 +11,7 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 3010ms - × a completion push ends the wait without waiting out an in-flight snapshot 3010ms + × a completion push ends the wait without waiting out an in-flight snapshot 3009ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -34,8 +34,8 @@ Received: "still waiting" Test Files 1 failed (1) Tests 1 failed | 13 skipped (14) - Start at 05:06:19 - Duration 3.49s (transform 192ms, setup 15ms, collect 325ms, tests 3.01s, environment 0ms, prepare 41ms) + Start at 05:18:01 + Duration 3.50s (transform 189ms, setup 17ms, collect 323ms, tests 3.01s, environment 0ms, prepare 41ms) exit=1 @@ -44,12 +44,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2062ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2061ms + ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2057ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2057ms Test Files 1 passed (1) Tests 1 passed | 13 skipped (14) - Start at 05:06:23 - Duration 2.53s (transform 182ms, setup 14ms, collect 311ms, tests 2.06s, environment 0ms, prepare 44ms) + Start at 05:18:05 + Duration 2.55s (transform 194ms, setup 17ms, collect 338ms, tests 2.06s, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-heartbeat.log b/evidence/run-read-timeout/mutation-heartbeat.log index d513e89b3..66ae0613b 100644 --- a/evidence/run-read-timeout/mutation-heartbeat.log +++ b/evidence/run-read-timeout/mutation-heartbeat.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/worker-lease.ts -Original SHA256: 44c13f9f61ac13cb8c0f872f4c22b1c212e020d05880ed080ea7205ba6007213 +Original SHA256: 4c58d4670422dc2169764af16eb082c55e72c5321ed9d6001af3607f03955fa3 Replaced: if (error instanceof JournalRequestTimeoutError && error.verb === 'step.heartbeat') { throw new WorkerLeaseLostError('renewal_expired', error.message, { cause: error }); @@ -14,7 +14,7 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 19ms - × a heartbeat timeout is lease loss rather than a worker body failure 18ms + × a heartbeat timeout is lease loss rather than a worker body failure 19ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -33,12 +33,12 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 05:06:26 - Duration 472ms (transform 167ms, setup 17ms, collect 290ms, tests 19ms, environment 0ms, prepare 42ms) + Start at 05:18:08 + Duration 437ms (transform 159ms, setup 17ms, collect 271ms, tests 19ms, environment 0ms, prepare 40ms) exit=1 -RESTORED SHA256: 44c13f9f61ac13cb8c0f872f4c22b1c212e020d05880ed080ea7205ba6007213 +RESTORED SHA256: 4c58d4670422dc2169764af16eb082c55e72c5321ed9d6001af3607f03955fa3 $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartbeat timeout' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk @@ -47,7 +47,7 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb Test Files 1 passed (1) Tests 1 passed (1) - Start at 05:06:27 - Duration 443ms (transform 165ms, setup 17ms, collect 284ms, tests 16ms, environment 0ms, prepare 29ms) + Start at 05:18:08 + Duration 471ms (transform 167ms, setup 16ms, collect 293ms, tests 16ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-reconnect.log b/evidence/run-read-timeout/mutation-reader-reconnect.log index f35b5c230..4fa93b87e 100644 --- a/evidence/run-read-timeout/mutation-reader-reconnect.log +++ b/evidence/run-read-timeout/mutation-reader-reconnect.log @@ -11,14 +11,14 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 1006ms - × reconnects the reader after it disconnects within the read budget 1006ms - → journal client: run.get read session was interrupted after 80 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 1009ms + × reconnects the reader after it disconnects within the read budget 1008ms + → journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 80 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:71:9 69| let last: unknown; 70| const exhausted = () => last instanceof JournalReadInterruptedError @@ -41,8 +41,8 @@ Caused by: Error: journal client: connection closed Test Files 1 failed (1) Tests 1 failed | 26 skipped (27) - Start at 05:06:03 - Duration 1.46s (transform 169ms, setup 17ms, collect 292ms, tests 1.01s, environment 0ms, prepare 41ms) + Start at 05:17:45 + Duration 1.48s (transform 172ms, setup 17ms, collect 302ms, tests 1.01s, environment 0ms, prepare 42ms) exit=1 @@ -51,11 +51,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 72ms + ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 88ms Test Files 1 passed (1) Tests 1 passed | 26 skipped (27) - Start at 05:06:05 - Duration 515ms (transform 168ms, setup 16ms, collect 294ms, tests 72ms, environment 0ms, prepare 41ms) + Start at 05:17:47 + Duration 531ms (transform 167ms, setup 17ms, collect 279ms, tests 88ms, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-setup-retry.log b/evidence/run-read-timeout/mutation-reader-setup-retry.log index e2ac4c707..316433edd 100644 --- a/evidence/run-read-timeout/mutation-reader-setup-retry.log +++ b/evidence/run-read-timeout/mutation-reader-setup-retry.log @@ -16,8 +16,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 76ms - × retries a reader setup that timed out instead of reading on the primary for good 75ms + ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 66ms + × retries a reader setup that timed out instead of reading on the primary for good 66ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -43,8 +43,8 @@ AssertionError: expected 1 to be 2 // Object.is equality Test Files 1 failed (1) Tests 1 failed | 26 skipped (27) - Start at 05:06:06 - Duration 540ms (transform 173ms, setup 17ms, collect 298ms, tests 76ms, environment 0ms, prepare 41ms) + Start at 05:17:47 + Duration 497ms (transform 171ms, setup 16ms, collect 292ms, tests 66ms, environment 0ms, prepare 36ms) exit=1 @@ -53,11 +53,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 75ms + ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 72ms Test Files 1 passed (1) Tests 1 passed | 26 skipped (27) - Start at 05:06:06 - Duration 526ms (transform 164ms, setup 17ms, collect 290ms, tests 75ms, environment 0ms, prepare 40ms) + Start at 05:17:48 + Duration 518ms (transform 161ms, setup 14ms, collect 286ms, tests 72ms, environment 0ms, prepare 42ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader.log b/evidence/run-read-timeout/mutation-reader.log index 5c80bf709..336be39d5 100644 --- a/evidence/run-read-timeout/mutation-reader.log +++ b/evidence/run-read-timeout/mutation-reader.log @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 109ms - × serves a bounded read while an unbounded command is in flight 108ms + ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 113ms + × serves a bounded read while an unbounded command is in flight 112ms → expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -37,8 +37,8 @@ AssertionError: expected true to be false // Object.is equality Test Files 1 failed (1) Tests 1 failed | 26 skipped (27) - Start at 05:06:01 - Duration 506ms (transform 154ms, setup 15ms, collect 263ms, tests 109ms, environment 0ms, prepare 28ms) + Start at 05:17:43 + Duration 562ms (transform 168ms, setup 14ms, collect 296ms, tests 113ms, environment 0ms, prepare 40ms) exit=1 @@ -47,11 +47,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 106ms + ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 107ms Test Files 1 passed (1) Tests 1 passed | 26 skipped (27) - Start at 05:06:02 - Duration 543ms (transform 166ms, setup 16ms, collect 288ms, tests 106ms, environment 0ms, prepare 40ms) + Start at 05:17:44 + Duration 568ms (transform 170ms, setup 15ms, collect 297ms, tests 107ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-root-parking.log b/evidence/run-read-timeout/mutation-root-parking.log index f812617ae..1b126ed9a 100644 --- a/evidence/run-read-timeout/mutation-root-parking.log +++ b/evidence/run-read-timeout/mutation-root-parking.log @@ -17,7 +17,7 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (36 tests | 3 failed | 33 skipped) 31ms + ❯ tests/authored-root.test.ts (36 tests | 3 failed | 33 skipped) 32ms × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 22ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) @@ -104,8 +104,8 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } Test Files 1 failed (1) Tests 3 failed | 33 skipped (36) - Start at 05:06:27 - Duration 1.30s (transform 662ms, setup 17ms, collect 1.11s, tests 31ms, environment 0ms, prepare 41ms) + Start at 05:18:09 + Duration 1.31s (transform 674ms, setup 17ms, collect 1.11s, tests 32ms, environment 0ms, prepare 41ms) exit=1 @@ -114,11 +114,11 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (36 tests | 33 skipped) 24ms + ✓ tests/authored-root.test.ts (36 tests | 33 skipped) 23ms Test Files 1 passed (1) Tests 3 passed | 33 skipped (36) - Start at 05:06:29 - Duration 1.37s (transform 701ms, setup 17ms, collect 1.18s, tests 24ms, environment 0ms, prepare 43ms) + Start at 05:18:11 + Duration 1.29s (transform 677ms, setup 18ms, collect 1.11s, tests 23ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-snapshot-cancel.log b/evidence/run-read-timeout/mutation-snapshot-cancel.log index 1757816db..8c9949b7c 100644 --- a/evidence/run-read-timeout/mutation-snapshot-cancel.log +++ b/evidence/run-read-timeout/mutation-snapshot-cancel.log @@ -11,8 +11,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2217ms - × cancels promptly while a lease snapshot read is in flight 2216ms + ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2215ms + × cancels promptly while a lease snapshot read is in flight 2214ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -31,8 +31,8 @@ AssertionError: expected 'still waiting' to be an instance of Error Test Files 1 failed (1) Tests 1 failed | 13 skipped (14) - Start at 05:06:13 - Duration 2.70s (transform 187ms, setup 17ms, collect 320ms, tests 2.22s, environment 0ms, prepare 40ms) + Start at 05:17:55 + Duration 2.68s (transform 188ms, setup 16ms, collect 308ms, tests 2.21s, environment 0ms, prepare 40ms) exit=1 @@ -41,12 +41,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2012ms - ✓ cancels promptly while a lease snapshot read is in flight 2011ms + ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2010ms + ✓ cancels promptly while a lease snapshot read is in flight 2009ms Test Files 1 passed (1) Tests 1 passed | 13 skipped (14) - Start at 05:06:16 - Duration 2.48s (transform 185ms, setup 17ms, collect 317ms, tests 2.01s, environment 0ms, prepare 40ms) + Start at 05:17:58 + Duration 2.49s (transform 201ms, setup 16ms, collect 346ms, tests 2.01s, environment 0ms, prepare 28ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-watch-cadence.log b/evidence/run-read-timeout/mutation-watch-cadence.log index 529925293..1e16924c7 100644 --- a/evidence/run-read-timeout/mutation-watch-cadence.log +++ b/evidence/run-read-timeout/mutation-watch-cadence.log @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2110ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2109ms + ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2108ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2108ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -30,8 +30,8 @@ AssertionError: expected 41 to be less than or equal to 1 Test Files 1 failed (1) Tests 1 failed | 13 skipped (14) - Start at 05:06:07 - Duration 2.60s (transform 192ms, setup 16ms, collect 332ms, tests 2.11s, environment 0ms, prepare 42ms) + Start at 05:17:49 + Duration 2.60s (transform 192ms, setup 17ms, collect 326ms, tests 2.11s, environment 0ms, prepare 41ms) exit=1 @@ -45,7 +45,7 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu Test Files 1 passed (1) Tests 1 passed | 13 skipped (14) - Start at 05:06:10 - Duration 2.59s (transform 185ms, setup 16ms, collect 309ms, tests 2.12s, environment 0ms, prepare 40ms) + Start at 05:17:52 + Duration 2.60s (transform 194ms, setup 17ms, collect 317ms, tests 2.12s, environment 0ms, prepare 42ms) exit=0 diff --git a/evidence/run-read-timeout/resume-live.log b/evidence/run-read-timeout/resume-live.log index 7ba13d37c..471f63f53 100644 --- a/evidence/run-read-timeout/resume-live.log +++ b/evidence/run-read-timeout/resume-live.log @@ -2,12 +2,12 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 943ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 943ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 921ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 920ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 05:07:35 - Duration 2.18s (transform 616ms, setup 19ms, collect 1.00s, tests 943ms, environment 0ms, prepare 58ms) + Start at 05:19:17 + Duration 2.17s (transform 616ms, setup 14ms, collect 1.01s, tests 921ms, environment 0ms, prepare 70ms) exit=0 diff --git a/evidence/run-read-timeout/verification.md b/evidence/run-read-timeout/verification.md index 982f1bf06..e7b8ff333 100644 --- a/evidence/run-read-timeout/verification.md +++ b/evidence/run-read-timeout/verification.md @@ -11,76 +11,77 @@ exit=0 ## final-focused.log ```text -$ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/kernel-target/debug/relayflowd npx vitest run tests/journal-client-read-timeout.test.ts tests/journal-client.test.ts tests/journal-client-completion.test.ts tests/journal-client-subscriptions.test.ts tests/running-step-watch.test.ts tests/heartbeat-timeout.test.ts tests/run-daemon-unresponsive.test.ts tests/authored-root.test.ts tests/classify-outcome.test.ts tests/cli.test.ts tests/direct-run-worker-lease.test.ts tests/resume-worker-lease.test.ts tests/worker-lease.test.ts tests/worker-lease-lost.test.ts tests/worker-lease-lost-live.test.ts tests/worker-lease-sweep.test.ts tests/run-read-load-live.test.ts tests/flow-executor-chain.test.ts tests/agent-transcript-live.test.ts tests/human-live.test.ts tests/reuse-summary-interruption.test.ts tests/memoization.test.ts tests/authored-verifier-read-budget.test.ts tests/communication-worker.test.ts tests/helper-effect-lease-loss.test.ts tests/authored-helpers.test.ts tests/authored-flow-slack.test.ts tests/effect-channel.test.ts --maxWorkers=1 --minWorkers=1 +$ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/kernel-target/debug/relayflowd npx vitest run tests/journal-client-read-timeout.test.ts tests/journal-client.test.ts tests/journal-client-completion.test.ts tests/journal-client-subscriptions.test.ts tests/running-step-watch.test.ts tests/heartbeat-timeout.test.ts tests/run-daemon-unresponsive.test.ts tests/authored-root.test.ts tests/classify-outcome.test.ts tests/cli.test.ts tests/direct-run-worker-lease.test.ts tests/resume-worker-lease.test.ts tests/worker-lease.test.ts tests/worker-lease-lost.test.ts tests/worker-lease-lost-live.test.ts tests/worker-lease-sweep.test.ts tests/run-read-load-live.test.ts tests/flow-executor-chain.test.ts tests/agent-transcript-live.test.ts tests/human-live.test.ts tests/reuse-summary-interruption.test.ts tests/memoization.test.ts tests/authored-verifier-read-budget.test.ts tests/communication-worker.test.ts tests/communication.test.ts tests/helper-effect-lease-loss.test.ts tests/authored-helpers.test.ts tests/authored-flow-slack.test.ts tests/effect-channel.test.ts --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5630ms - ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 436ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2047ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2048ms -(node:356923) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5331ms + ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 319ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2039ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2049ms +(node:414643) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (36 tests) 461ms - ✓ tests/journal-client.test.ts (17 tests) 83ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 6771ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 486ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 949ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2727ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 308ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 842ms - ✓ tests/journal-client-read-timeout.test.ts (27 tests) 1702ms - ✓ tests/authored-flow-slack.test.ts (7 tests) 1165ms - ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 445ms - ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 359ms - ✓ tests/running-step-watch.test.ts (14 tests) 12322ms + ✓ tests/authored-root.test.ts (36 tests) 456ms + ✓ tests/journal-client.test.ts (17 tests) 79ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 6978ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 464ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1062ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2739ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 351ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 793ms + ✓ tests/journal-client-read-timeout.test.ts (27 tests) 1801ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 1149ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 405ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 402ms + ✓ tests/running-step-watch.test.ts (14 tests) 12320ms ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2116ms ✓ cancels promptly while a lease snapshot read is in flight 2006ms ✓ a completion push ends the wait without waiting out an in-flight snapshot 2054ms ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms - ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2003ms - ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2002ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2574ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 661ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 589ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 666ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 657ms - ✓ tests/authored-helpers.test.ts (6 tests) 1859ms - ✓ runs every available provider through the real kernel and resumes completed effects without a second write 878ms - ✓ replays after SIGKILL before confirm with the same token and one successful completion 425ms - ✓ replays after SIGKILL before complete with the same token and one successful completion 325ms - ✓ tests/classify-outcome.test.ts (11 tests) 7426ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2010ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3008ms + ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2004ms + ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2001ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2517ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 655ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 529ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 681ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 652ms + ✓ tests/authored-helpers.test.ts (6 tests) 1855ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 801ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 411ms + ✓ replays after SIGKILL before complete with the same token and one successful completion 413ms + ✓ tests/classify-outcome.test.ts (11 tests) 7424ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2007ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3006ms ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2002ms - ✓ tests/human-live.test.ts (3 tests) 4300ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2560ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1082ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 658ms - ✓ tests/worker-lease.test.ts (7 tests) 19ms - ✓ tests/communication-worker.test.ts (18 tests) 1719ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 20ms - ✓ tests/effect-channel.test.ts (5 tests) 278ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 619ms + ✓ tests/human-live.test.ts (3 tests) 4576ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2824ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1017ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 734ms + ✓ tests/communication-worker.test.ts (19 tests) 1713ms + ✓ tests/worker-lease.test.ts (8 tests) 12ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 27ms + ✓ tests/communication.test.ts (10 tests) 14ms + ✓ tests/effect-channel.test.ts (5 tests) 277ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 707ms ✓ tests/run-daemon-unresponsive.test.ts (5 tests) 6ms ✓ tests/memoization.test.ts (58 tests) 64ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2440ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2056ms - ✓ drains read and watch promises before an authored flow completes 383ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2488ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2058ms + ✓ drains read and watch promises before an authored flow completes 429ms ✓ tests/reuse-summary-interruption.test.ts (6 tests) 4ms ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms - ✓ tests/journal-client-completion.test.ts (6 tests) 104ms + ✓ tests/journal-client-completion.test.ts (6 tests) 105ms ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms - ✓ tests/direct-run-worker-lease.test.ts (3 tests) 10ms - ✓ tests/helper-effect-lease-loss.test.ts (1 test) 223ms + ✓ tests/direct-run-worker-lease.test.ts (3 tests) 13ms + ✓ tests/helper-effect-lease-loss.test.ts (1 test) 232ms ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 47ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 14ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 17ms - Test Files 28 passed (28) - Tests 352 passed (352) - Start at 05:06:34 - Duration 60.19s (transform 1.26s, setup 150ms, collect 6.87s, tests 49.88s, environment 4ms, prepare 969ms) + Test Files 29 passed (29) + Tests 364 passed (364) + Start at 05:18:16 + Duration 60.69s (transform 1.13s, setup 158ms, collect 6.83s, tests 50.24s, environment 4ms, prepare 1.03s) exit=0 ``` @@ -92,13 +93,13 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 943ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 943ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 921ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 920ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 05:07:35 - Duration 2.18s (transform 616ms, setup 19ms, collect 1.00s, tests 943ms, environment 0ms, prepare 58ms) + Start at 05:19:17 + Duration 2.17s (transform 616ms, setup 14ms, collect 1.01s, tests 921ms, environment 0ms, prepare 70ms) exit=0 ``` @@ -120,8 +121,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 109ms - × serves a bounded read while an unbounded command is in flight 108ms + ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 113ms + × serves a bounded read while an unbounded command is in flight 112ms → expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -147,8 +148,8 @@ AssertionError: expected true to be false // Object.is equality Test Files 1 failed (1) Tests 1 failed | 26 skipped (27) - Start at 05:06:01 - Duration 506ms (transform 154ms, setup 15ms, collect 263ms, tests 109ms, environment 0ms, prepare 28ms) + Start at 05:17:43 + Duration 562ms (transform 168ms, setup 14ms, collect 296ms, tests 113ms, environment 0ms, prepare 40ms) exit=1 @@ -157,12 +158,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 106ms + ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 107ms Test Files 1 passed (1) Tests 1 passed | 26 skipped (27) - Start at 05:06:02 - Duration 543ms (transform 166ms, setup 16ms, collect 288ms, tests 106ms, environment 0ms, prepare 40ms) + Start at 05:17:44 + Duration 568ms (transform 170ms, setup 15ms, collect 297ms, tests 107ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -183,14 +184,14 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 1006ms - × reconnects the reader after it disconnects within the read budget 1006ms - → journal client: run.get read session was interrupted after 80 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 1009ms + × reconnects the reader after it disconnects within the read budget 1008ms + → journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 80 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:71:9 69| let last: unknown; 70| const exhausted = () => last instanceof JournalReadInterruptedError @@ -213,8 +214,8 @@ Caused by: Error: journal client: connection closed Test Files 1 failed (1) Tests 1 failed | 26 skipped (27) - Start at 05:06:03 - Duration 1.46s (transform 169ms, setup 17ms, collect 292ms, tests 1.01s, environment 0ms, prepare 41ms) + Start at 05:17:45 + Duration 1.48s (transform 172ms, setup 17ms, collect 302ms, tests 1.01s, environment 0ms, prepare 42ms) exit=1 @@ -223,12 +224,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 72ms + ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 88ms Test Files 1 passed (1) Tests 1 passed | 26 skipped (27) - Start at 05:06:05 - Duration 515ms (transform 168ms, setup 16ms, collect 294ms, tests 72ms, environment 0ms, prepare 41ms) + Start at 05:17:47 + Duration 531ms (transform 167ms, setup 17ms, collect 279ms, tests 88ms, environment 0ms, prepare 40ms) exit=0 ``` @@ -254,8 +255,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 76ms - × retries a reader setup that timed out instead of reading on the primary for good 75ms + ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 66ms + × retries a reader setup that timed out instead of reading on the primary for good 66ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -281,8 +282,8 @@ AssertionError: expected 1 to be 2 // Object.is equality Test Files 1 failed (1) Tests 1 failed | 26 skipped (27) - Start at 05:06:06 - Duration 540ms (transform 173ms, setup 17ms, collect 298ms, tests 76ms, environment 0ms, prepare 41ms) + Start at 05:17:47 + Duration 497ms (transform 171ms, setup 16ms, collect 292ms, tests 66ms, environment 0ms, prepare 36ms) exit=1 @@ -291,12 +292,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 75ms + ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 72ms Test Files 1 passed (1) Tests 1 passed | 26 skipped (27) - Start at 05:06:06 - Duration 526ms (transform 164ms, setup 17ms, collect 290ms, tests 75ms, environment 0ms, prepare 40ms) + Start at 05:17:48 + Duration 518ms (transform 161ms, setup 14ms, collect 286ms, tests 72ms, environment 0ms, prepare 42ms) exit=0 ``` @@ -316,8 +317,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2110ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2109ms + ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2108ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2108ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -336,8 +337,8 @@ AssertionError: expected 41 to be less than or equal to 1 Test Files 1 failed (1) Tests 1 failed | 13 skipped (14) - Start at 05:06:07 - Duration 2.60s (transform 192ms, setup 16ms, collect 332ms, tests 2.11s, environment 0ms, prepare 42ms) + Start at 05:17:49 + Duration 2.60s (transform 192ms, setup 17ms, collect 326ms, tests 2.11s, environment 0ms, prepare 41ms) exit=1 @@ -351,8 +352,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu Test Files 1 passed (1) Tests 1 passed | 13 skipped (14) - Start at 05:06:10 - Duration 2.59s (transform 185ms, setup 16ms, collect 309ms, tests 2.12s, environment 0ms, prepare 40ms) + Start at 05:17:52 + Duration 2.60s (transform 194ms, setup 17ms, collect 317ms, tests 2.12s, environment 0ms, prepare 42ms) exit=0 ``` @@ -373,8 +374,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2217ms - × cancels promptly while a lease snapshot read is in flight 2216ms + ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2215ms + × cancels promptly while a lease snapshot read is in flight 2214ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -393,8 +394,8 @@ AssertionError: expected 'still waiting' to be an instance of Error Test Files 1 failed (1) Tests 1 failed | 13 skipped (14) - Start at 05:06:13 - Duration 2.70s (transform 187ms, setup 17ms, collect 320ms, tests 2.22s, environment 0ms, prepare 40ms) + Start at 05:17:55 + Duration 2.68s (transform 188ms, setup 16ms, collect 308ms, tests 2.21s, environment 0ms, prepare 40ms) exit=1 @@ -403,13 +404,13 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2012ms - ✓ cancels promptly while a lease snapshot read is in flight 2011ms + ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2010ms + ✓ cancels promptly while a lease snapshot read is in flight 2009ms Test Files 1 passed (1) Tests 1 passed | 13 skipped (14) - Start at 05:06:16 - Duration 2.48s (transform 185ms, setup 17ms, collect 317ms, tests 2.01s, environment 0ms, prepare 40ms) + Start at 05:17:58 + Duration 2.49s (transform 201ms, setup 16ms, collect 346ms, tests 2.01s, environment 0ms, prepare 28ms) exit=0 ``` @@ -430,7 +431,7 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 3010ms - × a completion push ends the wait without waiting out an in-flight snapshot 3010ms + × a completion push ends the wait without waiting out an in-flight snapshot 3009ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -453,8 +454,8 @@ Received: "still waiting" Test Files 1 failed (1) Tests 1 failed | 13 skipped (14) - Start at 05:06:19 - Duration 3.49s (transform 192ms, setup 15ms, collect 325ms, tests 3.01s, environment 0ms, prepare 41ms) + Start at 05:18:01 + Duration 3.50s (transform 189ms, setup 17ms, collect 323ms, tests 3.01s, environment 0ms, prepare 41ms) exit=1 @@ -463,13 +464,13 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2062ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2061ms + ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2057ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2057ms Test Files 1 passed (1) Tests 1 passed | 13 skipped (14) - Start at 05:06:23 - Duration 2.53s (transform 182ms, setup 14ms, collect 311ms, tests 2.06s, environment 0ms, prepare 44ms) + Start at 05:18:05 + Duration 2.55s (transform 194ms, setup 17ms, collect 338ms, tests 2.06s, environment 0ms, prepare 41ms) exit=0 ``` @@ -478,7 +479,7 @@ exit=0 ```text Source: packages/sdk/src/worker-lease.ts -Original SHA256: 44c13f9f61ac13cb8c0f872f4c22b1c212e020d05880ed080ea7205ba6007213 +Original SHA256: 4c58d4670422dc2169764af16eb082c55e72c5321ed9d6001af3607f03955fa3 Replaced: if (error instanceof JournalRequestTimeoutError && error.verb === 'step.heartbeat') { throw new WorkerLeaseLostError('renewal_expired', error.message, { cause: error }); @@ -493,7 +494,7 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 19ms - × a heartbeat timeout is lease loss rather than a worker body failure 18ms + × a heartbeat timeout is lease loss rather than a worker body failure 19ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -512,12 +513,12 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 05:06:26 - Duration 472ms (transform 167ms, setup 17ms, collect 290ms, tests 19ms, environment 0ms, prepare 42ms) + Start at 05:18:08 + Duration 437ms (transform 159ms, setup 17ms, collect 271ms, tests 19ms, environment 0ms, prepare 40ms) exit=1 -RESTORED SHA256: 44c13f9f61ac13cb8c0f872f4c22b1c212e020d05880ed080ea7205ba6007213 +RESTORED SHA256: 4c58d4670422dc2169764af16eb082c55e72c5321ed9d6001af3607f03955fa3 $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartbeat timeout' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk @@ -526,8 +527,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb Test Files 1 passed (1) Tests 1 passed (1) - Start at 05:06:27 - Duration 443ms (transform 165ms, setup 17ms, collect 284ms, tests 16ms, environment 0ms, prepare 29ms) + Start at 05:18:08 + Duration 471ms (transform 167ms, setup 16ms, collect 293ms, tests 16ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -554,7 +555,7 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (36 tests | 3 failed | 33 skipped) 31ms + ❯ tests/authored-root.test.ts (36 tests | 3 failed | 33 skipped) 32ms × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 22ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) @@ -641,8 +642,8 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } Test Files 1 failed (1) Tests 3 failed | 33 skipped (36) - Start at 05:06:27 - Duration 1.30s (transform 662ms, setup 17ms, collect 1.11s, tests 31ms, environment 0ms, prepare 41ms) + Start at 05:18:09 + Duration 1.31s (transform 674ms, setup 17ms, collect 1.11s, tests 32ms, environment 0ms, prepare 41ms) exit=1 @@ -651,12 +652,12 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (36 tests | 33 skipped) 24ms + ✓ tests/authored-root.test.ts (36 tests | 33 skipped) 23ms Test Files 1 passed (1) Tests 3 passed | 33 skipped (36) - Start at 05:06:29 - Duration 1.37s (transform 701ms, setup 17ms, collect 1.18s, tests 24ms, environment 0ms, prepare 43ms) + Start at 05:18:11 + Duration 1.29s (transform 677ms, setup 18ms, collect 1.11s, tests 23ms, environment 0ms, prepare 41ms) exit=0 ``` diff --git a/packages/sdk/src/communication/local.ts b/packages/sdk/src/communication/local.ts index 60b7f4440..5fce83c62 100644 --- a/packages/sdk/src/communication/local.ts +++ b/packages/sdk/src/communication/local.ts @@ -1,5 +1,6 @@ import { onWorkerFailure } from '../worker-lease.js'; import { randomUUID } from 'node:crypto'; +import { FLOW_READ_BUDGET_MS } from '../journal-read-policy.js'; import { JournalClient } from '../journal-client.js'; import { AgentWorker } from '../worker.js'; import type { KernelRunSpec } from '../spec.js'; @@ -28,7 +29,8 @@ export async function attachCommunicationWorkers(spec: KernelRunSpec, socketPath if (step.type !== 'agent') continue; requireCommunicationCli(step.cli ?? spec.cli); if (step.surfaces?.workspace?.length) throw new Error('Local communication workers support stream surfaces only'); - const client = new JournalClient(socketPath); + // History reads on a retried attempt meet the same CPU load as flow reads. + const client = new JournalClient(socketPath, { readBudgetMs: FLOW_READ_BUDGET_MS }); const worker = new AgentWorker(client, { workerId: `communication-${randomUUID()}`, dataDir, environment, requiredStreams: [channelName(step.id, '$receipts')], pins: { workspace: [], streams: step.surfaces?.streams?.map(({ stream }) => ({ stream, read_offset: 0 })) ?? [] } }); diff --git a/packages/sdk/src/communication/worker.ts b/packages/sdk/src/communication/worker.ts index 4b67c2f34..fd1f25708 100644 --- a/packages/sdk/src/communication/worker.ts +++ b/packages/sdk/src/communication/worker.ts @@ -6,6 +6,7 @@ import type { JournalClient } from '../journal-client.js'; import type { StepDispatchEvent } from '../protocol.js'; import type { ResolvedKernelAgentStep } from '../resolved-cli-identity.js'; import { isLeaseLost, withWorkerLease } from '../worker-lease.js'; +import { isReadInterruptionError } from '../journal-read-policy.js'; import { workerInstruction } from '../worker-input.js'; import { resolveCliModel } from '../cli-adapter.js'; import { resolveAgentCwd } from '../agent-cwd.js'; @@ -29,8 +30,9 @@ export async function completeCommunicationDispatch(client: JournalClient, dispa output = await withWorkerLease(client, dispatch, signal => run(client, dispatch, instruction, spec, dataDir, signal, runRoot, environment)); } catch (error) { - // A lost lease belongs to the kernel's sweep and retry: never complete it here. - if (isLeaseLost(error)) throw error; + // A lost lease or an unanswered read belongs to the kernel's sweep and + // retry: never complete the attempt here as a worker failure. + if (isLeaseLost(error) || isReadInterruptionError(error)) throw error; completionReason = 'worker_error'; output = { error: error instanceof Error ? error.message : String(error) }; } diff --git a/packages/sdk/src/worker-lease.ts b/packages/sdk/src/worker-lease.ts index 9475b5428..575f69093 100644 --- a/packages/sdk/src/worker-lease.ts +++ b/packages/sdk/src/worker-lease.ts @@ -1,4 +1,5 @@ import { JournalProtocolError, JournalRequestTimeoutError, type JournalClient } from './journal-client.js'; +import { isReadInterruptionError } from './journal-read-policy.js'; import type { StepDispatchEvent } from './protocol.js'; export class WorkerLeaseLostError extends Error { @@ -22,7 +23,9 @@ export function isLeaseLost(error: unknown): boolean { export function onWorkerFailure(label: string, fatal: (error: unknown) => void) { return (error: unknown, dispatch?: StepDispatchEvent): void => { // Without the dispatch there is no attempt to hand back to the kernel: fail closed. - if (!isLeaseLost(error) || dispatch === undefined) { fatal(error); return; } + // A lost lease and an unanswered read both leave the attempt to the kernel, + // which sweeps the lease and retries it. + if (!(isLeaseLost(error) || isReadInterruptionError(error)) || dispatch === undefined) { fatal(error); return; } // The kernel owns this attempt's fate; its journal supplies the run outcome. // stderr keeps this diagnostic out of structured reports on stdout. process.emitWarning( diff --git a/packages/sdk/tests/communication-worker.test.ts b/packages/sdk/tests/communication-worker.test.ts index dc379c3e5..a5107bcda 100644 --- a/packages/sdk/tests/communication-worker.test.ts +++ b/packages/sdk/tests/communication-worker.test.ts @@ -108,3 +108,18 @@ it('leaves a heartbeat-lost attempt to the kernel instead of completing it as wo expect(isLeaseLost(error)).toBe(true); expect(f.client.stepComplete).not.toHaveBeenCalled(); }); +it('leaves a retried attempt whose history read was interrupted to the kernel', async () => { + const { JournalRequestTimeoutError } = await import('../src/journal-client.js'); + const { isReadInterruptionError } = await import('../src/journal-read-policy.js'); + const f = fixture(); + (f.client as unknown as { journalRead: () => Promise }).journalRead = async () => { + throw new JournalRequestTimeoutError('journal.read', 10, 3, 300_000, 300_000); + }; + const error = await completeCommunicationDispatch(f.client as unknown as JournalClient, { + run_id: 'run', step_id: 'agent', attempt: 2, idempotency_key: 'key', pins: {}, lease_id: 'lease', + lease_deadline_ms: Date.now() + 30_000, spec: { type: 'agent', cli: 'claude', instruction: '' }, + } as StepDispatchEvent, { type: 'relayflows.communication.v1', instruction: 'test', incoming: ['peer'], outgoing: [], timeoutMs: 1000 }, + '/tmp/data').catch(caught => caught); + expect(isReadInterruptionError(error)).toBe(true); + expect(f.client.stepComplete).not.toHaveBeenCalled(); +}); diff --git a/packages/sdk/tests/worker-lease.test.ts b/packages/sdk/tests/worker-lease.test.ts index 0cb168b81..6a769ac47 100644 --- a/packages/sdk/tests/worker-lease.test.ts +++ b/packages/sdk/tests/worker-lease.test.ts @@ -161,3 +161,15 @@ it('aborts the CLI when a later heartbeat response would revive an expired lease expect(client.stepComplete).not.toHaveBeenCalled(); expect(String(errors[0])).toContain('lease expired before renewal'); }); + +it('hands a worker read interruption back to the kernel instead of failing fatally', async () => { + const { onWorkerFailure } = await import('../src/worker-lease.js'); + const { JournalRequestTimeoutError } = await import('../src/journal-client.js'); + const fatal = vi.fn(); + const warn = vi.spyOn(process, 'emitWarning').mockImplementation(() => undefined); + onWorkerFailure('communication', fatal)(new JournalRequestTimeoutError('journal.read', 10, 3, 300_000, 300_000), + { run_id: 'r', step_id: 's', attempt: 2 } as never); + expect(fatal).not.toHaveBeenCalled(); + expect(warn).toHaveBeenCalled(); + warn.mockRestore(); +}); From a54d16cacf366df2384549b4f5401ecb78cfb843 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 05:37:06 -0700 Subject: [PATCH 31/34] fix(sdk): communication history reads stop on lease loss and shutdown - completeCommunicationDispatch scopes the client's budgeted reads to the attempt's lease signal, so a lost lease stops a delayed history read instead of letting it run beside the kernel's retry. - JournalClient.cancelReads() ends in-flight budgeted reads; the local communication workers call it before draining on close, so Ctrl-C does not wait out the read budget. Evidence re-captured at this tree. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: b8262468-ea26-4a6e-bee9-d193832e77dc --- evidence/run-read-timeout/final-focused.log | 115 ++++----- .../mutation-completion-aborts-snapshot.log | 16 +- .../run-read-timeout/mutation-heartbeat.log | 10 +- .../mutation-reader-reconnect.log | 22 +- .../mutation-reader-setup-retry.log | 18 +- evidence/run-read-timeout/mutation-reader.log | 18 +- .../mutation-root-parking.log | 12 +- .../mutation-snapshot-cancel.log | 16 +- .../mutation-watch-cadence.log | 8 +- evidence/run-read-timeout/resume-live.log | 8 +- evidence/run-read-timeout/verification.md | 243 +++++++++--------- packages/sdk/src/communication/local.ts | 3 + packages/sdk/src/communication/worker.ts | 8 +- packages/sdk/src/journal-client.ts | 5 + .../sdk/tests/communication-worker.test.ts | 10 + .../tests/journal-client-read-timeout.test.ts | 12 + 16 files changed, 278 insertions(+), 246 deletions(-) diff --git a/evidence/run-read-timeout/final-focused.log b/evidence/run-read-timeout/final-focused.log index 51bb845b8..db7edc2df 100644 --- a/evidence/run-read-timeout/final-focused.log +++ b/evidence/run-read-timeout/final-focused.log @@ -2,72 +2,71 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5331ms - ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 319ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2039ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2049ms -(node:414643) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5197ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2038ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2045ms +(node:497206) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (36 tests) 456ms - ✓ tests/journal-client.test.ts (17 tests) 79ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 6978ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 464ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1062ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2739ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 351ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 793ms - ✓ tests/journal-client-read-timeout.test.ts (27 tests) 1801ms - ✓ tests/authored-flow-slack.test.ts (7 tests) 1149ms - ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 405ms - ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 402ms - ✓ tests/running-step-watch.test.ts (14 tests) 12320ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2116ms - ✓ cancels promptly while a lease snapshot read is in flight 2006ms + ✓ tests/authored-root.test.ts (36 tests) 448ms + ✓ tests/journal-client.test.ts (17 tests) 77ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 6852ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 479ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1094ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2614ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 353ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 780ms + ✓ tests/journal-client-read-timeout.test.ts (28 tests) 1760ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 1167ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 415ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 413ms + ✓ tests/running-step-watch.test.ts (14 tests) 12324ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms + ✓ cancels promptly while a lease snapshot read is in flight 2005ms ✓ a completion push ends the wait without waiting out an in-flight snapshot 2054ms - ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms - ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2004ms - ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2001ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2517ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 655ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 529ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 681ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 652ms - ✓ tests/authored-helpers.test.ts (6 tests) 1855ms - ✓ runs every available provider through the real kernel and resumes completed effects without a second write 801ms - ✓ replays after SIGKILL before confirm with the same token and one successful completion 411ms - ✓ replays after SIGKILL before complete with the same token and one successful completion 413ms - ✓ tests/classify-outcome.test.ts (11 tests) 7424ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2007ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3006ms - ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2002ms - ✓ tests/human-live.test.ts (3 tests) 4576ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2824ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1017ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 734ms - ✓ tests/communication-worker.test.ts (19 tests) 1713ms - ✓ tests/worker-lease.test.ts (8 tests) 12ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 27ms + ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2003ms + ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2003ms + ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2002ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2775ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 700ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 678ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 697ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 698ms + ✓ tests/authored-helpers.test.ts (6 tests) 2483ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 1226ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 520ms + ✓ replays after SIGKILL before complete with the same token and one successful completion 468ms + ✓ tests/classify-outcome.test.ts (11 tests) 7423ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2008ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3007ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms + ✓ tests/human-live.test.ts (3 tests) 5036ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2906ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1236ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 894ms + ✓ tests/communication-worker.test.ts (20 tests) 1826ms + ✓ tests/worker-lease.test.ts (8 tests) 14ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 24ms ✓ tests/communication.test.ts (10 tests) 14ms - ✓ tests/effect-channel.test.ts (5 tests) 277ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 707ms + ✓ tests/effect-channel.test.ts (5 tests) 273ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 655ms ✓ tests/run-daemon-unresponsive.test.ts (5 tests) 6ms - ✓ tests/memoization.test.ts (58 tests) 64ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2488ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2058ms - ✓ drains read and watch promises before an authored flow completes 429ms + ✓ tests/memoization.test.ts (58 tests) 66ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2396ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2057ms + ✓ drains read and watch promises before an authored flow completes 339ms ✓ tests/reuse-summary-interruption.test.ts (6 tests) 4ms ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms - ✓ tests/journal-client-completion.test.ts (6 tests) 105ms - ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms - ✓ tests/direct-run-worker-lease.test.ts (3 tests) 13ms - ✓ tests/helper-effect-lease-loss.test.ts (1 test) 232ms + ✓ tests/journal-client-completion.test.ts (6 tests) 106ms + ✓ tests/resume-worker-lease.test.ts (3 tests) 8ms + ✓ tests/direct-run-worker-lease.test.ts (3 tests) 9ms + ✓ tests/helper-effect-lease-loss.test.ts (1 test) 236ms ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms - ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 47ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 17ms + ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 46ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms Test Files 29 passed (29) - Tests 364 passed (364) - Start at 05:18:16 - Duration 60.69s (transform 1.13s, setup 158ms, collect 6.83s, tests 50.24s, environment 4ms, prepare 1.03s) + Tests 366 passed (366) + Start at 05:36:01 + Duration 62.07s (transform 1.15s, setup 161ms, collect 7.02s, tests 51.25s, environment 4ms, prepare 1.05s) exit=0 diff --git a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log index b2ce240bc..751d78534 100644 --- a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log +++ b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 3010ms - × a completion push ends the wait without waiting out an in-flight snapshot 3009ms + ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 3011ms + × a completion push ends the wait without waiting out an in-flight snapshot 3011ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -34,8 +34,8 @@ Received: "still waiting" Test Files 1 failed (1) Tests 1 failed | 13 skipped (14) - Start at 05:18:01 - Duration 3.50s (transform 189ms, setup 17ms, collect 323ms, tests 3.01s, environment 0ms, prepare 41ms) + Start at 05:35:46 + Duration 3.48s (transform 192ms, setup 16ms, collect 318ms, tests 3.01s, environment 0ms, prepare 41ms) exit=1 @@ -44,12 +44,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2057ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2057ms + ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2062ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2062ms Test Files 1 passed (1) Tests 1 passed | 13 skipped (14) - Start at 05:18:05 - Duration 2.55s (transform 194ms, setup 17ms, collect 338ms, tests 2.06s, environment 0ms, prepare 41ms) + Start at 05:35:49 + Duration 2.53s (transform 188ms, setup 17ms, collect 321ms, tests 2.06s, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-heartbeat.log b/evidence/run-read-timeout/mutation-heartbeat.log index 66ae0613b..7022421ef 100644 --- a/evidence/run-read-timeout/mutation-heartbeat.log +++ b/evidence/run-read-timeout/mutation-heartbeat.log @@ -33,8 +33,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 05:18:08 - Duration 437ms (transform 159ms, setup 17ms, collect 271ms, tests 19ms, environment 0ms, prepare 40ms) + Start at 05:35:52 + Duration 473ms (transform 171ms, setup 17ms, collect 291ms, tests 19ms, environment 0ms, prepare 41ms) exit=1 @@ -43,11 +43,11 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 17ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 05:18:08 - Duration 471ms (transform 167ms, setup 16ms, collect 293ms, tests 16ms, environment 0ms, prepare 41ms) + Start at 05:35:53 + Duration 466ms (transform 171ms, setup 16ms, collect 286ms, tests 17ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-reconnect.log b/evidence/run-read-timeout/mutation-reader-reconnect.log index 4fa93b87e..7c021505c 100644 --- a/evidence/run-read-timeout/mutation-reader-reconnect.log +++ b/evidence/run-read-timeout/mutation-reader-reconnect.log @@ -11,14 +11,14 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 1009ms - × reconnects the reader after it disconnects within the read budget 1008ms - → journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ tests/journal-client-read-timeout.test.ts (28 tests | 1 failed | 27 skipped) 1007ms + × reconnects the reader after it disconnects within the read budget 1006ms + → journal client: run.get read session was interrupted after 76 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 76 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:71:9 69| let last: unknown; 70| const exhausted = () => last instanceof JournalReadInterruptedError @@ -40,9 +40,9 @@ Caused by: Error: journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 26 skipped (27) - Start at 05:17:45 - Duration 1.48s (transform 172ms, setup 17ms, collect 302ms, tests 1.01s, environment 0ms, prepare 42ms) + Tests 1 failed | 27 skipped (28) + Start at 05:35:29 + Duration 1.44s (transform 162ms, setup 17ms, collect 284ms, tests 1.01s, environment 0ms, prepare 40ms) exit=1 @@ -51,11 +51,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 88ms + ✓ tests/journal-client-read-timeout.test.ts (28 tests | 27 skipped) 81ms Test Files 1 passed (1) - Tests 1 passed | 26 skipped (27) - Start at 05:17:47 - Duration 531ms (transform 167ms, setup 17ms, collect 279ms, tests 88ms, environment 0ms, prepare 40ms) + Tests 1 passed | 27 skipped (28) + Start at 05:35:31 + Duration 531ms (transform 167ms, setup 17ms, collect 289ms, tests 81ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-setup-retry.log b/evidence/run-read-timeout/mutation-reader-setup-retry.log index 316433edd..593385663 100644 --- a/evidence/run-read-timeout/mutation-reader-setup-retry.log +++ b/evidence/run-read-timeout/mutation-reader-setup-retry.log @@ -16,8 +16,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 66ms - × retries a reader setup that timed out instead of reading on the primary for good 66ms + ❯ tests/journal-client-read-timeout.test.ts (28 tests | 1 failed | 27 skipped) 79ms + × retries a reader setup that timed out instead of reading on the primary for good 78ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -42,9 +42,9 @@ AssertionError: expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 26 skipped (27) - Start at 05:17:47 - Duration 497ms (transform 171ms, setup 16ms, collect 292ms, tests 66ms, environment 0ms, prepare 36ms) + Tests 1 failed | 27 skipped (28) + Start at 05:35:32 + Duration 524ms (transform 166ms, setup 17ms, collect 288ms, tests 79ms, environment 0ms, prepare 41ms) exit=1 @@ -53,11 +53,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 72ms + ✓ tests/journal-client-read-timeout.test.ts (28 tests | 27 skipped) 74ms Test Files 1 passed (1) - Tests 1 passed | 26 skipped (27) - Start at 05:17:48 - Duration 518ms (transform 161ms, setup 14ms, collect 286ms, tests 72ms, environment 0ms, prepare 42ms) + Tests 1 passed | 27 skipped (28) + Start at 05:35:33 + Duration 532ms (transform 173ms, setup 17ms, collect 300ms, tests 74ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader.log b/evidence/run-read-timeout/mutation-reader.log index 336be39d5..102a4c430 100644 --- a/evidence/run-read-timeout/mutation-reader.log +++ b/evidence/run-read-timeout/mutation-reader.log @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 113ms - × serves a bounded read while an unbounded command is in flight 112ms + ❯ tests/journal-client-read-timeout.test.ts (28 tests | 1 failed | 27 skipped) 109ms + × serves a bounded read while an unbounded command is in flight 108ms → expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -36,9 +36,9 @@ AssertionError: expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 26 skipped (27) - Start at 05:17:43 - Duration 562ms (transform 168ms, setup 14ms, collect 296ms, tests 113ms, environment 0ms, prepare 40ms) + Tests 1 failed | 27 skipped (28) + Start at 05:35:27 + Duration 569ms (transform 171ms, setup 16ms, collect 297ms, tests 109ms, environment 0ms, prepare 41ms) exit=1 @@ -47,11 +47,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 107ms + ✓ tests/journal-client-read-timeout.test.ts (28 tests | 27 skipped) 107ms Test Files 1 passed (1) - Tests 1 passed | 26 skipped (27) - Start at 05:17:44 - Duration 568ms (transform 170ms, setup 15ms, collect 297ms, tests 107ms, environment 0ms, prepare 41ms) + Tests 1 passed | 27 skipped (28) + Start at 05:35:28 + Duration 547ms (transform 167ms, setup 15ms, collect 293ms, tests 107ms, environment 0ms, prepare 30ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-root-parking.log b/evidence/run-read-timeout/mutation-root-parking.log index 1b126ed9a..65f612f01 100644 --- a/evidence/run-read-timeout/mutation-root-parking.log +++ b/evidence/run-read-timeout/mutation-root-parking.log @@ -17,7 +17,7 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (36 tests | 3 failed | 33 skipped) 32ms + ❯ tests/authored-root.test.ts (36 tests | 3 failed | 33 skipped) 31ms × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 22ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) @@ -104,8 +104,8 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } Test Files 1 failed (1) Tests 3 failed | 33 skipped (36) - Start at 05:18:09 - Duration 1.31s (transform 674ms, setup 17ms, collect 1.11s, tests 32ms, environment 0ms, prepare 41ms) + Start at 05:35:54 + Duration 1.30s (transform 672ms, setup 17ms, collect 1.11s, tests 31ms, environment 0ms, prepare 41ms) exit=1 @@ -114,11 +114,11 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (36 tests | 33 skipped) 23ms + ✓ tests/authored-root.test.ts (36 tests | 33 skipped) 25ms Test Files 1 passed (1) Tests 3 passed | 33 skipped (36) - Start at 05:18:11 - Duration 1.29s (transform 677ms, setup 18ms, collect 1.11s, tests 23ms, environment 0ms, prepare 41ms) + Start at 05:35:56 + Duration 1.30s (transform 683ms, setup 17ms, collect 1.11s, tests 25ms, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-snapshot-cancel.log b/evidence/run-read-timeout/mutation-snapshot-cancel.log index 8c9949b7c..88309d590 100644 --- a/evidence/run-read-timeout/mutation-snapshot-cancel.log +++ b/evidence/run-read-timeout/mutation-snapshot-cancel.log @@ -11,8 +11,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2215ms - × cancels promptly while a lease snapshot read is in flight 2214ms + ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2217ms + × cancels promptly while a lease snapshot read is in flight 2216ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -31,8 +31,8 @@ AssertionError: expected 'still waiting' to be an instance of Error Test Files 1 failed (1) Tests 1 failed | 13 skipped (14) - Start at 05:17:55 - Duration 2.68s (transform 188ms, setup 16ms, collect 308ms, tests 2.21s, environment 0ms, prepare 40ms) + Start at 05:35:40 + Duration 2.71s (transform 196ms, setup 17ms, collect 330ms, tests 2.22s, environment 0ms, prepare 41ms) exit=1 @@ -41,12 +41,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2010ms - ✓ cancels promptly while a lease snapshot read is in flight 2009ms + ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2011ms + ✓ cancels promptly while a lease snapshot read is in flight 2011ms Test Files 1 passed (1) Tests 1 passed | 13 skipped (14) - Start at 05:17:58 - Duration 2.49s (transform 201ms, setup 16ms, collect 346ms, tests 2.01s, environment 0ms, prepare 28ms) + Start at 05:35:43 + Duration 2.49s (transform 189ms, setup 17ms, collect 320ms, tests 2.01s, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-watch-cadence.log b/evidence/run-read-timeout/mutation-watch-cadence.log index 1e16924c7..0a7aa1cd7 100644 --- a/evidence/run-read-timeout/mutation-watch-cadence.log +++ b/evidence/run-read-timeout/mutation-watch-cadence.log @@ -30,8 +30,8 @@ AssertionError: expected 41 to be less than or equal to 1 Test Files 1 failed (1) Tests 1 failed | 13 skipped (14) - Start at 05:17:49 - Duration 2.60s (transform 192ms, setup 17ms, collect 326ms, tests 2.11s, environment 0ms, prepare 41ms) + Start at 05:35:34 + Duration 2.58s (transform 194ms, setup 17ms, collect 324ms, tests 2.11s, environment 0ms, prepare 42ms) exit=1 @@ -45,7 +45,7 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu Test Files 1 passed (1) Tests 1 passed | 13 skipped (14) - Start at 05:17:52 - Duration 2.60s (transform 194ms, setup 17ms, collect 317ms, tests 2.12s, environment 0ms, prepare 42ms) + Start at 05:35:37 + Duration 2.60s (transform 192ms, setup 16ms, collect 327ms, tests 2.12s, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/resume-live.log b/evidence/run-read-timeout/resume-live.log index 471f63f53..661fcf1d0 100644 --- a/evidence/run-read-timeout/resume-live.log +++ b/evidence/run-read-timeout/resume-live.log @@ -2,12 +2,12 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 921ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 920ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 901ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 901ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 05:19:17 - Duration 2.17s (transform 616ms, setup 14ms, collect 1.01s, tests 921ms, environment 0ms, prepare 70ms) + Start at 05:37:03 + Duration 2.12s (transform 614ms, setup 16ms, collect 977ms, tests 901ms, environment 0ms, prepare 69ms) exit=0 diff --git a/evidence/run-read-timeout/verification.md b/evidence/run-read-timeout/verification.md index e7b8ff333..b0b264630 100644 --- a/evidence/run-read-timeout/verification.md +++ b/evidence/run-read-timeout/verification.md @@ -15,73 +15,72 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5331ms - ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 319ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2039ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2049ms -(node:414643) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5197ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2038ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2045ms +(node:497206) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (36 tests) 456ms - ✓ tests/journal-client.test.ts (17 tests) 79ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 6978ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 464ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1062ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2739ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 351ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 793ms - ✓ tests/journal-client-read-timeout.test.ts (27 tests) 1801ms - ✓ tests/authored-flow-slack.test.ts (7 tests) 1149ms - ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 405ms - ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 402ms - ✓ tests/running-step-watch.test.ts (14 tests) 12320ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2116ms - ✓ cancels promptly while a lease snapshot read is in flight 2006ms + ✓ tests/authored-root.test.ts (36 tests) 448ms + ✓ tests/journal-client.test.ts (17 tests) 77ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 6852ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 479ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1094ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2614ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 353ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 780ms + ✓ tests/journal-client-read-timeout.test.ts (28 tests) 1760ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 1167ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 415ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 413ms + ✓ tests/running-step-watch.test.ts (14 tests) 12324ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms + ✓ cancels promptly while a lease snapshot read is in flight 2005ms ✓ a completion push ends the wait without waiting out an in-flight snapshot 2054ms - ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms - ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2004ms - ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2001ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2517ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 655ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 529ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 681ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 652ms - ✓ tests/authored-helpers.test.ts (6 tests) 1855ms - ✓ runs every available provider through the real kernel and resumes completed effects without a second write 801ms - ✓ replays after SIGKILL before confirm with the same token and one successful completion 411ms - ✓ replays after SIGKILL before complete with the same token and one successful completion 413ms - ✓ tests/classify-outcome.test.ts (11 tests) 7424ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2007ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3006ms - ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2002ms - ✓ tests/human-live.test.ts (3 tests) 4576ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2824ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1017ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 734ms - ✓ tests/communication-worker.test.ts (19 tests) 1713ms - ✓ tests/worker-lease.test.ts (8 tests) 12ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 27ms + ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2003ms + ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2003ms + ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2002ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2775ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 700ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 678ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 697ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 698ms + ✓ tests/authored-helpers.test.ts (6 tests) 2483ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 1226ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 520ms + ✓ replays after SIGKILL before complete with the same token and one successful completion 468ms + ✓ tests/classify-outcome.test.ts (11 tests) 7423ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2008ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3007ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms + ✓ tests/human-live.test.ts (3 tests) 5036ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2906ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1236ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 894ms + ✓ tests/communication-worker.test.ts (20 tests) 1826ms + ✓ tests/worker-lease.test.ts (8 tests) 14ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 24ms ✓ tests/communication.test.ts (10 tests) 14ms - ✓ tests/effect-channel.test.ts (5 tests) 277ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 707ms + ✓ tests/effect-channel.test.ts (5 tests) 273ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 655ms ✓ tests/run-daemon-unresponsive.test.ts (5 tests) 6ms - ✓ tests/memoization.test.ts (58 tests) 64ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2488ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2058ms - ✓ drains read and watch promises before an authored flow completes 429ms + ✓ tests/memoization.test.ts (58 tests) 66ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2396ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2057ms + ✓ drains read and watch promises before an authored flow completes 339ms ✓ tests/reuse-summary-interruption.test.ts (6 tests) 4ms ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms - ✓ tests/journal-client-completion.test.ts (6 tests) 105ms - ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms - ✓ tests/direct-run-worker-lease.test.ts (3 tests) 13ms - ✓ tests/helper-effect-lease-loss.test.ts (1 test) 232ms + ✓ tests/journal-client-completion.test.ts (6 tests) 106ms + ✓ tests/resume-worker-lease.test.ts (3 tests) 8ms + ✓ tests/direct-run-worker-lease.test.ts (3 tests) 9ms + ✓ tests/helper-effect-lease-loss.test.ts (1 test) 236ms ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms - ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 47ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 17ms + ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 46ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms Test Files 29 passed (29) - Tests 364 passed (364) - Start at 05:18:16 - Duration 60.69s (transform 1.13s, setup 158ms, collect 6.83s, tests 50.24s, environment 4ms, prepare 1.03s) + Tests 366 passed (366) + Start at 05:36:01 + Duration 62.07s (transform 1.15s, setup 161ms, collect 7.02s, tests 51.25s, environment 4ms, prepare 1.05s) exit=0 ``` @@ -93,13 +92,13 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 921ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 920ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 901ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 901ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 05:19:17 - Duration 2.17s (transform 616ms, setup 14ms, collect 1.01s, tests 921ms, environment 0ms, prepare 70ms) + Start at 05:37:03 + Duration 2.12s (transform 614ms, setup 16ms, collect 977ms, tests 901ms, environment 0ms, prepare 69ms) exit=0 ``` @@ -121,8 +120,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 113ms - × serves a bounded read while an unbounded command is in flight 112ms + ❯ tests/journal-client-read-timeout.test.ts (28 tests | 1 failed | 27 skipped) 109ms + × serves a bounded read while an unbounded command is in flight 108ms → expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -147,9 +146,9 @@ AssertionError: expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 26 skipped (27) - Start at 05:17:43 - Duration 562ms (transform 168ms, setup 14ms, collect 296ms, tests 113ms, environment 0ms, prepare 40ms) + Tests 1 failed | 27 skipped (28) + Start at 05:35:27 + Duration 569ms (transform 171ms, setup 16ms, collect 297ms, tests 109ms, environment 0ms, prepare 41ms) exit=1 @@ -158,12 +157,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 107ms + ✓ tests/journal-client-read-timeout.test.ts (28 tests | 27 skipped) 107ms Test Files 1 passed (1) - Tests 1 passed | 26 skipped (27) - Start at 05:17:44 - Duration 568ms (transform 170ms, setup 15ms, collect 297ms, tests 107ms, environment 0ms, prepare 41ms) + Tests 1 passed | 27 skipped (28) + Start at 05:35:28 + Duration 547ms (transform 167ms, setup 15ms, collect 293ms, tests 107ms, environment 0ms, prepare 30ms) exit=0 ``` @@ -184,14 +183,14 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 1009ms - × reconnects the reader after it disconnects within the read budget 1008ms - → journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ tests/journal-client-read-timeout.test.ts (28 tests | 1 failed | 27 skipped) 1007ms + × reconnects the reader after it disconnects within the read budget 1006ms + → journal client: run.get read session was interrupted after 76 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 78 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 76 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:71:9 69| let last: unknown; 70| const exhausted = () => last instanceof JournalReadInterruptedError @@ -213,9 +212,9 @@ Caused by: Error: journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 26 skipped (27) - Start at 05:17:45 - Duration 1.48s (transform 172ms, setup 17ms, collect 302ms, tests 1.01s, environment 0ms, prepare 42ms) + Tests 1 failed | 27 skipped (28) + Start at 05:35:29 + Duration 1.44s (transform 162ms, setup 17ms, collect 284ms, tests 1.01s, environment 0ms, prepare 40ms) exit=1 @@ -224,12 +223,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 88ms + ✓ tests/journal-client-read-timeout.test.ts (28 tests | 27 skipped) 81ms Test Files 1 passed (1) - Tests 1 passed | 26 skipped (27) - Start at 05:17:47 - Duration 531ms (transform 167ms, setup 17ms, collect 279ms, tests 88ms, environment 0ms, prepare 40ms) + Tests 1 passed | 27 skipped (28) + Start at 05:35:31 + Duration 531ms (transform 167ms, setup 17ms, collect 289ms, tests 81ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -255,8 +254,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (27 tests | 1 failed | 26 skipped) 66ms - × retries a reader setup that timed out instead of reading on the primary for good 66ms + ❯ tests/journal-client-read-timeout.test.ts (28 tests | 1 failed | 27 skipped) 79ms + × retries a reader setup that timed out instead of reading on the primary for good 78ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -281,9 +280,9 @@ AssertionError: expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 26 skipped (27) - Start at 05:17:47 - Duration 497ms (transform 171ms, setup 16ms, collect 292ms, tests 66ms, environment 0ms, prepare 36ms) + Tests 1 failed | 27 skipped (28) + Start at 05:35:32 + Duration 524ms (transform 166ms, setup 17ms, collect 288ms, tests 79ms, environment 0ms, prepare 41ms) exit=1 @@ -292,12 +291,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (27 tests | 26 skipped) 72ms + ✓ tests/journal-client-read-timeout.test.ts (28 tests | 27 skipped) 74ms Test Files 1 passed (1) - Tests 1 passed | 26 skipped (27) - Start at 05:17:48 - Duration 518ms (transform 161ms, setup 14ms, collect 286ms, tests 72ms, environment 0ms, prepare 42ms) + Tests 1 passed | 27 skipped (28) + Start at 05:35:33 + Duration 532ms (transform 173ms, setup 17ms, collect 300ms, tests 74ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -337,8 +336,8 @@ AssertionError: expected 41 to be less than or equal to 1 Test Files 1 failed (1) Tests 1 failed | 13 skipped (14) - Start at 05:17:49 - Duration 2.60s (transform 192ms, setup 17ms, collect 326ms, tests 2.11s, environment 0ms, prepare 41ms) + Start at 05:35:34 + Duration 2.58s (transform 194ms, setup 17ms, collect 324ms, tests 2.11s, environment 0ms, prepare 42ms) exit=1 @@ -352,8 +351,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu Test Files 1 passed (1) Tests 1 passed | 13 skipped (14) - Start at 05:17:52 - Duration 2.60s (transform 194ms, setup 17ms, collect 317ms, tests 2.12s, environment 0ms, prepare 42ms) + Start at 05:35:37 + Duration 2.60s (transform 192ms, setup 16ms, collect 327ms, tests 2.12s, environment 0ms, prepare 41ms) exit=0 ``` @@ -374,8 +373,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2215ms - × cancels promptly while a lease snapshot read is in flight 2214ms + ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2217ms + × cancels promptly while a lease snapshot read is in flight 2216ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -394,8 +393,8 @@ AssertionError: expected 'still waiting' to be an instance of Error Test Files 1 failed (1) Tests 1 failed | 13 skipped (14) - Start at 05:17:55 - Duration 2.68s (transform 188ms, setup 16ms, collect 308ms, tests 2.21s, environment 0ms, prepare 40ms) + Start at 05:35:40 + Duration 2.71s (transform 196ms, setup 17ms, collect 330ms, tests 2.22s, environment 0ms, prepare 41ms) exit=1 @@ -404,13 +403,13 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2010ms - ✓ cancels promptly while a lease snapshot read is in flight 2009ms + ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2011ms + ✓ cancels promptly while a lease snapshot read is in flight 2011ms Test Files 1 passed (1) Tests 1 passed | 13 skipped (14) - Start at 05:17:58 - Duration 2.49s (transform 201ms, setup 16ms, collect 346ms, tests 2.01s, environment 0ms, prepare 28ms) + Start at 05:35:43 + Duration 2.49s (transform 189ms, setup 17ms, collect 320ms, tests 2.01s, environment 0ms, prepare 40ms) exit=0 ``` @@ -430,8 +429,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 3010ms - × a completion push ends the wait without waiting out an in-flight snapshot 3009ms + ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 3011ms + × a completion push ends the wait without waiting out an in-flight snapshot 3011ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -454,8 +453,8 @@ Received: "still waiting" Test Files 1 failed (1) Tests 1 failed | 13 skipped (14) - Start at 05:18:01 - Duration 3.50s (transform 189ms, setup 17ms, collect 323ms, tests 3.01s, environment 0ms, prepare 41ms) + Start at 05:35:46 + Duration 3.48s (transform 192ms, setup 16ms, collect 318ms, tests 3.01s, environment 0ms, prepare 41ms) exit=1 @@ -464,13 +463,13 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2057ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2057ms + ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2062ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2062ms Test Files 1 passed (1) Tests 1 passed | 13 skipped (14) - Start at 05:18:05 - Duration 2.55s (transform 194ms, setup 17ms, collect 338ms, tests 2.06s, environment 0ms, prepare 41ms) + Start at 05:35:49 + Duration 2.53s (transform 188ms, setup 17ms, collect 321ms, tests 2.06s, environment 0ms, prepare 41ms) exit=0 ``` @@ -513,8 +512,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 05:18:08 - Duration 437ms (transform 159ms, setup 17ms, collect 271ms, tests 19ms, environment 0ms, prepare 40ms) + Start at 05:35:52 + Duration 473ms (transform 171ms, setup 17ms, collect 291ms, tests 19ms, environment 0ms, prepare 41ms) exit=1 @@ -523,12 +522,12 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 17ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 05:18:08 - Duration 471ms (transform 167ms, setup 16ms, collect 293ms, tests 16ms, environment 0ms, prepare 41ms) + Start at 05:35:53 + Duration 466ms (transform 171ms, setup 16ms, collect 286ms, tests 17ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -555,7 +554,7 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (36 tests | 3 failed | 33 skipped) 32ms + ❯ tests/authored-root.test.ts (36 tests | 3 failed | 33 skipped) 31ms × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 22ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) @@ -642,8 +641,8 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } Test Files 1 failed (1) Tests 3 failed | 33 skipped (36) - Start at 05:18:09 - Duration 1.31s (transform 674ms, setup 17ms, collect 1.11s, tests 32ms, environment 0ms, prepare 41ms) + Start at 05:35:54 + Duration 1.30s (transform 672ms, setup 17ms, collect 1.11s, tests 31ms, environment 0ms, prepare 41ms) exit=1 @@ -652,12 +651,12 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (36 tests | 33 skipped) 23ms + ✓ tests/authored-root.test.ts (36 tests | 33 skipped) 25ms Test Files 1 passed (1) Tests 3 passed | 33 skipped (36) - Start at 05:18:11 - Duration 1.29s (transform 677ms, setup 18ms, collect 1.11s, tests 23ms, environment 0ms, prepare 41ms) + Start at 05:35:56 + Duration 1.30s (transform 683ms, setup 17ms, collect 1.11s, tests 25ms, environment 0ms, prepare 40ms) exit=0 ``` diff --git a/packages/sdk/src/communication/local.ts b/packages/sdk/src/communication/local.ts index 5fce83c62..0b4dfa7e6 100644 --- a/packages/sdk/src/communication/local.ts +++ b/packages/sdk/src/communication/local.ts @@ -19,6 +19,9 @@ export async function attachCommunicationWorkers(spec: KernelRunSpec, socketPath let failure: unknown; const close = async () => { const outcomes = await Promise.allSettled(workers.map(async ({ client, worker }) => { + // Stop budgeted reads first: draining the worker must not wait out a + // delayed history read on shutdown. + client.cancelReads(); try { await worker.close(); } finally { client.close(); } })); const rejected = outcomes.find(result => result.status === 'rejected'); diff --git a/packages/sdk/src/communication/worker.ts b/packages/sdk/src/communication/worker.ts index fd1f25708..a485e9863 100644 --- a/packages/sdk/src/communication/worker.ts +++ b/packages/sdk/src/communication/worker.ts @@ -27,8 +27,12 @@ export async function completeCommunicationDispatch(client: JournalClient, dispa let output: unknown; let completionReason: 'success' | 'worker_error' = 'success'; try { - output = await withWorkerLease(client, dispatch, signal => - run(client, dispatch, instruction, spec, dataDir, signal, runRoot, environment)); + output = await withWorkerLease(client, dispatch, signal => { + // A lost lease stops this attempt's history reads instead of letting + // them run out the budget beside the kernel's retry. + const releaseReads = client.scopeReads?.(signal) ?? (() => {}); + return run(client, dispatch, instruction, spec, dataDir, signal, runRoot, environment).finally(releaseReads); + }); } catch (error) { // A lost lease or an unanswered read belongs to the kernel's sweep and // retry: never complete the attempt here as a worker failure. diff --git a/packages/sdk/src/journal-client.ts b/packages/sdk/src/journal-client.ts index ed1312399..527d03939 100644 --- a/packages/sdk/src/journal-client.ts +++ b/packages/sdk/src/journal-client.ts @@ -94,6 +94,11 @@ export class JournalClient extends JournalConnection { return this.budgeted?.scope(signal) ?? (() => {}); } + /** End every budgeted read now and refuse later ones; the session itself stays open. */ + cancelReads(): void { + this.budgeted?.close(new Error('journal client: reads canceled')); + } + // --- Typed verb methods (gate 1 minimal set, kernel DESIGN.md §5) -------- /** Handshake; version mismatch is a hard error. Records the daemon's additive features. */ diff --git a/packages/sdk/tests/communication-worker.test.ts b/packages/sdk/tests/communication-worker.test.ts index a5107bcda..555047a4d 100644 --- a/packages/sdk/tests/communication-worker.test.ts +++ b/packages/sdk/tests/communication-worker.test.ts @@ -123,3 +123,13 @@ it('leaves a retried attempt whose history read was interrupted to the kernel', expect(isReadInterruptionError(error)).toBe(true); expect(f.client.stepComplete).not.toHaveBeenCalled(); }); +it('scopes the attempt history reads to its lease and releases them after', async () => { + const f = fixture(); + const release = vi.fn(); + const scopeReads = vi.fn((_signal: AbortSignal) => release); + (f.client as unknown as { scopeReads: typeof scopeReads }).scopeReads = scopeReads; + await f.execute(); + expect(scopeReads).toHaveBeenCalledTimes(1); + expect(scopeReads.mock.calls[0]![0]).toBeInstanceOf(AbortSignal); + expect(release).toHaveBeenCalledTimes(1); +}); diff --git a/packages/sdk/tests/journal-client-read-timeout.test.ts b/packages/sdk/tests/journal-client-read-timeout.test.ts index c667343da..848d9bfeb 100644 --- a/packages/sdk/tests/journal-client-read-timeout.test.ts +++ b/packages/sdk/tests/journal-client-read-timeout.test.ts @@ -340,3 +340,15 @@ it('canceling a read during reader setup frees the queue for the next read at on await expect(client.runGet('run')).resolves.toMatchObject({ status: 'running' }); expect(performance.now() - begun).toBeLessThan(2_000); }); + +it('cancelReads ends in-flight budgeted reads at once, ahead of a draining shutdown', async () => { + let entered!: () => void; + const started = new Promise(resolve => { entered = resolve; }); + const client = await setup({ 'journal.read': () => entered() }, 60_000); + const pending = client.journalRead('run', 1); + await started; + const begun = performance.now(); + client.cancelReads(); + await expect(pending).rejects.toThrow(); + expect(performance.now() - begun).toBeLessThan(40); +}); From 76ff7279c436a01dfbe87ff87511c3cca3e8b8dd Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 05:48:36 -0700 Subject: [PATCH 32/34] fix(sdk): drop a reader whose read was canceled; canceled reads are interruptions - relayflowd answers one frame at a time per connection, so a canceled in-flight read still occupies its dedicated reader; the reader is now dropped on attempt cancellation and the next read opens a fresh session rather than queueing behind the abandoned request. - cancelReads() ends reads as JournalReadInterruptedError, so a communication worker torn down mid-history-read hands its attempt back to the kernel instead of completing it as worker_error. Evidence re-captured at this tree. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: b8262468-ea26-4a6e-bee9-d193832e77dc --- evidence/run-read-timeout/final-focused.log | 115 ++++---- .../mutation-completion-aborts-snapshot.log | 16 +- .../run-read-timeout/mutation-heartbeat.log | 14 +- .../mutation-reader-reconnect.log | 26 +- .../mutation-reader-setup-retry.log | 22 +- evidence/run-read-timeout/mutation-reader.log | 22 +- .../mutation-root-parking.log | 14 +- .../mutation-snapshot-cancel.log | 12 +- .../mutation-watch-cadence.log | 16 +- evidence/run-read-timeout/resume-live.log | 8 +- evidence/run-read-timeout/verification.md | 265 +++++++++--------- packages/sdk/src/journal-budgeted-reads.ts | 5 +- packages/sdk/src/journal-client.ts | 8 +- .../tests/journal-client-read-timeout.test.ts | 40 +++ 14 files changed, 316 insertions(+), 267 deletions(-) diff --git a/evidence/run-read-timeout/final-focused.log b/evidence/run-read-timeout/final-focused.log index db7edc2df..70b353e60 100644 --- a/evidence/run-read-timeout/final-focused.log +++ b/evidence/run-read-timeout/final-focused.log @@ -2,71 +2,72 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5197ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2038ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2045ms -(node:497206) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5306ms + ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 310ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2037ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2053ms +(node:556211) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (36 tests) 448ms - ✓ tests/journal-client.test.ts (17 tests) 77ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 6852ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 479ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1094ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2614ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 353ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 780ms - ✓ tests/journal-client-read-timeout.test.ts (28 tests) 1760ms - ✓ tests/authored-flow-slack.test.ts (7 tests) 1167ms - ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 415ms - ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 413ms - ✓ tests/running-step-watch.test.ts (14 tests) 12324ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms + ✓ tests/authored-root.test.ts (36 tests) 462ms + ✓ tests/journal-client.test.ts (17 tests) 80ms + ✓ tests/journal-client-read-timeout.test.ts (30 tests) 1811ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 7019ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 448ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 880ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2922ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 335ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 934ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 1280ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 448ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 443ms + ✓ tests/running-step-watch.test.ts (14 tests) 12322ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms ✓ cancels promptly while a lease snapshot read is in flight 2005ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2054ms - ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2003ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2053ms + ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2003ms - ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2002ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2775ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 700ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 678ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 697ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 698ms - ✓ tests/authored-helpers.test.ts (6 tests) 2483ms - ✓ runs every available provider through the real kernel and resumes completed effects without a second write 1226ms - ✓ replays after SIGKILL before confirm with the same token and one successful completion 520ms - ✓ replays after SIGKILL before complete with the same token and one successful completion 468ms - ✓ tests/classify-outcome.test.ts (11 tests) 7423ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2008ms + ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2003ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2600ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 556ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 660ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 700ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 684ms + ✓ tests/authored-helpers.test.ts (6 tests) 2048ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 1001ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 449ms + ✓ replays after SIGKILL before complete with the same token and one successful completion 345ms + ✓ tests/classify-outcome.test.ts (11 tests) 7425ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2010ms ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3007ms - ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms - ✓ tests/human-live.test.ts (3 tests) 5036ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2906ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1236ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 894ms - ✓ tests/communication-worker.test.ts (20 tests) 1826ms - ✓ tests/worker-lease.test.ts (8 tests) 14ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 24ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2002ms + ✓ tests/human-live.test.ts (3 tests) 5028ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 3047ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1181ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 799ms + ✓ tests/communication-worker.test.ts (20 tests) 1829ms + ✓ tests/worker-lease.test.ts (8 tests) 15ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 22ms ✓ tests/communication.test.ts (10 tests) 14ms - ✓ tests/effect-channel.test.ts (5 tests) 273ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 655ms - ✓ tests/run-daemon-unresponsive.test.ts (5 tests) 6ms - ✓ tests/memoization.test.ts (58 tests) 66ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2396ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2057ms - ✓ drains read and watch promises before an authored flow completes 339ms + ✓ tests/effect-channel.test.ts (5 tests) 306ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 739ms + ✓ tests/run-daemon-unresponsive.test.ts (5 tests) 9ms + ✓ tests/memoization.test.ts (58 tests) 85ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2498ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2063ms + ✓ drains read and watch promises before an authored flow completes 434ms ✓ tests/reuse-summary-interruption.test.ts (6 tests) 4ms - ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms - ✓ tests/journal-client-completion.test.ts (6 tests) 106ms + ✓ tests/worker-lease-sweep.test.ts (4 tests) 9ms + ✓ tests/journal-client-completion.test.ts (6 tests) 105ms ✓ tests/resume-worker-lease.test.ts (3 tests) 8ms - ✓ tests/direct-run-worker-lease.test.ts (3 tests) 9ms - ✓ tests/helper-effect-lease-loss.test.ts (1 test) 236ms - ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms - ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 46ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms + ✓ tests/direct-run-worker-lease.test.ts (3 tests) 13ms + ✓ tests/helper-effect-lease-loss.test.ts (1 test) 298ms + ✓ tests/journal-client-subscriptions.test.ts (1 test) 10ms + ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 84ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms Test Files 29 passed (29) - Tests 366 passed (366) - Start at 05:36:01 - Duration 62.07s (transform 1.15s, setup 161ms, collect 7.02s, tests 51.25s, environment 4ms, prepare 1.05s) + Tests 368 passed (368) + Start at 05:47:30 + Duration 63.12s (transform 1.16s, setup 176ms, collect 7.54s, tests 51.45s, environment 4ms, prepare 1.15s) exit=0 diff --git a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log index 751d78534..162c1f1cc 100644 --- a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log +++ b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 3011ms - × a completion push ends the wait without waiting out an in-flight snapshot 3011ms + ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 3013ms + × a completion push ends the wait without waiting out an in-flight snapshot 3012ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -34,8 +34,8 @@ Received: "still waiting" Test Files 1 failed (1) Tests 1 failed | 13 skipped (14) - Start at 05:35:46 - Duration 3.48s (transform 192ms, setup 16ms, collect 318ms, tests 3.01s, environment 0ms, prepare 41ms) + Start at 05:47:15 + Duration 3.49s (transform 190ms, setup 17ms, collect 322ms, tests 3.01s, environment 0ms, prepare 40ms) exit=1 @@ -44,12 +44,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2062ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2062ms + ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2061ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2060ms Test Files 1 passed (1) Tests 1 passed | 13 skipped (14) - Start at 05:35:49 - Duration 2.53s (transform 188ms, setup 17ms, collect 321ms, tests 2.06s, environment 0ms, prepare 41ms) + Start at 05:47:19 + Duration 2.55s (transform 193ms, setup 18ms, collect 325ms, tests 2.06s, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-heartbeat.log b/evidence/run-read-timeout/mutation-heartbeat.log index 7022421ef..df1e7c9da 100644 --- a/evidence/run-read-timeout/mutation-heartbeat.log +++ b/evidence/run-read-timeout/mutation-heartbeat.log @@ -13,8 +13,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 19ms - × a heartbeat timeout is lease loss rather than a worker body failure 19ms + ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 21ms + × a heartbeat timeout is lease loss rather than a worker body failure 20ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -33,8 +33,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 05:35:52 - Duration 473ms (transform 171ms, setup 17ms, collect 291ms, tests 19ms, environment 0ms, prepare 41ms) + Start at 05:47:21 + Duration 467ms (transform 161ms, setup 16ms, collect 286ms, tests 21ms, environment 0ms, prepare 41ms) exit=1 @@ -43,11 +43,11 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/heartbeat-timeout.test.ts (1 test) 17ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 05:35:53 - Duration 466ms (transform 171ms, setup 16ms, collect 286ms, tests 17ms, environment 0ms, prepare 41ms) + Start at 05:47:22 + Duration 471ms (transform 165ms, setup 17ms, collect 290ms, tests 16ms, environment 0ms, prepare 43ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-reconnect.log b/evidence/run-read-timeout/mutation-reader-reconnect.log index 7c021505c..9482d60b7 100644 --- a/evidence/run-read-timeout/mutation-reader-reconnect.log +++ b/evidence/run-read-timeout/mutation-reader-reconnect.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: 668d4d45dd732da1c653985dda4c2c3ddb115dab7462c0c1187c3e820f779b6a +Original SHA256: 06ce5e0ba9248b0aae4d5973b91ad24b60c5333724bc53e395200f5c121f1510 Replaced: this.drop(reader); @@ -11,14 +11,14 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (28 tests | 1 failed | 27 skipped) 1007ms - × reconnects the reader after it disconnects within the read budget 1006ms - → journal client: run.get read session was interrupted after 76 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ tests/journal-client-read-timeout.test.ts (30 tests | 1 failed | 29 skipped) 1008ms + × reconnects the reader after it disconnects within the read budget 1007ms + → journal client: run.get read session was interrupted after 77 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 76 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 77 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:71:9 69| let last: unknown; 70| const exhausted = () => last instanceof JournalReadInterruptedError @@ -40,22 +40,22 @@ Caused by: Error: journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 27 skipped (28) - Start at 05:35:29 - Duration 1.44s (transform 162ms, setup 17ms, collect 284ms, tests 1.01s, environment 0ms, prepare 40ms) + Tests 1 failed | 29 skipped (30) + Start at 05:46:59 + Duration 1.47s (transform 174ms, setup 16ms, collect 303ms, tests 1.01s, environment 0ms, prepare 40ms) exit=1 -RESTORED SHA256: 668d4d45dd732da1c653985dda4c2c3ddb115dab7462c0c1187c3e820f779b6a +RESTORED SHA256: 06ce5e0ba9248b0aae4d5973b91ad24b60c5333724bc53e395200f5c121f1510 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'reconnects the reader after it disconnects' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (28 tests | 27 skipped) 81ms + ✓ tests/journal-client-read-timeout.test.ts (30 tests | 29 skipped) 70ms Test Files 1 passed (1) - Tests 1 passed | 27 skipped (28) - Start at 05:35:31 - Duration 531ms (transform 167ms, setup 17ms, collect 289ms, tests 81ms, environment 0ms, prepare 41ms) + Tests 1 passed | 29 skipped (30) + Start at 05:47:00 + Duration 485ms (transform 173ms, setup 16ms, collect 284ms, tests 70ms, environment 0ms, prepare 28ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-setup-retry.log b/evidence/run-read-timeout/mutation-reader-setup-retry.log index 593385663..fec4a23b5 100644 --- a/evidence/run-read-timeout/mutation-reader-setup-retry.log +++ b/evidence/run-read-timeout/mutation-reader-setup-retry.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: 668d4d45dd732da1c653985dda4c2c3ddb115dab7462c0c1187c3e820f779b6a +Original SHA256: 06ce5e0ba9248b0aae4d5973b91ad24b60c5333724bc53e395200f5c121f1510 Replaced: if (this.reader === reader) { this.reader = undefined; @@ -16,8 +16,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (28 tests | 1 failed | 27 skipped) 79ms - × retries a reader setup that timed out instead of reading on the primary for good 78ms + ❯ tests/journal-client-read-timeout.test.ts (30 tests | 1 failed | 29 skipped) 81ms + × retries a reader setup that timed out instead of reading on the primary for good 80ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -42,22 +42,22 @@ AssertionError: expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 27 skipped (28) - Start at 05:35:32 - Duration 524ms (transform 166ms, setup 17ms, collect 288ms, tests 79ms, environment 0ms, prepare 41ms) + Tests 1 failed | 29 skipped (30) + Start at 05:47:01 + Duration 536ms (transform 172ms, setup 17ms, collect 295ms, tests 81ms, environment 0ms, prepare 40ms) exit=1 -RESTORED SHA256: 668d4d45dd732da1c653985dda4c2c3ddb115dab7462c0c1187c3e820f779b6a +RESTORED SHA256: 06ce5e0ba9248b0aae4d5973b91ad24b60c5333724bc53e395200f5c121f1510 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'retries a reader setup that timed out' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (28 tests | 27 skipped) 74ms + ✓ tests/journal-client-read-timeout.test.ts (30 tests | 29 skipped) 70ms Test Files 1 passed (1) - Tests 1 passed | 27 skipped (28) - Start at 05:35:33 - Duration 532ms (transform 173ms, setup 17ms, collect 300ms, tests 74ms, environment 0ms, prepare 41ms) + Tests 1 passed | 29 skipped (30) + Start at 05:47:02 + Duration 540ms (transform 176ms, setup 17ms, collect 301ms, tests 70ms, environment 0ms, prepare 42ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader.log b/evidence/run-read-timeout/mutation-reader.log index 102a4c430..10c1f8ff2 100644 --- a/evidence/run-read-timeout/mutation-reader.log +++ b/evidence/run-read-timeout/mutation-reader.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: 668d4d45dd732da1c653985dda4c2c3ddb115dab7462c0c1187c3e820f779b6a +Original SHA256: 06ce5e0ba9248b0aae4d5973b91ad24b60c5333724bc53e395200f5c121f1510 Replaced: return await reader.requestOnce(verb, params, remaining, attemptSignal); With: @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (28 tests | 1 failed | 27 skipped) 109ms - × serves a bounded read while an unbounded command is in flight 108ms + ❯ tests/journal-client-read-timeout.test.ts (30 tests | 1 failed | 29 skipped) 113ms + × serves a bounded read while an unbounded command is in flight 113ms → expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -36,22 +36,22 @@ AssertionError: expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 27 skipped (28) - Start at 05:35:27 - Duration 569ms (transform 171ms, setup 16ms, collect 297ms, tests 109ms, environment 0ms, prepare 41ms) + Tests 1 failed | 29 skipped (30) + Start at 05:46:57 + Duration 554ms (transform 172ms, setup 16ms, collect 296ms, tests 113ms, environment 0ms, prepare 31ms) exit=1 -RESTORED SHA256: 668d4d45dd732da1c653985dda4c2c3ddb115dab7462c0c1187c3e820f779b6a +RESTORED SHA256: 06ce5e0ba9248b0aae4d5973b91ad24b60c5333724bc53e395200f5c121f1510 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'serves a bounded read' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (28 tests | 27 skipped) 107ms + ✓ tests/journal-client-read-timeout.test.ts (30 tests | 29 skipped) 106ms Test Files 1 passed (1) - Tests 1 passed | 27 skipped (28) - Start at 05:35:28 - Duration 547ms (transform 167ms, setup 15ms, collect 293ms, tests 107ms, environment 0ms, prepare 30ms) + Tests 1 passed | 29 skipped (30) + Start at 05:46:58 + Duration 567ms (transform 178ms, setup 16ms, collect 302ms, tests 106ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-root-parking.log b/evidence/run-read-timeout/mutation-root-parking.log index 65f612f01..52bdfcf38 100644 --- a/evidence/run-read-timeout/mutation-root-parking.log +++ b/evidence/run-read-timeout/mutation-root-parking.log @@ -17,8 +17,8 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (36 tests | 3 failed | 33 skipped) 31ms - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 22ms + ❯ tests/authored-root.test.ts (36 tests | 3 failed | 33 skipped) 32ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 24ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms @@ -104,8 +104,8 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } Test Files 1 failed (1) Tests 3 failed | 33 skipped (36) - Start at 05:35:54 - Duration 1.30s (transform 672ms, setup 17ms, collect 1.11s, tests 31ms, environment 0ms, prepare 41ms) + Start at 05:47:23 + Duration 1.28s (transform 671ms, setup 17ms, collect 1.10s, tests 32ms, environment 0ms, prepare 41ms) exit=1 @@ -114,11 +114,11 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (36 tests | 33 skipped) 25ms + ✓ tests/authored-root.test.ts (36 tests | 33 skipped) 22ms Test Files 1 passed (1) Tests 3 passed | 33 skipped (36) - Start at 05:35:56 - Duration 1.30s (transform 683ms, setup 17ms, collect 1.11s, tests 25ms, environment 0ms, prepare 40ms) + Start at 05:47:25 + Duration 1.28s (transform 669ms, setup 16ms, collect 1.10s, tests 22ms, environment 0ms, prepare 43ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-snapshot-cancel.log b/evidence/run-read-timeout/mutation-snapshot-cancel.log index 88309d590..c3ca52223 100644 --- a/evidence/run-read-timeout/mutation-snapshot-cancel.log +++ b/evidence/run-read-timeout/mutation-snapshot-cancel.log @@ -31,8 +31,8 @@ AssertionError: expected 'still waiting' to be an instance of Error Test Files 1 failed (1) Tests 1 failed | 13 skipped (14) - Start at 05:35:40 - Duration 2.71s (transform 196ms, setup 17ms, collect 330ms, tests 2.22s, environment 0ms, prepare 41ms) + Start at 05:47:09 + Duration 2.69s (transform 189ms, setup 16ms, collect 320ms, tests 2.22s, environment 0ms, prepare 40ms) exit=1 @@ -41,12 +41,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2011ms - ✓ cancels promptly while a lease snapshot read is in flight 2011ms + ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2010ms + ✓ cancels promptly while a lease snapshot read is in flight 2010ms Test Files 1 passed (1) Tests 1 passed | 13 skipped (14) - Start at 05:35:43 - Duration 2.49s (transform 189ms, setup 17ms, collect 320ms, tests 2.01s, environment 0ms, prepare 40ms) + Start at 05:47:12 + Duration 2.49s (transform 190ms, setup 17ms, collect 321ms, tests 2.01s, environment 0ms, prepare 42ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-watch-cadence.log b/evidence/run-read-timeout/mutation-watch-cadence.log index 0a7aa1cd7..ea3ce0f5a 100644 --- a/evidence/run-read-timeout/mutation-watch-cadence.log +++ b/evidence/run-read-timeout/mutation-watch-cadence.log @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2108ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2108ms + ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2111ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2110ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -30,8 +30,8 @@ AssertionError: expected 41 to be less than or equal to 1 Test Files 1 failed (1) Tests 1 failed | 13 skipped (14) - Start at 05:35:34 - Duration 2.58s (transform 194ms, setup 17ms, collect 324ms, tests 2.11s, environment 0ms, prepare 42ms) + Start at 05:47:03 + Duration 2.58s (transform 188ms, setup 16ms, collect 319ms, tests 2.11s, environment 0ms, prepare 40ms) exit=1 @@ -40,12 +40,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2118ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms + ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2119ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms Test Files 1 passed (1) Tests 1 passed | 13 skipped (14) - Start at 05:35:37 - Duration 2.60s (transform 192ms, setup 16ms, collect 327ms, tests 2.12s, environment 0ms, prepare 41ms) + Start at 05:47:06 + Duration 2.60s (transform 189ms, setup 16ms, collect 322ms, tests 2.12s, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/resume-live.log b/evidence/run-read-timeout/resume-live.log index 661fcf1d0..ad157d6da 100644 --- a/evidence/run-read-timeout/resume-live.log +++ b/evidence/run-read-timeout/resume-live.log @@ -2,12 +2,12 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 901ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 901ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 1004ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 1003ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 05:37:03 - Duration 2.12s (transform 614ms, setup 16ms, collect 977ms, tests 901ms, environment 0ms, prepare 69ms) + Start at 05:48:34 + Duration 2.27s (transform 578ms, setup 20ms, collect 942ms, tests 1.00s, environment 0ms, prepare 66ms) exit=0 diff --git a/evidence/run-read-timeout/verification.md b/evidence/run-read-timeout/verification.md index b0b264630..092e7608a 100644 --- a/evidence/run-read-timeout/verification.md +++ b/evidence/run-read-timeout/verification.md @@ -15,72 +15,73 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5197ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2038ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2045ms -(node:497206) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5306ms + ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 310ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2037ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2053ms +(node:556211) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (36 tests) 448ms - ✓ tests/journal-client.test.ts (17 tests) 77ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 6852ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 479ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1094ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2614ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 353ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 780ms - ✓ tests/journal-client-read-timeout.test.ts (28 tests) 1760ms - ✓ tests/authored-flow-slack.test.ts (7 tests) 1167ms - ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 415ms - ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 413ms - ✓ tests/running-step-watch.test.ts (14 tests) 12324ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms + ✓ tests/authored-root.test.ts (36 tests) 462ms + ✓ tests/journal-client.test.ts (17 tests) 80ms + ✓ tests/journal-client-read-timeout.test.ts (30 tests) 1811ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 7019ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 448ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 880ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2922ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 335ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 934ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 1280ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 448ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 443ms + ✓ tests/running-step-watch.test.ts (14 tests) 12322ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms ✓ cancels promptly while a lease snapshot read is in flight 2005ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2054ms - ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2003ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2053ms + ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2003ms - ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2002ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2775ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 700ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 678ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 697ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 698ms - ✓ tests/authored-helpers.test.ts (6 tests) 2483ms - ✓ runs every available provider through the real kernel and resumes completed effects without a second write 1226ms - ✓ replays after SIGKILL before confirm with the same token and one successful completion 520ms - ✓ replays after SIGKILL before complete with the same token and one successful completion 468ms - ✓ tests/classify-outcome.test.ts (11 tests) 7423ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2008ms + ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2003ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2600ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 556ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 660ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 700ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 684ms + ✓ tests/authored-helpers.test.ts (6 tests) 2048ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 1001ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 449ms + ✓ replays after SIGKILL before complete with the same token and one successful completion 345ms + ✓ tests/classify-outcome.test.ts (11 tests) 7425ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2010ms ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3007ms - ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms - ✓ tests/human-live.test.ts (3 tests) 5036ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2906ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1236ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 894ms - ✓ tests/communication-worker.test.ts (20 tests) 1826ms - ✓ tests/worker-lease.test.ts (8 tests) 14ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 24ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2002ms + ✓ tests/human-live.test.ts (3 tests) 5028ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 3047ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1181ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 799ms + ✓ tests/communication-worker.test.ts (20 tests) 1829ms + ✓ tests/worker-lease.test.ts (8 tests) 15ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 22ms ✓ tests/communication.test.ts (10 tests) 14ms - ✓ tests/effect-channel.test.ts (5 tests) 273ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 655ms - ✓ tests/run-daemon-unresponsive.test.ts (5 tests) 6ms - ✓ tests/memoization.test.ts (58 tests) 66ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2396ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2057ms - ✓ drains read and watch promises before an authored flow completes 339ms + ✓ tests/effect-channel.test.ts (5 tests) 306ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 739ms + ✓ tests/run-daemon-unresponsive.test.ts (5 tests) 9ms + ✓ tests/memoization.test.ts (58 tests) 85ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2498ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2063ms + ✓ drains read and watch promises before an authored flow completes 434ms ✓ tests/reuse-summary-interruption.test.ts (6 tests) 4ms - ✓ tests/worker-lease-sweep.test.ts (4 tests) 8ms - ✓ tests/journal-client-completion.test.ts (6 tests) 106ms + ✓ tests/worker-lease-sweep.test.ts (4 tests) 9ms + ✓ tests/journal-client-completion.test.ts (6 tests) 105ms ✓ tests/resume-worker-lease.test.ts (3 tests) 8ms - ✓ tests/direct-run-worker-lease.test.ts (3 tests) 9ms - ✓ tests/helper-effect-lease-loss.test.ts (1 test) 236ms - ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms - ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 46ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 15ms + ✓ tests/direct-run-worker-lease.test.ts (3 tests) 13ms + ✓ tests/helper-effect-lease-loss.test.ts (1 test) 298ms + ✓ tests/journal-client-subscriptions.test.ts (1 test) 10ms + ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 84ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms Test Files 29 passed (29) - Tests 366 passed (366) - Start at 05:36:01 - Duration 62.07s (transform 1.15s, setup 161ms, collect 7.02s, tests 51.25s, environment 4ms, prepare 1.05s) + Tests 368 passed (368) + Start at 05:47:30 + Duration 63.12s (transform 1.16s, setup 176ms, collect 7.54s, tests 51.45s, environment 4ms, prepare 1.15s) exit=0 ``` @@ -92,13 +93,13 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 901ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 901ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 1004ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 1003ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 05:37:03 - Duration 2.12s (transform 614ms, setup 16ms, collect 977ms, tests 901ms, environment 0ms, prepare 69ms) + Start at 05:48:34 + Duration 2.27s (transform 578ms, setup 20ms, collect 942ms, tests 1.00s, environment 0ms, prepare 66ms) exit=0 ``` @@ -109,7 +110,7 @@ The mutation sections below were regenerated by `cd packages/sdk && python3 ../. ```text Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: 668d4d45dd732da1c653985dda4c2c3ddb115dab7462c0c1187c3e820f779b6a +Original SHA256: 06ce5e0ba9248b0aae4d5973b91ad24b60c5333724bc53e395200f5c121f1510 Replaced: return await reader.requestOnce(verb, params, remaining, attemptSignal); With: @@ -120,8 +121,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (28 tests | 1 failed | 27 skipped) 109ms - × serves a bounded read while an unbounded command is in flight 108ms + ❯ tests/journal-client-read-timeout.test.ts (30 tests | 1 failed | 29 skipped) 113ms + × serves a bounded read while an unbounded command is in flight 113ms → expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -146,23 +147,23 @@ AssertionError: expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 27 skipped (28) - Start at 05:35:27 - Duration 569ms (transform 171ms, setup 16ms, collect 297ms, tests 109ms, environment 0ms, prepare 41ms) + Tests 1 failed | 29 skipped (30) + Start at 05:46:57 + Duration 554ms (transform 172ms, setup 16ms, collect 296ms, tests 113ms, environment 0ms, prepare 31ms) exit=1 -RESTORED SHA256: 668d4d45dd732da1c653985dda4c2c3ddb115dab7462c0c1187c3e820f779b6a +RESTORED SHA256: 06ce5e0ba9248b0aae4d5973b91ad24b60c5333724bc53e395200f5c121f1510 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'serves a bounded read' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (28 tests | 27 skipped) 107ms + ✓ tests/journal-client-read-timeout.test.ts (30 tests | 29 skipped) 106ms Test Files 1 passed (1) - Tests 1 passed | 27 skipped (28) - Start at 05:35:28 - Duration 547ms (transform 167ms, setup 15ms, collect 293ms, tests 107ms, environment 0ms, prepare 30ms) + Tests 1 passed | 29 skipped (30) + Start at 05:46:58 + Duration 567ms (transform 178ms, setup 16ms, collect 302ms, tests 106ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -171,7 +172,7 @@ exit=0 ```text Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: 668d4d45dd732da1c653985dda4c2c3ddb115dab7462c0c1187c3e820f779b6a +Original SHA256: 06ce5e0ba9248b0aae4d5973b91ad24b60c5333724bc53e395200f5c121f1510 Replaced: this.drop(reader); @@ -183,14 +184,14 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (28 tests | 1 failed | 27 skipped) 1007ms - × reconnects the reader after it disconnects within the read budget 1006ms - → journal client: run.get read session was interrupted after 76 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ tests/journal-client-read-timeout.test.ts (30 tests | 1 failed | 29 skipped) 1008ms + × reconnects the reader after it disconnects within the read budget 1007ms + → journal client: run.get read session was interrupted after 77 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 76 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 77 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:71:9 69| let last: unknown; 70| const exhausted = () => last instanceof JournalReadInterruptedError @@ -212,23 +213,23 @@ Caused by: Error: journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 27 skipped (28) - Start at 05:35:29 - Duration 1.44s (transform 162ms, setup 17ms, collect 284ms, tests 1.01s, environment 0ms, prepare 40ms) + Tests 1 failed | 29 skipped (30) + Start at 05:46:59 + Duration 1.47s (transform 174ms, setup 16ms, collect 303ms, tests 1.01s, environment 0ms, prepare 40ms) exit=1 -RESTORED SHA256: 668d4d45dd732da1c653985dda4c2c3ddb115dab7462c0c1187c3e820f779b6a +RESTORED SHA256: 06ce5e0ba9248b0aae4d5973b91ad24b60c5333724bc53e395200f5c121f1510 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'reconnects the reader after it disconnects' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (28 tests | 27 skipped) 81ms + ✓ tests/journal-client-read-timeout.test.ts (30 tests | 29 skipped) 70ms Test Files 1 passed (1) - Tests 1 passed | 27 skipped (28) - Start at 05:35:31 - Duration 531ms (transform 167ms, setup 17ms, collect 289ms, tests 81ms, environment 0ms, prepare 41ms) + Tests 1 passed | 29 skipped (30) + Start at 05:47:00 + Duration 485ms (transform 173ms, setup 16ms, collect 284ms, tests 70ms, environment 0ms, prepare 28ms) exit=0 ``` @@ -237,7 +238,7 @@ exit=0 ```text Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: 668d4d45dd732da1c653985dda4c2c3ddb115dab7462c0c1187c3e820f779b6a +Original SHA256: 06ce5e0ba9248b0aae4d5973b91ad24b60c5333724bc53e395200f5c121f1510 Replaced: if (this.reader === reader) { this.reader = undefined; @@ -254,8 +255,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (28 tests | 1 failed | 27 skipped) 79ms - × retries a reader setup that timed out instead of reading on the primary for good 78ms + ❯ tests/journal-client-read-timeout.test.ts (30 tests | 1 failed | 29 skipped) 81ms + × retries a reader setup that timed out instead of reading on the primary for good 80ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -280,23 +281,23 @@ AssertionError: expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 27 skipped (28) - Start at 05:35:32 - Duration 524ms (transform 166ms, setup 17ms, collect 288ms, tests 79ms, environment 0ms, prepare 41ms) + Tests 1 failed | 29 skipped (30) + Start at 05:47:01 + Duration 536ms (transform 172ms, setup 17ms, collect 295ms, tests 81ms, environment 0ms, prepare 40ms) exit=1 -RESTORED SHA256: 668d4d45dd732da1c653985dda4c2c3ddb115dab7462c0c1187c3e820f779b6a +RESTORED SHA256: 06ce5e0ba9248b0aae4d5973b91ad24b60c5333724bc53e395200f5c121f1510 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'retries a reader setup that timed out' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (28 tests | 27 skipped) 74ms + ✓ tests/journal-client-read-timeout.test.ts (30 tests | 29 skipped) 70ms Test Files 1 passed (1) - Tests 1 passed | 27 skipped (28) - Start at 05:35:33 - Duration 532ms (transform 173ms, setup 17ms, collect 300ms, tests 74ms, environment 0ms, prepare 41ms) + Tests 1 passed | 29 skipped (30) + Start at 05:47:02 + Duration 540ms (transform 176ms, setup 17ms, collect 301ms, tests 70ms, environment 0ms, prepare 42ms) exit=0 ``` @@ -316,8 +317,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2108ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2108ms + ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2111ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2110ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -336,8 +337,8 @@ AssertionError: expected 41 to be less than or equal to 1 Test Files 1 failed (1) Tests 1 failed | 13 skipped (14) - Start at 05:35:34 - Duration 2.58s (transform 194ms, setup 17ms, collect 324ms, tests 2.11s, environment 0ms, prepare 42ms) + Start at 05:47:03 + Duration 2.58s (transform 188ms, setup 16ms, collect 319ms, tests 2.11s, environment 0ms, prepare 40ms) exit=1 @@ -346,13 +347,13 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2118ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms + ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2119ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms Test Files 1 passed (1) Tests 1 passed | 13 skipped (14) - Start at 05:35:37 - Duration 2.60s (transform 192ms, setup 16ms, collect 327ms, tests 2.12s, environment 0ms, prepare 41ms) + Start at 05:47:06 + Duration 2.60s (transform 189ms, setup 16ms, collect 322ms, tests 2.12s, environment 0ms, prepare 41ms) exit=0 ``` @@ -393,8 +394,8 @@ AssertionError: expected 'still waiting' to be an instance of Error Test Files 1 failed (1) Tests 1 failed | 13 skipped (14) - Start at 05:35:40 - Duration 2.71s (transform 196ms, setup 17ms, collect 330ms, tests 2.22s, environment 0ms, prepare 41ms) + Start at 05:47:09 + Duration 2.69s (transform 189ms, setup 16ms, collect 320ms, tests 2.22s, environment 0ms, prepare 40ms) exit=1 @@ -403,13 +404,13 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2011ms - ✓ cancels promptly while a lease snapshot read is in flight 2011ms + ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2010ms + ✓ cancels promptly while a lease snapshot read is in flight 2010ms Test Files 1 passed (1) Tests 1 passed | 13 skipped (14) - Start at 05:35:43 - Duration 2.49s (transform 189ms, setup 17ms, collect 320ms, tests 2.01s, environment 0ms, prepare 40ms) + Start at 05:47:12 + Duration 2.49s (transform 190ms, setup 17ms, collect 321ms, tests 2.01s, environment 0ms, prepare 42ms) exit=0 ``` @@ -429,8 +430,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 3011ms - × a completion push ends the wait without waiting out an in-flight snapshot 3011ms + ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 3013ms + × a completion push ends the wait without waiting out an in-flight snapshot 3012ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -453,8 +454,8 @@ Received: "still waiting" Test Files 1 failed (1) Tests 1 failed | 13 skipped (14) - Start at 05:35:46 - Duration 3.48s (transform 192ms, setup 16ms, collect 318ms, tests 3.01s, environment 0ms, prepare 41ms) + Start at 05:47:15 + Duration 3.49s (transform 190ms, setup 17ms, collect 322ms, tests 3.01s, environment 0ms, prepare 40ms) exit=1 @@ -463,13 +464,13 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2062ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2062ms + ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2061ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2060ms Test Files 1 passed (1) Tests 1 passed | 13 skipped (14) - Start at 05:35:49 - Duration 2.53s (transform 188ms, setup 17ms, collect 321ms, tests 2.06s, environment 0ms, prepare 41ms) + Start at 05:47:19 + Duration 2.55s (transform 193ms, setup 18ms, collect 325ms, tests 2.06s, environment 0ms, prepare 41ms) exit=0 ``` @@ -492,8 +493,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 19ms - × a heartbeat timeout is lease loss rather than a worker body failure 19ms + ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 21ms + × a heartbeat timeout is lease loss rather than a worker body failure 20ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -512,8 +513,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 05:35:52 - Duration 473ms (transform 171ms, setup 17ms, collect 291ms, tests 19ms, environment 0ms, prepare 41ms) + Start at 05:47:21 + Duration 467ms (transform 161ms, setup 16ms, collect 286ms, tests 21ms, environment 0ms, prepare 41ms) exit=1 @@ -522,12 +523,12 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/heartbeat-timeout.test.ts (1 test) 17ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 05:35:53 - Duration 466ms (transform 171ms, setup 16ms, collect 286ms, tests 17ms, environment 0ms, prepare 41ms) + Start at 05:47:22 + Duration 471ms (transform 165ms, setup 17ms, collect 290ms, tests 16ms, environment 0ms, prepare 43ms) exit=0 ``` @@ -554,8 +555,8 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (36 tests | 3 failed | 33 skipped) 31ms - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 22ms + ❯ tests/authored-root.test.ts (36 tests | 3 failed | 33 skipped) 32ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 24ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms @@ -641,8 +642,8 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } Test Files 1 failed (1) Tests 3 failed | 33 skipped (36) - Start at 05:35:54 - Duration 1.30s (transform 672ms, setup 17ms, collect 1.11s, tests 31ms, environment 0ms, prepare 41ms) + Start at 05:47:23 + Duration 1.28s (transform 671ms, setup 17ms, collect 1.10s, tests 32ms, environment 0ms, prepare 41ms) exit=1 @@ -651,12 +652,12 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (36 tests | 33 skipped) 25ms + ✓ tests/authored-root.test.ts (36 tests | 33 skipped) 22ms Test Files 1 passed (1) Tests 3 passed | 33 skipped (36) - Start at 05:35:56 - Duration 1.30s (transform 683ms, setup 17ms, collect 1.11s, tests 25ms, environment 0ms, prepare 40ms) + Start at 05:47:25 + Duration 1.28s (transform 669ms, setup 16ms, collect 1.10s, tests 22ms, environment 0ms, prepare 43ms) exit=0 ``` diff --git a/packages/sdk/src/journal-budgeted-reads.ts b/packages/sdk/src/journal-budgeted-reads.ts index 46ac0dde9..75cbaed3d 100644 --- a/packages/sdk/src/journal-budgeted-reads.ts +++ b/packages/sdk/src/journal-budgeted-reads.ts @@ -59,7 +59,10 @@ export class BudgetedReads { try { return await reader.requestOnce(verb, params, remaining, attemptSignal); } catch (error) { - if (attemptSignal.aborted || error instanceof JournalRequestTimeoutError + // relayflowd answers one frame at a time per connection, so a canceled + // request still occupies this reader: drop it for a fresh session. + if (attemptSignal.aborted) { this.drop(reader); throw error; } + if (error instanceof JournalRequestTimeoutError || error instanceof JournalProtocolError || error instanceof JournalFrameError) throw error; // Only the dedicated reader's transport failed; reconnect within the budget. this.drop(reader); diff --git a/packages/sdk/src/journal-client.ts b/packages/sdk/src/journal-client.ts index 527d03939..4eebb0233 100644 --- a/packages/sdk/src/journal-client.ts +++ b/packages/sdk/src/journal-client.ts @@ -94,9 +94,13 @@ export class JournalClient extends JournalConnection { return this.budgeted?.scope(signal) ?? (() => {}); } - /** End every budgeted read now and refuse later ones; the session itself stays open. */ + /** + * End every budgeted read now and refuse later ones; the session itself stays + * open. The reads end as read interruptions, so a worker hands its attempt + * back to the kernel rather than completing it as a failure. + */ cancelReads(): void { - this.budgeted?.close(new Error('journal client: reads canceled')); + this.budgeted?.close(new Error('journal client: reads canceled'), true); } // --- Typed verb methods (gate 1 minimal set, kernel DESIGN.md §5) -------- diff --git a/packages/sdk/tests/journal-client-read-timeout.test.ts b/packages/sdk/tests/journal-client-read-timeout.test.ts index 848d9bfeb..f5560be25 100644 --- a/packages/sdk/tests/journal-client-read-timeout.test.ts +++ b/packages/sdk/tests/journal-client-read-timeout.test.ts @@ -352,3 +352,43 @@ it('cancelReads ends in-flight budgeted reads at once, ahead of a draining shutd await expect(pending).rejects.toThrow(); expect(performance.now() - begun).toBeLessThan(40); }); + +it('drops a reader whose in-flight read was canceled, so the next read gets a fresh session', async () => { + let reads = 0; + const sockets: unknown[] = []; + let firstRead!: () => void; + const started = new Promise(resolve => { firstRead = resolve; }); + const path = sockPath(); + // Serialized per connection, like relayflowd: a hung request blocks its socket. + const server = startLoopback(path, { + hello: sendOk, + 'run.get': ctx => { reads += 1; sockets.push(ctx.socket); if (reads === 1) firstRead(); else sendResult(ctx, { status: 'running' }); }, + }, { serialize: true }); + servers.push(server); + await once(server, 'listening'); + const client = new JournalClient(path, { requestTimeoutMs: 20_000, readBudgetMs: 60_000 }); + clients.push(client); + await client.connect(); + const lease = new AbortController(); + const release = client.scopeReads(lease.signal); + const first = client.runGet('run'); + await started; + lease.abort(new Error('root lease lost')); + await expect(first).rejects.toThrow('root lease lost'); + release(); + const begun = performance.now(); + await expect(client.runGet('run')).resolves.toMatchObject({ status: 'running' }); + expect(performance.now() - begun).toBeLessThan(2_000); + expect(sockets[1]).not.toBe(sockets[0]); +}); + +it('canceled reads are typed read interruptions, so workers hand attempts back', async () => { + let entered!: () => void; + const started = new Promise(resolve => { entered = resolve; }); + const client = await setup({ 'journal.read': () => entered() }, 60_000); + const pending = client.journalRead('run', 1); + await started; + client.cancelReads(); + const error = await pending.catch(caught => caught); + expect(isReadInterruptionError(error)).toBe(true); +}); From 59cc8c424de897ed89d27376f89b222b5a906365 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 06:07:14 -0700 Subject: [PATCH 33/34] fix(sdk): budget exhaustion aborts the queued read work, not just the caller The read budget timer rejected the caller but left the attempt running, so a reader handshake stalled near the end of the budget held the serialized queue until the full request timeout and starved the next read. Exhausting the budget now aborts the attempt signal too, which drops a pending reader setup just like a cancellation. Evidence re-captured at this tree. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: b8262468-ea26-4a6e-bee9-d193832e77dc --- evidence/run-read-timeout/final-focused.log | 122 ++++---- .../mutation-completion-aborts-snapshot.log | 14 +- .../run-read-timeout/mutation-heartbeat.log | 12 +- .../mutation-reader-reconnect.log | 26 +- .../mutation-reader-setup-retry.log | 18 +- evidence/run-read-timeout/mutation-reader.log | 18 +- .../mutation-root-parking.log | 14 +- .../mutation-snapshot-cancel.log | 14 +- .../mutation-watch-cadence.log | 16 +- evidence/run-read-timeout/resume-live.log | 8 +- evidence/run-read-timeout/verification.md | 262 +++++++++--------- packages/sdk/src/journal-read-policy.ts | 14 +- .../tests/journal-client-read-timeout.test.ts | 18 ++ 13 files changed, 289 insertions(+), 267 deletions(-) diff --git a/evidence/run-read-timeout/final-focused.log b/evidence/run-read-timeout/final-focused.log index 70b353e60..8d0d9d0c8 100644 --- a/evidence/run-read-timeout/final-focused.log +++ b/evidence/run-read-timeout/final-focused.log @@ -2,72 +2,72 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5306ms - ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 310ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2037ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2053ms -(node:556211) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5259ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2035ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2037ms +(node:703660) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (36 tests) 462ms - ✓ tests/journal-client.test.ts (17 tests) 80ms - ✓ tests/journal-client-read-timeout.test.ts (30 tests) 1811ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 7019ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 448ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 880ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2922ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 335ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 934ms - ✓ tests/authored-flow-slack.test.ts (7 tests) 1280ms - ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 448ms - ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 443ms - ✓ tests/running-step-watch.test.ts (14 tests) 12322ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms - ✓ cancels promptly while a lease snapshot read is in flight 2005ms + ✓ tests/authored-root.test.ts (36 tests) 454ms + ✓ tests/journal-client-read-timeout.test.ts (31 tests) 2103ms + ✓ an exhausted budget during reader setup frees the queue for the next read 302ms + ✓ tests/journal-client.test.ts (17 tests) 86ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 7518ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 651ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1074ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2745ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 327ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 826ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 1181ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 421ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 416ms + ✓ tests/running-step-watch.test.ts (14 tests) 12314ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2115ms + ✓ cancels promptly while a lease snapshot read is in flight 2006ms ✓ a completion push ends the wait without waiting out an in-flight snapshot 2053ms - ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms - ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2003ms - ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2003ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2600ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 556ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 660ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 700ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 684ms - ✓ tests/authored-helpers.test.ts (6 tests) 2048ms - ✓ runs every available provider through the real kernel and resumes completed effects without a second write 1001ms - ✓ replays after SIGKILL before confirm with the same token and one successful completion 449ms - ✓ replays after SIGKILL before complete with the same token and one successful completion 345ms - ✓ tests/classify-outcome.test.ts (11 tests) 7425ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2010ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3007ms - ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2002ms - ✓ tests/human-live.test.ts (3 tests) 5028ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 3047ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1181ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 799ms - ✓ tests/communication-worker.test.ts (20 tests) 1829ms - ✓ tests/worker-lease.test.ts (8 tests) 15ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 22ms - ✓ tests/communication.test.ts (10 tests) 14ms - ✓ tests/effect-channel.test.ts (5 tests) 306ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 739ms - ✓ tests/run-daemon-unresponsive.test.ts (5 tests) 9ms - ✓ tests/memoization.test.ts (58 tests) 85ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2498ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2063ms - ✓ drains read and watch promises before an authored flow completes 434ms + ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2003ms + ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2002ms + ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2002ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2581ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 671ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 591ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 646ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 671ms + ✓ tests/authored-helpers.test.ts (6 tests) 1883ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 882ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 390ms + ✓ replays after SIGKILL before complete with the same token and one successful completion 389ms + ✓ tests/classify-outcome.test.ts (11 tests) 7415ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2005ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3004ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2000ms + ✓ tests/human-live.test.ts (3 tests) 4571ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2744ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1038ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 787ms + ✓ tests/communication-worker.test.ts (20 tests) 1815ms + ✓ tests/worker-lease.test.ts (8 tests) 12ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 20ms + ✓ tests/communication.test.ts (10 tests) 17ms + ✓ tests/effect-channel.test.ts (5 tests) 259ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 632ms + ✓ tests/run-daemon-unresponsive.test.ts (5 tests) 6ms + ✓ tests/memoization.test.ts (58 tests) 65ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2413ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2055ms + ✓ drains read and watch promises before an authored flow completes 358ms ✓ tests/reuse-summary-interruption.test.ts (6 tests) 4ms - ✓ tests/worker-lease-sweep.test.ts (4 tests) 9ms + ✓ tests/worker-lease-sweep.test.ts (4 tests) 7ms ✓ tests/journal-client-completion.test.ts (6 tests) 105ms - ✓ tests/resume-worker-lease.test.ts (3 tests) 8ms - ✓ tests/direct-run-worker-lease.test.ts (3 tests) 13ms - ✓ tests/helper-effect-lease-loss.test.ts (1 test) 298ms - ✓ tests/journal-client-subscriptions.test.ts (1 test) 10ms - ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 84ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms + ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms + ✓ tests/direct-run-worker-lease.test.ts (3 tests) 7ms + ✓ tests/helper-effect-lease-loss.test.ts (1 test) 228ms + ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms + ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 31ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 17ms Test Files 29 passed (29) - Tests 368 passed (368) - Start at 05:47:30 - Duration 63.12s (transform 1.16s, setup 176ms, collect 7.54s, tests 51.45s, environment 4ms, prepare 1.15s) + Tests 369 passed (369) + Start at 06:06:10 + Duration 61.49s (transform 1.15s, setup 153ms, collect 6.89s, tests 51.02s, environment 4ms, prepare 1.03s) exit=0 diff --git a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log index 162c1f1cc..7e40aec1d 100644 --- a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log +++ b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log @@ -10,7 +10,7 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 3013ms + ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 3012ms × a completion push ends the wait without waiting out an in-flight snapshot 3012ms → expected 'still waiting' to be 'returned' // Object.is equality @@ -34,8 +34,8 @@ Received: "still waiting" Test Files 1 failed (1) Tests 1 failed | 13 skipped (14) - Start at 05:47:15 - Duration 3.49s (transform 190ms, setup 17ms, collect 322ms, tests 3.01s, environment 0ms, prepare 40ms) + Start at 06:05:54 + Duration 3.52s (transform 201ms, setup 17ms, collect 341ms, tests 3.01s, environment 0ms, prepare 43ms) exit=1 @@ -44,12 +44,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2061ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2060ms + ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2060ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2059ms Test Files 1 passed (1) Tests 1 passed | 13 skipped (14) - Start at 05:47:19 - Duration 2.55s (transform 193ms, setup 18ms, collect 325ms, tests 2.06s, environment 0ms, prepare 41ms) + Start at 06:05:58 + Duration 2.53s (transform 188ms, setup 16ms, collect 312ms, tests 2.06s, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-heartbeat.log b/evidence/run-read-timeout/mutation-heartbeat.log index df1e7c9da..7dfacded0 100644 --- a/evidence/run-read-timeout/mutation-heartbeat.log +++ b/evidence/run-read-timeout/mutation-heartbeat.log @@ -13,8 +13,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 21ms - × a heartbeat timeout is lease loss rather than a worker body failure 20ms + ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 20ms + × a heartbeat timeout is lease loss rather than a worker body failure 19ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -33,8 +33,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 05:47:21 - Duration 467ms (transform 161ms, setup 16ms, collect 286ms, tests 21ms, environment 0ms, prepare 41ms) + Start at 06:06:01 + Duration 467ms (transform 164ms, setup 14ms, collect 286ms, tests 20ms, environment 0ms, prepare 42ms) exit=1 @@ -47,7 +47,7 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb Test Files 1 passed (1) Tests 1 passed (1) - Start at 05:47:22 - Duration 471ms (transform 165ms, setup 17ms, collect 290ms, tests 16ms, environment 0ms, prepare 43ms) + Start at 06:06:02 + Duration 449ms (transform 164ms, setup 17ms, collect 280ms, tests 16ms, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-reconnect.log b/evidence/run-read-timeout/mutation-reader-reconnect.log index 9482d60b7..920a7ce8b 100644 --- a/evidence/run-read-timeout/mutation-reader-reconnect.log +++ b/evidence/run-read-timeout/mutation-reader-reconnect.log @@ -11,14 +11,14 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (30 tests | 1 failed | 29 skipped) 1008ms - × reconnects the reader after it disconnects within the read budget 1007ms - → journal client: run.get read session was interrupted after 77 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ tests/journal-client-read-timeout.test.ts (31 tests | 1 failed | 30 skipped) 1007ms + × reconnects the reader after it disconnects within the read budget 1006ms + → journal client: run.get read session was interrupted after 81 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 77 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 81 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:71:9 69| let last: unknown; 70| const exhausted = () => last instanceof JournalReadInterruptedError @@ -26,13 +26,13 @@ JournalReadInterruptedError: journal client: run.get read session was interrupte | ^ 72| : new JournalRequestTimeoutError(verb, timeoutMs, attempts, perf… 73| // A caller's cancellation settles this read and drains its queued… - ❯ Timeout. src/journal-read-policy.ts:82:39 + ❯ Timeout. src/journal-read-policy.ts:86:48 Caused by: Error: journal client: not connected (run.get): journal client: connection closed ❯ src/journal-connection.ts:183:16 ❯ JournalClient.requestOnce src/journal-connection.ts:176:12 ❯ src/journal-budgeted-reads.ts:60:29 - ❯ src/journal-read-policy.ts:94:18 + ❯ src/journal-read-policy.ts:98:18 Caused by: Error: journal client: connection closed ❯ Socket. src/journal-connection.ts:89:26 @@ -40,9 +40,9 @@ Caused by: Error: journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 29 skipped (30) - Start at 05:46:59 - Duration 1.47s (transform 174ms, setup 16ms, collect 303ms, tests 1.01s, environment 0ms, prepare 40ms) + Tests 1 failed | 30 skipped (31) + Start at 06:05:38 + Duration 1.47s (transform 180ms, setup 17ms, collect 308ms, tests 1.01s, environment 0ms, prepare 40ms) exit=1 @@ -51,11 +51,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (30 tests | 29 skipped) 70ms + ✓ tests/journal-client-read-timeout.test.ts (31 tests | 30 skipped) 86ms Test Files 1 passed (1) - Tests 1 passed | 29 skipped (30) - Start at 05:47:00 - Duration 485ms (transform 173ms, setup 16ms, collect 284ms, tests 70ms, environment 0ms, prepare 28ms) + Tests 1 passed | 30 skipped (31) + Start at 06:05:40 + Duration 543ms (transform 170ms, setup 17ms, collect 283ms, tests 86ms, environment 0ms, prepare 45ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-setup-retry.log b/evidence/run-read-timeout/mutation-reader-setup-retry.log index fec4a23b5..a91c90591 100644 --- a/evidence/run-read-timeout/mutation-reader-setup-retry.log +++ b/evidence/run-read-timeout/mutation-reader-setup-retry.log @@ -16,8 +16,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (30 tests | 1 failed | 29 skipped) 81ms - × retries a reader setup that timed out instead of reading on the primary for good 80ms + ❯ tests/journal-client-read-timeout.test.ts (31 tests | 1 failed | 30 skipped) 76ms + × retries a reader setup that timed out instead of reading on the primary for good 76ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -42,9 +42,9 @@ AssertionError: expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 29 skipped (30) - Start at 05:47:01 - Duration 536ms (transform 172ms, setup 17ms, collect 295ms, tests 81ms, environment 0ms, prepare 40ms) + Tests 1 failed | 30 skipped (31) + Start at 06:05:41 + Duration 535ms (transform 179ms, setup 16ms, collect 302ms, tests 76ms, environment 0ms, prepare 41ms) exit=1 @@ -53,11 +53,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (30 tests | 29 skipped) 70ms + ✓ tests/journal-client-read-timeout.test.ts (31 tests | 30 skipped) 67ms Test Files 1 passed (1) - Tests 1 passed | 29 skipped (30) - Start at 05:47:02 - Duration 540ms (transform 176ms, setup 17ms, collect 301ms, tests 70ms, environment 0ms, prepare 42ms) + Tests 1 passed | 30 skipped (31) + Start at 06:05:42 + Duration 527ms (transform 174ms, setup 17ms, collect 298ms, tests 67ms, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader.log b/evidence/run-read-timeout/mutation-reader.log index 10c1f8ff2..c814cd942 100644 --- a/evidence/run-read-timeout/mutation-reader.log +++ b/evidence/run-read-timeout/mutation-reader.log @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (30 tests | 1 failed | 29 skipped) 113ms - × serves a bounded read while an unbounded command is in flight 113ms + ❯ tests/journal-client-read-timeout.test.ts (31 tests | 1 failed | 30 skipped) 113ms + × serves a bounded read while an unbounded command is in flight 112ms → expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -36,9 +36,9 @@ AssertionError: expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 29 skipped (30) - Start at 05:46:57 - Duration 554ms (transform 172ms, setup 16ms, collect 296ms, tests 113ms, environment 0ms, prepare 31ms) + Tests 1 failed | 30 skipped (31) + Start at 06:05:36 + Duration 588ms (transform 178ms, setup 16ms, collect 303ms, tests 113ms, environment 0ms, prepare 41ms) exit=1 @@ -47,11 +47,11 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (30 tests | 29 skipped) 106ms + ✓ tests/journal-client-read-timeout.test.ts (31 tests | 30 skipped) 107ms Test Files 1 passed (1) - Tests 1 passed | 29 skipped (30) - Start at 05:46:58 - Duration 567ms (transform 178ms, setup 16ms, collect 302ms, tests 106ms, environment 0ms, prepare 41ms) + Tests 1 passed | 30 skipped (31) + Start at 06:05:37 + Duration 570ms (transform 180ms, setup 17ms, collect 302ms, tests 107ms, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-root-parking.log b/evidence/run-read-timeout/mutation-root-parking.log index 52bdfcf38..84520c9f6 100644 --- a/evidence/run-read-timeout/mutation-root-parking.log +++ b/evidence/run-read-timeout/mutation-root-parking.log @@ -17,8 +17,8 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (36 tests | 3 failed | 33 skipped) 32ms - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 24ms + ❯ tests/authored-root.test.ts (36 tests | 3 failed | 33 skipped) 31ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 22ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms @@ -104,8 +104,8 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } Test Files 1 failed (1) Tests 3 failed | 33 skipped (36) - Start at 05:47:23 - Duration 1.28s (transform 671ms, setup 17ms, collect 1.10s, tests 32ms, environment 0ms, prepare 41ms) + Start at 06:06:03 + Duration 1.31s (transform 677ms, setup 16ms, collect 1.12s, tests 31ms, environment 0ms, prepare 41ms) exit=1 @@ -114,11 +114,11 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (36 tests | 33 skipped) 22ms + ✓ tests/authored-root.test.ts (36 tests | 33 skipped) 23ms Test Files 1 passed (1) Tests 3 passed | 33 skipped (36) - Start at 05:47:25 - Duration 1.28s (transform 669ms, setup 16ms, collect 1.10s, tests 22ms, environment 0ms, prepare 43ms) + Start at 06:06:05 + Duration 1.31s (transform 683ms, setup 16ms, collect 1.12s, tests 23ms, environment 0ms, prepare 40ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-snapshot-cancel.log b/evidence/run-read-timeout/mutation-snapshot-cancel.log index c3ca52223..dd99d1f36 100644 --- a/evidence/run-read-timeout/mutation-snapshot-cancel.log +++ b/evidence/run-read-timeout/mutation-snapshot-cancel.log @@ -11,8 +11,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2217ms - × cancels promptly while a lease snapshot read is in flight 2216ms + ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2218ms + × cancels promptly while a lease snapshot read is in flight 2217ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -31,8 +31,8 @@ AssertionError: expected 'still waiting' to be an instance of Error Test Files 1 failed (1) Tests 1 failed | 13 skipped (14) - Start at 05:47:09 - Duration 2.69s (transform 189ms, setup 16ms, collect 320ms, tests 2.22s, environment 0ms, prepare 40ms) + Start at 06:05:48 + Duration 2.71s (transform 191ms, setup 17ms, collect 325ms, tests 2.22s, environment 0ms, prepare 41ms) exit=1 @@ -41,12 +41,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2010ms + ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2011ms ✓ cancels promptly while a lease snapshot read is in flight 2010ms Test Files 1 passed (1) Tests 1 passed | 13 skipped (14) - Start at 05:47:12 - Duration 2.49s (transform 190ms, setup 17ms, collect 321ms, tests 2.01s, environment 0ms, prepare 42ms) + Start at 06:05:52 + Duration 2.50s (transform 191ms, setup 16ms, collect 328ms, tests 2.01s, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-watch-cadence.log b/evidence/run-read-timeout/mutation-watch-cadence.log index ea3ce0f5a..65de3288f 100644 --- a/evidence/run-read-timeout/mutation-watch-cadence.log +++ b/evidence/run-read-timeout/mutation-watch-cadence.log @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2111ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2110ms + ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2110ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2109ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -30,8 +30,8 @@ AssertionError: expected 41 to be less than or equal to 1 Test Files 1 failed (1) Tests 1 failed | 13 skipped (14) - Start at 05:47:03 - Duration 2.58s (transform 188ms, setup 16ms, collect 319ms, tests 2.11s, environment 0ms, prepare 40ms) + Start at 06:05:43 + Duration 2.60s (transform 189ms, setup 17ms, collect 325ms, tests 2.11s, environment 0ms, prepare 41ms) exit=1 @@ -40,12 +40,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2119ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms + ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2117ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms Test Files 1 passed (1) Tests 1 passed | 13 skipped (14) - Start at 05:47:06 - Duration 2.60s (transform 189ms, setup 16ms, collect 322ms, tests 2.12s, environment 0ms, prepare 41ms) + Start at 06:05:46 + Duration 2.60s (transform 191ms, setup 16ms, collect 325ms, tests 2.12s, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/resume-live.log b/evidence/run-read-timeout/resume-live.log index ad157d6da..17434b389 100644 --- a/evidence/run-read-timeout/resume-live.log +++ b/evidence/run-read-timeout/resume-live.log @@ -2,12 +2,12 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 1004ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 1003ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 876ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 876ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 05:48:34 - Duration 2.27s (transform 578ms, setup 20ms, collect 942ms, tests 1.00s, environment 0ms, prepare 66ms) + Start at 06:07:12 + Duration 2.15s (transform 629ms, setup 15ms, collect 1.02s, tests 876ms, environment 0ms, prepare 52ms) exit=0 diff --git a/evidence/run-read-timeout/verification.md b/evidence/run-read-timeout/verification.md index 092e7608a..bcfa02d10 100644 --- a/evidence/run-read-timeout/verification.md +++ b/evidence/run-read-timeout/verification.md @@ -15,73 +15,73 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5306ms - ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 310ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2037ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2053ms -(node:556211) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5259ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2035ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2037ms +(node:703660) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (36 tests) 462ms - ✓ tests/journal-client.test.ts (17 tests) 80ms - ✓ tests/journal-client-read-timeout.test.ts (30 tests) 1811ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 7019ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 448ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 880ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2922ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 335ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 934ms - ✓ tests/authored-flow-slack.test.ts (7 tests) 1280ms - ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 448ms - ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 443ms - ✓ tests/running-step-watch.test.ts (14 tests) 12322ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms - ✓ cancels promptly while a lease snapshot read is in flight 2005ms + ✓ tests/authored-root.test.ts (36 tests) 454ms + ✓ tests/journal-client-read-timeout.test.ts (31 tests) 2103ms + ✓ an exhausted budget during reader setup frees the queue for the next read 302ms + ✓ tests/journal-client.test.ts (17 tests) 86ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 7518ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 651ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1074ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2745ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 327ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 826ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 1181ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 421ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 416ms + ✓ tests/running-step-watch.test.ts (14 tests) 12314ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2115ms + ✓ cancels promptly while a lease snapshot read is in flight 2006ms ✓ a completion push ends the wait without waiting out an in-flight snapshot 2053ms - ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2004ms - ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2003ms - ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2003ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2600ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 556ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 660ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 700ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 684ms - ✓ tests/authored-helpers.test.ts (6 tests) 2048ms - ✓ runs every available provider through the real kernel and resumes completed effects without a second write 1001ms - ✓ replays after SIGKILL before confirm with the same token and one successful completion 449ms - ✓ replays after SIGKILL before complete with the same token and one successful completion 345ms - ✓ tests/classify-outcome.test.ts (11 tests) 7425ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2010ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3007ms - ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2002ms - ✓ tests/human-live.test.ts (3 tests) 5028ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 3047ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1181ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 799ms - ✓ tests/communication-worker.test.ts (20 tests) 1829ms - ✓ tests/worker-lease.test.ts (8 tests) 15ms - ✓ tests/worker-lease-lost.test.ts (17 tests) 22ms - ✓ tests/communication.test.ts (10 tests) 14ms - ✓ tests/effect-channel.test.ts (5 tests) 306ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 739ms - ✓ tests/run-daemon-unresponsive.test.ts (5 tests) 9ms - ✓ tests/memoization.test.ts (58 tests) 85ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2498ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2063ms - ✓ drains read and watch promises before an authored flow completes 434ms + ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2003ms + ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2002ms + ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2002ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2581ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 671ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 591ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 646ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 671ms + ✓ tests/authored-helpers.test.ts (6 tests) 1883ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 882ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 390ms + ✓ replays after SIGKILL before complete with the same token and one successful completion 389ms + ✓ tests/classify-outcome.test.ts (11 tests) 7415ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2005ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3004ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2000ms + ✓ tests/human-live.test.ts (3 tests) 4571ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2744ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1038ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 787ms + ✓ tests/communication-worker.test.ts (20 tests) 1815ms + ✓ tests/worker-lease.test.ts (8 tests) 12ms + ✓ tests/worker-lease-lost.test.ts (17 tests) 20ms + ✓ tests/communication.test.ts (10 tests) 17ms + ✓ tests/effect-channel.test.ts (5 tests) 259ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 632ms + ✓ tests/run-daemon-unresponsive.test.ts (5 tests) 6ms + ✓ tests/memoization.test.ts (58 tests) 65ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2413ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2055ms + ✓ drains read and watch promises before an authored flow completes 358ms ✓ tests/reuse-summary-interruption.test.ts (6 tests) 4ms - ✓ tests/worker-lease-sweep.test.ts (4 tests) 9ms + ✓ tests/worker-lease-sweep.test.ts (4 tests) 7ms ✓ tests/journal-client-completion.test.ts (6 tests) 105ms - ✓ tests/resume-worker-lease.test.ts (3 tests) 8ms - ✓ tests/direct-run-worker-lease.test.ts (3 tests) 13ms - ✓ tests/helper-effect-lease-loss.test.ts (1 test) 298ms - ✓ tests/journal-client-subscriptions.test.ts (1 test) 10ms - ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 84ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 16ms + ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms + ✓ tests/direct-run-worker-lease.test.ts (3 tests) 7ms + ✓ tests/helper-effect-lease-loss.test.ts (1 test) 228ms + ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms + ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 31ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 17ms Test Files 29 passed (29) - Tests 368 passed (368) - Start at 05:47:30 - Duration 63.12s (transform 1.16s, setup 176ms, collect 7.54s, tests 51.45s, environment 4ms, prepare 1.15s) + Tests 369 passed (369) + Start at 06:06:10 + Duration 61.49s (transform 1.15s, setup 153ms, collect 6.89s, tests 51.02s, environment 4ms, prepare 1.03s) exit=0 ``` @@ -93,13 +93,13 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 1004ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 1003ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 876ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 876ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 05:48:34 - Duration 2.27s (transform 578ms, setup 20ms, collect 942ms, tests 1.00s, environment 0ms, prepare 66ms) + Start at 06:07:12 + Duration 2.15s (transform 629ms, setup 15ms, collect 1.02s, tests 876ms, environment 0ms, prepare 52ms) exit=0 ``` @@ -121,8 +121,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (30 tests | 1 failed | 29 skipped) 113ms - × serves a bounded read while an unbounded command is in flight 113ms + ❯ tests/journal-client-read-timeout.test.ts (31 tests | 1 failed | 30 skipped) 113ms + × serves a bounded read while an unbounded command is in flight 112ms → expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -147,9 +147,9 @@ AssertionError: expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 29 skipped (30) - Start at 05:46:57 - Duration 554ms (transform 172ms, setup 16ms, collect 296ms, tests 113ms, environment 0ms, prepare 31ms) + Tests 1 failed | 30 skipped (31) + Start at 06:05:36 + Duration 588ms (transform 178ms, setup 16ms, collect 303ms, tests 113ms, environment 0ms, prepare 41ms) exit=1 @@ -158,12 +158,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (30 tests | 29 skipped) 106ms + ✓ tests/journal-client-read-timeout.test.ts (31 tests | 30 skipped) 107ms Test Files 1 passed (1) - Tests 1 passed | 29 skipped (30) - Start at 05:46:58 - Duration 567ms (transform 178ms, setup 16ms, collect 302ms, tests 106ms, environment 0ms, prepare 41ms) + Tests 1 passed | 30 skipped (31) + Start at 06:05:37 + Duration 570ms (transform 180ms, setup 17ms, collect 302ms, tests 107ms, environment 0ms, prepare 40ms) exit=0 ``` @@ -184,14 +184,14 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (30 tests | 1 failed | 29 skipped) 1008ms - × reconnects the reader after it disconnects within the read budget 1007ms - → journal client: run.get read session was interrupted after 77 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ tests/journal-client-read-timeout.test.ts (31 tests | 1 failed | 30 skipped) 1007ms + × reconnects the reader after it disconnects within the read budget 1006ms + → journal client: run.get read session was interrupted after 81 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 77 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed +JournalReadInterruptedError: journal client: run.get read session was interrupted after 81 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ❯ exhausted src/journal-read-policy.ts:71:9 69| let last: unknown; 70| const exhausted = () => last instanceof JournalReadInterruptedError @@ -199,13 +199,13 @@ JournalReadInterruptedError: journal client: run.get read session was interrupte | ^ 72| : new JournalRequestTimeoutError(verb, timeoutMs, attempts, perf… 73| // A caller's cancellation settles this read and drains its queued… - ❯ Timeout. src/journal-read-policy.ts:82:39 + ❯ Timeout. src/journal-read-policy.ts:86:48 Caused by: Error: journal client: not connected (run.get): journal client: connection closed ❯ src/journal-connection.ts:183:16 ❯ JournalClient.requestOnce src/journal-connection.ts:176:12 ❯ src/journal-budgeted-reads.ts:60:29 - ❯ src/journal-read-policy.ts:94:18 + ❯ src/journal-read-policy.ts:98:18 Caused by: Error: journal client: connection closed ❯ Socket. src/journal-connection.ts:89:26 @@ -213,9 +213,9 @@ Caused by: Error: journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 29 skipped (30) - Start at 05:46:59 - Duration 1.47s (transform 174ms, setup 16ms, collect 303ms, tests 1.01s, environment 0ms, prepare 40ms) + Tests 1 failed | 30 skipped (31) + Start at 06:05:38 + Duration 1.47s (transform 180ms, setup 17ms, collect 308ms, tests 1.01s, environment 0ms, prepare 40ms) exit=1 @@ -224,12 +224,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (30 tests | 29 skipped) 70ms + ✓ tests/journal-client-read-timeout.test.ts (31 tests | 30 skipped) 86ms Test Files 1 passed (1) - Tests 1 passed | 29 skipped (30) - Start at 05:47:00 - Duration 485ms (transform 173ms, setup 16ms, collect 284ms, tests 70ms, environment 0ms, prepare 28ms) + Tests 1 passed | 30 skipped (31) + Start at 06:05:40 + Duration 543ms (transform 170ms, setup 17ms, collect 283ms, tests 86ms, environment 0ms, prepare 45ms) exit=0 ``` @@ -255,8 +255,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (30 tests | 1 failed | 29 skipped) 81ms - × retries a reader setup that timed out instead of reading on the primary for good 80ms + ❯ tests/journal-client-read-timeout.test.ts (31 tests | 1 failed | 30 skipped) 76ms + × retries a reader setup that timed out instead of reading on the primary for good 76ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -281,9 +281,9 @@ AssertionError: expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 29 skipped (30) - Start at 05:47:01 - Duration 536ms (transform 172ms, setup 17ms, collect 295ms, tests 81ms, environment 0ms, prepare 40ms) + Tests 1 failed | 30 skipped (31) + Start at 06:05:41 + Duration 535ms (transform 179ms, setup 16ms, collect 302ms, tests 76ms, environment 0ms, prepare 41ms) exit=1 @@ -292,12 +292,12 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (30 tests | 29 skipped) 70ms + ✓ tests/journal-client-read-timeout.test.ts (31 tests | 30 skipped) 67ms Test Files 1 passed (1) - Tests 1 passed | 29 skipped (30) - Start at 05:47:02 - Duration 540ms (transform 176ms, setup 17ms, collect 301ms, tests 70ms, environment 0ms, prepare 42ms) + Tests 1 passed | 30 skipped (31) + Start at 06:05:42 + Duration 527ms (transform 174ms, setup 17ms, collect 298ms, tests 67ms, environment 0ms, prepare 40ms) exit=0 ``` @@ -317,8 +317,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2111ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2110ms + ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2110ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2109ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -337,8 +337,8 @@ AssertionError: expected 41 to be less than or equal to 1 Test Files 1 failed (1) Tests 1 failed | 13 skipped (14) - Start at 05:47:03 - Duration 2.58s (transform 188ms, setup 16ms, collect 319ms, tests 2.11s, environment 0ms, prepare 40ms) + Start at 06:05:43 + Duration 2.60s (transform 189ms, setup 17ms, collect 325ms, tests 2.11s, environment 0ms, prepare 41ms) exit=1 @@ -347,13 +347,13 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2119ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2118ms + ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2117ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms Test Files 1 passed (1) Tests 1 passed | 13 skipped (14) - Start at 05:47:06 - Duration 2.60s (transform 189ms, setup 16ms, collect 322ms, tests 2.12s, environment 0ms, prepare 41ms) + Start at 06:05:46 + Duration 2.60s (transform 191ms, setup 16ms, collect 325ms, tests 2.12s, environment 0ms, prepare 41ms) exit=0 ``` @@ -374,8 +374,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2217ms - × cancels promptly while a lease snapshot read is in flight 2216ms + ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2218ms + × cancels promptly while a lease snapshot read is in flight 2217ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -394,8 +394,8 @@ AssertionError: expected 'still waiting' to be an instance of Error Test Files 1 failed (1) Tests 1 failed | 13 skipped (14) - Start at 05:47:09 - Duration 2.69s (transform 189ms, setup 16ms, collect 320ms, tests 2.22s, environment 0ms, prepare 40ms) + Start at 06:05:48 + Duration 2.71s (transform 191ms, setup 17ms, collect 325ms, tests 2.22s, environment 0ms, prepare 41ms) exit=1 @@ -404,13 +404,13 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2010ms + ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2011ms ✓ cancels promptly while a lease snapshot read is in flight 2010ms Test Files 1 passed (1) Tests 1 passed | 13 skipped (14) - Start at 05:47:12 - Duration 2.49s (transform 190ms, setup 17ms, collect 321ms, tests 2.01s, environment 0ms, prepare 42ms) + Start at 06:05:52 + Duration 2.50s (transform 191ms, setup 16ms, collect 328ms, tests 2.01s, environment 0ms, prepare 41ms) exit=0 ``` @@ -430,7 +430,7 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 3013ms + ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 3012ms × a completion push ends the wait without waiting out an in-flight snapshot 3012ms → expected 'still waiting' to be 'returned' // Object.is equality @@ -454,8 +454,8 @@ Received: "still waiting" Test Files 1 failed (1) Tests 1 failed | 13 skipped (14) - Start at 05:47:15 - Duration 3.49s (transform 190ms, setup 17ms, collect 322ms, tests 3.01s, environment 0ms, prepare 40ms) + Start at 06:05:54 + Duration 3.52s (transform 201ms, setup 17ms, collect 341ms, tests 3.01s, environment 0ms, prepare 43ms) exit=1 @@ -464,13 +464,13 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2061ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2060ms + ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2060ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2059ms Test Files 1 passed (1) Tests 1 passed | 13 skipped (14) - Start at 05:47:19 - Duration 2.55s (transform 193ms, setup 18ms, collect 325ms, tests 2.06s, environment 0ms, prepare 41ms) + Start at 06:05:58 + Duration 2.53s (transform 188ms, setup 16ms, collect 312ms, tests 2.06s, environment 0ms, prepare 40ms) exit=0 ``` @@ -493,8 +493,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 21ms - × a heartbeat timeout is lease loss rather than a worker body failure 20ms + ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 20ms + × a heartbeat timeout is lease loss rather than a worker body failure 19ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -513,8 +513,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 05:47:21 - Duration 467ms (transform 161ms, setup 16ms, collect 286ms, tests 21ms, environment 0ms, prepare 41ms) + Start at 06:06:01 + Duration 467ms (transform 164ms, setup 14ms, collect 286ms, tests 20ms, environment 0ms, prepare 42ms) exit=1 @@ -527,8 +527,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb Test Files 1 passed (1) Tests 1 passed (1) - Start at 05:47:22 - Duration 471ms (transform 165ms, setup 17ms, collect 290ms, tests 16ms, environment 0ms, prepare 43ms) + Start at 06:06:02 + Duration 449ms (transform 164ms, setup 17ms, collect 280ms, tests 16ms, environment 0ms, prepare 40ms) exit=0 ``` @@ -555,8 +555,8 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/authored-root.test.ts (36 tests | 3 failed | 33 skipped) 32ms - × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 24ms + ❯ tests/authored-root.test.ts (36 tests | 3 failed | 33 skipped) 31ms + × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=false) 22ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) × durable authored root > leaves the root resumable after a read timeout without waiting for redispatch (Node frame=true) 4ms @@ -642,8 +642,8 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } Test Files 1 failed (1) Tests 3 failed | 33 skipped (36) - Start at 05:47:23 - Duration 1.28s (transform 671ms, setup 17ms, collect 1.10s, tests 32ms, environment 0ms, prepare 41ms) + Start at 06:06:03 + Duration 1.31s (transform 677ms, setup 16ms, collect 1.12s, tests 31ms, environment 0ms, prepare 41ms) exit=1 @@ -652,12 +652,12 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/authored-root.test.ts (36 tests | 33 skipped) 22ms + ✓ tests/authored-root.test.ts (36 tests | 33 skipped) 23ms Test Files 1 passed (1) Tests 3 passed | 33 skipped (36) - Start at 05:47:25 - Duration 1.28s (transform 669ms, setup 16ms, collect 1.10s, tests 22ms, environment 0ms, prepare 43ms) + Start at 06:06:05 + Duration 1.31s (transform 683ms, setup 16ms, collect 1.12s, tests 23ms, environment 0ms, prepare 40ms) exit=0 ``` diff --git a/packages/sdk/src/journal-read-policy.ts b/packages/sdk/src/journal-read-policy.ts index 6538ec2f7..9fc506131 100644 --- a/packages/sdk/src/journal-read-policy.ts +++ b/packages/sdk/src/journal-read-policy.ts @@ -72,6 +72,10 @@ export class JournalReadPolicy { : new JournalRequestTimeoutError(verb, timeoutMs, attempts, performance.now() - started, budgetMs); // A caller's cancellation settles this read and drains its queued attempt. const signal = caller === undefined ? this.closed.signal : AbortSignal.any([this.closed.signal, caller]); + // Budget exhaustion also aborts the queued work, so a stalled setup or + // request does not hold the serialized queue after its caller gave up. + const budget = new AbortController(); + const attemptSignal = AbortSignal.any([signal, budget.signal]); let timer: ReturnType | undefined; let onClose: () => void = () => {}; let deliver: (value: T) => void = () => {}; @@ -79,27 +83,27 @@ export class JournalReadPolicy { const response = new Promise((resolve, reject) => { deliver = resolve; fail = reject; - timer = setTimeout(() => reject(exhausted()), budgetMs); + timer = setTimeout(() => { const error = exhausted(); budget.abort(error); reject(error); }, budgetMs); onClose = () => reject(signal.reason); signal.addEventListener('abort', onClose, { once: true }); if (signal.aborted) onClose(); }); const job = this.tail.then(async () => { while (true) { - signal.throwIfAborted(); + attemptSignal.throwIfAborted(); const remaining = deadline - performance.now(); if (remaining <= 0) throw exhausted(); attempts += 1; try { - return await request(Math.min(remaining, timeoutMs * 2 ** Math.max(0, attempts - 2)), signal); + return await request(Math.min(remaining, timeoutMs * 2 ** Math.max(0, attempts - 2)), attemptSignal); } catch (error) { - signal.throwIfAborted(); + attemptSignal.throwIfAborted(); if (!(error instanceof JournalRequestTimeoutError || error instanceof JournalReadInterruptedError)) throw error; last = error; const left = deadline - performance.now(); if (left <= 0) throw exhausted(); // Jitter plus escalating bounds keep a slow daemon from collecting a retry storm. - await sleep(Math.min(left, Math.min(1000, timeoutMs / 4) * (0.5 + Math.random())), undefined, { signal }); + await sleep(Math.min(left, Math.min(1000, timeoutMs / 4) * (0.5 + Math.random())), undefined, { signal: attemptSignal }); } } }); diff --git a/packages/sdk/tests/journal-client-read-timeout.test.ts b/packages/sdk/tests/journal-client-read-timeout.test.ts index f5560be25..b85f09897 100644 --- a/packages/sdk/tests/journal-client-read-timeout.test.ts +++ b/packages/sdk/tests/journal-client-read-timeout.test.ts @@ -392,3 +392,21 @@ it('canceled reads are typed read interruptions, so workers hand attempts back', const error = await pending.catch(caught => caught); expect(isReadInterruptionError(error)).toBe(true); }); + +it('an exhausted budget during reader setup frees the queue for the next read', async () => { + let hellos = 0; + const path = sockPath(); + const server = startLoopback(path, { + hello: ctx => { if (++hellos > 1) sendOk(ctx); }, // the first reader handshake stalls + 'run.get': ctx => sendResult(ctx, { status: 'running' }), + }, { serialize: true }); + servers.push(server); + await once(server, 'listening'); + const client = new JournalClient(path, { requestTimeoutMs: 20_000, readBudgetMs: 300 }); + clients.push(client); + await client.connect(); + await expect(client.runGet('run')).rejects.toMatchObject({ readBudgetMs: 300 }); + const begun = performance.now(); + await expect(client.runGet('run')).resolves.toMatchObject({ status: 'running' }); + expect(performance.now() - begun).toBeLessThan(2_000); +}); From 850d7d51611fa580f995aa864f42e69f024a4536 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Thu, 8 Oct 2026 06:17:55 -0700 Subject: [PATCH 34/34] fix(sdk): handshake transport errors are read interruptions; timeouts get a fresh reader - A socket error (ECONNRESET, ...) during a budgeted post-admission hello now surfaces as a non-retryable JournalReadInterruptedError, so the authored child and verifier park the root instead of terminalizing it; the session is gone, so the policy does not retry it. - A timed-out read drops its dedicated reader before retrying: relayflowd answers one frame at a time per connection, so the abandoned request would otherwise stall every retry on that reader. Evidence re-captured at this tree. Co-Authored-By: Claude Opus 5.5 (1M context) Session-Id: b8262468-ea26-4a6e-bee9-d193832e77dc --- evidence/run-read-timeout/final-focused.log | 118 ++++---- .../mutation-completion-aborts-snapshot.log | 16 +- .../run-read-timeout/mutation-heartbeat.log | 12 +- .../mutation-reader-reconnect.log | 40 +-- .../mutation-reader-setup-retry.log | 22 +- evidence/run-read-timeout/mutation-reader.log | 20 +- .../mutation-root-parking.log | 16 +- .../mutation-snapshot-cancel.log | 16 +- .../mutation-watch-cadence.log | 14 +- evidence/run-read-timeout/resume-live.log | 8 +- evidence/run-read-timeout/verification.md | 282 +++++++++--------- packages/sdk/src/journal-budgeted-reads.ts | 14 +- packages/sdk/src/journal-read-policy.ts | 9 +- .../tests/journal-client-read-timeout.test.ts | 25 ++ 14 files changed, 327 insertions(+), 285 deletions(-) diff --git a/evidence/run-read-timeout/final-focused.log b/evidence/run-read-timeout/final-focused.log index 8d0d9d0c8..e9d5dcb40 100644 --- a/evidence/run-read-timeout/final-focused.log +++ b/evidence/run-read-timeout/final-focused.log @@ -2,72 +2,74 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5259ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2035ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2037ms -(node:703660) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5583ms + ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 350ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2045ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2045ms +(node:783010) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (36 tests) 454ms - ✓ tests/journal-client-read-timeout.test.ts (31 tests) 2103ms - ✓ an exhausted budget during reader setup frees the queue for the next read 302ms - ✓ tests/journal-client.test.ts (17 tests) 86ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 7518ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 651ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1074ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2745ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 327ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 826ms - ✓ tests/authored-flow-slack.test.ts (7 tests) 1181ms - ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 421ms - ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 416ms - ✓ tests/running-step-watch.test.ts (14 tests) 12314ms + ✓ tests/authored-root.test.ts (36 tests) 458ms + ✓ tests/journal-client-read-timeout.test.ts (33 tests) 2247ms + ✓ an exhausted budget during reader setup frees the queue for the next read 303ms + ✓ tests/journal-client.test.ts (17 tests) 80ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 6820ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 422ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 999ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2666ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 355ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 889ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 1271ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 458ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 436ms + ✓ tests/running-step-watch.test.ts (14 tests) 12309ms ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2115ms - ✓ cancels promptly while a lease snapshot read is in flight 2006ms + ✓ cancels promptly while a lease snapshot read is in flight 2004ms ✓ a completion push ends the wait without waiting out an in-flight snapshot 2053ms - ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2003ms + ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2002ms ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2002ms - ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2002ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2581ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 671ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 591ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 646ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 671ms - ✓ tests/authored-helpers.test.ts (6 tests) 1883ms - ✓ runs every available provider through the real kernel and resumes completed effects without a second write 882ms - ✓ replays after SIGKILL before confirm with the same token and one successful completion 390ms - ✓ replays after SIGKILL before complete with the same token and one successful completion 389ms - ✓ tests/classify-outcome.test.ts (11 tests) 7415ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2005ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3004ms - ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2000ms - ✓ tests/human-live.test.ts (3 tests) 4571ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2744ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1038ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 787ms - ✓ tests/communication-worker.test.ts (20 tests) 1815ms + ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2001ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2755ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 662ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 686ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 792ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 614ms + ✓ tests/authored-helpers.test.ts (6 tests) 4017ms + ✓ lowers the named acceptance helpers and Slack to confirmed journal effects 352ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 2254ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 726ms + ✓ replays after SIGKILL before complete with the same token and one successful completion 621ms + ✓ tests/classify-outcome.test.ts (11 tests) 7422ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2010ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3005ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms + ✓ tests/human-live.test.ts (3 tests) 4840ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2929ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1217ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 693ms + ✓ tests/communication-worker.test.ts (20 tests) 1826ms ✓ tests/worker-lease.test.ts (8 tests) 12ms ✓ tests/worker-lease-lost.test.ts (17 tests) 20ms - ✓ tests/communication.test.ts (10 tests) 17ms - ✓ tests/effect-channel.test.ts (5 tests) 259ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 632ms + ✓ tests/communication.test.ts (10 tests) 14ms + ✓ tests/effect-channel.test.ts (5 tests) 282ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 714ms ✓ tests/run-daemon-unresponsive.test.ts (5 tests) 6ms - ✓ tests/memoization.test.ts (58 tests) 65ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2413ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2055ms - ✓ drains read and watch promises before an authored flow completes 358ms - ✓ tests/reuse-summary-interruption.test.ts (6 tests) 4ms - ✓ tests/worker-lease-sweep.test.ts (4 tests) 7ms - ✓ tests/journal-client-completion.test.ts (6 tests) 105ms - ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms - ✓ tests/direct-run-worker-lease.test.ts (3 tests) 7ms - ✓ tests/helper-effect-lease-loss.test.ts (1 test) 228ms - ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms - ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 31ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 17ms + ✓ tests/memoization.test.ts (58 tests) 66ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2979ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2064ms + ✓ drains read and watch promises before an authored flow completes 914ms + ✓ tests/reuse-summary-interruption.test.ts (6 tests) 5ms + ✓ tests/worker-lease-sweep.test.ts (4 tests) 6ms + ✓ tests/journal-client-completion.test.ts (6 tests) 109ms + ✓ tests/resume-worker-lease.test.ts (3 tests) 10ms + ✓ tests/direct-run-worker-lease.test.ts (3 tests) 11ms + ✓ tests/helper-effect-lease-loss.test.ts (1 test) 280ms + ✓ tests/journal-client-subscriptions.test.ts (1 test) 10ms + ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 59ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 14ms Test Files 29 passed (29) - Tests 369 passed (369) - Start at 06:06:10 - Duration 61.49s (transform 1.15s, setup 153ms, collect 6.89s, tests 51.02s, environment 4ms, prepare 1.03s) + Tests 371 passed (371) + Start at 06:16:46 + Duration 66.56s (transform 1.62s, setup 189ms, collect 8.28s, tests 54.22s, environment 4ms, prepare 1.15s) exit=0 diff --git a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log index 7e40aec1d..437214d3e 100644 --- a/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log +++ b/evidence/run-read-timeout/mutation-completion-aborts-snapshot.log @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 3012ms - × a completion push ends the wait without waiting out an in-flight snapshot 3012ms + ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 3014ms + × a completion push ends the wait without waiting out an in-flight snapshot 3013ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -34,8 +34,8 @@ Received: "still waiting" Test Files 1 failed (1) Tests 1 failed | 13 skipped (14) - Start at 06:05:54 - Duration 3.52s (transform 201ms, setup 17ms, collect 341ms, tests 3.01s, environment 0ms, prepare 43ms) + Start at 06:16:29 + Duration 3.51s (transform 191ms, setup 16ms, collect 329ms, tests 3.01s, environment 0ms, prepare 41ms) exit=1 @@ -44,12 +44,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2060ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2059ms + ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2062ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2061ms Test Files 1 passed (1) Tests 1 passed | 13 skipped (14) - Start at 06:05:58 - Duration 2.53s (transform 188ms, setup 16ms, collect 312ms, tests 2.06s, environment 0ms, prepare 40ms) + Start at 06:16:33 + Duration 2.57s (transform 204ms, setup 17ms, collect 350ms, tests 2.06s, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-heartbeat.log b/evidence/run-read-timeout/mutation-heartbeat.log index 7dfacded0..e94f0a2c0 100644 --- a/evidence/run-read-timeout/mutation-heartbeat.log +++ b/evidence/run-read-timeout/mutation-heartbeat.log @@ -13,8 +13,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 20ms - × a heartbeat timeout is lease loss rather than a worker body failure 19ms + ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 19ms + × a heartbeat timeout is lease loss rather than a worker body failure 18ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -33,8 +33,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 06:06:01 - Duration 467ms (transform 164ms, setup 14ms, collect 286ms, tests 20ms, environment 0ms, prepare 42ms) + Start at 06:16:36 + Duration 460ms (transform 168ms, setup 17ms, collect 294ms, tests 19ms, environment 0ms, prepare 42ms) exit=1 @@ -47,7 +47,7 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb Test Files 1 passed (1) Tests 1 passed (1) - Start at 06:06:02 - Duration 449ms (transform 164ms, setup 17ms, collect 280ms, tests 16ms, environment 0ms, prepare 40ms) + Start at 06:16:37 + Duration 435ms (transform 164ms, setup 16ms, collect 277ms, tests 16ms, environment 0ms, prepare 32ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-reconnect.log b/evidence/run-read-timeout/mutation-reader-reconnect.log index 920a7ce8b..75cf57455 100644 --- a/evidence/run-read-timeout/mutation-reader-reconnect.log +++ b/evidence/run-read-timeout/mutation-reader-reconnect.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: 06ce5e0ba9248b0aae4d5973b91ad24b60c5333724bc53e395200f5c121f1510 +Original SHA256: fe56924a3b780e35507350c7cef74a4cb2815fee0588cf952542cb8119332710 Replaced: this.drop(reader); @@ -11,28 +11,28 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (31 tests | 1 failed | 30 skipped) 1007ms + ❯ tests/journal-client-read-timeout.test.ts (33 tests | 1 failed | 32 skipped) 1007ms × reconnects the reader after it disconnects within the read budget 1006ms - → journal client: run.get read session was interrupted after 81 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + → journal client: run.get read session was interrupted after 76 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 81 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed - ❯ exhausted src/journal-read-policy.ts:71:9 - 69| let last: unknown; - 70| const exhausted = () => last instanceof JournalReadInterruptedError - 71| ? new JournalReadInterruptedError(verb, attempts, performance.no… +JournalReadInterruptedError: journal client: run.get read session was interrupted after 76 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ exhausted src/journal-read-policy.ts:75:9 + 73| let last: unknown; + 74| const exhausted = () => last instanceof JournalReadInterruptedError + 75| ? new JournalReadInterruptedError(verb, attempts, performance.no… | ^ - 72| : new JournalRequestTimeoutError(verb, timeoutMs, attempts, perf… - 73| // A caller's cancellation settles this read and drains its queued… - ❯ Timeout. src/journal-read-policy.ts:86:48 + 76| : new JournalRequestTimeoutError(verb, timeoutMs, attempts, perf… + 77| // A caller's cancellation settles this read and drains its queued… + ❯ Timeout. src/journal-read-policy.ts:90:48 Caused by: Error: journal client: not connected (run.get): journal client: connection closed ❯ src/journal-connection.ts:183:16 ❯ JournalClient.requestOnce src/journal-connection.ts:176:12 ❯ src/journal-budgeted-reads.ts:60:29 - ❯ src/journal-read-policy.ts:98:18 + ❯ src/journal-read-policy.ts:102:18 Caused by: Error: journal client: connection closed ❯ Socket. src/journal-connection.ts:89:26 @@ -40,22 +40,22 @@ Caused by: Error: journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 30 skipped (31) - Start at 06:05:38 - Duration 1.47s (transform 180ms, setup 17ms, collect 308ms, tests 1.01s, environment 0ms, prepare 40ms) + Tests 1 failed | 32 skipped (33) + Start at 06:16:11 + Duration 1.49s (transform 189ms, setup 18ms, collect 324ms, tests 1.01s, environment 0ms, prepare 46ms) exit=1 -RESTORED SHA256: 06ce5e0ba9248b0aae4d5973b91ad24b60c5333724bc53e395200f5c121f1510 +RESTORED SHA256: fe56924a3b780e35507350c7cef74a4cb2815fee0588cf952542cb8119332710 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'reconnects the reader after it disconnects' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (31 tests | 30 skipped) 86ms + ✓ tests/journal-client-read-timeout.test.ts (33 tests | 32 skipped) 91ms Test Files 1 passed (1) - Tests 1 passed | 30 skipped (31) - Start at 06:05:40 - Duration 543ms (transform 170ms, setup 17ms, collect 283ms, tests 86ms, environment 0ms, prepare 45ms) + Tests 1 passed | 32 skipped (33) + Start at 06:16:13 + Duration 559ms (transform 180ms, setup 18ms, collect 295ms, tests 91ms, environment 0ms, prepare 45ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader-setup-retry.log b/evidence/run-read-timeout/mutation-reader-setup-retry.log index a91c90591..e72403820 100644 --- a/evidence/run-read-timeout/mutation-reader-setup-retry.log +++ b/evidence/run-read-timeout/mutation-reader-setup-retry.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: 06ce5e0ba9248b0aae4d5973b91ad24b60c5333724bc53e395200f5c121f1510 +Original SHA256: fe56924a3b780e35507350c7cef74a4cb2815fee0588cf952542cb8119332710 Replaced: if (this.reader === reader) { this.reader = undefined; @@ -16,8 +16,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (31 tests | 1 failed | 30 skipped) 76ms - × retries a reader setup that timed out instead of reading on the primary for good 76ms + ❯ tests/journal-client-read-timeout.test.ts (33 tests | 1 failed | 32 skipped) 84ms + × retries a reader setup that timed out instead of reading on the primary for good 83ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -42,22 +42,22 @@ AssertionError: expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 30 skipped (31) - Start at 06:05:41 - Duration 535ms (transform 179ms, setup 16ms, collect 302ms, tests 76ms, environment 0ms, prepare 41ms) + Tests 1 failed | 32 skipped (33) + Start at 06:16:14 + Duration 535ms (transform 173ms, setup 17ms, collect 299ms, tests 84ms, environment 0ms, prepare 42ms) exit=1 -RESTORED SHA256: 06ce5e0ba9248b0aae4d5973b91ad24b60c5333724bc53e395200f5c121f1510 +RESTORED SHA256: fe56924a3b780e35507350c7cef74a4cb2815fee0588cf952542cb8119332710 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'retries a reader setup that timed out' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (31 tests | 30 skipped) 67ms + ✓ tests/journal-client-read-timeout.test.ts (33 tests | 32 skipped) 74ms Test Files 1 passed (1) - Tests 1 passed | 30 skipped (31) - Start at 06:05:42 - Duration 527ms (transform 174ms, setup 17ms, collect 298ms, tests 67ms, environment 0ms, prepare 40ms) + Tests 1 passed | 32 skipped (33) + Start at 06:16:15 + Duration 534ms (transform 180ms, setup 17ms, collect 303ms, tests 74ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-reader.log b/evidence/run-read-timeout/mutation-reader.log index c814cd942..c4822afb2 100644 --- a/evidence/run-read-timeout/mutation-reader.log +++ b/evidence/run-read-timeout/mutation-reader.log @@ -1,5 +1,5 @@ Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: 06ce5e0ba9248b0aae4d5973b91ad24b60c5333724bc53e395200f5c121f1510 +Original SHA256: fe56924a3b780e35507350c7cef74a4cb2815fee0588cf952542cb8119332710 Replaced: return await reader.requestOnce(verb, params, remaining, attemptSignal); With: @@ -10,7 +10,7 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (31 tests | 1 failed | 30 skipped) 113ms + ❯ tests/journal-client-read-timeout.test.ts (33 tests | 1 failed | 32 skipped) 113ms × serves a bounded read while an unbounded command is in flight 112ms → expected true to be false // Object.is equality @@ -36,22 +36,22 @@ AssertionError: expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 30 skipped (31) - Start at 06:05:36 - Duration 588ms (transform 178ms, setup 16ms, collect 303ms, tests 113ms, environment 0ms, prepare 41ms) + Tests 1 failed | 32 skipped (33) + Start at 06:16:09 + Duration 589ms (transform 185ms, setup 18ms, collect 300ms, tests 113ms, environment 0ms, prepare 44ms) exit=1 -RESTORED SHA256: 06ce5e0ba9248b0aae4d5973b91ad24b60c5333724bc53e395200f5c121f1510 +RESTORED SHA256: fe56924a3b780e35507350c7cef74a4cb2815fee0588cf952542cb8119332710 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'serves a bounded read' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (31 tests | 30 skipped) 107ms + ✓ tests/journal-client-read-timeout.test.ts (33 tests | 32 skipped) 106ms Test Files 1 passed (1) - Tests 1 passed | 30 skipped (31) - Start at 06:05:37 - Duration 570ms (transform 180ms, setup 17ms, collect 302ms, tests 107ms, environment 0ms, prepare 40ms) + Tests 1 passed | 32 skipped (33) + Start at 06:16:10 + Duration 542ms (transform 172ms, setup 15ms, collect 309ms, tests 106ms, environment 0ms, prepare 28ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-root-parking.log b/evidence/run-read-timeout/mutation-root-parking.log index 84520c9f6..342ecd71d 100644 --- a/evidence/run-read-timeout/mutation-root-parking.log +++ b/evidence/run-read-timeout/mutation-root-parking.log @@ -25,8 +25,8 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r → expected AuthoredFlowExecutionError: daemon_unresp… { …(7) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 4ms - → expected JournalReadInterruptedError: journal clie… { …(4) } to match object { code: 'daemon_unresponsive', …(2) } -(5 matching properties omitted from actual) + → expected JournalReadInterruptedError: journal clie… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } +(6 matching properties omitted from actual) ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 3 ⎯⎯⎯⎯⎯⎯⎯ @@ -79,8 +79,8 @@ AssertionError: expected AuthoredFlowExecutionError: daemon_unresp… { …(7) } ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[2/3]⎯ FAIL tests/authored-root.test.ts > durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy -AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } to match object { code: 'daemon_unresponsive', …(2) } -(5 matching properties omitted from actual) +AssertionError: expected JournalReadInterruptedError: journal clie… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } +(6 matching properties omitted from actual) - Expected + Received @@ -104,8 +104,8 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } Test Files 1 failed (1) Tests 3 failed | 33 skipped (36) - Start at 06:06:03 - Duration 1.31s (transform 677ms, setup 16ms, collect 1.12s, tests 31ms, environment 0ms, prepare 41ms) + Start at 06:16:37 + Duration 1.33s (transform 681ms, setup 15ms, collect 1.12s, tests 31ms, environment 0ms, prepare 44ms) exit=1 @@ -118,7 +118,7 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r Test Files 1 passed (1) Tests 3 passed | 33 skipped (36) - Start at 06:06:05 - Duration 1.31s (transform 683ms, setup 16ms, collect 1.12s, tests 23ms, environment 0ms, prepare 40ms) + Start at 06:16:39 + Duration 1.29s (transform 674ms, setup 14ms, collect 1.11s, tests 23ms, environment 0ms, prepare 41ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-snapshot-cancel.log b/evidence/run-read-timeout/mutation-snapshot-cancel.log index dd99d1f36..d94ca66ae 100644 --- a/evidence/run-read-timeout/mutation-snapshot-cancel.log +++ b/evidence/run-read-timeout/mutation-snapshot-cancel.log @@ -11,8 +11,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2218ms - × cancels promptly while a lease snapshot read is in flight 2217ms + ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2224ms + × cancels promptly while a lease snapshot read is in flight 2223ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -31,8 +31,8 @@ AssertionError: expected 'still waiting' to be an instance of Error Test Files 1 failed (1) Tests 1 failed | 13 skipped (14) - Start at 06:05:48 - Duration 2.71s (transform 191ms, setup 17ms, collect 325ms, tests 2.22s, environment 0ms, prepare 41ms) + Start at 06:16:22 + Duration 3.35s (transform 482ms, setup 30ms, collect 848ms, tests 2.22s, environment 0ms, prepare 75ms) exit=1 @@ -41,12 +41,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2011ms - ✓ cancels promptly while a lease snapshot read is in flight 2010ms + ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2009ms + ✓ cancels promptly while a lease snapshot read is in flight 2008ms Test Files 1 passed (1) Tests 1 passed | 13 skipped (14) - Start at 06:05:52 - Duration 2.50s (transform 191ms, setup 16ms, collect 328ms, tests 2.01s, environment 0ms, prepare 41ms) + Start at 06:16:26 + Duration 2.60s (transform 223ms, setup 20ms, collect 393ms, tests 2.01s, environment 0ms, prepare 50ms) exit=0 diff --git a/evidence/run-read-timeout/mutation-watch-cadence.log b/evidence/run-read-timeout/mutation-watch-cadence.log index 65de3288f..d4d5dc347 100644 --- a/evidence/run-read-timeout/mutation-watch-cadence.log +++ b/evidence/run-read-timeout/mutation-watch-cadence.log @@ -10,8 +10,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2110ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2109ms + ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2109ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2108ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -30,8 +30,8 @@ AssertionError: expected 41 to be less than or equal to 1 Test Files 1 failed (1) Tests 1 failed | 13 skipped (14) - Start at 06:05:43 - Duration 2.60s (transform 189ms, setup 17ms, collect 325ms, tests 2.11s, environment 0ms, prepare 41ms) + Start at 06:16:16 + Duration 2.59s (transform 194ms, setup 16ms, collect 322ms, tests 2.11s, environment 0ms, prepare 41ms) exit=1 @@ -41,11 +41,11 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2117ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2116ms Test Files 1 passed (1) Tests 1 passed | 13 skipped (14) - Start at 06:05:46 - Duration 2.60s (transform 191ms, setup 16ms, collect 325ms, tests 2.12s, environment 0ms, prepare 41ms) + Start at 06:16:19 + Duration 2.56s (transform 186ms, setup 14ms, collect 298ms, tests 2.12s, environment 0ms, prepare 33ms) exit=0 diff --git a/evidence/run-read-timeout/resume-live.log b/evidence/run-read-timeout/resume-live.log index 17434b389..7169d526d 100644 --- a/evidence/run-read-timeout/resume-live.log +++ b/evidence/run-read-timeout/resume-live.log @@ -2,12 +2,12 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 876ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 876ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 860ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 860ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 06:07:12 - Duration 2.15s (transform 629ms, setup 15ms, collect 1.02s, tests 876ms, environment 0ms, prepare 52ms) + Start at 06:17:53 + Duration 2.17s (transform 643ms, setup 20ms, collect 1.02s, tests 860ms, environment 0ms, prepare 82ms) exit=0 diff --git a/evidence/run-read-timeout/verification.md b/evidence/run-read-timeout/verification.md index bcfa02d10..3fa45b5d5 100644 --- a/evidence/run-read-timeout/verification.md +++ b/evidence/run-read-timeout/verification.md @@ -15,73 +15,75 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/cli.test.ts (71 tests) 5259ms - ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2035ms - ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2037ms -(node:703660) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. + ✓ tests/cli.test.ts (71 tests) 5583ms + ✓ flows check CLI > resolves a bare PATH-resolved claude with no declared model, in an isolated PATH 350ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 2045ms + ✓ flows run/resume CLI over the journal protocol > follows a worker wait past a locally expired lease until the daemon settles it 2045ms +(node:783010) [FLOWS_ROOT_LEASE_LOST] Warning: authored root run_id=root-run attempt=1: lease_conflict: attempt has no active worker lease. Waiting for the kernel to retry it. (Use `node --trace-warnings ...` to show where the warning was created) - ✓ tests/authored-root.test.ts (36 tests) 454ms - ✓ tests/journal-client-read-timeout.test.ts (31 tests) 2103ms - ✓ an exhausted budget during reader setup frees the queue for the next read 302ms - ✓ tests/journal-client.test.ts (17 tests) 86ms - ✓ tests/flow-executor-chain.test.ts (14 tests) 7518ms - ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 651ms - ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 1074ms - ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2745ms - ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 327ms - ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 826ms - ✓ tests/authored-flow-slack.test.ts (7 tests) 1181ms - ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 421ms - ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 416ms - ✓ tests/running-step-watch.test.ts (14 tests) 12314ms + ✓ tests/authored-root.test.ts (36 tests) 458ms + ✓ tests/journal-client-read-timeout.test.ts (33 tests) 2247ms + ✓ an exhausted budget during reader setup frees the queue for the next read 303ms + ✓ tests/journal-client.test.ts (17 tests) 80ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 6820ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 422ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 999ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 2666ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 355ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 889ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 1271ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 458ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 436ms + ✓ tests/running-step-watch.test.ts (14 tests) 12309ms ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2115ms - ✓ cancels promptly while a lease snapshot read is in flight 2006ms + ✓ cancels promptly while a lease snapshot read is in flight 2004ms ✓ a completion push ends the wait without waiting out an in-flight snapshot 2053ms - ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2003ms + ✓ a completion that aborts the snapshot is not an error even if a retry starts at once 2002ms ✓ a malformed watch frame during an in-flight snapshot is an error, not a completion 2002ms - ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2002ms - ✓ tests/agent-transcript-live.test.ts (4 tests) 2581ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 671ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 591ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 646ms - ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 671ms - ✓ tests/authored-helpers.test.ts (6 tests) 1883ms - ✓ runs every available provider through the real kernel and resumes completed effects without a second write 882ms - ✓ replays after SIGKILL before confirm with the same token and one successful completion 390ms - ✓ replays after SIGKILL before complete with the same token and one successful completion 389ms - ✓ tests/classify-outcome.test.ts (11 tests) 7415ms - ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2005ms - ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3004ms - ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2000ms - ✓ tests/human-live.test.ts (3 tests) 4571ms - ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2744ms - ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1038ms - ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 787ms - ✓ tests/communication-worker.test.ts (20 tests) 1815ms + ✓ a watch dropped during an in-flight snapshot is reported at once, not after the read budget 2001ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 2755ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 662ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 686ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 792ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 614ms + ✓ tests/authored-helpers.test.ts (6 tests) 4017ms + ✓ lowers the named acceptance helpers and Slack to confirmed journal effects 352ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 2254ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 726ms + ✓ replays after SIGKILL before complete with the same token and one successful completion 621ms + ✓ tests/classify-outcome.test.ts (11 tests) 7422ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2010ms + ✓ the remedy on a worker park > follows a step through a retry backoff longer than the unclassified bound 3005ms + ✓ the remedy on a worker park > follows a run.start outcome that is already running on a retried attempt 2001ms + ✓ tests/human-live.test.ts (3 tests) 4840ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 2929ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1217ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 693ms + ✓ tests/communication-worker.test.ts (20 tests) 1826ms ✓ tests/worker-lease.test.ts (8 tests) 12ms ✓ tests/worker-lease-lost.test.ts (17 tests) 20ms - ✓ tests/communication.test.ts (10 tests) 17ms - ✓ tests/effect-channel.test.ts (5 tests) 259ms - ✓ tests/worker-lease-lost-live.test.ts (3 tests) 632ms + ✓ tests/communication.test.ts (10 tests) 14ms + ✓ tests/effect-channel.test.ts (5 tests) 282ms + ✓ tests/worker-lease-lost-live.test.ts (3 tests) 714ms ✓ tests/run-daemon-unresponsive.test.ts (5 tests) 6ms - ✓ tests/memoization.test.ts (58 tests) 65ms - ✓ tests/run-read-load-live.test.ts (2 tests) 2413ms - ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2055ms - ✓ drains read and watch promises before an authored flow completes 358ms - ✓ tests/reuse-summary-interruption.test.ts (6 tests) 4ms - ✓ tests/worker-lease-sweep.test.ts (4 tests) 7ms - ✓ tests/journal-client-completion.test.ts (6 tests) 105ms - ✓ tests/resume-worker-lease.test.ts (3 tests) 7ms - ✓ tests/direct-run-worker-lease.test.ts (3 tests) 7ms - ✓ tests/helper-effect-lease-loss.test.ts (1 test) 228ms - ✓ tests/journal-client-subscriptions.test.ts (1 test) 8ms - ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 31ms - ✓ tests/heartbeat-timeout.test.ts (1 test) 17ms + ✓ tests/memoization.test.ts (58 tests) 66ms + ✓ tests/run-read-load-live.test.ts (2 tests) 2979ms + ✓ completes a CPU-saturating deterministic flow with reads in flight and preserves its journal 2064ms + ✓ drains read and watch promises before an authored flow completes 914ms + ✓ tests/reuse-summary-interruption.test.ts (6 tests) 5ms + ✓ tests/worker-lease-sweep.test.ts (4 tests) 6ms + ✓ tests/journal-client-completion.test.ts (6 tests) 109ms + ✓ tests/resume-worker-lease.test.ts (3 tests) 10ms + ✓ tests/direct-run-worker-lease.test.ts (3 tests) 11ms + ✓ tests/helper-effect-lease-loss.test.ts (1 test) 280ms + ✓ tests/journal-client-subscriptions.test.ts (1 test) 10ms + ✓ tests/authored-verifier-read-budget.test.ts (2 tests) 59ms + ✓ tests/heartbeat-timeout.test.ts (1 test) 14ms Test Files 29 passed (29) - Tests 369 passed (369) - Start at 06:06:10 - Duration 61.49s (transform 1.15s, setup 153ms, collect 6.89s, tests 51.02s, environment 4ms, prepare 1.03s) + Tests 371 passed (371) + Start at 06:16:46 + Duration 66.56s (transform 1.62s, setup 189ms, collect 8.28s, tests 54.22s, environment 4ms, prepare 1.15s) exit=0 ``` @@ -93,13 +95,13 @@ $ cd packages/sdk && RELAYFLOWD_BIN=/home/khaliqgant/Projects/AgentWorkforce/flo RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/read-timeout-resume-live.test.ts (1 test) 876ms - ✓ parks an unreadable authored root and resumes without repeating its journaled effect 876ms + ✓ tests/read-timeout-resume-live.test.ts (1 test) 860ms + ✓ parks an unreadable authored root and resumes without repeating its journaled effect 860ms Test Files 1 passed (1) Tests 1 passed (1) - Start at 06:07:12 - Duration 2.15s (transform 629ms, setup 15ms, collect 1.02s, tests 876ms, environment 0ms, prepare 52ms) + Start at 06:17:53 + Duration 2.17s (transform 643ms, setup 20ms, collect 1.02s, tests 860ms, environment 0ms, prepare 82ms) exit=0 ``` @@ -110,7 +112,7 @@ The mutation sections below were regenerated by `cd packages/sdk && python3 ../. ```text Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: 06ce5e0ba9248b0aae4d5973b91ad24b60c5333724bc53e395200f5c121f1510 +Original SHA256: fe56924a3b780e35507350c7cef74a4cb2815fee0588cf952542cb8119332710 Replaced: return await reader.requestOnce(verb, params, remaining, attemptSignal); With: @@ -121,7 +123,7 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (31 tests | 1 failed | 30 skipped) 113ms + ❯ tests/journal-client-read-timeout.test.ts (33 tests | 1 failed | 32 skipped) 113ms × serves a bounded read while an unbounded command is in flight 112ms → expected true to be false // Object.is equality @@ -147,23 +149,23 @@ AssertionError: expected true to be false // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 30 skipped (31) - Start at 06:05:36 - Duration 588ms (transform 178ms, setup 16ms, collect 303ms, tests 113ms, environment 0ms, prepare 41ms) + Tests 1 failed | 32 skipped (33) + Start at 06:16:09 + Duration 589ms (transform 185ms, setup 18ms, collect 300ms, tests 113ms, environment 0ms, prepare 44ms) exit=1 -RESTORED SHA256: 06ce5e0ba9248b0aae4d5973b91ad24b60c5333724bc53e395200f5c121f1510 +RESTORED SHA256: fe56924a3b780e35507350c7cef74a4cb2815fee0588cf952542cb8119332710 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'serves a bounded read' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (31 tests | 30 skipped) 107ms + ✓ tests/journal-client-read-timeout.test.ts (33 tests | 32 skipped) 106ms Test Files 1 passed (1) - Tests 1 passed | 30 skipped (31) - Start at 06:05:37 - Duration 570ms (transform 180ms, setup 17ms, collect 302ms, tests 107ms, environment 0ms, prepare 40ms) + Tests 1 passed | 32 skipped (33) + Start at 06:16:10 + Duration 542ms (transform 172ms, setup 15ms, collect 309ms, tests 106ms, environment 0ms, prepare 28ms) exit=0 ``` @@ -172,7 +174,7 @@ exit=0 ```text Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: 06ce5e0ba9248b0aae4d5973b91ad24b60c5333724bc53e395200f5c121f1510 +Original SHA256: fe56924a3b780e35507350c7cef74a4cb2815fee0588cf952542cb8119332710 Replaced: this.drop(reader); @@ -184,28 +186,28 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (31 tests | 1 failed | 30 skipped) 1007ms + ❯ tests/journal-client-read-timeout.test.ts (33 tests | 1 failed | 32 skipped) 1007ms × reconnects the reader after it disconnects within the read budget 1006ms - → journal client: run.get read session was interrupted after 81 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + → journal client: run.get read session was interrupted after 76 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ FAIL tests/journal-client-read-timeout.test.ts > reconnects the reader after it disconnects within the read budget -JournalReadInterruptedError: journal client: run.get read session was interrupted after 81 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed - ❯ exhausted src/journal-read-policy.ts:71:9 - 69| let last: unknown; - 70| const exhausted = () => last instanceof JournalReadInterruptedError - 71| ? new JournalReadInterruptedError(verb, attempts, performance.no… +JournalReadInterruptedError: journal client: run.get read session was interrupted after 76 attempts in 1000ms (read budget 1000ms): journal client: not connected (run.get): journal client: connection closed + ❯ exhausted src/journal-read-policy.ts:75:9 + 73| let last: unknown; + 74| const exhausted = () => last instanceof JournalReadInterruptedError + 75| ? new JournalReadInterruptedError(verb, attempts, performance.no… | ^ - 72| : new JournalRequestTimeoutError(verb, timeoutMs, attempts, perf… - 73| // A caller's cancellation settles this read and drains its queued… - ❯ Timeout. src/journal-read-policy.ts:86:48 + 76| : new JournalRequestTimeoutError(verb, timeoutMs, attempts, perf… + 77| // A caller's cancellation settles this read and drains its queued… + ❯ Timeout. src/journal-read-policy.ts:90:48 Caused by: Error: journal client: not connected (run.get): journal client: connection closed ❯ src/journal-connection.ts:183:16 ❯ JournalClient.requestOnce src/journal-connection.ts:176:12 ❯ src/journal-budgeted-reads.ts:60:29 - ❯ src/journal-read-policy.ts:98:18 + ❯ src/journal-read-policy.ts:102:18 Caused by: Error: journal client: connection closed ❯ Socket. src/journal-connection.ts:89:26 @@ -213,23 +215,23 @@ Caused by: Error: journal client: connection closed ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 30 skipped (31) - Start at 06:05:38 - Duration 1.47s (transform 180ms, setup 17ms, collect 308ms, tests 1.01s, environment 0ms, prepare 40ms) + Tests 1 failed | 32 skipped (33) + Start at 06:16:11 + Duration 1.49s (transform 189ms, setup 18ms, collect 324ms, tests 1.01s, environment 0ms, prepare 46ms) exit=1 -RESTORED SHA256: 06ce5e0ba9248b0aae4d5973b91ad24b60c5333724bc53e395200f5c121f1510 +RESTORED SHA256: fe56924a3b780e35507350c7cef74a4cb2815fee0588cf952542cb8119332710 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'reconnects the reader after it disconnects' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (31 tests | 30 skipped) 86ms + ✓ tests/journal-client-read-timeout.test.ts (33 tests | 32 skipped) 91ms Test Files 1 passed (1) - Tests 1 passed | 30 skipped (31) - Start at 06:05:40 - Duration 543ms (transform 170ms, setup 17ms, collect 283ms, tests 86ms, environment 0ms, prepare 45ms) + Tests 1 passed | 32 skipped (33) + Start at 06:16:13 + Duration 559ms (transform 180ms, setup 18ms, collect 295ms, tests 91ms, environment 0ms, prepare 45ms) exit=0 ``` @@ -238,7 +240,7 @@ exit=0 ```text Source: packages/sdk/src/journal-budgeted-reads.ts -Original SHA256: 06ce5e0ba9248b0aae4d5973b91ad24b60c5333724bc53e395200f5c121f1510 +Original SHA256: fe56924a3b780e35507350c7cef74a4cb2815fee0588cf952542cb8119332710 Replaced: if (this.reader === reader) { this.reader = undefined; @@ -255,8 +257,8 @@ $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/journal-client-read-timeout.test.ts (31 tests | 1 failed | 30 skipped) 76ms - × retries a reader setup that timed out instead of reading on the primary for good 76ms + ❯ tests/journal-client-read-timeout.test.ts (33 tests | 1 failed | 32 skipped) 84ms + × retries a reader setup that timed out instead of reading on the primary for good 83ms → expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -281,23 +283,23 @@ AssertionError: expected 1 to be 2 // Object.is equality ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ Test Files 1 failed (1) - Tests 1 failed | 30 skipped (31) - Start at 06:05:41 - Duration 535ms (transform 179ms, setup 16ms, collect 302ms, tests 76ms, environment 0ms, prepare 41ms) + Tests 1 failed | 32 skipped (33) + Start at 06:16:14 + Duration 535ms (transform 173ms, setup 17ms, collect 299ms, tests 84ms, environment 0ms, prepare 42ms) exit=1 -RESTORED SHA256: 06ce5e0ba9248b0aae4d5973b91ad24b60c5333724bc53e395200f5c121f1510 +RESTORED SHA256: fe56924a3b780e35507350c7cef74a4cb2815fee0588cf952542cb8119332710 $ cd packages/sdk && npx vitest run tests/journal-client-read-timeout.test.ts -t 'retries a reader setup that timed out' --maxWorkers=1 --minWorkers=1 RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/journal-client-read-timeout.test.ts (31 tests | 30 skipped) 67ms + ✓ tests/journal-client-read-timeout.test.ts (33 tests | 32 skipped) 74ms Test Files 1 passed (1) - Tests 1 passed | 30 skipped (31) - Start at 06:05:42 - Duration 527ms (transform 174ms, setup 17ms, collect 298ms, tests 67ms, environment 0ms, prepare 40ms) + Tests 1 passed | 32 skipped (33) + Start at 06:16:15 + Duration 534ms (transform 180ms, setup 17ms, collect 303ms, tests 74ms, environment 0ms, prepare 41ms) exit=0 ``` @@ -317,8 +319,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2110ms - × uses pushes for completion with lease-cadence reads and releases its watcher 2109ms + ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2109ms + × uses pushes for completion with lease-cadence reads and releases its watcher 2108ms → expected 41 to be less than or equal to 1 ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -337,8 +339,8 @@ AssertionError: expected 41 to be less than or equal to 1 Test Files 1 failed (1) Tests 1 failed | 13 skipped (14) - Start at 06:05:43 - Duration 2.60s (transform 189ms, setup 17ms, collect 325ms, tests 2.11s, environment 0ms, prepare 41ms) + Start at 06:16:16 + Duration 2.59s (transform 194ms, setup 16ms, collect 322ms, tests 2.11s, environment 0ms, prepare 41ms) exit=1 @@ -348,12 +350,12 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'uses pu RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2117ms - ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2117ms + ✓ uses pushes for completion with lease-cadence reads and releases its watcher 2116ms Test Files 1 passed (1) Tests 1 passed | 13 skipped (14) - Start at 06:05:46 - Duration 2.60s (transform 191ms, setup 16ms, collect 325ms, tests 2.12s, environment 0ms, prepare 41ms) + Start at 06:16:19 + Duration 2.56s (transform 186ms, setup 14ms, collect 298ms, tests 2.12s, environment 0ms, prepare 33ms) exit=0 ``` @@ -374,8 +376,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2218ms - × cancels promptly while a lease snapshot read is in flight 2217ms + ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 2224ms + × cancels promptly while a lease snapshot read is in flight 2223ms → expected 'still waiting' to be an instance of Error ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -394,8 +396,8 @@ AssertionError: expected 'still waiting' to be an instance of Error Test Files 1 failed (1) Tests 1 failed | 13 skipped (14) - Start at 06:05:48 - Duration 2.71s (transform 191ms, setup 17ms, collect 325ms, tests 2.22s, environment 0ms, prepare 41ms) + Start at 06:16:22 + Duration 3.35s (transform 482ms, setup 30ms, collect 848ms, tests 2.22s, environment 0ms, prepare 75ms) exit=1 @@ -404,13 +406,13 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'cancels RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2011ms - ✓ cancels promptly while a lease snapshot read is in flight 2010ms + ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2009ms + ✓ cancels promptly while a lease snapshot read is in flight 2008ms Test Files 1 passed (1) Tests 1 passed | 13 skipped (14) - Start at 06:05:52 - Duration 2.50s (transform 191ms, setup 16ms, collect 328ms, tests 2.01s, environment 0ms, prepare 41ms) + Start at 06:16:26 + Duration 2.60s (transform 223ms, setup 20ms, collect 393ms, tests 2.01s, environment 0ms, prepare 50ms) exit=0 ``` @@ -430,8 +432,8 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 3012ms - × a completion push ends the wait without waiting out an in-flight snapshot 3012ms + ❯ tests/running-step-watch.test.ts (14 tests | 1 failed | 13 skipped) 3014ms + × a completion push ends the wait without waiting out an in-flight snapshot 3013ms → expected 'still waiting' to be 'returned' // Object.is equality ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -454,8 +456,8 @@ Received: "still waiting" Test Files 1 failed (1) Tests 1 failed | 13 skipped (14) - Start at 06:05:54 - Duration 3.52s (transform 201ms, setup 17ms, collect 341ms, tests 3.01s, environment 0ms, prepare 43ms) + Start at 06:16:29 + Duration 3.51s (transform 191ms, setup 16ms, collect 329ms, tests 3.01s, environment 0ms, prepare 41ms) exit=1 @@ -464,13 +466,13 @@ $ cd packages/sdk && npx vitest run tests/running-step-watch.test.ts -t 'a compl RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2060ms - ✓ a completion push ends the wait without waiting out an in-flight snapshot 2059ms + ✓ tests/running-step-watch.test.ts (14 tests | 13 skipped) 2062ms + ✓ a completion push ends the wait without waiting out an in-flight snapshot 2061ms Test Files 1 passed (1) Tests 1 passed | 13 skipped (14) - Start at 06:05:58 - Duration 2.53s (transform 188ms, setup 16ms, collect 312ms, tests 2.06s, environment 0ms, prepare 40ms) + Start at 06:16:33 + Duration 2.57s (transform 204ms, setup 17ms, collect 350ms, tests 2.06s, environment 0ms, prepare 41ms) exit=0 ``` @@ -493,8 +495,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb RUN v2.1.9 /home/khaliqgant/Projects/AgentWorkforce/flows-worktrees/pr613/packages/sdk - ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 20ms - × a heartbeat timeout is lease loss rather than a worker body failure 19ms + ❯ tests/heartbeat-timeout.test.ts (1 test | 1 failed) 19ms + × a heartbeat timeout is lease loss rather than a worker body failure 18ms → expected JournalRequestTimeoutError: journal clien… { …(5) } to be an instance of WorkerLeaseLostError ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ @@ -513,8 +515,8 @@ AssertionError: expected JournalRequestTimeoutError: journal clien… { …(5) } Test Files 1 failed (1) Tests 1 failed (1) - Start at 06:06:01 - Duration 467ms (transform 164ms, setup 14ms, collect 286ms, tests 20ms, environment 0ms, prepare 42ms) + Start at 06:16:36 + Duration 460ms (transform 168ms, setup 17ms, collect 294ms, tests 19ms, environment 0ms, prepare 42ms) exit=1 @@ -527,8 +529,8 @@ $ cd packages/sdk && npx vitest run tests/heartbeat-timeout.test.ts -t 'a heartb Test Files 1 passed (1) Tests 1 passed (1) - Start at 06:06:02 - Duration 449ms (transform 164ms, setup 17ms, collect 280ms, tests 16ms, environment 0ms, prepare 40ms) + Start at 06:16:37 + Duration 435ms (transform 164ms, setup 16ms, collect 277ms, tests 16ms, environment 0ms, prepare 32ms) exit=0 ``` @@ -563,8 +565,8 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r → expected AuthoredFlowExecutionError: daemon_unresp… { …(7) } to match object { code: 'daemon_unresponsive', …(2) } (6 matching properties omitted from actual) × durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy 4ms - → expected JournalReadInterruptedError: journal clie… { …(4) } to match object { code: 'daemon_unresponsive', …(2) } -(5 matching properties omitted from actual) + → expected JournalReadInterruptedError: journal clie… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } +(6 matching properties omitted from actual) ⎯⎯⎯⎯⎯⎯⎯ Failed Tests 3 ⎯⎯⎯⎯⎯⎯⎯ @@ -617,8 +619,8 @@ AssertionError: expected AuthoredFlowExecutionError: daemon_unresp… { …(7) } ⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[2/3]⎯ FAIL tests/authored-root.test.ts > durable authored root > leaves the root resumable when the read session disconnects while the root worker is healthy -AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } to match object { code: 'daemon_unresponsive', …(2) } -(5 matching properties omitted from actual) +AssertionError: expected JournalReadInterruptedError: journal clie… { …(5) } to match object { code: 'daemon_unresponsive', …(2) } +(6 matching properties omitted from actual) - Expected + Received @@ -642,8 +644,8 @@ AssertionError: expected JournalReadInterruptedError: journal clie… { …(4) } Test Files 1 failed (1) Tests 3 failed | 33 skipped (36) - Start at 06:06:03 - Duration 1.31s (transform 677ms, setup 16ms, collect 1.12s, tests 31ms, environment 0ms, prepare 41ms) + Start at 06:16:37 + Duration 1.33s (transform 681ms, setup 15ms, collect 1.12s, tests 31ms, environment 0ms, prepare 44ms) exit=1 @@ -656,8 +658,8 @@ $ cd packages/sdk && npx vitest run tests/authored-root.test.ts -t 'leaves the r Test Files 1 passed (1) Tests 3 passed | 33 skipped (36) - Start at 06:06:05 - Duration 1.31s (transform 683ms, setup 16ms, collect 1.12s, tests 23ms, environment 0ms, prepare 40ms) + Start at 06:16:39 + Duration 1.29s (transform 674ms, setup 14ms, collect 1.11s, tests 23ms, environment 0ms, prepare 41ms) exit=0 ``` diff --git a/packages/sdk/src/journal-budgeted-reads.ts b/packages/sdk/src/journal-budgeted-reads.ts index 75cbaed3d..01ffbc4f3 100644 --- a/packages/sdk/src/journal-budgeted-reads.ts +++ b/packages/sdk/src/journal-budgeted-reads.ts @@ -62,8 +62,10 @@ export class BudgetedReads { // relayflowd answers one frame at a time per connection, so a canceled // request still occupies this reader: drop it for a fresh session. if (attemptSignal.aborted) { this.drop(reader); throw error; } - if (error instanceof JournalRequestTimeoutError - || error instanceof JournalProtocolError || error instanceof JournalFrameError) throw error; + // A timed-out request still occupies this sequential connection: retry + // on a fresh reader rather than queueing behind it. + if (error instanceof JournalRequestTimeoutError) { this.drop(reader); throw error; } + if (error instanceof JournalProtocolError || error instanceof JournalFrameError) throw error; // Only the dedicated reader's transport failed; reconnect within the budget. this.drop(reader); throw new JournalReadInterruptedError(verb, 1, performance.now() - started, undefined, { cause: error }); @@ -74,7 +76,13 @@ export class BudgetedReads { /** The primary session's own `hello`, retried within the same budget as reads. */ handshake(params: VerbContract['hello']['params'], timeoutMs: number): Promise { return this.policy.read('hello', timeoutMs, this.budgetMs, - (bound, attemptSignal) => this.primary.requestOnce('hello', params, bound, attemptSignal), this.lifecycle); + (bound, attemptSignal) => this.primary.requestOnce('hello', params, bound, attemptSignal).catch(error => { + if (attemptSignal.aborted || error instanceof JournalRequestTimeoutError + || error instanceof JournalProtocolError || error instanceof JournalFrameError) throw error; + // The primary's own transport failed (ECONNRESET, ...): a read + // interruption, not a body failure, and not retryable on this session. + throw new JournalReadInterruptedError('hello', 1, 0, undefined, { cause: error, retryable: false }); + }), this.lifecycle); } private session(): Promise { diff --git a/packages/sdk/src/journal-read-policy.ts b/packages/sdk/src/journal-read-policy.ts index 9fc506131..14b403b5a 100644 --- a/packages/sdk/src/journal-read-policy.ts +++ b/packages/sdk/src/journal-read-policy.ts @@ -31,7 +31,7 @@ export class JournalReadInterruptedError extends Error { readonly attempts = 1, readonly elapsedMs = 0, readonly readBudgetMs?: number, - options?: ErrorOptions, + options?: ErrorOptions & { retryable?: boolean }, ) { const cause = options?.cause instanceof Error ? `: ${options.cause.message}` : ''; super(readBudgetMs === undefined @@ -39,7 +39,11 @@ export class JournalReadInterruptedError extends Error { : `journal client: ${verb} read session was interrupted after ${attempts} attempts in ${Math.round(elapsedMs)}ms (read budget ${readBudgetMs}ms)${cause}`, options); this.name = 'JournalReadInterruptedError'; + this.retryable = options?.retryable !== false; } + + /** False when the session it ran on is gone, so retrying it cannot help. */ + readonly retryable: boolean; } /** A read that could not be answered — by timeout or lost read transport — not a run failure. */ @@ -98,7 +102,8 @@ export class JournalReadPolicy { return await request(Math.min(remaining, timeoutMs * 2 ** Math.max(0, attempts - 2)), attemptSignal); } catch (error) { attemptSignal.throwIfAborted(); - if (!(error instanceof JournalRequestTimeoutError || error instanceof JournalReadInterruptedError)) throw error; + if (!(error instanceof JournalRequestTimeoutError + || (error instanceof JournalReadInterruptedError && error.retryable))) throw error; last = error; const left = deadline - performance.now(); if (left <= 0) throw exhausted(); diff --git a/packages/sdk/tests/journal-client-read-timeout.test.ts b/packages/sdk/tests/journal-client-read-timeout.test.ts index b85f09897..c815b082a 100644 --- a/packages/sdk/tests/journal-client-read-timeout.test.ts +++ b/packages/sdk/tests/journal-client-read-timeout.test.ts @@ -410,3 +410,28 @@ it('an exhausted budget during reader setup frees the queue for the next read', await expect(client.runGet('run')).resolves.toMatchObject({ status: 'running' }); expect(performance.now() - begun).toBeLessThan(2_000); }); + +it('a transport error during a budgeted handshake is a read interruption, not a body failure', async () => { + const client = await handshakeClient({ hello: ctx => { ctx.socket.destroy(); } }, 60_000); + const begun = performance.now(); + const error = await client.hello('flows-authored-node').catch(caught => caught); + expect(isReadInterruptionError(error)).toBe(true); + expect(performance.now() - begun).toBeLessThan(2_000); // the session is gone; no futile retries +}); + +it('retries a timed-out read on a fresh reader instead of queueing behind it', async () => { + let reads = 0; + const sockets: unknown[] = []; + const path = sockPath(); + const server = startLoopback(path, { + hello: sendOk, + 'run.get': ctx => { reads += 1; sockets.push(ctx.socket); if (reads > 1) sendResult(ctx, { status: 'running' }); }, + }, { serialize: true }); + servers.push(server); + await once(server, 'listening'); + const client = new JournalClient(path, { requestTimeoutMs: 100, readBudgetMs: 10_000 }); + clients.push(client); + await client.connect(); + await expect(client.runGet('run')).resolves.toMatchObject({ status: 'running' }); + expect(sockets[1]).not.toBe(sockets[0]); +});