From e4c4c80c844aa2ac4e654062909ad1fbf1a84776 Mon Sep 17 00:00:00 2001 From: Miya Date: Fri, 11 Sep 2026 10:36:14 +0200 Subject: [PATCH 1/4] feat(surface,sdk,kernel): budget header and spend attribution Session-Id: 01a08f7f-7b46-78c3-84c7-097476f69e4a Session-Id: efeda5df-9b7c-48d4-b2ce-957f5bef0a82 Session-Id: efeda5df-9b7c-48d4-b2ce-957f5bef0a82 Session-Id: efeda5df-9b7c-48d4-b2ce-957f5bef0a82 --- docs/BUDGET.md | 61 +++ docs/SURFACE.md | 2 +- evidence/spec-G-budget/PR.md | 36 ++ evidence/spec-G-budget/README.md | 97 +++++ evidence/spec-G-budget/cargo-test.log | 361 ++++++++++++++++++ evidence/spec-G-budget/claude-auth-status.log | 8 + evidence/spec-G-budget/sdk-focused.log | 23 ++ .../spec-G-budget/sdk-npm-test-earlier.log | 214 +++++++++++ evidence/spec-G-budget/sdk-npm-test.log | 193 ++++++++++ evidence/spec-G-budget/smoke-journal.json | 48 +++ evidence/spec-G-budget/smoke.log | 2 + evidence/spec-G-budget/smoke.spec.json | 1 + evidence/spec-G-budget/surface-test.log | 9 + kernel/relayflowd-core/src/entry.rs | 24 +- kernel/relayflowd-core/src/machine.rs | 13 + kernel/relayflowd-core/src/machine/budget.rs | 29 ++ kernel/relayflowd-core/src/memory.rs | 9 +- kernel/relayflowd-core/src/spec.rs | 48 +++ kernel/relayflowd-core/src/state.rs | 38 +- kernel/relayflowd-core/src/state/budget.rs | 29 +- kernel/relayflowd-core/src/state/memory.rs | 6 +- kernel/relayflowd/src/engine.rs | 23 +- kernel/relayflowd/src/engine/drive.rs | 10 + kernel/relayflowd/src/engine/remote.rs | 16 +- kernel/relayflowd/tests/budget_gate.rs | 172 +++++++++ packages/sdk/src/authored-budget.ts | 69 ++++ packages/sdk/src/authored-flow-error.ts | 2 + packages/sdk/src/authored-flow-executor.ts | 15 +- packages/sdk/src/authored-worker-step.ts | 13 +- packages/sdk/src/budget-preflight.ts | 31 ++ packages/sdk/src/budget.ts | 49 +++ packages/sdk/src/cli/direct-run.ts | 9 +- packages/sdk/src/compile.ts | 30 +- packages/sdk/src/failure-kinds.ts | 2 + packages/sdk/src/index.ts | 3 + packages/sdk/src/llm-worker.ts | 6 +- packages/sdk/src/model-pricing.ts | 15 + packages/sdk/src/preflight.ts | 13 +- packages/sdk/src/protocol.ts | 7 + packages/sdk/src/spec.ts | 12 +- packages/sdk/src/validate.ts | 9 +- packages/sdk/src/worker-cli.ts | 12 +- packages/sdk/src/worker-spend.ts | 15 + packages/sdk/src/worker-usage.ts | 48 +++ packages/sdk/src/worker.ts | 6 +- packages/sdk/tests/budget-attribution.test.ts | 53 +++ .../sdk/tests/budget-authored-live.test.ts | 36 ++ packages/sdk/tests/budget-preflight.test.ts | 53 +++ packages/sdk/tests/preflight.test.ts | 2 + packages/surface/src/flow.ts | 15 +- testdata/budget-guarded.flow.yaml | 12 + 51 files changed, 1951 insertions(+), 58 deletions(-) create mode 100644 docs/BUDGET.md create mode 100644 evidence/spec-G-budget/PR.md create mode 100644 evidence/spec-G-budget/README.md create mode 100644 evidence/spec-G-budget/cargo-test.log create mode 100644 evidence/spec-G-budget/claude-auth-status.log create mode 100644 evidence/spec-G-budget/sdk-focused.log create mode 100644 evidence/spec-G-budget/sdk-npm-test-earlier.log create mode 100644 evidence/spec-G-budget/sdk-npm-test.log create mode 100644 evidence/spec-G-budget/smoke-journal.json create mode 100644 evidence/spec-G-budget/smoke.log create mode 100644 evidence/spec-G-budget/smoke.spec.json create mode 100644 evidence/spec-G-budget/surface-test.log create mode 100644 kernel/relayflowd-core/src/machine/budget.rs create mode 100644 kernel/relayflowd/tests/budget_gate.rs create mode 100644 packages/sdk/src/authored-budget.ts create mode 100644 packages/sdk/src/budget-preflight.ts create mode 100644 packages/sdk/src/budget.ts create mode 100644 packages/sdk/src/model-pricing.ts create mode 100644 packages/sdk/src/worker-spend.ts create mode 100644 packages/sdk/src/worker-usage.ts create mode 100644 packages/sdk/tests/budget-attribution.test.ts create mode 100644 packages/sdk/tests/budget-authored-live.test.ts create mode 100644 packages/sdk/tests/budget-preflight.test.ts create mode 100644 testdata/budget-guarded.flow.yaml diff --git a/docs/BUDGET.md b/docs/BUDGET.md new file mode 100644 index 000000000..4f2dd9e4d --- /dev/null +++ b/docs/BUDGET.md @@ -0,0 +1,61 @@ +# Budget headers and spend + +`budget` is optional. A flow may declare `"$0.10/run"`, `"$20/day"`, or +`{ tokens: 10000, dollars: 0.10, wallclock: "2m" }`. Objects default to a run +window. Tokens count input plus output; wallclock sums attempt durations from +journaled start to completion. Duration units are `ms`, `s`, `m`, `h`, and `d`. +Limits are non-negative; dollars support up to six decimal places. + +A day is a UTC calendar day within a run. Completed spend resets for admission +at the next UTC day; unrelated runs do not share a global account. Header +syntax errors refuse as `budget_syntax_invalid`. Declared models without a +frozen price refuse as `budget_missing_price`; dollar budgets also require a +model on each worker step. Existing project model allowlist checks still apply. + +Every newly written `step.completed` includes: + +```json +"spend": { + "tokens_input": 1000, + "tokens_output": 200, + "dollars": 0.006, + "wallclock_ms": 25 +} +``` + +The frozen table in `packages/sdk/src/model-pricing.ts` quotes dollars per +million tokens. Workers compute integer microdollars: +`inputTokens * inputPrice + outputTokens * outputPrice`. The existing `budget` +field retains exact decimal dollars; only the journal's `spend.dollars` becomes +a JSON number. Memory-injected costs retain their existing single charge. +Deterministic steps have zero model tokens and dollars, with measured duration. + +The kernel checks accumulated spend before each new attempt. Equality is +permitted. Crossing a limit keeps that completion valid and refuses the next +start with `run.completed.completionReason: "budget_exceeded"`. Running peers +may finish; no new peers start. A final successful step may cross a limit and +still complete its run successfully. Replay reconstructs accounting from the +journal, including retries and prior epochs. + +The internal TypeScript executor currently lowers each authored step to its +own kernel run. For budgeted flows it serializes admission and carries exact +journaled costs into the next run's `budget.prior_spend`. Its generated terminal +marker does not consume the author's step budget. This does not add a durable +TypeScript root or change that runner's existing resume contract. + +Raw Claude/Codex adapters request structured output to extract usage. A custom +wrapper may return an explicit result envelope after its execution handshake: + +```json +{"protocol":"relayflows-agent-cli-v1-result","output":"answer","usage":{"input_tokens":1000,"output_tokens":200}} +``` + +A priced model with missing or malformed usage produces a journaled worker +error. Legacy `maxTokensIn` / `maxTokensOut` / `maxDollars` envelopes keep their +worker-supplied pricing contract, including existing synthetic test models. +New surface headers carry `pricing: "frozen"` through compiled artifacts so +checking a compiled flow preserves the same missing-price refusal. + +`testdata/budget-guarded.flow.yaml` is a local smoke using a zero-duration budget: +its first completed command exceeds the ceiling and its dependent command +never starts. diff --git a/docs/SURFACE.md b/docs/SURFACE.md index 52c77c7df..09812de16 100644 --- a/docs/SURFACE.md +++ b/docs/SURFACE.md @@ -87,7 +87,7 @@ No process runs between events: the handler wakes, executes to its next await, p [the generator notes](../packages/surface/src/helpers/README.md). 4. **`{{prev}}` / return-value chaining.** Output flows downward implicitly; naming steps is for reaching back, not bookkeeping. -5. **Headers are optional escalation.** identity, memory, budget, tools appear only when used. The empty header is the common case. +5. **Headers are optional escalation.** identity, memory, budget, tools appear only when used. The empty header is the common case. [Budget headers and spend](BUDGET.md) specifies parsing, prices, journal attribution, and admission limits. 6. **Agent definitions escalate by composition** — and a reusable agent *is* a flow: ```yaml - agent: Review this diff for security issues. # 1. anonymous diff --git a/evidence/spec-G-budget/PR.md b/evidence/spec-G-budget/PR.md new file mode 100644 index 000000000..77a429cd4 --- /dev/null +++ b/evidence/spec-G-budget/PR.md @@ -0,0 +1,36 @@ +Title: feat(surface,sdk,kernel): budget header + spend attribution (SURFACE §2 rule 5) + +The `budget:` header was declared in surface examples but never enforced. This +change accepts string and object headers, refuses invalid syntax and missing +model prices before execution, and records tokens, dollars, and elapsed attempt +time on every completion. Once completed spend crosses a limit, the kernel +preserves completed work and refuses the next start with `budget_exceeded`. + +Parent issue: the lead will insert the issue link when opening this PR. + +The SDK uses the frozen model-pricing table and integer microdollars; the kernel +keeps exact decimal accounting through replay. UTC day windows, total-token and +wallclock limits are supported. The internal authored executor carries journaled +spend between its existing per-step kernel runs. Legacy explicit envelopes keep +their worker-supplied price contract. See [the budget contract](docs/BUDGET.md). + +Fixture: [budget-guarded.flow.yaml](testdata/budget-guarded.flow.yaml). Its +`measured` command completes; `guarded` never starts. Actual journal excerpt: + +```json +{"entry_type":"step.completed","step_id":"measured","payload":{"completionReason":"success","spend":{"tokens_input":0,"tokens_output":0,"dollars":0,"wallclock_ms":19}}} +{"entry_type":"run.completed","payload":{"completionReason":"budget_exceeded"}} +``` + +Validation commands and complete captured output are in +[evidence/spec-G-budget](evidence/spec-G-budget/README.md). Kernel workspace, +surface tests, and focused SDK tests pass. Plain `npm test` remains blocked by +this host's missing Claude login for the existing live analyzer acceptance test; +that failure has not been skipped or weakened. A timing-sensitive existing +lease-wait test failed in one full run and passed in the subsequent focused run. +The two new exhaustive refusal scenarios were added with lead approval; the +existing assertion is unchanged. + +The lead owns PR creation, CI verification, and bot review because GitHub +credentials on the worker return HTTP 401. Do not mark ready for merge until +those checks and the live analyzer acceptance requirement are satisfied. diff --git a/evidence/spec-G-budget/README.md b/evidence/spec-G-budget/README.md new file mode 100644 index 000000000..3a9c4e939 --- /dev/null +++ b/evidence/spec-G-budget/README.md @@ -0,0 +1,97 @@ +# Spec G verification and handoff + +This branch implements budget headers, frozen SDK pricing, per-completion spend, +and kernel admission limits. The lead approved adding two scenarios to the +existing exhaustive refusal test without changing its assertion, and preserving +legacy explicit-budget synthetic model behavior (Relay message +`224082264437923840`). The same message directs the worker to push and hand off +PR creation because fleet GitHub credentials return HTTP 401. + +## Commands and captured output + +Only trailing whitespace was normalized in the captured text logs for git. + +From `kernel`: + +```sh +PATH=/Users/khaliqgant/.cargo/bin:$PATH cargo test --workspace +``` + +Exit 0. Full literal output: [cargo-test.log](cargo-test.log). New integration +coverage output: + +```text +running 3 tests +test daily_windows_reset_and_exact_limits_do_not_refuse ... ok +test crossing_completion_is_durable_and_next_step_is_refused ... ok +test deterministic_spend_and_wallclock_limit_gate_parallel_batch_starts ... ok +``` + +From `packages/sdk`: + +```sh +PATH=/Users/khaliqgant/.cargo/bin:$PATH npm test +``` + +The full suite is not green on this host. Full output is retained in +[sdk-npm-test.log](sdk-npm-test.log): + +```text + Test Files 1 failed | 53 passed | 1 skipped (55) + Tests 1 failed | 984 passed | 3 skipped (988) +``` + +The real analyzer requires a Claude login; +`claude auth status` returned exit 1; its full output is +[claude-auth-status.log](claude-auth-status.log). + +An [earlier full run](sdk-npm-test-earlier.log) also failed the existing lease-wait timing assertion. The +entire CLI test file subsequently passed in the focused rerun below. Neither +assertion was modified and the analyzer was not skipped with an environment +flag. + +```sh +PATH=/Users/khaliqgant/.cargo/bin:$PATH ./node_modules/.bin/vitest run tests/cli.test.ts tests/budget-preflight.test.ts tests/budget-attribution.test.ts tests/budget-authored-live.test.ts tests/preflight.test.ts +``` + +Final focused output: [sdk-focused.log](sdk-focused.log). + +```text + Test Files 5 passed (5) + Tests 109 passed (109) +``` + +From `packages/surface`, `npm test` could not launch because `bun` is absent. +Its build, test typecheck, and test commands were run directly: + +```sh +npm run build +./node_modules/.bin/tsc -p tsconfig.test.json +./node_modules/.bin/vitest run +``` + +All three exited 0; test output: [surface-test.log](surface-test.log). + +```text + Test Files 1 passed (1) + Tests 7 passed (7) +``` + +## Declarative smoke + +The fixture `testdata/budget-guarded.flow.yaml` was compiled with the SDK's +`compileYaml` and `toKernelSpec` into `/tmp/spec-g-smoke.json`, then run from the +repository root: + +```sh +kernel/target/debug/relayflowd --data-dir /tmp/spec-g-smoke-data run /tmp/spec-g-smoke.json +``` + +Exit 1 is the expected budget refusal. The emitted spec and captured outcome are +[smoke.spec.json](smoke.spec.json) and [smoke.log](smoke.log). The actual SQLite +journal excerpts are [smoke-journal.json](smoke-journal.json): `measured` completed +successfully with 19ms spend; `guarded` never started; the run completed with +`budget_exceeded`. + +PR creation, CI checks and review-bot triage are handed to the lead. This evidence +does not claim required CI passed or that the branch is ready for merge. diff --git a/evidence/spec-G-budget/cargo-test.log b/evidence/spec-G-budget/cargo-test.log new file mode 100644 index 000000000..59c8b6ba4 --- /dev/null +++ b/evidence/spec-G-budget/cargo-test.log @@ -0,0 +1,361 @@ + Compiling relayflowd-core v0.1.0 (/Users/khaliqgant/flows-spec-G-budget/kernel/relayflowd-core) + Compiling relayflowd-journal v0.1.0 (/Users/khaliqgant/flows-spec-G-budget/kernel/relayflowd-journal) + Compiling relayflowd v0.1.0 (/Users/khaliqgant/flows-spec-G-budget/kernel/relayflowd) + Finished `test` profile [unoptimized + debuginfo] target(s) in 5.96s + Running unittests src/lib.rs (target/debug/deps/relayflowd-3287862779f564ca) + +running 40 tests +test engine::remote::worker_failure_detail_tests::a_null_or_blank_output_yields_no_detail ... ok +test engine::remote::worker_failure_detail_tests::a_string_output_is_carried_verbatim_and_trimmed ... ok +test engine::remote::worker_failure_detail_tests::an_output_at_the_boundary_is_not_truncated ... ok +test engine::remote::worker_failure_detail_tests::a_non_string_output_is_rendered_rather_than_dropped ... ok +test engine::remote::worker_failure_detail_tests::truncation_does_not_split_a_multi_byte_char ... ok +test server::client::tests::resume_waits_while_the_heartbeat_renewed_lease_is_live ... ok +test engine::boot_identity_tests::every_engine_in_this_process_shares_one_boot_id ... ok +test server::liveness::tests::sweep_id_buckets_by_the_interval ... ok +test server::channels::tests::unknown_verb_never_falls_through_to_receive ... ok +test exec_det::tests::timeout_has_an_explicit_completion_reason ... ok +test exec_det::tests::captures_deterministic_output ... ok +test engine::wake::claim_guard_tests::a_disarmed_guard_leaves_the_claim_alone ... ok +test server::tests::agent::contract::an_agent_worker_attaching_without_pins_is_refused_at_attach ... ok +test server::liveness::tests::sweep_pass_latches_after_journaling_and_next_bucket_is_empty ... ok +test server::liveness::tests::sweep_pass_healthy_subscription_is_a_noop ... ok +test engine::wake::claim_guard_tests::an_armed_guard_releases_the_claim_when_dropped ... ok +test server::tests::agent::contract::an_oversized_trajectory_tail_is_refused_at_step_complete ... ok +test engine::wake::claim_guard_tests::a_panic_between_claim_and_register_still_releases ... ok +test engine::wake::claim_guard_tests::a_guard_only_releases_its_own_run ... ok +test server::tests::agent::contract::agent_without_a_compatible_worker_parks_without_starting ... ok +test server::tests::hello_enforces_protocol_version ... ok +test server::tests::agent::contract::an_agent_worker_missing_a_declared_surface_parks_the_run_instead_of_erroring ... ok +test server::tests::run_resume_asks_the_registry_instead_of_treating_an_orphan_file_as_a_run ... ok +test server::tests::a_failed_disconnect_journal_append_is_retained_and_retried_not_dropped ... ok +test server::tests::agent::contract::an_llm_completion_claiming_an_effect_fails_closed_with_the_reason_journaled ... ok +test server::tests::agent::pins::reset_worker_reporting_a_revision_other_than_its_pin_fails_closed_as_worker_error ... ok +test server::tests::run_start_fails_closed_on_an_unknown_verification_key ... ok +test socket_path::tests::deep_data_dir_produces_short_socket_path ... ok +test socket_path::tests::different_data_dirs_yield_different_sockets ... ok +test socket_path::tests::relative_and_absolute_data_dirs_agree ... ok +test socket_path::tests::same_data_dir_yields_same_socket ... ok +test server::tests::agent::contract::a_replacement_worker_that_never_reported_the_pinned_surface_is_not_dispatched_to ... ok +test server::tests::run_resume_refuses_a_journal_that_never_recorded_its_run ... ok +test server::tests::run_resume_adopts_a_real_journal_whose_registry_row_is_missing ... ok +test server::tests::run_resume_refuses_a_valid_journal_that_belongs_to_another_run ... ok +test server::tests::agent::pins::consecutive_agent_steps_on_different_surfaces_each_start_from_their_own_pins ... ok +test server::tests::stopped_heartbeats_past_the_deadline_journal_lease_expired_and_release_the_step ... ok +test exec_det::tests::timeout_kills_the_whole_process_group ... ok +test server::tests::agent::pins::a_replacement_worker_at_a_different_revision_is_not_dispatched_the_stale_pins ... ok +test server::tests::an_entry_appended_during_watch_registration_is_delivered_exactly_once ... ok + +test result: ok. 40 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.57s + + Running unittests src/main.rs (target/debug/deps/relayflowd-ae3ceb16e5d803ec) + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/budget_gate.rs (target/debug/deps/budget_gate-48be15f3361f11b0) + +running 3 tests +test daily_windows_reset_and_exact_limits_do_not_refuse ... ok +test crossing_completion_is_durable_and_next_step_is_refused ... ok +test deterministic_spend_and_wallclock_limit_gate_parallel_batch_starts ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.05s + + Running tests/crash_resume.rs (target/debug/deps/crash_resume-4619905d2726c36c) + +running 40 tests +test agent::resume_without_a_worker_parks_immediately_instead_of_timing_out ... ok +test concurrency::cancel_closes_the_lease_and_rejects_a_late_completion ... ok +test concurrency::cancel_and_completion_race_has_one_terminal_fact ... ok +test agent::rung_c_sigkill_after_final_effect_replays_results_without_redispatch ... ok +test channels::channels_reject_foreign_workers_stale_attempts_and_invalid_acknowledgements ... ok +test agent::rung_c_sigkill_between_agent_completion_and_final_effect_memoizes_the_agent ... ok +test agent::rung_c_crash_between_effect_election_and_the_provider_call_performs_it_exactly_once ... ok +test agent::rung_c_reset_sigkill_mid_edit_restores_pins_dedupes_effect_and_explains_attempts ... ok +test concurrency::concurrent_resumes_lease_exactly_one_attempt ... ok +test concurrency::run_start_dispatches_every_independent_lane_before_any_completion ... ok +test concurrency::live_resume_leaves_an_active_lease_running ... ok +test concurrency::server_restart_recovers_every_parallel_lease_without_duplicate_success ... ok +test agent::rung_c_sigkill_boundaries_resume_only_unfinished_steps_via_real_cli ... ok +test llm::serve_plumbs_watch_events_and_replayable_stream_verbs ... ok +test llm::failing_llm_verification_schedules_a_durable_retry_and_succeeds ... ok +test llm::llm_verification_exhaustion_is_a_declared_failure_kind ... ok +test llm::completed_llm_output_is_memoized_when_serve_dies_during_the_next_step ... ok +test llm::sigkill_after_the_final_rung_b_effect_resumes_without_redispatching_llm ... ok +test pin_projection::rejected_completion_cannot_forge_inspect_retry_pins_over_the_real_socket ... ok +test memory::memory_sigkill_after_injection_replays_pack_and_charges_it_once ... ok +test llm::worker_killed_while_holding_a_lease_is_explained_and_released_on_cli_resume ... ok +test llm::sigkill_under_serve_mid_llm_releases_the_lease_and_finishes_via_cli_resume ... ok +test protocol_admission::every_mutating_run_verb_refuses_terminal_before_changing_state ... ok +test parallel_lifecycle::terminal_failure_drains_or_explains_every_live_sibling ... ok +test sigkill_after_cancel_request_resumes_to_one_canceled_fact ... ok +test sigkill_mid_step_replaces_and_explains_the_dead_attempt ... ok +test llm::sigkill_sweep_covers_before_and_between_the_rung_b_steps ... ok +test parallel_lifecycle::overlapping_agent_conflict_survives_server_crash_and_resume ... ok +test sigkill_under_serve_resumes_the_socket_started_run ... ok +test surface_identity::aliases_are_rejected_and_external_ancestors_serialize_over_real_sockets ... ok +test parallel_lifecycle::overlapping_agent_lanes_serialize_while_disjoint_lanes_merge_in_either_order ... ok +test worker_capacity::default_capacity_one_reopens_only_after_durable_completion_or_crash ... ok +test worker_capacity::two_workers_receive_a_deterministic_fair_capacity_bounded_batch ... ok +test placement::declared_placement_keeps_one_source_tree_across_resume ... ok +test workspace_identity::workspace_aliases_are_refused_and_canonical_subtrees_serialize_over_real_sockets ... ok +test channels::channels_sigkill_resume_redelivers_unacked_messages_with_exactly_once_effects ... ok +test sigkill_sweep_covers_every_hello_step_boundary ... ok +test placement::sigkill_before_first_step_preserves_the_submitted_workspace ... ok +test placement::sigkill_mid_step_keeps_the_route_and_source_tree ... ok +test parallel_lifecycle::renewed_parallel_leases_survive_the_original_grant_and_remain_distinct ... ok + +test result: ok. 40 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 37.34s + + Running tests/daemon_lifecycle.rs (target/debug/deps/daemon_lifecycle-e483798b2ec9e14a) + +running 6 tests +test connection_file_is_published_only_after_the_socket_is_live ... ok +test clean_shutdown_removes_advertisement_and_socket ... ok +test deep_data_dir_still_binds ... ok +test sigkill_leaves_a_stale_file_with_a_dead_pid ... ok +test a_second_serve_on_a_served_data_dir_refuses_and_the_first_keeps_serving ... ok +test a_sigkilled_daemons_successor_starts_cleanly ... ok + +test result: ok. 6 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.05s + + Running tests/event_wake.rs (target/debug/deps/event_wake-bc602276d1cebcd0) + +running 3 tests +test two_racing_deliveries_of_one_event_produce_exactly_one_run ... ok +test matching_event_wakes_once_with_fresh_context ... ok +test a_resumed_run_dispatches_the_original_wake_context ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.03s + + Running tests/hn_monitor_integration.rs (target/debug/deps/hn_monitor_integration-6b22693ad979dd80) + +running 1 test +test hn_story_event_wakes_monitor_once_with_story_context ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.04s + + Running tests/input_binding.rs (target/debug/deps/input_binding-4608eb1f1e2c7549) + +running 2 tests +test binding_schema_is_additive_and_fails_closed ... ok +test sigkill_before_consumer_resolves_original_journal_output_without_reexecuting_source ... ok + +test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.09s + + Running tests/invalid_schema_preflight.rs (target/debug/deps/invalid_schema_preflight-2c352b9351d5acf8) + +running 3 tests +test invalid_json_schema_is_refused_before_journal_or_command ... ok +test unbounded_json_schema_is_refused_before_journal_or_command ... ok +test legitimately_recursive_json_schema_still_starts ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 3.65s + + Running tests/memory.rs (target/debug/deps/memory-a74848a8274a290e) + +running 5 tests +test rejected_journal_fact_releases_reservation_and_never_dispatches ... ok +test llm_dispatch_receives_same_pack_after_resume_without_provider ... ok +test replay_and_resume_need_no_provider_and_script_receives_recorded_pack ... ok +test semantic_retry_reuses_memory_without_a_second_charge ... ok +test over_budget_and_provider_errors_fail_without_dispatch_or_charge ... ok + +test result: ok. 5 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.04s + + Running tests/memory_epoch.rs (target/debug/deps/memory_epoch-edee30a47cad81b5) + +running 1 test +test epoch_carries_pack_and_exact_charge_and_refuses_duplicate_injection ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.03s + + Running tests/parallel_driver.rs (target/debug/deps/parallel_driver-74d9d10149c534bd) + +running 4 tests +test stop_after_one_holds_for_an_independent_deterministic_batch ... ok +test pause_before_second_independent_step_holds_the_driver_boundary ... ok +test backpressured_or_mismatched_lane_does_not_drop_a_later_dispatch ... ok +test crash_boundaries_resume_the_real_driver_with_one_effect_per_lane ... ok + +test result: ok. 4 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.07s + + Running tests/placement_pins.rs (target/debug/deps/placement_pins-893a8773fbe94aee) + +running 3 tests +test unsupported_local_pty_is_refused_before_an_earlier_step_can_run ... ok +test default_worker_pins_the_declared_worktree_base_commit_and_refuses_missing_source ... ok +test a_resumed_attempt_keeps_the_original_pin_after_the_worktree_head_moves ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.24s + + Running tests/placement_routing.rs (target/debug/deps/placement_routing-aabf5947f3ca76b4) + +running 3 tests +test a_failed_routing_append_never_starts_or_dispatches_work ... ok +test crash_between_routing_and_start_does_not_redecide ... ok +test worker_retry_consumes_the_original_routing_fact ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s + + Running tests/routing_diagnostics.rs (target/debug/deps/routing_diagnostics-ddddee3a427ee9ae) + +running 2 tests +test duplicate_routes_have_a_distinct_diagnostic_and_leave_the_original_fact_intact ... ok +test malformed_routes_name_the_same_field_at_append_replay_and_epoch_replay ... ok + +test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s + + Running tests/spec_review_routing.rs (target/debug/deps/spec_review_routing-0731f97e90d67407) + +running 4 tests +test attempt_scoped_route_is_rejected_at_append_and_replay ... ok +test malformed_epoch_routes_are_rejected_before_commit ... ok +test epoch_cannot_drop_or_replace_a_durable_route ... ok +test workspace_pin_peels_tags_and_refuses_non_commit_objects ... ok + +test result: ok. 4 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.21s + + Running tests/subscription_liveness.rs (target/debug/deps/subscription_liveness-9506d11f678a9586) + +running 3 tests +test submit_event_upserts_subscription_row_and_sweep_flags_it_stale_after_budget ... ok +test stale_transition_is_journaled_as_subscription_stale_entry_in_the_last_known_run ... ok +test a_fresh_arrival_re_arms_the_latch_and_the_next_silence_can_stale_again ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s + + Running unittests src/lib.rs (target/debug/deps/relayflowd_core-b1fe3b3250e9e7a2) + +running 60 tests +test clock::tests::simulated_clock_is_explicitly_advanced ... ok +test channel::tests::malformed_payloads_and_invalid_new_channel_appends_leave_state_unchanged ... ok +test channel::tests::forged_deliveries_and_acknowledgements_fail_closed ... ok +test journal::tests::memory_journal_assigns_sequences_and_rolls_epochs ... ok +test channel::tests::send_retry_is_stable_and_conflicting_content_is_rejected ... ok +test channel::tests::delivery_replay_and_independent_acknowledged_offsets ... ok +test machine::parallel_tests::machine_starts_every_runnable_step_in_authored_order ... ok +test machine::parallel_tests::every_declared_mutable_surface_participates_in_conflict_selection ... ok +test machine::parallel_tests::external_ancestor_and_descendant_paths_conflict_but_siblings_do_not ... ok +test machine::tests::all_backing_off_steps_return_timers ... ok +test machine::parallel_tests::workspace_ancestor_and_descendant_paths_conflict_but_siblings_do_not ... ok +test machine::parallel_tests::failed_run_drains_open_siblings_before_terminal_entry ... ok +test machine::parallel_tests::parallel_lanes_do_not_cross_the_dependency_barrier_early ... ok +test machine::tests::every_reason_label_matches_its_serialized_form ... ok +test machine::tests::durable_cancel_request_outranks_crash_recovery ... ok +test machine::tests::crashed_attempt_does_not_consume_an_iteration ... ok +test machine::tests::cancel_request_closes_the_active_lease_before_the_terminal_fact ... ok +test machine::parallel_tests::disjoint_agent_lanes_merge_pins_in_either_completion_order ... ok +test machine::parallel_tests::overlapping_agent_surfaces_are_serialized_in_authored_order ... ok +test machine::tests::machine_starts_runnable_step_with_stable_effect_key ... ok +test machine::tests::repeated_cancel_request_is_idempotent ... ok +test machine::tests::manual_recovery_parks_needs_human_and_never_redispatches ... ok +test machine::tests::verification_failure_schedules_a_durable_retry ... ok +test machine::tests::successful_memo_is_never_scheduled_again ... ok +test machine::tests::worker_reported_failure_without_detail_still_records_a_verification ... ok +test machine::parallel_tests::crash_resume_preserves_each_parallel_lease_exactly_once ... ok +test machine::tests::reset_recovery_dispatches_the_original_pinned_revision ... ok +test memory::tests::caps_compare_exact_decimals_and_each_token_dimension ... ok +test machine::tests::inspect_recovery_injects_the_dirty_pin_completion_reason_and_tail ... ok +test retry::tests::jitter_is_repeatable_and_bounded ... ok +test machine::tests::every_failed_run_terminates_with_declared_completion_reasons ... ok +test schema::tests::in_document_uri_references_resolve_to_the_node_they_name ... ok +test spec::tests::a_misspelled_step_level_key_is_a_parse_error ... ok +test schema::tests::refusal_names_the_cycle_it_found ... ok +test spec::tests::a_misspelled_verification_gate_key_is_a_parse_error_not_a_dropped_gate ... ok +test spec::tests::cycles_are_rejected ... ok +test spec::tests::preflight_data_is_fail_closed ... ok +test spec::tests::external_surface_paths_must_have_one_canonical_spelling ... ok +test spec::tests::spec_version_is_semver_and_gated ... ok +test spec::tests::unknown_root_and_nested_fields_are_rejected ... ok +test schema::tests::references_the_bound_leaves_opaque_are_refused_by_the_engine ... ok +test spec::tests::zero_agent_flow_is_valid ... ok +test state::budget::tests::adds_costs_exactly_beyond_machine_decimal_precision ... ok +test state::budget::tests::overflow_and_malformed_cost_leave_total_unchanged ... ok +test spec::tests::workspace_mounts_and_worktrees_must_have_one_canonical_spelling ... ok +test state::tests::budget_decimal_strings_add_without_floats ... ok +test state::tests::a_completion_that_omits_a_surface_does_not_drop_it_from_the_pin_chain ... ok +test state::tests::journal_replays_data_gate_verdict_without_rerunning_completed_code ... ok +test verify::tests::an_unbounded_schema_in_a_journal_fails_its_gate_instead_of_aborting ... ok +test state::tests::end_pin_chain_is_enforced_and_a_broken_chain_is_a_hard_error ... ok +test verify::tests::deterministic_output_requires_successful_exit_and_content ... ok +test spec::tests::the_full_ladder_parses_in_the_one_dialect ... ok +test verify::tests::json_schema_is_a_control_gate ... ok +test schema::tests::a_property_named_ref_is_not_a_reference ... ok +test schema::tests::shared_declarations_and_boolean_schemas_are_validated ... ok +test schema::tests::every_accepted_corpus_schema_is_accepted ... ok +test schema::tests::every_refused_corpus_schema_compiles_but_is_refused_by_the_bound ... ok +test spec::tests::sdk_boundary_rejects_a_10_000_step_cycle_with_a_typed_error ... ok +test spec::tests::sdk_boundary_accepts_a_valid_10_000_step_reverse_chain ... ok +test schema::tests::deeply_nested_schemas_do_not_overflow_the_checker ... ok + +test result: ok. 60 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.60s + + Running tests/spec_parity.rs (target/debug/deps/spec_parity-bbda6cf1e1cf1c19) + +running 9 tests +test placement_requirements_have_identical_canonical_bytes_and_hash ... ok +test the_kernel_parses_the_deterministic_rung_and_stamps_the_same_hash ... ok +test the_kernel_parses_the_event_triggered_spec_and_stamps_the_same_hash ... ok +test step_memory_has_identical_canonical_bytes_and_hash ... ok +test the_kernel_parses_the_rung_c_agent_spec_and_stamps_the_same_hash ... ok +test memory_declaration_acceptance_matches_the_sdk_corpus ... ok +test placement_declaration_acceptance_matches_the_sdk_corpus ... ok +test the_kernel_parses_the_sdk_compiled_spec_and_stamps_the_same_hash ... ok +test the_kernel_parses_the_rung_b_spec_and_stamps_the_same_hash ... ok + +test result: ok. 9 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s + + Running unittests src/lib.rs (target/debug/deps/relayflowd_journal-d13cb7954335385c) + +running 28 tests +test registry::tests::a_same_boot_claim_with_no_run_yet_is_a_duplicate_not_wreckage ... ok +test registry::tests::a_registered_run_dedupes_across_boots ... ok +test registry::tests::a_previous_boots_claim_with_no_run_is_repaired ... ok +test registry::tests::releasing_a_claim_lets_the_same_boot_retry ... ok +test registry::tests::registry_is_a_rebuildable_run_locator ... ok +test registry::tests::a_pre_migration_registry_gains_boot_id_and_its_claims_are_repairable ... ok +test channel::tests::stale_attempts_and_raw_forged_acknowledgements_cannot_change_offsets ... ok +test subscriptions::tests::detect_without_latch_stays_available_for_the_next_sweep ... ok +test registry::tests::releasing_is_scoped_to_the_claiming_run ... ok +test subscriptions::tests::last_run_for_subscription_returns_none_before_first_arrival ... ok +test subscriptions::tests::latch_is_a_no_op_if_a_fresh_event_arrived_between_detect_and_latch ... ok +test subscriptions::tests::last_run_for_subscription_returns_the_lex_greatest_ulid_regardless_of_insertion ... ok +test subscriptions::tests::prune_sweep_claims_deletes_only_rows_older_than_cutoff ... ok +test channel::tests::channels_cross_segment_boundaries_and_terminal_runs_reject_mutations ... ok +test subscriptions::tests::sweep_election_gives_the_first_caller_the_result_and_second_gets_empty ... ok +test subscriptions::tests::sweep_ignores_subscriptions_whose_silence_is_still_within_budget ... ok +test subscriptions::tests::sweep_does_not_re_emit_the_same_stale_row_on_a_later_tick ... ok +test subscriptions::tests::sweep_marks_row_stale_when_silence_exceeds_budget ... ok +test subscriptions::tests::upsert_after_stale_re_arms_and_next_silence_can_re_emit ... ok +test subscriptions::tests::upsert_is_idempotent_across_bumps_and_preserves_event_type_updates ... ok +test tests::an_unconfirmed_election_is_reclaimed_by_the_next_attempt_not_treated_as_done ... ok +test tests::append_is_durable_and_monotonic_after_reopen ... ok +test channel::tests::failed_channel_writes_never_expose_delivery_or_advance_acknowledged_offset ... ok +test tests::failed_commit_is_returned_not_swallowed ... ok +test tests::terminal_run_refuses_every_later_entry_atomically ... ok +test tests::rollover_is_atomic_scaffolding_for_epoch_resume ... ok +test tests::effects_are_deduplicated_at_the_journal_boundary ... ok +test channel::tests::independent_connections_serialize_send_receive_and_acknowledgement ... ok + +test result: ok. 28 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.13s + + Doc-tests relayflowd + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Doc-tests relayflowd_core + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Doc-tests relayflowd_journal + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s diff --git a/evidence/spec-G-budget/claude-auth-status.log b/evidence/spec-G-budget/claude-auth-status.log new file mode 100644 index 000000000..8439cac2a --- /dev/null +++ b/evidence/spec-G-budget/claude-auth-status.log @@ -0,0 +1,8 @@ +{ + "loggedIn": false, + "authMethod": "none", + "apiProvider": "firstParty", + "analyticsDisabled": false, + "projectsDirectory": "/Users/khaliqgant/.claude/projects", + "configDirectory": "/Users/khaliqgant/.claude" +} diff --git a/evidence/spec-G-budget/sdk-focused.log b/evidence/spec-G-budget/sdk-focused.log new file mode 100644 index 000000000..764729802 --- /dev/null +++ b/evidence/spec-G-budget/sdk-focused.log @@ -0,0 +1,23 @@ + + RUN v2.1.9 /Users/khaliqgant/flows-spec-G-budget/packages/sdk + + ✓ tests/budget-attribution.test.ts (4 tests) 3ms + ✓ tests/budget-preflight.test.ts (13 tests) 6ms + ✓ tests/preflight.test.ts (27 tests) 26ms + ✓ tests/budget-authored-live.test.ts (2 tests) 267ms + ✓ tests/cli.test.ts (63 tests) 9479ms + ✓ flows check CLI > binds a checked relative wrapper to the flow directory for worker execution 459ms + ✓ flows check CLI > refuses a typo model before probing or contacting relayflowd 593ms + ✓ flows check CLI > resolves a project CLI path relative to the flows.json that declares it 315ms + ✓ flows check CLI > maps every input refusal path to its declared kind without raw exceptions 799ms + ✓ flows run/resume CLI over the journal protocol > reports a needs_human agent step as parked for human recovery 605ms + ✓ flows run/resume CLI over the journal protocol > classifies a typed hello refusal as a protocol error, not an unreachable daemon 721ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 696ms + ✓ flows run/resume CLI over the journal protocol > bounds a worker wait by its lease and reports what it is waiting for 626ms + ✓ flows run/resume CLI over the journal protocol > resumes a parked run from snapshot step types without reading journal sequence one 532ms + ✓ flows run/resume CLI over the journal protocol > maps only run_not_found resumes to exit 2 1821ms + + Test Files 5 passed (5) + Tests 109 passed (109) + Start at 10:32:46 + Duration 10.07s (transform 473ms, setup 0ms, collect 1.31s, tests 9.78s, environment 0ms, prepare 230ms) diff --git a/evidence/spec-G-budget/sdk-npm-test-earlier.log b/evidence/spec-G-budget/sdk-npm-test-earlier.log new file mode 100644 index 000000000..29b5c6dab --- /dev/null +++ b/evidence/spec-G-budget/sdk-npm-test-earlier.log @@ -0,0 +1,214 @@ + +> @relayflows/sdk@2.0.8 test +> sh scripts/test.sh + + +> @relayflows/sdk@2.0.8 test:prep +> ( cd ../../kernel && sh ../ops/cargo.sh build ) && ( [ ! -d ../../testdata/preflight ] || find ../../testdata/preflight -name '*-cli' -type f -exec chmod +x {} + ) + + Compiling relayflowd-core v0.1.0 (/Users/khaliqgant/flows-spec-G-budget/kernel/relayflowd-core) + Compiling relayflowd-journal v0.1.0 (/Users/khaliqgant/flows-spec-G-budget/kernel/relayflowd-journal) + Compiling relayflowd v0.1.0 (/Users/khaliqgant/flows-spec-G-budget/kernel/relayflowd) + Finished `dev` profile [unoptimized + debuginfo] target(s) in 3.96s + +> @relayflows/sdk@2.0.8 typecheck +> tsc --noEmit && tsc -p tsconfig.type-tests.json + + +> @relayflows/sdk@2.0.8 build +> tsc && node scripts/make-cli-executable.mjs + + +> @relayflows/sdk@2.0.8 typecheck:tests +> tsc -p tsconfig.tests.json + + + RUN v2.1.9 /Users/khaliqgant/flows-spec-G-budget/packages/sdk + +stdout | tests/live-kernel.test.ts +LIVE_KERNEL relayflowd=/Users/khaliqgant/.relayflows-toolchain/target/3804940860/debug/relayflowd +LIVE_KERNEL flows=/Users/khaliqgant/flows-spec-G-budget/packages/sdk/dist/cli.js + + ✓ tests/journal-client.test.ts (14 tests) 70ms + ✓ tests/daemon-lifecycle.test.ts (42 tests) 29ms + ✓ tests/validate.test.ts (68 tests) 23ms + ✓ tests/preflight.test.ts (27 tests) 52ms + ✓ tests/observer-link.test.ts (39 tests) 326ms + ✓ tests/tick-source.test.ts (33 tests) 12ms + ✓ tests/cloud-run.test.ts (47 tests) 119ms + ✓ tests/authored-flow.test.ts (24 tests) 656ms + ✓ tests/verb-field-lint.test.ts (78 tests) 262ms + ✓ tests/gate-contract.test.ts (20 tests) 79ms + ✓ tests/cli-hn-monitor.test.ts (16 tests) 104ms + ✓ tests/authored-flow-lifecycle-executor.test.ts (27 tests) 527ms + ✓ tests/backlog-picker.test.ts (14 tests) 70ms + ✓ tests/tick-runner.test.ts (22 tests) 828ms + ✓ tests/authored-flow-operation.test.ts (23 tests) 324ms + ✓ tests/backlog-picker-flow.test.ts (6 tests) 549ms + ✓ tests/work-package-consumer.test.ts (13 tests) 207ms + ✓ tests/spec-parity.test.ts (31 tests) 187ms + ✓ tests/worker-lease.test.ts (7 tests) 8ms + ✓ tests/typed-output.test.ts (14 tests) 158ms + ✓ tests/model-selection.test.ts (10 tests) 12ms + ✓ tests/relayflowd-path.test.ts (10 tests) 2ms + ✓ tests/json-schema-bound.test.ts (71 tests) 1448ms + ✓ JSON Schema termination bound > walks a deep schema with an explicit stack rather than recursion 1175ms + ✓ tests/yaml-local-agent-live.test.ts (7 tests) 2683ms + ✓ YAML --local-agent through the built CLI and real daemon > runs with the checked step CLI and model and journals done 422ms + ✓ YAML --local-agent through the built CLI and real daemon > runs with the checked named CLI and model and journals done 387ms + ✓ YAML --local-agent through the built CLI and real daemon > runs with the checked flow CLI and model and journals done 448ms + ✓ YAML --local-agent through the built CLI and real daemon > runs with the checked project CLI and model and journals done 429ms + ✓ YAML --local-agent through the built CLI and real daemon > reports the agent process failure 405ms + ✓ tests/local-dev-ux.test.ts (8 tests) 12ms + ✓ tests/dependency-validation.test.ts (6 tests) 273ms + ✓ tests/input-binding.test.ts (12 tests) 121ms + ✓ tests/deterministic-llm.test.ts (5 tests) 38ms + ✓ tests/bin.test.ts (7 tests) 603ms + ✓ tests/hn-poller.test.ts (6 tests) 3ms + ✓ tests/stop-process-group.test.ts (6 tests) 8298ms + ✓ every stop reaches the process group, not just the direct child > exits the run after an execution-timeout stop 976ms + ✓ every stop reaches the process group, not just the direct child > exits the run after a protocol terminate stop 1107ms + ✓ every stop reaches the process group, not just the direct child > kills a SIGTERM-deaf grandchild after a protocol terminate stop 2288ms + ✓ every stop reaches the process group, not just the direct child > kills a SIGTERM-deaf grandchild after an execution-timeout stop 2444ms + ✓ every stop reaches the process group, not just the direct child > holds the loop open long enough for the escalation to run 1172ms + ✓ every stop reaches the process group, not just the direct child > terminate() forces a group that outlives SIGTERM 310ms + ✓ tests/budget-attribution.test.ts (4 tests) 2ms + ✓ tests/direct-input.test.ts (4 tests) 7469ms + ✓ direct .flow.ts input through the built CLI and live runtime > executes inline and file JSON input through relayflowd 3207ms + ✓ direct .flow.ts input through the built CLI and live runtime > refuses missing and malformed input before contacting relayflowd 2587ms + ✓ direct .flow.ts input through the built CLI and live runtime > does not import or execute authored code before daemon availability 874ms + ✓ direct .flow.ts input through the built CLI and live runtime > refuses oversized file input before contacting relayflowd 801ms + ✓ tests/dir-watcher-poller.test.ts (6 tests) 2ms + ✓ tests/work-package-validator.test.ts (7 tests) 3ms + ✓ tests/hello-deterministic.test.ts (5 tests) 12ms + ✓ tests/budget-preflight.test.ts (13 tests) 5ms + ↓ tests/real-cli-adapters.test.ts (3 tests | 3 skipped) + ✓ tests/parse-json-output.test.ts (7 tests) 1ms + ✓ tests/cli-adapter.test.ts (3 tests) 2ms + ✓ tests/journal-client-completion.test.ts (4 tests) 99ms + ✓ tests/direct-run-failure.test.ts (6 tests) 2ms + ✓ tests/budget-authored-live.test.ts (2 tests) 189ms + ✓ tests/daemon-lifecycle-live.test.ts (9 tests) 9560ms + ✓ flows run against a data dir with no daemon (§6 test 7) > cold start spawns exactly one daemon, the run succeeds, and the daemon outlives the CLI 909ms + ✓ flows run against a data dir with no daemon (§6 test 7) > polls, bounded, for a daemon that holds the lock before it binds 1866ms + ✓ flows run against a data dir with no daemon (§6 test 7) > attaches to a serving daemon that has not published a connection file 1079ms + ✓ flows run against a data dir with no daemon (§6 test 7) > a second run attaches to the daemon the first one started, spawning nothing 1091ms + ✓ flows run against a data dir with no daemon (§6 test 7) > detects a stale connection file left by a hard kill and starts a fresh daemon 1748ms + ✓ concurrent invocations against one empty data dir (§6 test 15) > ends with exactly one daemon owning the socket, and both runs succeed 995ms + ✓ refusals from a spawn that cannot produce a daemon > names relayflowd_not_found rather than falling through to PATH 716ms + ✓ refusals from a spawn that cannot produce a daemon > names daemon_start_failed and quotes the daemon log when startup dies 720ms + ✓ refusals from a spawn that cannot produce a daemon > refuses a daemon speaking another protocol version instead of binding over it 436ms + ✓ tests/placement.test.ts (54 tests) 7ms + ✓ tests/memory.test.ts (18 tests) 4ms + ✓ tests/worker-platform.test.ts (1 test) 2ms + ✓ tests/classify-outcome.test.ts (2 tests) 2224ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2068ms + ✓ tests/flow-executor-chain.test.ts (12 tests) 10484ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 1272ms + ✓ flow executor LLM and output-binding chain > fails invalid LLM output before the next step: not JSON 1156ms + ✓ flow executor LLM and output-binding chain > fails invalid LLM output before the next step: {"message":7} 1079ms + ✓ flow executor LLM and output-binding chain > retains tagged-template text output 756ms + ✓ flow executor LLM and output-binding chain > preserves JSON values without promoting them to process wrappers: null 943ms + ✓ flow executor LLM and output-binding chain > preserves JSON values without promoting them to process wrappers: [1,2] 571ms + ✓ flow executor LLM and output-binding chain > preserves JSON values without promoting them to process wrappers: "hello" 622ms + ✓ flow executor LLM and output-binding chain > runs the authored LLM path through the built flows CLI 973ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 654ms + ✓ flow executor LLM and output-binding chain > journals a missing optional field as a failure before the consuming command executes 606ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 1772ms + ✓ tests/cli-progress-wait.test.ts (2 tests) 1662ms + ✓ run starts the wait clock on its first observed lease 657ms + ✓ resume starts the wait clock on its first observed lease 1004ms + ✓ tests/worker-cli-abort.test.ts (2 tests) 2649ms + ✓ stops claude and its process group when lease ownership is lost 1393ms + ✓ stops wrapper.mjs and its process group when lease ownership is lost 1255ms + ❯ tests/cli.test.ts (63 tests | 1 failed) 13468ms + ✓ flows check CLI > binds a checked relative wrapper to the flow directory for worker execution 937ms + ✓ flows check CLI > uses the raw Claude adapter model flag instead of accepting auth status as model proof 512ms + ✓ flows check CLI > uses Codex login status and reports a rejected model as unavailable, not unauthenticated 751ms + ✓ flows check CLI > refuses a nonconforming custom wrapper without calling it an authentication failure 823ms + ✓ flows check CLI > accepts an exact allowlisted named-agent model and probes that model 729ms + ✓ flows check CLI > checks the same named-agent contract from declarative JSON 829ms + ✓ flows check CLI > refuses a typo model before probing or contacting relayflowd 448ms + ✓ flows check CLI > distinguishes an allowlisted but inaccessible model from broken auth 493ms + ✓ flows check CLI > resolves a project CLI path relative to the flows.json that declares it 464ms + ✓ flows check CLI > maps every input refusal path to its declared kind without raw exceptions 983ms + ✓ flows run/resume CLI over the journal protocol > parses run options, submits the kernel dialect, and exits 0 on success 611ms + ✓ flows run/resume CLI over the journal protocol > exits 3 and names the parked llm step 599ms + ✓ flows run/resume CLI over the journal protocol > reports a needs_human agent step as parked for human recovery 592ms + ✓ flows run/resume CLI over the journal protocol > classifies a typed hello refusal as a protocol error, not an unreachable daemon 610ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 672ms + × flows run/resume CLI over the journal protocol > bounds a worker wait by its lease and reports what it is waiting for 641ms + → expected +0 to be 1 // Object.is equality + ✓ flows run/resume CLI over the journal protocol > resumes a parked run from snapshot step types without reading journal sequence one 384ms + ✓ flows run/resume CLI over the journal protocol > maps only run_not_found resumes to exit 2 1764ms + ✓ tests/worker-cli.test.ts (13 tests) 23091ms + ✓ custom wrapper execution identity > refuses a wrapper symlink retarget before delivering private values 897ms + ✓ custom wrapper execution identity > bounds wrapper execution after acknowledgement 710ms + ✓ custom wrapper execution identity > bounds captured wrapper output 762ms + ✓ custom wrapper execution identity > refuses a duplicate execute protocol frame 740ms + ✓ custom wrapper execution bounds are reader-owned > resolves when a conforming wrapper leaks a stdio pipe to a background helper 2348ms + ✓ custom wrapper execution bounds are reader-owned > resolves when the leaked helper inherits stderr only 1926ms + ✓ custom wrapper execution bounds are reader-owned > resolves when a wrapper leaks a stdio pipe and exits before identifying 3255ms + ✓ custom wrapper execution bounds are reader-owned > journals a completionReason at the default bound when a wrapper leaks a stdio pipe 11258ms + ✓ custom wrapper execution bounds are reader-owned > accepts the same over-8KiB payload whether or not it coalesces with the execute token 457ms + ✓ tests/local-agent-live.test.ts (5 tests) 39743ms + ✓ built CLI local agent against a real daemon > dispatches through the wrapper and keeps --json stdout report-shaped 1367ms + ✓ built CLI local agent against a real daemon > runs beyond the initial 30-second lease without a second invocation 35809ms + ✓ built CLI local agent against a real daemon > renders actual agent completion in text output 870ms + ✓ built CLI local agent against a real daemon > returns a failed run when the agent process fails 778ms + ✓ built CLI local agent against a real daemon > refuses a workspace it cannot pin before invoking the agent 918ms +stdout | tests/live-kernel.test.ts > surface resume after a real daemon kill > resumes a three-step run with each successful completion exactly once +LIVE_KERNEL kill -9 pid=90664 run=01M27SEJVGNF1Y8C2SP68CEDSX while step=two state=Running + + ❯ tests/live-kernel.test.ts (30 tests | 1 failed) 53621ms + ✓ built flows CLI against live relayflowd > runs rung (a), parks rung (b), and keeps JSON report-shaped 5226ms + ✓ built flows CLI against live relayflowd > allows a deterministic run to exceed the bounded request timeout 32243ms + ✓ built flows CLI against live relayflowd > follows a live worker dispatch through flows run 1096ms + ✓ built flows CLI against live relayflowd > f.agent lowers to a real agent step and dispatches through a live worker 380ms + ✓ built flows CLI against live relayflowd > f.agent's default flowPath anchors on cwd, not cwd's parent 430ms + ✓ built flows CLI against live relayflowd > can always get a parked run to a late-attaching worker 5570ms + ✓ built flows CLI against live relayflowd > reports a real manual-recovery NeedsHuman state as parked 820ms + ✓ built flows CLI against live relayflowd > AgentWorker passes a declared model to an identified wrapper as RELAYFLOW_MODEL 337ms + × built flows CLI against live relayflowd > hn-monitor analyze-story reaches done through the real Claude analyzer CLI 1404ms + → LIVE_ANALYZER_UNAVAILABLE: "/Users/khaliqgant/flows-spec-G-budget/testdata/preflight/analyze-story-claude-cli auth status" exited 1: analyze-story-claude-cli: "claude -p --model claude-haiku-4-5-20251001" exited 1: — failing because gate-2 acceptance requires the real analyzer to execute. Set RELAYFLOWS_ALLOW_ANALYZER_SKIP=1 only if this run is not gate evidence. + ✓ built flows CLI against live relayflowd > preflights before journaling and names an unreachable socket 1505ms + ✓ built flows CLI against live relayflowd > starts exactly one daemon when two runs race for one empty data dir 967ms + ✓ surface resume after a real daemon kill > resumes a three-step run with each successful completion exactly once 1528ms + +⎯⎯⎯⎯⎯⎯⎯ Failed Tests 2 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/cli.test.ts > flows run/resume CLI over the journal protocol > bounds a worker wait by its lease and reports what it is waiting for +AssertionError: expected +0 to be 1 // Object.is equality + +- Expected ++ Received + +- 1 ++ 0 + + ❯ tests/cli.test.ts:887:18 + 885| ], output.io); + 886| + 887| expect(code).toBe(1); + | ^ + 888| expect(output.stderr.join('\n')).toContain('WAITING [worker_lease]… + 889| expect(output.stderr.join('\n')).toContain(`until ${leaseDeadlineM… + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/2]⎯ + + FAIL tests/live-kernel.test.ts > built flows CLI against live relayflowd > hn-monitor analyze-story reaches done through the real Claude analyzer CLI +Error: LIVE_ANALYZER_UNAVAILABLE: "/Users/khaliqgant/flows-spec-G-budget/testdata/preflight/analyze-story-claude-cli auth status" exited 1: analyze-story-claude-cli: "claude -p --model claude-haiku-4-5-20251001" exited 1: — failing because gate-2 acceptance requires the real analyzer to execute. Set RELAYFLOWS_ALLOW_ANALYZER_SKIP=1 only if this run is not gate evidence. + ❯ tests/live-kernel.test.ts:1186:15 + 1184| const notice = `LIVE_ANALYZER_UNAVAILABLE: ${readiness.detail}`; + 1185| if (process.env['RELAYFLOWS_ALLOW_ANALYZER_SKIP'] !== '1') { + 1186| throw new Error( + | ^ + 1187| `${notice} — failing because gate-2 acceptance requires the … + 1188| + 'Set RELAYFLOWS_ALLOW_ANALYZER_SKIP=1 only if this run is … + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[2/2]⎯ + + Test Files 2 failed | 52 passed | 1 skipped (55) + Tests 2 failed | 983 passed | 3 skipped (988) + Start at 10:29:59 + Duration 54.17s (transform 962ms, setup 0ms, collect 4.39s, tests 182.39s, environment 6ms, prepare 1.78s) diff --git a/evidence/spec-G-budget/sdk-npm-test.log b/evidence/spec-G-budget/sdk-npm-test.log new file mode 100644 index 000000000..d299b7abf --- /dev/null +++ b/evidence/spec-G-budget/sdk-npm-test.log @@ -0,0 +1,193 @@ + +> @relayflows/sdk@2.0.8 test +> sh scripts/test.sh + + +> @relayflows/sdk@2.0.8 test:prep +> ( cd ../../kernel && sh ../ops/cargo.sh build ) && ( [ ! -d ../../testdata/preflight ] || find ../../testdata/preflight -name '*-cli' -type f -exec chmod +x {} + ) + + Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.28s + +> @relayflows/sdk@2.0.8 typecheck +> tsc --noEmit && tsc -p tsconfig.type-tests.json + + +> @relayflows/sdk@2.0.8 build +> tsc && node scripts/make-cli-executable.mjs + + +> @relayflows/sdk@2.0.8 typecheck:tests +> tsc -p tsconfig.tests.json + + + RUN v2.1.9 /Users/khaliqgant/flows-spec-G-budget/packages/sdk + + ✓ tests/journal-client.test.ts (14 tests) 68ms +stdout | tests/live-kernel.test.ts +LIVE_KERNEL relayflowd=/Users/khaliqgant/.relayflows-toolchain/target/3804940860/debug/relayflowd +LIVE_KERNEL flows=/Users/khaliqgant/flows-spec-G-budget/packages/sdk/dist/cli.js + + ✓ tests/validate.test.ts (68 tests) 23ms + ✓ tests/daemon-lifecycle.test.ts (42 tests) 68ms + ✓ tests/preflight.test.ts (27 tests) 48ms + ✓ tests/observer-link.test.ts (39 tests) 259ms + ✓ tests/tick-source.test.ts (33 tests) 64ms + ✓ tests/cloud-run.test.ts (47 tests) 122ms + ✓ tests/authored-flow.test.ts (24 tests) 659ms + ✓ tests/verb-field-lint.test.ts (78 tests) 291ms + ✓ tests/authored-flow-lifecycle-executor.test.ts (27 tests) 506ms + ✓ tests/cli-hn-monitor.test.ts (16 tests) 97ms + ✓ tests/gate-contract.test.ts (20 tests) 163ms + ✓ tests/backlog-picker.test.ts (14 tests) 101ms + ✓ tests/tick-runner.test.ts (22 tests) 943ms + ✓ tests/authored-flow-operation.test.ts (23 tests) 348ms + ✓ tests/backlog-picker-flow.test.ts (6 tests) 768ms + ✓ backlog-picker flow > does not emit a stale entry left by a previous run 373ms + ✓ tests/work-package-consumer.test.ts (13 tests) 267ms + ✓ tests/spec-parity.test.ts (31 tests) 304ms + ✓ tests/worker-lease.test.ts (7 tests) 14ms + ✓ tests/typed-output.test.ts (14 tests) 155ms + ✓ tests/model-selection.test.ts (10 tests) 12ms + ✓ tests/relayflowd-path.test.ts (10 tests) 5ms + ✓ tests/json-schema-bound.test.ts (71 tests) 1983ms + ✓ JSON Schema termination bound > walks a deep schema with an explicit stack rather than recursion 1653ms + ✓ tests/yaml-local-agent-live.test.ts (7 tests) 2794ms + ✓ YAML --local-agent through the built CLI and real daemon > runs with the checked step CLI and model and journals done 533ms + ✓ YAML --local-agent through the built CLI and real daemon > runs with the checked named CLI and model and journals done 476ms + ✓ YAML --local-agent through the built CLI and real daemon > runs with the checked flow CLI and model and journals done 421ms + ✓ YAML --local-agent through the built CLI and real daemon > runs with the checked project CLI and model and journals done 392ms + ✓ YAML --local-agent through the built CLI and real daemon > reports the agent process failure 406ms + ✓ tests/local-dev-ux.test.ts (8 tests) 9ms + ✓ tests/direct-input.test.ts (4 tests) 6554ms + ✓ direct .flow.ts input through the built CLI and live runtime > executes inline and file JSON input through relayflowd 2508ms + ✓ direct .flow.ts input through the built CLI and live runtime > refuses missing and malformed input before contacting relayflowd 2330ms + ✓ direct .flow.ts input through the built CLI and live runtime > does not import or execute authored code before daemon availability 694ms + ✓ direct .flow.ts input through the built CLI and live runtime > refuses oversized file input before contacting relayflowd 1021ms + ✓ tests/input-binding.test.ts (12 tests) 105ms + ✓ tests/dependency-validation.test.ts (6 tests) 297ms + ✓ tests/stop-process-group.test.ts (6 tests) 7894ms + ✓ every stop reaches the process group, not just the direct child > exits the run after an execution-timeout stop 992ms + ✓ every stop reaches the process group, not just the direct child > exits the run after a protocol terminate stop 1143ms + ✓ every stop reaches the process group, not just the direct child > kills a SIGTERM-deaf grandchild after a protocol terminate stop 2228ms + ✓ every stop reaches the process group, not just the direct child > kills a SIGTERM-deaf grandchild after an execution-timeout stop 2055ms + ✓ every stop reaches the process group, not just the direct child > holds the loop open long enough for the escalation to run 1160ms + ✓ every stop reaches the process group, not just the direct child > terminate() forces a group that outlives SIGTERM 315ms + ✓ tests/deterministic-llm.test.ts (5 tests) 36ms + ✓ tests/budget-attribution.test.ts (4 tests) 2ms + ✓ tests/hn-poller.test.ts (6 tests) 3ms + ✓ tests/dir-watcher-poller.test.ts (6 tests) 2ms + ✓ tests/hello-deterministic.test.ts (5 tests) 10ms + ✓ tests/work-package-validator.test.ts (7 tests) 4ms + ✓ tests/budget-preflight.test.ts (13 tests) 8ms + ✓ tests/bin.test.ts (7 tests) 644ms + ↓ tests/real-cli-adapters.test.ts (3 tests | 3 skipped) + ✓ tests/parse-json-output.test.ts (7 tests) 1ms + ✓ tests/cli-adapter.test.ts (3 tests) 2ms + ✓ tests/journal-client-completion.test.ts (4 tests) 98ms + ✓ tests/direct-run-failure.test.ts (6 tests) 2ms + ✓ tests/budget-authored-live.test.ts (2 tests) 187ms + ✓ tests/flow-executor-chain.test.ts (12 tests) 10086ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 1501ms + ✓ flow executor LLM and output-binding chain > fails invalid LLM output before the next step: not JSON 862ms + ✓ flow executor LLM and output-binding chain > fails invalid LLM output before the next step: {"message":7} 1030ms + ✓ flow executor LLM and output-binding chain > retains tagged-template text output 697ms + ✓ flow executor LLM and output-binding chain > preserves JSON values without promoting them to process wrappers: null 882ms + ✓ flow executor LLM and output-binding chain > preserves JSON values without promoting them to process wrappers: [1,2] 721ms + ✓ flow executor LLM and output-binding chain > preserves JSON values without promoting them to process wrappers: "hello" 560ms + ✓ flow executor LLM and output-binding chain > runs the authored LLM path through the built flows CLI 1219ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 659ms + ✓ flow executor LLM and output-binding chain > journals a missing optional field as a failure before the consuming command executes 561ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 1335ms + ✓ tests/classify-outcome.test.ts (2 tests) 2197ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2041ms + ✓ tests/placement.test.ts (54 tests) 7ms + ✓ tests/daemon-lifecycle-live.test.ts (9 tests) 10644ms + ✓ flows run against a data dir with no daemon (§6 test 7) > cold start spawns exactly one daemon, the run succeeds, and the daemon outlives the CLI 1003ms + ✓ flows run against a data dir with no daemon (§6 test 7) > polls, bounded, for a daemon that holds the lock before it binds 1855ms + ✓ flows run against a data dir with no daemon (§6 test 7) > attaches to a serving daemon that has not published a connection file 668ms + ✓ flows run against a data dir with no daemon (§6 test 7) > a second run attaches to the daemon the first one started, spawning nothing 1637ms + ✓ flows run against a data dir with no daemon (§6 test 7) > detects a stale connection file left by a hard kill and starts a fresh daemon 2075ms + ✓ concurrent invocations against one empty data dir (§6 test 15) > ends with exactly one daemon owning the socket, and both runs succeed 979ms + ✓ refusals from a spawn that cannot produce a daemon > names relayflowd_not_found rather than falling through to PATH 796ms + ✓ refusals from a spawn that cannot produce a daemon > names daemon_start_failed and quotes the daemon log when startup dies 853ms + ✓ refusals from a spawn that cannot produce a daemon > refuses a daemon speaking another protocol version instead of binding over it 778ms + ✓ tests/memory.test.ts (18 tests) 5ms + ✓ tests/cli-progress-wait.test.ts (2 tests) 1189ms + ✓ run starts the wait clock on its first observed lease 641ms + ✓ resume starts the wait clock on its first observed lease 547ms + ✓ tests/worker-platform.test.ts (1 test) 2ms + ✓ tests/worker-cli-abort.test.ts (2 tests) 2589ms + ✓ stops claude and its process group when lease ownership is lost 1266ms + ✓ stops wrapper.mjs and its process group when lease ownership is lost 1323ms + ✓ tests/cli.test.ts (63 tests) 14605ms + ✓ flows check CLI > binds a checked relative wrapper to the flow directory for worker execution 944ms + ✓ flows check CLI > uses the raw Claude adapter model flag instead of accepting auth status as model proof 482ms + ✓ flows check CLI > uses Codex login status and reports a rejected model as unavailable, not unauthenticated 936ms + ✓ flows check CLI > refuses a nonconforming custom wrapper without calling it an authentication failure 530ms + ✓ flows check CLI > accepts an exact allowlisted named-agent model and probes that model 519ms + ✓ flows check CLI > checks the same named-agent contract from declarative JSON 516ms + ✓ flows check CLI > refuses a typo model before probing or contacting relayflowd 621ms + ✓ flows check CLI > resolves a project CLI path relative to the flows.json that declares it 617ms + ✓ flows check CLI > maps every input refusal path to its declared kind without raw exceptions 1876ms + ✓ flows run/resume CLI over the journal protocol > parses run options, submits the kernel dialect, and exits 0 on success 719ms + ✓ flows run/resume CLI over the journal protocol > exits 1 and emits the declared completionReason for a failed run 539ms + ✓ flows run/resume CLI over the journal protocol > exits 3 and names the parked llm step 622ms + ✓ flows run/resume CLI over the journal protocol > reports a needs_human agent step as parked for human recovery 552ms + ✓ flows run/resume CLI over the journal protocol > classifies a typed hello refusal as a protocol error, not an unreachable daemon 304ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 571ms + ✓ flows run/resume CLI over the journal protocol > bounds a worker wait by its lease and reports what it is waiting for 597ms + ✓ flows run/resume CLI over the journal protocol > resumes a parked run from snapshot step types without reading journal sequence one 720ms + ✓ flows run/resume CLI over the journal protocol > maps only run_not_found resumes to exit 2 1902ms + ✓ tests/worker-cli.test.ts (13 tests) 23025ms + ✓ custom wrapper execution identity > passes an explicit safe environment at identification and execution 341ms + ✓ custom wrapper execution identity > refuses a wrapper symlink retarget before delivering private values 823ms + ✓ custom wrapper execution identity > bounds wrapper execution after acknowledgement 720ms + ✓ custom wrapper execution identity > bounds captured wrapper output 848ms + ✓ custom wrapper execution identity > refuses a duplicate execute protocol frame 529ms + ✓ custom wrapper execution bounds are reader-owned > resolves when a conforming wrapper leaks a stdio pipe to a background helper 2361ms + ✓ custom wrapper execution bounds are reader-owned > resolves when the leaked helper inherits stderr only 1950ms + ✓ custom wrapper execution bounds are reader-owned > resolves when a wrapper leaks a stdio pipe and exits before identifying 3254ms + ✓ custom wrapper execution bounds are reader-owned > journals a completionReason at the default bound when a wrapper leaks a stdio pipe 11255ms + ✓ custom wrapper execution bounds are reader-owned > accepts the same over-8KiB payload whether or not it coalesces with the execute token 430ms + ✓ tests/local-agent-live.test.ts (5 tests) 39723ms + ✓ built CLI local agent against a real daemon > dispatches through the wrapper and keeps --json stdout report-shaped 1209ms + ✓ built CLI local agent against a real daemon > runs beyond the initial 30-second lease without a second invocation 36039ms + ✓ built CLI local agent against a real daemon > renders actual agent completion in text output 881ms + ✓ built CLI local agent against a real daemon > returns a failed run when the agent process fails 793ms + ✓ built CLI local agent against a real daemon > refuses a workspace it cannot pin before invoking the agent 801ms +stdout | tests/live-kernel.test.ts > surface resume after a real daemon kill > resumes a three-step run with each successful completion exactly once +LIVE_KERNEL kill -9 pid=99452 run=01M27SNDND6GMEMVJVKTEMTSF0 while step=two state=Running + + ❯ tests/live-kernel.test.ts (30 tests | 1 failed) 53714ms + ✓ built flows CLI against live relayflowd > runs rung (a), parks rung (b), and keeps JSON report-shaped 5232ms + ✓ built flows CLI against live relayflowd > allows a deterministic run to exceed the bounded request timeout 32295ms + ✓ built flows CLI against live relayflowd > follows a live worker dispatch through flows run 862ms + ✓ built flows CLI against live relayflowd > f.agent lowers to a real agent step and dispatches through a live worker 304ms + ✓ built flows CLI against live relayflowd > f.agent's default flowPath anchors on cwd, not cwd's parent 372ms + ✓ built flows CLI against live relayflowd > can always get a parked run to a late-attaching worker 5564ms + ✓ built flows CLI against live relayflowd > reports a real manual-recovery NeedsHuman state as parked 823ms + ✓ built flows CLI against live relayflowd > AgentWorker passes a declared model to an identified wrapper as RELAYFLOW_MODEL 407ms + × built flows CLI against live relayflowd > hn-monitor analyze-story reaches done through the real Claude analyzer CLI 1534ms + → LIVE_ANALYZER_UNAVAILABLE: "/Users/khaliqgant/flows-spec-G-budget/testdata/preflight/analyze-story-claude-cli auth status" exited 1: analyze-story-claude-cli: "claude -p --model claude-haiku-4-5-20251001" exited 1: — failing because gate-2 acceptance requires the real analyzer to execute. Set RELAYFLOWS_ALLOW_ANALYZER_SKIP=1 only if this run is not gate evidence. + ✓ built flows CLI against live relayflowd > preflights before journaling and names an unreachable socket 1643ms + ✓ built flows CLI against live relayflowd > starts exactly one daemon when two runs race for one empty data dir 879ms + ✓ surface resume after a real daemon kill > resumes a three-step run with each successful completion exactly once 1669ms + +⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/live-kernel.test.ts > built flows CLI against live relayflowd > hn-monitor analyze-story reaches done through the real Claude analyzer CLI +Error: LIVE_ANALYZER_UNAVAILABLE: "/Users/khaliqgant/flows-spec-G-budget/testdata/preflight/analyze-story-claude-cli auth status" exited 1: analyze-story-claude-cli: "claude -p --model claude-haiku-4-5-20251001" exited 1: — failing because gate-2 acceptance requires the real analyzer to execute. Set RELAYFLOWS_ALLOW_ANALYZER_SKIP=1 only if this run is not gate evidence. + ❯ tests/live-kernel.test.ts:1186:15 + 1184| const notice = `LIVE_ANALYZER_UNAVAILABLE: ${readiness.detail}`; + 1185| if (process.env['RELAYFLOWS_ALLOW_ANALYZER_SKIP'] !== '1') { + 1186| throw new Error( + | ^ + 1187| `${notice} — failing because gate-2 acceptance requires the … + 1188| + 'Set RELAYFLOWS_ALLOW_ANALYZER_SKIP=1 only if this run is … + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ + + Test Files 1 failed | 53 passed | 1 skipped (55) + Tests 1 failed | 984 passed | 3 skipped (988) + Start at 10:33:43 + Duration 54.37s (transform 989ms, setup 0ms, collect 4.98s, tests 183.70s, environment 6ms, prepare 2.02s) diff --git a/evidence/spec-G-budget/smoke-journal.json b/evidence/spec-G-budget/smoke-journal.json new file mode 100644 index 000000000..221eb96ee --- /dev/null +++ b/evidence/spec-G-budget/smoke-journal.json @@ -0,0 +1,48 @@ +[ + { + "entry_type": "step.completed", + "step_id": "measured", + "payload": { + "budget": { + "dollars": "0", + "tokens_in": 0, + "tokens_out": 0 + }, + "completed_by": "kernel", + "completionReason": "success", + "disposition": "step_done", + "effects": [], + "end_pins": null, + "next_attempt_at_ms": null, + "output": { + "exit_code": 0, + "stderr_tail": "", + "stdout_tail": "" + }, + "spend": { + "dollars": 0, + "tokens_input": 0, + "tokens_output": 0, + "wallclock_ms": 19 + }, + "verification": { + "detail": "all gates passed", + "gate": "exit_code", + "verdict": "pass" + } + } + }, + { + "entry_type": "run.completed", + "step_id": null, + "payload": { + "budget_total": { + "dollars": "0", + "tokens_in": 0, + "tokens_out": 0 + }, + "completionReason": "budget_exceeded", + "failed_step_id": null + } + } +] diff --git a/evidence/spec-G-budget/smoke.log b/evidence/spec-G-budget/smoke.log new file mode 100644 index 000000000..f29c767f3 --- /dev/null +++ b/evidence/spec-G-budget/smoke.log @@ -0,0 +1,2 @@ +{"run_id":"01M27SE6KVHAK8G7Y2CHVK0HRF","status":"failed","completion_reason":"budget_exceeded","completed_steps":1} +Error: run 01M27SE6KVHAK8G7Y2CHVK0HRF failed diff --git a/evidence/spec-G-budget/smoke.spec.json b/evidence/spec-G-budget/smoke.spec.json new file mode 100644 index 000000000..2fdf844cf --- /dev/null +++ b/evidence/spec-G-budget/smoke.spec.json @@ -0,0 +1 @@ +{"version":"0.1.0","name":"budget-guarded","steps":[{"id":"measured","depends_on":[],"max_iterations":1,"retry":{"initial_backoff_ms":100,"max_backoff_ms":60000,"multiplier":2,"jitter_percent":20},"verification":{},"type":"deterministic","command":"sleep 0.01"},{"id":"guarded","depends_on":["measured"],"max_iterations":1,"retry":{"initial_backoff_ms":100,"max_backoff_ms":60000,"multiplier":2,"jitter_percent":20},"verification":{},"type":"deterministic","command":"printf 'budget should refuse this step'"}],"budget":{"pricing":"frozen","max_wallclock_ms":0}} diff --git a/evidence/spec-G-budget/surface-test.log b/evidence/spec-G-budget/surface-test.log new file mode 100644 index 000000000..263f20305 --- /dev/null +++ b/evidence/spec-G-budget/surface-test.log @@ -0,0 +1,9 @@ + + RUN v2.1.9 /Users/khaliqgant/flows-spec-G-budget/packages/surface + + ✓ tests/flow.test.ts (7 tests) 3ms + + Test Files 1 passed (1) + Tests 7 passed (7) + Start at 10:30:40 + Duration 228ms (transform 21ms, setup 0ms, collect 21ms, tests 3ms, environment 0ms, prepare 32ms) diff --git a/kernel/relayflowd-core/src/entry.rs b/kernel/relayflowd-core/src/entry.rs index 075a2a218..645f0cf62 100644 --- a/kernel/relayflowd-core/src/entry.rs +++ b/kernel/relayflowd-core/src/entry.rs @@ -133,6 +133,18 @@ pub struct JournalEntry { pub payload: Value, } +pub fn journal_dollars(value: &str) -> Result { + let (whole, fraction) = value.split_once('.').unwrap_or((value, "")); + let whole = whole.trim_start_matches('0'); + let whole = if whole.is_empty() { "0" } else { whole }; + let normalized = if fraction.is_empty() { + whole.to_owned() + } else { + format!("{whole}.{fraction}") + }; + normalized.parse() +} + impl JournalEntry { pub fn new( entry_type: EntryType, @@ -142,6 +154,16 @@ impl JournalEntry { at_ms: i64, payload: T, ) -> Self { + let mut payload = serde_json::to_value(payload).expect("journal payload must serialize"); + if entry_type == EntryType::StepCompleted { + let budget: Budget = + serde_json::from_value(payload["budget"].clone()).unwrap_or_default(); + payload["spend"] = serde_json::json!({ + "tokens_input": budget.tokens_in, "tokens_output": budget.tokens_out, + "dollars": journal_dollars(&budget.dollars).expect("valid journal dollars"), + "wallclock_ms": 0, + }); + } Self { seq: 0, segment_id: 0, @@ -150,7 +172,7 @@ impl JournalEntry { step_id, attempt, at_ms, - payload: serde_json::to_value(payload).expect("journal payload must serialize"), + payload, } } } diff --git a/kernel/relayflowd-core/src/machine.rs b/kernel/relayflowd-core/src/machine.rs index 9c2f8501d..0a0282814 100644 --- a/kernel/relayflowd-core/src/machine.rs +++ b/kernel/relayflowd-core/src/machine.rs @@ -17,6 +17,8 @@ use crate::{ const LEASE_DURATION_MS: i64 = 30_000; +mod budget; +pub use budget::exceeded as budget_exceeded; mod cancel; use cancel::cancel_run_actions; pub use cancel::request_cancel_action; @@ -115,6 +117,17 @@ pub fn next_actions(state: &RunState, now_ms: i64) -> Vec { return complete_run_actions(state, RunCompletionReason::Success, None, now_ms); } + if budget::exceeded(state, now_ms) { + if state + .steps + .values() + .any(|runtime| matches!(runtime.state, StepState::Running { .. })) + { + return Vec::new(); + } + return complete_run_actions(state, RunCompletionReason::BudgetExceeded, None, now_ms); + } + // Wake every retry whose deterministic timer is due before starting work. // Recovery can put several crashed parallel lanes into the same zero-delay // backoff; waking just one would let an already-runnable peer start and park diff --git a/kernel/relayflowd-core/src/machine/budget.rs b/kernel/relayflowd-core/src/machine/budget.rs new file mode 100644 index 000000000..092dcb68b --- /dev/null +++ b/kernel/relayflowd-core/src/machine/budget.rs @@ -0,0 +1,29 @@ +use crate::{Budget, RunState}; +use std::cmp::Ordering; + +/// Completion facts charge the envelope; already-running work remains valid. +pub fn exceeded(state: &RunState, now_ms: i64) -> bool { + let Some(limit) = &state.spec.budget else { + return false; + }; + let empty = Budget::default(); + let (spent, wallclock) = if limit.window.is_some() { + if state.budget_day == Some(now_ms.div_euclid(86_400_000)) { + (&state.daily_budget, state.daily_wallclock_ms) + } else { + (&empty, 0) + } + } else { + (&state.budget, state.wallclock_ms) + }; + limit.max_tokens_in.is_some_and(|n| spent.tokens_in > n) + || limit.max_tokens_out.is_some_and(|n| spent.tokens_out > n) + || limit.max_tokens.is_some_and(|n| { + u128::from(spent.tokens_in) + u128::from(spent.tokens_out) > u128::from(n) + }) + || limit.max_wallclock_ms.is_some_and(|n| wallclock > n) + || limit + .max_dollars + .as_deref() + .is_some_and(|n| crate::memory::decimal_cmp(&spent.dollars, n) == Ordering::Greater) +} diff --git a/kernel/relayflowd-core/src/memory.rs b/kernel/relayflowd-core/src/memory.rs index 6b912297a..06cbf2ea3 100644 --- a/kernel/relayflowd-core/src/memory.rs +++ b/kernel/relayflowd-core/src/memory.rs @@ -32,6 +32,11 @@ pub struct MemoryInjectedPayload { impl MemorySpec { pub fn validate(&self) -> Result<(), String> { + if self.budget.pricing.is_some() || self.budget.prior_spend.is_some() + || self.budget.max_tokens.is_some() || self.budget.max_wallclock_ms.is_some() + || self.budget.window.is_some() { + return Err("memory budgets accept only max_tokens_in, max_tokens_out, max_dollars".into()); + } if self.query.trim().is_empty() { return Err("query must be a non-empty string".into()); } @@ -73,7 +78,7 @@ impl MemorySpec { } } -fn valid_decimal(value: &str) -> bool { +pub fn valid_decimal(value: &str) -> bool { let (whole, fraction) = value .split_once('.') .map_or((value, None), |(w, f)| (w, Some(f))); @@ -83,7 +88,7 @@ fn valid_decimal(value: &str) -> bool { } /// Exact decimal comparison without floating point or fixed-width scaling. -fn decimal_cmp(left: &str, right: &str) -> Ordering { +pub(crate) fn decimal_cmp(left: &str, right: &str) -> Ordering { let parts = |value: &str| { let (whole, fraction) = value.split_once('.').unwrap_or((value, "")); ( diff --git a/kernel/relayflowd-core/src/spec.rs b/kernel/relayflowd-core/src/spec.rs index a2c15d8f3..c4e549e83 100644 --- a/kernel/relayflowd-core/src/spec.rs +++ b/kernel/relayflowd-core/src/spec.rs @@ -74,6 +74,21 @@ impl RunSpec { return Err(SpecError::EmptyCli); } + if let Some(budget) = &self.budget { + if budget + .max_dollars + .as_deref() + .is_some_and(|v| !crate::memory::valid_decimal(v)) + || budget + .prior_spend + .as_ref() + .is_some_and(|p| !crate::memory::valid_decimal(&p.dollars)) + { + return Err(SpecError::Malformed( + "budget dollars must be non-negative decimal strings".into(), + )); + } + } let mut trigger_ids = BTreeSet::new(); for trigger in &self.triggers { if trigger.id.trim().is_empty() @@ -525,6 +540,16 @@ impl TriggerSpec { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct BudgetSpec { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pricing: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub prior_spend: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_wallclock_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub window: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub max_tokens_in: Option, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -533,6 +558,29 @@ pub struct BudgetSpec { pub max_dollars: Option, } +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum BudgetWindow { + Day, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum BudgetPricing { + Frozen, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct PriorSpend { + pub tokens_in: u64, + pub tokens_out: u64, + pub dollars: String, + pub wallclock_ms: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub day: Option, +} + /// v0 verification gates (kernel DESIGN.md §4): `exit_code == 0` is implicit /// for deterministic steps; these two are optional and combinable. Unknown /// keys are a parse error — verification is control flow, and a dropped gate diff --git a/kernel/relayflowd-core/src/state.rs b/kernel/relayflowd-core/src/state.rs index 9575fd854..e14f034a1 100644 --- a/kernel/relayflowd-core/src/state.rs +++ b/kernel/relayflowd-core/src/state.rs @@ -13,10 +13,11 @@ use crate::{ }; mod budget; +#[cfg(test)] +use budget::add_budget; mod memory; mod pins; mod routing; -use budget::add_budget; #[derive(Debug, Clone, PartialEq)] pub enum StepState { @@ -69,6 +70,10 @@ pub struct RunState { pub steps: BTreeMap, pub memo: BTreeMap, pub budget: Budget, + pub wallclock_ms: u64, + pub budget_day: Option, + pub daily_budget: Budget, + pub daily_wallclock_ms: u64, pub completion: Option, /// Durable cancellation intent. Once present, scheduling can only close /// live work and append the terminal canceled fact. @@ -110,12 +115,32 @@ impl RunState { spec, memo: BTreeMap::new(), budget: Budget::default(), + wallclock_ms: 0, + budget_day: None, + daily_budget: Budget::default(), + daily_wallclock_ms: 0, completion: None, cancel_requested: None, current_pins: None, routing: BTreeMap::new(), }; + if let Some(prior) = state + .spec + .budget + .as_ref() + .and_then(|b| b.prior_spend.as_ref()) + { + state.budget = Budget { + tokens_in: prior.tokens_in, + tokens_out: prior.tokens_out, + dollars: prior.dollars.clone(), + }; + state.wallclock_ms = prior.wallclock_ms; + state.budget_day = prior.day; + state.daily_budget = state.budget.clone(); + state.daily_wallclock_ms = prior.wallclock_ms; + } for entry in entries { if entry.run_id != state.run_id { return Err(StateError::WrongRun(entry.run_id.clone())); @@ -251,7 +276,11 @@ impl RunState { fn apply_step_completed(&mut self, entry: &JournalEntry) -> Result<(), StateError> { let payload: StepCompletedPayload = decode(entry)?; - add_budget(&mut self.budget, &payload.budget)?; + self.charge_budget( + &payload.budget, + entry.payload["spend"]["wallclock_ms"].as_u64().unwrap_or(0), + entry.at_ms, + )?; let step_id = entry .step_id .clone() @@ -321,6 +350,9 @@ impl RunState { self.validate_routing(&payload.routing)?; self.routing = payload.routing; self.memo.clear(); + if self.spec.budget.is_some() && self.budget_day.is_some() && self.budget != payload.budget_spent { + return Err(StateError::BudgetSummaryMismatch); + } self.budget = payload.budget_spent; for runtime in self.steps.values_mut() { *runtime = StepRuntime { @@ -464,6 +496,8 @@ pub enum StateError { }, #[error("invalid non-negative decimal dollar amount {0:?}")] InvalidDollars(String), + #[error("epoch budget differs from recorded spend")] + BudgetSummaryMismatch, #[error("budget token total overflow")] BudgetOverflow, } diff --git a/kernel/relayflowd-core/src/state/budget.rs b/kernel/relayflowd-core/src/state/budget.rs index d12f52097..f3e25d4bc 100644 --- a/kernel/relayflowd-core/src/state/budget.rs +++ b/kernel/relayflowd-core/src/state/budget.rs @@ -1,6 +1,6 @@ use crate::entry::Budget; -use super::StateError; +use super::{RunState, StateError}; pub(super) fn add_budget(total: &mut Budget, value: &Budget) -> Result<(), StateError> { let tokens_in = total @@ -133,3 +133,30 @@ mod tests { assert_eq!(total, original); } } + +impl RunState { + pub(super) fn charge_budget( + &mut self, + value: &Budget, + duration: u64, + at_ms: i64, + ) -> Result<(), StateError> { + add_budget(&mut self.budget, value)?; + self.wallclock_ms = self + .wallclock_ms + .checked_add(duration) + .ok_or(StateError::BudgetOverflow)?; + let day = at_ms.div_euclid(86_400_000); + if self.budget_day != Some(day) { + self.budget_day = Some(day); + self.daily_budget = Budget::default(); + self.daily_wallclock_ms = 0; + } + add_budget(&mut self.daily_budget, value)?; + self.daily_wallclock_ms = self + .daily_wallclock_ms + .checked_add(duration) + .ok_or(StateError::BudgetOverflow)?; + Ok(()) + } +} diff --git a/kernel/relayflowd-core/src/state/memory.rs b/kernel/relayflowd-core/src/state/memory.rs index d3334dadf..db1fdce4f 100644 --- a/kernel/relayflowd-core/src/state/memory.rs +++ b/kernel/relayflowd-core/src/state/memory.rs @@ -1,4 +1,4 @@ -use super::{RunState, StateError, budget::add_budget, decode}; +use super::{RunState, StateError, decode}; use crate::{JournalEntry, MemoryInjectedPayload, StepState}; impl RunState { @@ -43,9 +43,7 @@ impl RunState { detail: "memory must be injected once by an active attempt".into(), }); } - let mut total = self.budget.clone(); - add_budget(&mut total, &payload.budget)?; - self.budget = total; + self.charge_budget(&payload.budget, 0, entry.at_ms)?; self.steps.get_mut(step_id).expect("validated step").memory = Some(payload); Ok(()) } diff --git a/kernel/relayflowd/src/engine.rs b/kernel/relayflowd/src/engine.rs index 8b62721cd..a005306c3 100644 --- a/kernel/relayflowd/src/engine.rs +++ b/kernel/relayflowd/src/engine.rs @@ -318,9 +318,15 @@ impl Engine { fn load_state(&self, journal: &SqliteJournal, spec: RunSpec) -> Result { let segment = journal.current_segment().map_err(|error| anyhow!(error))?; - let entries = journal - .scan_segment(segment) - .map_err(|error| anyhow!(error))?; + // Window and elapsed-time counters are reconstructed from completion + // facts across epochs, including facts predating this SDK version. + let entries = if spec.budget.is_some() { + journal.scan_all().map_err(|error| anyhow!(error))? + } else { + journal + .scan_segment(segment) + .map_err(|error| anyhow!(error))? + }; RunState::fold(journal.run_id(), spec, &entries).context("fold run journal") } @@ -332,6 +338,17 @@ impl Engine { self.ensure_journal_mutable(journal)?; let mut entry = entry.clone(); self.stamp_completion(journal, &mut entry)?; + if entry.entry_type == EntryType::StepCompleted { + let start = journal.scan_all()?.into_iter().rev().find(|e| { + e.entry_type == EntryType::StepAttemptStarted + && e.step_id == entry.step_id + && e.attempt == entry.attempt + }); + if let Some(start) = start { + entry.payload["spend"]["wallclock_ms"] = + serde_json::json!(entry.at_ms.saturating_sub(start.at_ms).max(0)); + } + } let persisted = journal.append(&entry).map_err(|error| anyhow!(error))?; if let Some(observer) = &self.observer { observer.appended(&persisted); diff --git a/kernel/relayflowd/src/engine/drive.rs b/kernel/relayflowd/src/engine/drive.rs index 8781005e0..2005b9e05 100644 --- a/kernel/relayflowd/src/engine/drive.rs +++ b/kernel/relayflowd/src/engine/drive.rs @@ -54,6 +54,16 @@ impl Engine { match action { Action::Append(mut entry) => { if entry.entry_type == relayflowd_core::EntryType::StepAttemptStarted { + // A deterministic peer may have completed since this batch + // was elected. Re-fold before admitting the next start. + if spec.budget.is_some() + && relayflowd_core::machine::budget_exceeded( + &self.load_state(&journal, spec.clone())?, + self.clock.now_ms(), + ) + { + break; + } let step_id = entry .step_id .as_deref() diff --git a/kernel/relayflowd/src/engine/remote.rs b/kernel/relayflowd/src/engine/remote.rs index 89051670c..3956b7251 100644 --- a/kernel/relayflowd/src/engine/remote.rs +++ b/kernel/relayflowd/src/engine/remote.rs @@ -38,6 +38,11 @@ impl Engine { step_id: &str, completion: OutOfBandCompletion, ) -> Result { + if !relayflowd_core::memory::valid_decimal(&completion.budget.dollars) + || relayflowd_core::journal_dollars(&completion.budget.dollars).is_err() + { + bail!("step completion usage.dollars must be a non-negative decimal string"); + } let mut journal = self.open_run(run_id)?; let spec = journal.run_spec().context("read run spec")?; let state = self.load_state(&journal, spec.clone())?; @@ -380,7 +385,11 @@ fn worker_failure_detail(output: &Value) -> Option { // and slicing it by byte index would panic on multi-byte input. Some(match trimmed.char_indices().nth(MAX_CHARS) { None => trimmed.to_owned(), - Some((cut, _)) => format!("{}… ({} bytes truncated)", &trimmed[..cut], trimmed.len() - cut), + Some((cut, _)) => format!( + "{}… ({} bytes truncated)", + &trimmed[..cut], + trimmed.len() - cut + ), }) } @@ -433,7 +442,10 @@ mod worker_failure_detail_tests { // 3000 three-byte chars = 9000 bytes. let output = json!("€".repeat(3000)); let detail = worker_failure_detail(&output).expect("detail for a long output"); - assert!(detail.contains('…'), "expected a truncation marker, got {detail:?}"); + assert!( + detail.contains('…'), + "expected a truncation marker, got {detail:?}" + ); // Cut at 2000 CHARS = 6000 bytes, so 3000 bytes remain. assert!( detail.contains("3000 bytes truncated"), diff --git a/kernel/relayflowd/tests/budget_gate.rs b/kernel/relayflowd/tests/budget_gate.rs new file mode 100644 index 000000000..4f810c849 --- /dev/null +++ b/kernel/relayflowd/tests/budget_gate.rs @@ -0,0 +1,172 @@ +use relayflowd::{ + Engine, OutOfBandCompletion, + worker::{DispatchOutcome, JournalObserver, StepDispatch, StepDispatcher}, +}; +use relayflowd_core::{ + Budget, CompletionReason, EntryType, JournalEntry, RunCompletionReason, RunSpec, StepType, +}; +use serde_json::json; +use std::sync::{Arc, Mutex}; + +#[derive(Default)] +struct Worker(Mutex>); +impl JournalObserver for Worker { + fn appended(&self, _: &JournalEntry) {} +} +impl StepDispatcher for Worker { + fn executor(&self, _: StepType) -> Option { + Some("mock".into()) + } + fn available(&self, _: StepType) -> bool { + true + } + fn dispatch(&self, d: StepDispatch) -> anyhow::Result { + self.0.lock().unwrap().push(d); + Ok(DispatchOutcome::Dispatched) + } +} + +#[test] +fn crossing_completion_is_durable_and_next_step_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let worker = Arc::new(Worker::default()); + let engine = Engine::with_runtime(dir.path(), worker.clone(), worker.clone()); + let spec = RunSpec::parse(&json!({"budget":{"max_dollars":"0.001"},"steps":[ + {"id":"first","type":"llm","prompt":"answer"}, + {"id":"second","type":"deterministic","command":"printf must-not-run","depends_on":["first"]} + ]})).unwrap(); + let started = engine.start(spec, "test", None).unwrap(); + let d = worker.0.lock().unwrap()[0].clone(); + let outcome = engine + .complete_out_of_band( + &started.run_id, + "first", + OutOfBandCompletion { + attempt: d.attempt, + idempotency_key: d.idempotency_key, + completion_reason: CompletionReason::Success, + output: json!("answer"), + budget: Budget { + tokens_in: 1000, + tokens_out: 200, + dollars: "0.006000".into(), + }, + completed_by: "mock".into(), + started_pins: None, + end_pins: None, + effects: vec![], + trajectory_tail: None, + }, + ) + .unwrap(); + assert_eq!( + outcome.completion_reason, + Some(RunCompletionReason::BudgetExceeded) + ); + let reopened = Engine::new(dir.path()); + let entries = reopened.journal_entries(&started.run_id, 1, 100).unwrap(); + let completed = entries + .iter() + .find(|e| e.entry_type == EntryType::StepCompleted) + .unwrap(); + assert_eq!(completed.payload["completionReason"], "success"); + assert_eq!(completed.payload["output"], "answer"); + assert_eq!(completed.payload["spend"]["tokens_input"], 1000); + assert_eq!(completed.payload["spend"]["tokens_output"], 200); + assert_eq!(completed.payload["spend"]["dollars"], json!(0.006)); + assert!(completed.payload["spend"]["wallclock_ms"].is_u64()); + assert!( + !entries + .iter() + .any(|e| e.entry_type == EntryType::StepAttemptStarted + && e.step_id.as_deref() == Some("second")) + ); + assert_eq!( + reopened + .resume(&started.run_id, None) + .unwrap() + .completion_reason, + outcome.completion_reason + ); +} + +#[test] +fn deterministic_spend_and_wallclock_limit_gate_parallel_batch_starts() { + let dir = tempfile::tempdir().unwrap(); + let engine = Engine::new(dir.path()); + let spec = RunSpec::parse(&json!({"budget":{"max_wallclock_ms":0},"steps":[ + {"id":"first","type":"deterministic","command":"sleep 0.01"}, + {"id":"second","type":"deterministic","command":"printf must-not-run"} + ]})) + .unwrap(); + let outcome = engine.start(spec, "test", None).unwrap(); + assert_eq!( + outcome.completion_reason, + Some(RunCompletionReason::BudgetExceeded) + ); + let entries = engine.journal_entries(&outcome.run_id, 1, 100).unwrap(); + let completions: Vec<_> = entries + .iter() + .filter(|e| e.entry_type == EntryType::StepCompleted) + .collect(); + assert_eq!(completions.len(), 1); + assert_eq!(completions[0].payload["spend"]["tokens_input"], 0); + assert!( + completions[0].payload["spend"]["wallclock_ms"] + .as_u64() + .unwrap() + > 0 + ); +} + +#[test] +fn daily_windows_reset_and_exact_limits_do_not_refuse() { + use relayflowd_core::{Action, AttemptResult, RunState, completion_actions, next_actions}; + for (window, limit, now, exceeded) in [ + ("day", "0.005", 10, true), + ("day", "0.005", 86_400_000, false), + ("day", "0.006", 10, false), + ] { + let spec = RunSpec::parse( + &json!({"budget":{"max_dollars":limit,"window":window},"steps":[ + {"id":"first","type":"llm","prompt":"answer"}, + {"id":"second","type":"deterministic","command":":","depends_on":["first"]} + ]}), + ) + .unwrap(); + let initial = RunState::fold("run", spec.clone(), &[]).unwrap(); + let mut entries: Vec<_> = next_actions(&initial, 0) + .into_iter() + .filter_map(|a| { + if let Action::Append(e) = a { + Some(e) + } else { + None + } + }) + .collect(); + let mut result = AttemptResult::successful(json!("answer"), "mock"); + result.budget = Budget { + tokens_in: 1000, + tokens_out: 200, + dollars: "0.006".into(), + }; + entries.extend( + completion_actions("run", &spec.steps[0], 1, 0, result, 1) + .into_iter() + .filter_map(|a| { + if let Action::Append(e) = a { + Some(e) + } else { + None + } + }), + ); + let replay = RunState::fold("run", spec, &entries).unwrap(); + assert_eq!( + relayflowd_core::machine::budget_exceeded(&replay, now), + exceeded + ); + assert_eq!(replay.memo["first"], json!("answer")); + } +} diff --git a/packages/sdk/src/authored-budget.ts b/packages/sdk/src/authored-budget.ts new file mode 100644 index 000000000..491a02698 --- /dev/null +++ b/packages/sdk/src/authored-budget.ts @@ -0,0 +1,69 @@ +import { BudgetSyntaxError, parseBudget, toKernelBudget } from './budget.js'; +import { AuthoredFlowExecutionError } from './authored-flow-error.js'; +import type { JournalClient } from './journal-client.js'; +import type { RunOutcome } from './protocol.js'; +import type { KernelBudgetSpec, KernelRunSpec } from './spec.js'; + +/** Serialized admission for the internal authored runner's separate step runs. */ +export class AuthoredBudget { + private readonly limit: KernelBudgetSpec | undefined; + private failed = false; + private tail: Promise = Promise.resolve(); + private charges: { input: bigint; output: bigint; micro: bigint; ms: bigint; day: number }[] = []; + + constructor(header: unknown) { + try { + this.limit = header === undefined ? undefined : toKernelBudget(parseBudget(header)); + } catch (error) { + if (error instanceof BudgetSyntaxError) throw new AuthoredFlowExecutionError('budget_syntax_invalid', error.message); + throw error; + } + } + + async execute(journal: JournalClient, spec: KernelRunSpec, consume: (outcome: RunOutcome) => Promise): Promise { + if (this.limit === undefined) return consume(await journal.runStart(spec)); + const previous = this.tail; + let release!: () => void; + this.tail = new Promise(resolve => { release = resolve; }); + await previous; + try { + if (this.failed) throw new AuthoredFlowExecutionError('step_failed', 'A prior budgeted step did not finish successfully.'); + // Window selection follows journal timestamps, never the SDK host clock. + const day = this.charges.at(-1)?.day; + const total = this.charges.filter(c => this.limit!.window !== 'day' || c.day === day) + .reduce((s, c) => ({ input: s.input + c.input, output: s.output + c.output, micro: s.micro + c.micro, ms: s.ms + c.ms }), + { input: 0n, output: 0n, micro: 0n, ms: 0n }); + const exactNumber = (n: bigint) => { if (n > BigInt(Number.MAX_SAFE_INTEGER)) throw new Error('budget counter overflow'); return Number(n); }; + const outcome = await journal.runStart({ ...spec, budget: { ...this.limit, prior_spend: { + tokens_in: exactNumber(total.input), tokens_out: exactNumber(total.output), + dollars: `${total.micro / 1_000_000n}.${String(total.micro % 1_000_000n).padStart(6, '0')}`, + wallclock_ms: exactNumber(total.ms), ...(this.limit.window === 'day' && day !== undefined ? { day } : {}), + } } }); + try { + if (outcome.completion_reason === 'budget_exceeded') throw new AuthoredFlowExecutionError('step_failed', 'Flow budget exceeded before the next step.', 'budget_exceeded', outcome.run_id); + return await consume(outcome); + } finally { + let seq = 1; + for (;;) { + const { entries } = await journal.journalRead(outcome.run_id, seq); + if (entries.length === 0) break; + for (const raw of entries) { + const e = raw as {seq: number; entry_type: string; at_ms: number; payload: {budget?: {tokens_in: number; tokens_out: number; dollars: string}; spend?: {wallclock_ms: number}}}; + seq = e.seq + 1; + if (!['step.completed', 'memory.injected'].includes(e.entry_type)) continue; + const b = e.payload.budget; + if (b === undefined) throw new Error('journal completion missing budget'); + const [whole, fraction = ''] = b.dollars.split('.'); + if (fraction.length > 6 && /[1-9]/.test(fraction.slice(6))) throw new Error('budget accounting requires microdollar precision'); + this.charges.push({input: BigInt(b.tokens_in), output: BigInt(b.tokens_out), + micro: BigInt(whole!) * 1_000_000n + BigInt(fraction.slice(0, 6).padEnd(6, '0')), + ms: BigInt(e.payload.spend?.wallclock_ms ?? 0), day: Math.floor(e.at_ms / 86_400_000)}); + } + } + } + } catch (error) { + this.failed = true; + throw error; + } finally { release(); } + } +} diff --git a/packages/sdk/src/authored-flow-error.ts b/packages/sdk/src/authored-flow-error.ts index 728a24c46..e3eabc899 100644 --- a/packages/sdk/src/authored-flow-error.ts +++ b/packages/sdk/src/authored-flow-error.ts @@ -6,6 +6,8 @@ import type { export type AuthoredFlowExecutionErrorCode = | 'helper_slack.credential_missing' | 'helper_slack.mount_required' + | 'budget_syntax_invalid' + | 'budget_missing_price' | 'agent_cli_unresolved' | 'agent_parked' | 'llm_cli_unresolved' diff --git a/packages/sdk/src/authored-flow-executor.ts b/packages/sdk/src/authored-flow-executor.ts index 64a5a7005..190f4a851 100644 --- a/packages/sdk/src/authored-flow-executor.ts +++ b/packages/sdk/src/authored-flow-executor.ts @@ -6,6 +6,7 @@ import { snapshotJsonValue } from './json-value.js'; import type { SlackCall } from './slack-writeback.js'; import { checkMcpHeader, McpPreflightError } from './cli/check-typescript.js'; import { buildMcpProxy, runMcpEffect } from './authored-mcp.js'; +import { AuthoredBudget } from './authored-budget.js'; import { authoredWorkerRunner } from './authored-worker-step.js'; import { readSuccessfulOutput, isSurfaceRunCompletionReason } from './authored-step-output.js'; import { @@ -84,7 +85,7 @@ type JournalStepUsesStepCompletionReason = Assert< * * This is deliberately not exported by the SDK package: without a durable * authored root, it is not a resumable public runner. The seam is narrow: an - * empty-header flow may await `f.run`, `f.llm`, and `f.agent` steps and must + * flow with an optional budget may await `f.run`, `f.llm`, and `f.agent` steps and must * finish with `f.done("success")`. Each step and the terminal marker is * a compiled spec submitted through * `JournalClient`; values are read back from `step.completed` journal entries. @@ -134,7 +135,7 @@ export async function executeAuthoredFlow( ...(options.onWait !== undefined ? { onWait: options.onWait } : {}), }; const definition = getDefinition(handle); - const headerFields = Object.keys(definition.header).filter(key => key !== 'tools'); + const headerFields = Object.keys(definition.header).filter(key => key !== 'tools' && key !== 'budget'); if (definition.header.tools && Object.keys(definition.header.tools).some(key => !['slack', 'mcp'].includes(key))) headerFields.push('tools'); if (definition.header.tools?.relayfile !== undefined) headerFields.push('tools.relayfile'); const helperPreflight = checkSlackHelpers(definition); @@ -149,6 +150,7 @@ export async function executeAuthoredFlow( const checkedMcp = await checkMcpHeader(definition, flowPath); if (!checkedMcp.report.ok) throw new McpPreflightError(checkedMcp.report); + const budget = new AuthoredBudget(definition.header.budget); const journalSteps: AuthoredFlowJournalStep[] = []; const authoredSteps: AuthoredFlowOperation[] = []; const lifecycle = new AuthoredFlowLifecycle(); @@ -158,17 +160,18 @@ export async function executeAuthoredFlow( const lowerDeterministic = async ( id: string, command: string, + terminal = false, ): Promise => { const spec = toKernelSpec(compileSpec({ version: SPEC_SCHEMA_VERSION, name: `${definition.name}/${id}`, steps: [{ id, type: 'deterministic', command }], })); - const outcome = await journal.runStart(spec); - return readSuccessfulOutput(journal, outcome, id, journalSteps); + if (terminal) return readSuccessfulOutput(journal, await journal.runStart(spec), id, journalSteps); + return budget.execute(journal, spec, outcome => readSuccessfulOutput(journal, outcome, id, journalSteps)); }; - const worker = authoredWorkerRunner(definition, journal, flowPath, journalSteps, waitOptions, localAgentStream); + const worker = authoredWorkerRunner(definition, journal, flowPath, journalSteps, waitOptions, localAgentStream, budget, definition.header.budget); function llmOperation(strings: TemplateStringsArray, ...values: unknown[]): Step; function llmOperation(prompt: string, options: LlmOptions): Step; @@ -337,7 +340,7 @@ export async function executeAuthoredFlow( lifecycle.close(); } - await lowerDeterministic(`complete-${nextStep}`, ':'); + await lowerDeterministic(`complete-${nextStep}`, ':', true); return Object.freeze({ name: definition.name, completionReason: requestedCompletion, diff --git a/packages/sdk/src/authored-worker-step.ts b/packages/sdk/src/authored-worker-step.ts index 389f449f4..01c965b44 100644 --- a/packages/sdk/src/authored-worker-step.ts +++ b/packages/sdk/src/authored-worker-step.ts @@ -1,3 +1,5 @@ +import type { AuthoredBudget } from './authored-budget.js'; +import { parseBudget } from './budget.js'; import type { AgentOptions, AgentResult, LlmOptions } from '@relayflows/surface'; import { toKernelSpec } from './compile.js'; import { checkAuthoredFlow } from './cli/check.js'; @@ -16,11 +18,11 @@ const WORKSPACE_PERMISSION_ANNOTATION = /:\s*(readonly|readwrite)\s*$/i; export function authoredWorkerRunner( definition: { name: string }, journal: JournalClient, flowPath: string, journalSteps: AuthoredFlowJournalStep[], waitOptions: RunLifecycleOptions, - localAgentStream?: string, + localAgentStream?: string, budget?: AuthoredBudget, headerBudget?: unknown, ) { async function run(step: StepSpec): Promise { const id = step.id; - const authoring: FlowSpec = { version: SPEC_SCHEMA_VERSION, name: `${definition.name}/${id}`, steps: [step] }; + const authoring: FlowSpec = { version: SPEC_SCHEMA_VERSION, name: `${definition.name}/${id}`, steps: [step], ...(headerBudget === undefined ? {} : { budget: parseBudget(headerBudget) }) }; // The kernel never resolves a `cli` on its own — every declarative // `flows run`/`flows check` binds it first via this exact function // (cli/check.ts), searching for the nearest flows.json from `flowPath` @@ -33,14 +35,15 @@ export function authoredWorkerRunner( diagnostic.severity === 'refusal', ); throw new AuthoredFlowExecutionError( - step.type === 'llm' ? 'llm_cli_unresolved' : 'agent_cli_unresolved', + refusal?.kind === 'budget_missing_price' || refusal?.kind === 'budget_syntax_invalid' ? refusal.kind + : step.type === 'llm' ? 'llm_cli_unresolved' : 'agent_cli_unresolved', refusal?.message ?? `flow "${definition.name}" step "${id}": no CLI could be resolved for f.${step.type} ` + `(searched for flows.json from "${flowPath}")`, ); } const spec = toKernelSpec(resolved); - const outcome = await journal.runStart(spec); + const consume = async (outcome: import('./protocol.js').RunOutcome) => { // Reuse the declarative CLI's own wait/classification (cli/run.ts) rather // than a hand-rolled poll: `step.completed` and the run's own terminal // state are appended as two SEPARATE actions (kernel/relayflowd-core/src/machine.rs @@ -75,6 +78,8 @@ export function authoredWorkerRunner( ); } return readCompletedStepOutput(journal, outcome.run_id, id, journalSteps); + }; + return budget === undefined ? consume(await journal.runStart(spec)) : budget.execute(journal, spec, consume); } return { diff --git a/packages/sdk/src/budget-preflight.ts b/packages/sdk/src/budget-preflight.ts new file mode 100644 index 000000000..449f14633 --- /dev/null +++ b/packages/sdk/src/budget-preflight.ts @@ -0,0 +1,31 @@ +import type { CompiledFlowSpec } from './compile.js'; +import type { PreflightRefusal } from './preflight.js'; +import { MODEL_PRICING } from './model-pricing.js'; + +/** Legacy explicit envelopes keep their worker-supplied pricing contract. */ +export function budgetDiagnostics(flow: CompiledFlowSpec): PreflightRefusal[] { + if (flow.budget?.pricing !== 'frozen') return []; + const diagnostics: PreflightRefusal[] = []; + const declared = [ + ...Object.entries(flow.agents ?? {}).map(([agent, d]) => ({ agent, model: d.model })), + ...flow.steps.flatMap(s => s.type !== 'deterministic' && s.model !== undefined + ? [{ stepId: s.id, model: s.model }] : []), + ]; + for (const step of flow.steps) { + if (step.type === 'deterministic' || flow.budget.maxDollars === undefined) continue; + const model = step.model ?? (step.type === 'agent' && step.agent !== undefined + ? flow.agents?.[step.agent]?.model : undefined); + if (model === undefined) diagnostics.push({ + severity: 'refusal', kind: 'budget_missing_price', stepId: step.id, + message: `Step "${step.id}" needs a declared, priced model for its dollar budget.`, + }); + } + for (const declaration of declared) { + if (Object.hasOwn(MODEL_PRICING, declaration.model)) continue; + diagnostics.push({ + severity: 'refusal', kind: 'budget_missing_price', ...declaration, + message: `Model "${declaration.model}" has no frozen price for budget accounting.`, + }); + } + return diagnostics; +} diff --git a/packages/sdk/src/budget.ts b/packages/sdk/src/budget.ts new file mode 100644 index 000000000..264723d95 --- /dev/null +++ b/packages/sdk/src/budget.ts @@ -0,0 +1,49 @@ +import type { BudgetSpec, KernelBudgetSpec } from './spec.js'; + +export type HeaderBudget = string | { tokens?: number; dollars?: number; wallclock?: string }; + +export class BudgetSyntaxError extends Error { + constructor() { super('budget_syntax_invalid: expected $/run, $/day, or { tokens, dollars, wallclock }'); } +} + +/** Normalize surface sugar; existing explicit envelopes remain supported. */ +export function parseBudget(value: unknown): BudgetSpec { + if (typeof value === 'string') { + const match = /^\$(\d+(?:\.\d{1,6})?)\/(run|day)$/.exec(value); + if (!match) throw new BudgetSyntaxError(); + return { pricing: 'frozen', maxDollars: match[1]!, ...(match[2] === 'day' ? { window: 'day' as const } : {}) }; + } + if (typeof value !== 'object' || value === null || Array.isArray(value)) throw new BudgetSyntaxError(); + const b = value as Record; + const legacy = ['maxTokensIn', 'maxTokensOut', 'maxTokens', 'maxDollars', 'maxWallclockMs', 'window', 'pricing']; + if (Object.keys(b).some(k => legacy.includes(k))) { + if (Object.keys(b).some(k => !legacy.includes(k))) throw new BudgetSyntaxError(); + return b as BudgetSpec; + } + if (Object.keys(b).some(k => !['tokens', 'dollars', 'wallclock'].includes(k))) throw new BudgetSyntaxError(); + if (b.tokens !== undefined && (!Number.isSafeInteger(b.tokens) || (b.tokens as number) < 0)) throw new BudgetSyntaxError(); + if (b.dollars !== undefined && (typeof b.dollars !== 'number' || !Number.isFinite(b.dollars) || b.dollars < 0 || !/^\d+(?:\.\d{1,6})?$/.test(String(b.dollars)))) throw new BudgetSyntaxError(); + let ms: number | undefined; + if (b.wallclock !== undefined) { + const match = typeof b.wallclock === 'string' && /^(\d+)(ms|s|m|h|d)$/.exec(b.wallclock); + if (!match) throw new BudgetSyntaxError(); + ms = Number(match[1]) * ({ms:1,s:1000,m:60000,h:3600000,d:86400000}[match[2]!]!); + if (!Number.isSafeInteger(ms)) throw new BudgetSyntaxError(); + } + return { pricing: 'frozen', ...(b.tokens === undefined ? {} : {maxTokens: b.tokens as number}), + ...(b.dollars === undefined ? {} : {maxDollars: String(b.dollars)}), + ...(ms === undefined ? {} : {maxWallclockMs: ms}) }; +} + +/** Lower a normalized envelope to the existing kernel protocol. */ +export function toKernelBudget(budget: BudgetSpec): KernelBudgetSpec { + return { + ...(budget.pricing !== undefined ? { pricing: budget.pricing } : {}), + ...(budget.maxTokens !== undefined ? { max_tokens: budget.maxTokens } : {}), + ...(budget.maxWallclockMs !== undefined ? { max_wallclock_ms: budget.maxWallclockMs } : {}), + ...(budget.window !== undefined ? { window: budget.window } : {}), + ...(budget.maxTokensIn !== undefined ? { max_tokens_in: budget.maxTokensIn } : {}), + ...(budget.maxTokensOut !== undefined ? { max_tokens_out: budget.maxTokensOut } : {}), + ...(budget.maxDollars !== undefined ? { max_dollars: budget.maxDollars } : {}), + }; +} diff --git a/packages/sdk/src/cli/direct-run.ts b/packages/sdk/src/cli/direct-run.ts index 1987317d3..d9bc82652 100644 --- a/packages/sdk/src/cli/direct-run.ts +++ b/packages/sdk/src/cli/direct-run.ts @@ -120,6 +120,8 @@ export async function runDirectFlow( || (error instanceof AuthoredFlowExecutionError && (error.code === 'helper_slack.credential_missing' || error.code === 'helper_slack.mount_required' + || error.code === 'budget_syntax_invalid' + || error.code === 'budget_missing_price' || error.code === 'unsupported_header' || error.code === 'agent_cli_unresolved' || error.code === 'llm_cli_unresolved' @@ -128,7 +130,12 @@ export async function runDirectFlow( exitCode: 2, report: { ...fromCheckReport('run', inputFailureReport({ - kind: error instanceof AuthoredFlowExecutionError && (error.code === 'helper_slack.credential_missing' || error.code === 'helper_slack.mount_required') ? error.code : 'invalid_spec', + kind: error instanceof AuthoredFlowExecutionError && ( + error.code === 'helper_slack.credential_missing' + || error.code === 'helper_slack.mount_required' + || error.code === 'budget_syntax_invalid' + || error.code === 'budget_missing_price' + ) ? error.code : 'invalid_spec', message: error.message, }, path)), socketPath, diff --git a/packages/sdk/src/compile.ts b/packages/sdk/src/compile.ts index 1bbf5efb2..701f04107 100644 --- a/packages/sdk/src/compile.ts +++ b/packages/sdk/src/compile.ts @@ -15,6 +15,7 @@ // `tests/spec_parity.rs` pin both sides to the same `testdata/` fixture. import { parse as parseYaml } from 'yaml'; +import { parseBudget, toKernelBudget } from './budget.js'; import { bindingDependencies } from './input-binding.js'; import type { AgentStepSpec, @@ -53,7 +54,7 @@ export class CompileError extends Error { * Compile a YAML string into a validated authoring `FlowSpec`. * Throws `CompileError` on a YAML parse error or any validation failure. */ -export function compileYaml(yaml: string): FlowSpec { +export function compileYaml(yaml: string): CompiledFlowSpec { const parsed = parseYaml(yaml); if (parsed === null || typeof parsed !== 'object') { throw new CompileError(['YAML: expected a mapping at the top level']); @@ -70,7 +71,9 @@ export function compileYamlToCanonicalJson(yaml: string): string { * Validate a parsed spec object and apply authoring defaults, returning a * normalized `FlowSpec`. Throws `CompileError` on validation failure. */ -export function compileSpec(spec: unknown): FlowSpec { +export type CompiledFlowSpec = Omit & { budget?: import('./spec.js').BudgetSpec }; + +export function compileSpec(spec: unknown): CompiledFlowSpec { let snapshot: unknown; try { snapshot = snapshotJsonValue(spec, 'spec'); @@ -79,15 +82,18 @@ export function compileSpec(spec: unknown): FlowSpec { error instanceof Error ? error.message : 'spec: expected JSON-compatible data', ]); } + if (snapshot !== null && typeof snapshot === 'object' && !Array.isArray(snapshot) && 'budget' in snapshot && snapshot.budget !== undefined) { + snapshot = { ...snapshot, budget: parseBudget(snapshot.budget) }; + } const validation: ValidationResult = validateSpec(snapshot); if (!validation.ok) throw new CompileError(validation.errors); - const input = snapshot as FlowSpec; + const input = snapshot as CompiledFlowSpec; // Preserve named declarations and selectors through authoring normalization. // They are resolved exactly once at the kernel boundary, after public // preflight has validated every declaration with truthful provenance. const steps = input.steps.map(compileStep); - const flow: FlowSpec = { + const flow: CompiledFlowSpec = { version: input.version, ...(input.name !== undefined ? { name: input.name } : {}), ...(input.description !== undefined ? { description: input.description } : {}), @@ -256,15 +262,7 @@ export function toKernelSpec(flow: FlowSpec): KernelRunSpec { // look correct. See ops/reviews/20260903-pr139-repair-0903.md section 10. ...(compiled.triggers?.length ? { triggers: compiled.triggers.map(toKernelTrigger) } : {}), steps: compiled.steps.map((step) => toKernelStep(resolveNamedAgent(step, compiled.agents))), - ...(compiled.budget !== undefined - ? { - budget: { - ...(compiled.budget.maxTokensIn !== undefined ? { max_tokens_in: compiled.budget.maxTokensIn } : {}), - ...(compiled.budget.maxTokensOut !== undefined ? { max_tokens_out: compiled.budget.maxTokensOut } : {}), - ...(compiled.budget.maxDollars !== undefined ? { max_dollars: compiled.budget.maxDollars } : {}), - }, - } - : {}), + ...(compiled.budget !== undefined ? { budget: toKernelBudget(compiled.budget) } : {}), }; } @@ -451,8 +449,12 @@ function kernelMemoryToAuthoring(value: unknown, at: string): unknown { } function kernelBudgetToAuthoring(value: unknown, at: string): unknown { - const budget = requireKernelObject(value, ['max_tokens_in', 'max_tokens_out', 'max_dollars'], at); + const budget = requireKernelObject(value, ['max_tokens_in', 'max_tokens_out', 'max_dollars', 'max_tokens', 'max_wallclock_ms', 'window', 'pricing'], at); return { + ...(budget['pricing'] !== undefined ? { pricing: budget['pricing'] } : {}), + ...(budget['max_tokens'] !== undefined ? { maxTokens: budget['max_tokens'] } : {}), + ...(budget['max_wallclock_ms'] !== undefined ? { maxWallclockMs: budget['max_wallclock_ms'] } : {}), + ...(budget['window'] !== undefined ? { window: budget['window'] } : {}), ...(budget['max_tokens_in'] !== undefined ? { maxTokensIn: budget['max_tokens_in'] } : {}), ...(budget['max_tokens_out'] !== undefined ? { maxTokensOut: budget['max_tokens_out'] } : {}), ...(budget['max_dollars'] !== undefined ? { maxDollars: budget['max_dollars'] } : {}), diff --git a/packages/sdk/src/failure-kinds.ts b/packages/sdk/src/failure-kinds.ts index 4cdafa202..db6f48dfa 100644 --- a/packages/sdk/src/failure-kinds.ts +++ b/packages/sdk/src/failure-kinds.ts @@ -6,6 +6,8 @@ const PREFLIGHT_ENVIRONMENT_FAILURE_KINDS = [ 'helper_slack.mount_required', 'mcp_undeclared_server', 'mcp_unreachable', + 'budget_syntax_invalid', + 'budget_missing_price', 'cli_missing', 'cli_unauthenticated', 'cli_unresolved', diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index 102770910..962df481d 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -45,6 +45,9 @@ export type { VerificationSpec, WorkspaceSurface, } from './spec.js'; +export { MODEL_PRICING } from './model-pricing.js'; +export type { HeaderBudget } from './budget.js'; +export type { StepSpend } from './protocol.js'; export { SPEC_SCHEMA_VERSION } from './spec.js'; export { CloudFlowError, type CloudConnectionOptions } from './cloud-http.js'; diff --git a/packages/sdk/src/llm-worker.ts b/packages/sdk/src/llm-worker.ts index 6d9094f65..6a287d092 100644 --- a/packages/sdk/src/llm-worker.ts +++ b/packages/sdk/src/llm-worker.ts @@ -1,3 +1,5 @@ +import { workerSpend } from './worker-spend.js'; +import type { WorkerCliResult } from './worker-cli.js'; import { EventEmitter } from 'node:events'; import type { JournalClient } from './journal-client.js'; import type { CompletionReason, StepDispatchEvent } from './protocol.js'; @@ -47,10 +49,11 @@ export class LlmWorker extends EventEmitter { const schema = spec.verification?.json_schema; const prompt = schema === undefined ? spec.prompt : `${spec.prompt}\n\nReturn only a JSON value matching this JSON Schema (no Markdown fences):\n${JSON.stringify(schema)}`; - const result = await withWorkerLease(this.client, dispatch, signal => + const completed: WorkerCliResult = await withWorkerLease(this.client, dispatch, signal => typeof spec.cli === 'string' && typeof spec.prompt === 'string' ? runAgentCli(spec.cli, workerInstruction(prompt, dispatch), dispatch.wake_context, spec.model, undefined, signal, 'llm') : Promise.resolve({ exit_code: null, stdout_tail: '', stderr_tail: 'llm step has no declared CLI' })); + const { result, usage } = workerSpend(completed, spec.model); let reason: CompletionReason = result.exit_code === 0 ? 'success' : 'worker_error'; let output: unknown = result.stdout_tail; let detail = result.stderr_tail; @@ -72,6 +75,7 @@ export class LlmWorker extends EventEmitter { await this.client.stepComplete(dispatch.run_id, dispatch.step_id, dispatch.attempt, dispatch.idempotency_key, reason, { output, + usage, ...(reason === 'success' ? {} : { trajectory_tail: { error: detail } }), }); } diff --git a/packages/sdk/src/model-pricing.ts b/packages/sdk/src/model-pricing.ts new file mode 100644 index 000000000..d828b6fdf --- /dev/null +++ b/packages/sdk/src/model-pricing.ts @@ -0,0 +1,15 @@ +/** Frozen dollars per million tokens. Each rate is an integer microdollar per token. */ +export const MODEL_PRICING: Readonly>> = Object.freeze({ + 'claude-sonnet-4-6': Object.freeze({ input: 3, output: 15 }), + 'claude-opus-4-7': Object.freeze({ input: 15, output: 75 }), + 'codex-medium': Object.freeze({ input: 2, output: 8 }), + 'codex-large': Object.freeze({ input: 5, output: 20 }), +}); + +export function pricedUsage(model: string | undefined, input = 0, output = 0) { + if (![input, output].every(n => Number.isSafeInteger(n) && n >= 0)) throw new Error('Invalid token usage'); + const price = model === undefined || !Object.hasOwn(MODEL_PRICING, model) ? undefined : MODEL_PRICING[model]; + if (model !== undefined && price === undefined && input + output > 0) throw new Error(`budget_missing_price: ${model}`); + const micro = BigInt(input) * BigInt(price?.input ?? 0) + BigInt(output) * BigInt(price?.output ?? 0); + return { tokens_in: input, tokens_out: output, dollars: `${micro / 1_000_000n}.${String(micro % 1_000_000n).padStart(6, '0')}` }; +} diff --git a/packages/sdk/src/preflight.ts b/packages/sdk/src/preflight.ts index 2e8f74258..7620d09f9 100644 --- a/packages/sdk/src/preflight.ts +++ b/packages/sdk/src/preflight.ts @@ -1,5 +1,7 @@ import type { FlowSpec, StepSpec, TriggerSpec, McpServerConfig } from './spec.js'; import { McpError, openMcpSession, type McpDiagnostic } from './mcp-client.js'; +import { BudgetSyntaxError } from './budget.js'; +import { budgetDiagnostics } from './budget-preflight.js'; import { acceptsAnyOutput, inspectStepGate, type StepGateInspection } from './gate-contract.js'; import { compileSpec, CompileError } from './compile.js'; import type { @@ -112,7 +114,7 @@ export interface PreflightResult { export function preflight(flow: FlowSpec, options: PreflightOptions & { mcpServers: readonly string[] }): Promise; export function preflight(flow: FlowSpec, options: PreflightOptions & { mcpServers?: undefined }): PreflightResult; export function preflight(flow: FlowSpec, options: PreflightOptions): PreflightResult | Promise; -export function preflight(flow: FlowSpec, options: PreflightOptions): PreflightResult | Promise { +export function preflight(flow: unknown, options: PreflightOptions): PreflightResult | Promise { const result = preflightSync(flow, options); if (options.mcpServers === undefined) return result; return probeMcp(result, options); @@ -135,7 +137,7 @@ async function probeMcp(result: PreflightResult, options: PreflightOptions): Pro return { ...result, ok: !result.diagnostics.some(d => d.severity === 'refusal'), mcpTools: Object.freeze(inventory) }; } -function preflightSync(flow: FlowSpec, options: PreflightOptions): PreflightResult { +function preflightSync(flow: unknown, options: PreflightOptions): PreflightResult { // Compile before touching any environment fact. `compileSpec` snapshots raw // input into inert data, validates it against the closed authoring schema, // and lowers `output` sugar into its json_schema gate — so the gate plan @@ -143,7 +145,7 @@ function preflightSync(flow: FlowSpec, options: PreflightOptions): PreflightResu // inspection ever reads a live accessor. The failure is a named refusal // rather than a thrown error (RFC covenant 2), which is the contract main // settled for this boundary. - let compiled: FlowSpec; + let compiled: import('./compile.js').CompiledFlowSpec; try { compiled = compileSpec(flow); } catch (error) { @@ -156,7 +158,7 @@ function preflightSync(flow: FlowSpec, options: PreflightOptions): PreflightResu resolutions: [], diagnostics: [{ severity: 'refusal', - kind: 'invalid_spec', + kind: error instanceof BudgetSyntaxError ? 'budget_syntax_invalid' : 'invalid_spec', message: `Relayflow spec is invalid: ${errors.join('; ')}`, errors, }], @@ -168,11 +170,12 @@ function preflightSync(flow: FlowSpec, options: PreflightOptions): PreflightResu const cliProbeResults = new Map(); diagnostics.push(...unknownModelDiagnostics(compiled, options)); - for (const server of new Set(options.mcpServers)) { + for (const server of new Set(options.mcpServers ?? [])) { if (options.mcp !== undefined && Object.hasOwn(options.mcp, server)) continue; diagnostics.push({ severity: 'refusal', kind: 'mcp_undeclared_server', server, message: `MCP server "${server}" is not declared in the nearest flows.json mcp map.` }); } + diagnostics.push(...budgetDiagnostics(compiled)); // Resolve the complete flow before touching any environment fact. A later // statically unresolved CLI makes the whole submission impossible, so no // earlier command, provider/model, or trigger probe may run first. diff --git a/packages/sdk/src/protocol.ts b/packages/sdk/src/protocol.ts index e92fd6731..0679b5eb6 100644 --- a/packages/sdk/src/protocol.ts +++ b/packages/sdk/src/protocol.ts @@ -69,6 +69,13 @@ export interface HelloResult { server: string; } +export interface StepSpend { + tokens_input: number; + tokens_output: number; + dollars: number; + wallclock_ms: number; +} + export interface RunStartParams { reuse_from_run_id?: string; /** diff --git a/packages/sdk/src/spec.ts b/packages/sdk/src/spec.ts index 39444b497..895ddeb21 100644 --- a/packages/sdk/src/spec.ts +++ b/packages/sdk/src/spec.ts @@ -100,6 +100,11 @@ export interface PermissionsSpec { * are integers. */ export interface BudgetSpec { + /** Set when normalizing the surface budget header. Legacy explicit envelopes accept worker-supplied prices. */ + pricing?: 'frozen'; + maxTokens?: number; + maxWallclockMs?: number; + window?: 'day'; maxTokensIn?: number; maxTokensOut?: number; /** Decimal string, e.g. "1.50". */ @@ -288,7 +293,7 @@ export interface FlowSpec { /** Declarations checked by preflight; gate 1 never dispatches them. */ triggers?: TriggerSpec[]; steps: StepSpec[]; - budget?: BudgetSpec; + budget?: BudgetSpec | import('./budget.js').HeaderBudget; } /** Current spec schema version emitted by this SDK. */ @@ -370,6 +375,11 @@ export interface KernelAgentStep extends KernelStepCommon { export type KernelStepSpec = KernelDeterministicStep | KernelLlmStep | KernelAgentStep; export interface KernelBudgetSpec { + pricing?: 'frozen'; + prior_spend?: { tokens_in: number; tokens_out: number; dollars: string; wallclock_ms: number; day?: number }; + max_tokens?: number; + max_wallclock_ms?: number; + window?: 'day'; max_tokens_in?: number; max_tokens_out?: number; max_dollars?: string; diff --git a/packages/sdk/src/validate.ts b/packages/sdk/src/validate.ts index a5cb0184a..bce5be7a3 100644 --- a/packages/sdk/src/validate.ts +++ b/packages/sdk/src/validate.ts @@ -69,7 +69,7 @@ function isCanonicalPathSurface(value: unknown): value is string { // unknown keys (AGENTS.md rule 4; RFC covenant 2): a typo'd key like // `depends_on` must be an error naming the nearest valid key, never a // silently discarded field — silently dropping `dependsOn` loses ordering. -const BUDGET_KEYS = ['maxTokensIn', 'maxTokensOut', 'maxDollars'] as const; +const BUDGET_KEYS = ['maxTokensIn', 'maxTokensOut', 'maxDollars', 'maxTokens', 'maxWallclockMs', 'window', 'pricing'] as const; const VERIFICATION_KEYS: Record = { exit_code: ['type', 'expect'], output_contains: ['type', 'value'], @@ -210,8 +210,13 @@ class Validator { this.fail(`${at}: expected an object`); return; } - this.checkKeys(b, BUDGET_KEYS, at); + this.checkKeys(b, at === 'spec.budget' ? BUDGET_KEYS : ['maxTokensIn', 'maxTokensOut', 'maxDollars'], at); const budget = b as BudgetSpec; + for (const key of ['maxTokens', 'maxWallclockMs'] as const) { + if (budget[key] !== undefined && (!Number.isSafeInteger(budget[key]) || budget[key]! < 0)) this.fail(`${at}.${key}: expected a safe non-negative integer`); + } + if (budget.pricing !== undefined && budget.pricing !== 'frozen') this.fail(`${at}.pricing: expected frozen`); + if (budget.window !== undefined && budget.window !== 'day') this.fail(`${at}.window: expected day`); if ( budget.maxTokensIn !== undefined && !isNonNegInt(budget.maxTokensIn) diff --git a/packages/sdk/src/worker-cli.ts b/packages/sdk/src/worker-cli.ts index f72fd2313..f5d7ffb8d 100644 --- a/packages/sdk/src/worker-cli.ts +++ b/packages/sdk/src/worker-cli.ts @@ -1,3 +1,4 @@ +import { decodeProviderResult, decodeWrapperResult, requirePricedUsage } from './worker-usage.js'; import { spawn } from 'node:child_process'; import { childStop, ownsProcessGroup } from './child-stop.js'; import { @@ -23,6 +24,8 @@ export const WAKE_CONTEXT_ENV = 'RELAYFLOW_WAKE_CONTEXT'; export const MODEL_ENV = 'RELAYFLOW_MODEL'; export interface WorkerCliResult { + tokens_input?: number; + tokens_output?: number; exit_code: number | null; stdout_tail: string; stderr_tail: string; @@ -44,7 +47,7 @@ export async function runAgentCli( const kind = cliAdapterKind(cli); if (kind === 'relayflows-wrapper-v1') { - return runWrapperSession( + return requirePricedUsage(decodeWrapperResult(await runWrapperSession( cli, instruction, wakeContext, @@ -52,7 +55,7 @@ export async function runAgentCli( wrapperEnvironment(process.env), wrapperLimits, signal, - ); + )), model); } const env: NodeJS.ProcessEnv = { ...process.env }; @@ -73,7 +76,10 @@ export async function runAgentCli( } if (invocation.modelEnv !== undefined) env[MODEL_ENV] = invocation.modelEnv; - return spawnInvocation(cli, invocation, env, signal); + // Structured provider output carries the authoritative token counts. + const args = [...invocation.args]; + args.splice(args.length - 1, 0, ...(kind === 'claude' ? ['--output-format', 'json'] : ['--json'])); + return requirePricedUsage(decodeProviderResult(await spawnInvocation(cli, { ...invocation, args }, env, signal), kind), model); } function spawnInvocation( diff --git a/packages/sdk/src/worker-spend.ts b/packages/sdk/src/worker-spend.ts new file mode 100644 index 000000000..b6375da63 --- /dev/null +++ b/packages/sdk/src/worker-spend.ts @@ -0,0 +1,15 @@ +import { pricedUsage } from './model-pricing.js'; +import type { WorkerCliResult } from './worker-cli.js'; + +/** Pricing failures are journaled worker errors, never dropped completions. */ +export function workerSpend(result: WorkerCliResult, model?: string) { + try { return { result, usage: pricedUsage(model, result.tokens_input, result.tokens_output) }; } + catch (error) { + return { + result: { ...result, exit_code: null, stderr_tail: error instanceof Error ? error.message : 'Invalid model usage' }, + usage: pricedUsage(undefined, + Number.isSafeInteger(result.tokens_input) && result.tokens_input! >= 0 ? result.tokens_input : 0, + Number.isSafeInteger(result.tokens_output) && result.tokens_output! >= 0 ? result.tokens_output : 0), + }; + } +} diff --git a/packages/sdk/src/worker-usage.ts b/packages/sdk/src/worker-usage.ts new file mode 100644 index 000000000..860e7efea --- /dev/null +++ b/packages/sdk/src/worker-usage.ts @@ -0,0 +1,48 @@ +import type { WorkerCliResult } from './worker-cli.js'; +import { MODEL_PRICING } from './model-pricing.js'; + +type RecordValue = Record; +const record = (value: unknown): value is RecordValue => typeof value === 'object' && value !== null && !Array.isArray(value); +const count = (value: unknown): value is number => typeof value === 'number' && Number.isSafeInteger(value) && value >= 0; + +function invalid(result: WorkerCliResult, detail: string): WorkerCliResult { + return { ...result, exit_code: null, stderr_tail: `${result.stderr_tail}\n${detail}`.trim() }; +} + +function usageResult(result: WorkerCliResult, usage: unknown, output: unknown): WorkerCliResult { + if (output === undefined) return invalid(result, 'Provider completion is missing its output.'); + if (!record(usage) || !count(usage.input_tokens) || !count(usage.output_tokens)) { + return invalid(result, 'Provider completion has invalid token usage.'); + } + return { ...result, stdout_tail: typeof output === 'string' ? output : JSON.stringify(output), + tokens_input: usage.input_tokens, tokens_output: usage.output_tokens }; +} + +/** Provider envelopes are execution metadata, never the authored output. */ +export function decodeProviderResult(result: WorkerCliResult, kind: 'claude' | 'codex'): WorkerCliResult { + const frames: RecordValue[] = []; + for (const line of result.stdout_tail.split('\n')) { + try { const frame: unknown = JSON.parse(line); if (record(frame)) frames.push(frame); } catch { /* Plain text remains output. */ } + } + const terminal = [...frames].reverse().find(f => kind === 'claude' ? f.type === 'result' : f.type === 'turn.completed'); + if (terminal === undefined) return result; + const text = kind === 'claude' ? terminal.result : frames + .flatMap(f => f.type === 'item.completed' && record(f.item) && f.item.type === 'agent_message' && typeof f.item.text === 'string' ? [f.item.text] : []).join('\n'); + return usageResult(result, terminal.usage, text); +} + +/** Optional wrapper result envelope; existing opaque text output stays valid. */ +export function decodeWrapperResult(result: WorkerCliResult): WorkerCliResult { + let value: unknown; + try { value = JSON.parse(result.stdout_tail); } catch { return result; } + if (!record(value) || value.protocol !== 'relayflows-agent-cli-v1-result') return result; + return usageResult(result, value.usage, value.output); +} + +export function requirePricedUsage(result: WorkerCliResult, model?: string): WorkerCliResult { + if (model !== undefined && Object.hasOwn(MODEL_PRICING, model) + && (result.tokens_input === undefined || result.tokens_output === undefined)) { + return invalid(result, 'Priced model completion is missing token usage.'); + } + return result; +} diff --git a/packages/sdk/src/worker.ts b/packages/sdk/src/worker.ts index 67cc5ad15..ff0dad015 100644 --- a/packages/sdk/src/worker.ts +++ b/packages/sdk/src/worker.ts @@ -1,3 +1,5 @@ +import { workerSpend } from './worker-spend.js'; +import type { WorkerCliResult } from './worker-cli.js'; import { EventEmitter } from 'node:events'; import type { JournalClient } from './journal-client.js'; import type { Pins, StepDispatchEvent } from './protocol.js'; @@ -92,10 +94,11 @@ export class AgentWorker extends EventEmitter { private async execute(dispatch: StepDispatchEvent): Promise { const spec = dispatch.spec as Partial; - const result = await withWorkerLease(this.client, dispatch, signal => + const completed: WorkerCliResult = await withWorkerLease(this.client, dispatch, signal => typeof spec.cli === 'string' && typeof spec.instruction === 'string' ? runAgentCli(spec.cli, workerInstruction(spec.instruction, dispatch), dispatch.wake_context, spec.model, undefined, signal) : Promise.resolve({ exit_code: null, stdout_tail: '', stderr_tail: 'agent step has no declared CLI' })); + const { result, usage } = workerSpend(completed, spec.model); const completionReason = result.exit_code === 0 ? 'success' : 'worker_error'; // Output shape: if the CLI's stdout parses as JSON, promote THAT @@ -121,6 +124,7 @@ export class AgentWorker extends EventEmitter { completionReason, { output, + usage, started_pins: dispatch.pins, end_pins: dispatch.pins, }, diff --git a/packages/sdk/tests/budget-attribution.test.ts b/packages/sdk/tests/budget-attribution.test.ts new file mode 100644 index 000000000..be628d77d --- /dev/null +++ b/packages/sdk/tests/budget-attribution.test.ts @@ -0,0 +1,53 @@ +import { EventEmitter } from 'node:events'; +import { describe, expect, it, vi } from 'vitest'; +import { decodeProviderResult, decodeWrapperResult, requirePricedUsage } from '../src/worker-usage.js'; +import { pricedUsage, MODEL_PRICING } from '../src/model-pricing.js'; +import { AuthoredBudget } from '../src/authored-budget.js'; +import type { JournalClient } from '../src/journal-client.js'; +import type { StepDispatchEvent } from '../src/protocol.js'; +import { LlmWorker } from '../src/llm-worker.js'; + +vi.mock('../src/worker-cli.js', () => ({runAgentCli: vi.fn(async () => ({exit_code: 0, stdout_tail: 'answer', stderr_tail: '', tokens_input: 1000, tokens_output: 200}))})); +vi.mock('../src/worker-lease.js', () => ({withWorkerLease: (_c: unknown, _d: unknown, run: (s: AbortSignal) => unknown) => run(new AbortController().signal)})); + +describe('budget attribution', () => { + it('threads mocked provider tokens through the worker completion', async () => { + const client = Object.assign(new EventEmitter(), {workerAttach: vi.fn(), stepComplete: vi.fn()}); + const worker = new LlmWorker(client as unknown as JournalClient, 'test'); + await worker.attach(); + client.emit('step.dispatch', {run_id:'r', step_id:'s', step_type:'llm', attempt:1, idempotency_key:'k', + spec:{type:'llm', prompt:'hello', cli:'claude', model:'claude-sonnet-4-6'}} as StepDispatchEvent); + await worker.close(); + expect(client.stepComplete.mock.calls[0]?.[5]).toMatchObject({output:'answer', usage:{tokens_in:1000, tokens_out:200, dollars:'0.006000'}}); + }); + it('extracts provider usage while preserving the authored output', () => { + for (const [kind, stdout] of [ + ['claude', JSON.stringify({type:'result', result:'answer', usage:{input_tokens:1000, output_tokens:200}})], + ['codex', [JSON.stringify({type:'item.completed', item:{type:'agent_message', text:'answer'}}), JSON.stringify({type:'turn.completed', usage:{input_tokens:1000, output_tokens:200}})].join('\n')], + ] as const) { + const result = decodeProviderResult({exit_code:0, stdout_tail:stdout, stderr_tail:''}, kind); + expect(result).toMatchObject({stdout_tail:'answer', tokens_input:1000, tokens_output:200}); + expect(pricedUsage('claude-sonnet-4-6', result.tokens_input, result.tokens_output).dollars).toBe('0.006000'); + } + expect(Object.isFrozen(MODEL_PRICING)).toBe(true); + expect(Object.isFrozen(MODEL_PRICING['codex-large'])).toBe(true); + }); + it('accepts explicit wrapper usage and refuses missing or malformed priced usage', () => { + const base = {exit_code: 0, stdout_tail: 'answer', stderr_tail: ''}; + expect(requirePricedUsage(base, 'codex-medium').exit_code).toBeNull(); + expect(decodeWrapperResult({...base, stdout_tail: JSON.stringify({protocol:'relayflows-agent-cli-v1-result', usage:{input_tokens:5, output_tokens:2}})}).exit_code).toBeNull(); + const wrapped = decodeWrapperResult({...base, stdout_tail: JSON.stringify({protocol:'relayflows-agent-cli-v1-result', output:'answer', usage:{input_tokens:5, output_tokens:2}})}); + expect(requirePricedUsage(wrapped, 'codex-medium')).toMatchObject({exit_code:0, stdout_tail:'answer', tokens_input:5, tokens_output:2}); + expect(decodeProviderResult({...base, stdout_tail: JSON.stringify({type:'result', result:'answer', usage:{input_tokens:-1, output_tokens:2}})}, 'claude').exit_code).toBeNull(); + }); + it('carries exact completed spend into the next authored step kernel run', async () => { + const budget = new AuthoredBudget('$0.001/run'); + const client = {runStart: vi.fn(async () => ({run_id:'r', status:'completed', completion_reason:'success', completed_steps:1})), + journalRead: vi.fn(async (_run: string, seq: number) => ({entries: seq === 1 ? [{seq:1, entry_type:'step.completed', at_ms:0, + payload:{budget:{tokens_in:1000, tokens_out:200, dollars:'0.006000'}, spend:{wallclock_ms:25}}}] : []}))}; + const spec = {version:'0.1.0', steps:[]}; + await budget.execute(client as unknown as JournalClient, spec, async () => 'answer'); + await budget.execute(client as unknown as JournalClient, spec, async () => 'answer'); + expect((client.runStart.mock.calls as unknown as [{budget:unknown}][])[1]?.[0].budget).toMatchObject({max_dollars:'0.001', prior_spend:{tokens_in:1000, tokens_out:200, dollars:'0.006000', wallclock_ms:25}}); + }); +}); diff --git a/packages/sdk/tests/budget-authored-live.test.ts b/packages/sdk/tests/budget-authored-live.test.ts new file mode 100644 index 000000000..2edbf10b7 --- /dev/null +++ b/packages/sdk/tests/budget-authored-live.test.ts @@ -0,0 +1,36 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { flow } from '@relayflows/surface'; +import { executeAuthoredFlow } from '../src/authored-flow-executor.js'; +import { chainFixture } from './flow-chain-fixture.js'; + +const cleanup: Array<() => Promise> = []; +afterEach(async () => { for (const close of cleanup.splice(0)) await close(); }); + +describe('authored budgets through the live kernel', () => { + it('retains the first completion and journals refusal of the next authored step', async () => { + const fixture = chainFixture(); + cleanup.push(() => fixture.close()); + const client = await fixture.connect(); + const start = vi.spyOn(client, 'runStart'); + const handle = flow('budgeted', {budget: {wallclock: '0ms'}}, async f => { + await f.run('sleep 0.01'); + await f.run('printf must-not-run'); + f.done('success'); + }); + await expect(executeAuthoredFlow(handle, client)).rejects.toMatchObject({completionReason:'budget_exceeded'}); + expect(start).toHaveBeenCalledTimes(2); + const first = await start.mock.results[0]!.value; + const refused = await start.mock.results[1]!.value; + const entries = (await client.journalRead(first.run_id,1)).entries; + expect(entries).toEqual(expect.arrayContaining([expect.objectContaining({entry_type:'step.completed',payload:expect.objectContaining({completionReason:'success',spend:expect.objectContaining({dollars:0})})})])); + const refusal = (await client.journalRead(refused.run_id,1)).entries; + expect(refusal).toEqual(expect.arrayContaining([expect.objectContaining({entry_type:'run.completed',payload:expect.objectContaining({completionReason:'budget_exceeded'})})])); + expect(refusal).not.toEqual(expect.arrayContaining([expect.objectContaining({entry_type:'step.attempt_started'})])); + }); + it('permits done after the last valid step crosses the limit', async () => { + const fixture = chainFixture(); cleanup.push(() => fixture.close()); + const client = await fixture.connect(); + const handle = flow('last', {budget:{wallclock:'0ms'}}, async f => { await f.run('sleep 0.01'); f.done('success'); }); + expect((await executeAuthoredFlow(handle,client)).completionReason).toBe('success'); + }); +}); diff --git a/packages/sdk/tests/budget-preflight.test.ts b/packages/sdk/tests/budget-preflight.test.ts new file mode 100644 index 000000000..aeb72baf7 --- /dev/null +++ b/packages/sdk/tests/budget-preflight.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it, vi } from 'vitest'; +import { preflight } from '../src/preflight.js'; +import { compileSpec, toKernelSpec, kernelToAuthoring } from '../src/compile.js'; +import { flow } from '@relayflows/surface'; +import { getFlowDefinition } from '@relayflows/surface/runtime'; + +const options = () => ({models: ['claude-sonnet-4-6', 'unknown'], probes: { + cli: vi.fn(() => ({ exists: true, authenticated: true, modelAvailable: true })), + executor: () => true, command: () => true, +}}); +const spec = (budget: unknown, model = 'claude-sonnet-4-6') => ({ version: '0.1.0', budget, + steps: [{ id: 'ask', type: 'llm', cli: 'claude', model, prompt: 'hello' }] }); + +describe('budget preflight', () => { + it('accepts the string header and lowers an exact daily envelope', () => { + expect(preflight(spec('$20/day'), options()).ok).toBe(true); + expect(toKernelSpec(compileSpec(spec('$0.10/run'))).budget).toEqual({ pricing: 'frozen', max_dollars: '0.10' }); + expect(toKernelSpec(compileSpec(spec('$20/day'))).budget).toEqual({ pricing: 'frozen', max_dollars: '20', window: 'day' }); + }); + it('accepts object limits and snapshots the surface header', () => { + const budget = { tokens: 100, dollars: 0.1, wallclock: '2m' }; + const handle = flow('budget', {budget}, async f => { f.done('success'); }); + budget.tokens = 999; + expect(getFlowDefinition(handle).header.budget).toEqual({ tokens: 100, dollars: 0.1, wallclock: '2m' }); + expect(preflight(spec(budget), options()).ok).toBe(true); + expect(toKernelSpec(compileSpec(spec(budget))).budget).toEqual({pricing: 'frozen', max_tokens: 999, max_dollars: '0.1', max_wallclock_ms: 120000}); + }); + it('refuses missing windows before any environment probe', () => { + const o = options(); + const result = preflight(spec('$20'), o); + expect(result.diagnostics.map(d => d.kind)).toEqual(['budget_syntax_invalid']); + expect(o.probes.cli).not.toHaveBeenCalled(); + }); + it('refuses an unpriced declared model before probing', () => { + const o = options(); + const result = preflight(spec('$20/run', 'unknown'), o); + expect(result.diagnostics.map(d => d.kind)).toEqual(['budget_missing_price']); + expect(o.probes.cli).not.toHaveBeenCalled(); + }); + it('retains frozen pricing when checking a compiled artifact', () => { + expect(preflight(kernelToAuthoring(toKernelSpec(compileSpec(spec('$20/run', 'unknown')))), options()).diagnostics) + .toEqual(expect.arrayContaining([expect.objectContaining({kind: 'budget_missing_price'})])); + }); + it('requires a model when declaring a dollar budget', () => { + const input = spec('$20/run'); + const { model, ...step } = input.steps[0]!; + expect(preflight({...input, steps:[step]}, options()).diagnostics) + .toEqual(expect.arrayContaining([expect.objectContaining({kind: 'budget_missing_price'})])); + }); + it.each(['$1/week', '-$1/run', '$1.0000001/run', {tokens: -1}, {wallclock: 'soon'}, {dollars: Infinity}, {typo: 2}])('refuses malformed budget %j', budget => { + expect(preflight(spec(budget), options()).ok).toBe(false); + }); +}); diff --git a/packages/sdk/tests/preflight.test.ts b/packages/sdk/tests/preflight.test.ts index c01150543..bf26ab44a 100644 --- a/packages/sdk/tests/preflight.test.ts +++ b/packages/sdk/tests/preflight.test.ts @@ -370,6 +370,8 @@ describe('preflight: CLI resolution and refusal predicates', () => { probes: probes(), mcpServers: ['missing-binary'], mcp: { 'missing-binary': { command: '/nonexistent-mcp-test-binary' } }, }), + preflight({ ...flow({ id: 'a', type: 'deterministic', command: 'x' }), budget: '$20' }, { probes: probes() }), + preflight({ ...flow({ id: 'a', type: 'llm', prompt: 'p', cli: 'x', model: 'unpriced' }), budget: '$20/run' }, { models: ['unpriced'], probes: probes() }), preflight({ version: '0.1.0', steps: [{ id: 'a', type: 'deterministic', command: 'x', prompt: 'cross-verb' }], diff --git a/packages/surface/src/flow.ts b/packages/surface/src/flow.ts index d5eb95f77..4013f7358 100644 --- a/packages/surface/src/flow.ts +++ b/packages/surface/src/flow.ts @@ -6,7 +6,7 @@ export interface FlowHeader { use?: string[]; identity?: string; memory?: { script?: boolean; agent?: boolean }; - budget?: string; + budget?: string | { tokens?: number; dollars?: number; wallclock?: string }; tools?: { slack?: boolean; relayfile?: string[]; mcp?: string[] }; workspace?: string; } @@ -17,7 +17,7 @@ export interface ReadonlyFlowHeader { readonly use?: readonly string[]; readonly identity?: string; readonly memory?: Readonly<{ script?: boolean; agent?: boolean }>; - readonly budget?: string; + readonly budget?: string | Readonly<{ tokens?: number; dollars?: number; wallclock?: string }>; readonly tools?: Readonly<{ slack?: boolean; relayfile?: readonly string[]; @@ -146,7 +146,7 @@ function freezeHeader(header: FlowHeader): ReadonlyFlowHeader { ...(header.use === undefined ? {} : { use: Object.freeze([...header.use]) }), ...(header.identity === undefined ? {} : { identity: header.identity }), ...(memory === undefined ? {} : { memory }), - ...(header.budget === undefined ? {} : { budget: header.budget }), + ...(header.budget === undefined ? {} : { budget: typeof header.budget === "string" ? header.budget : Object.freeze({ ...header.budget }) }), ...(tools === undefined ? {} : { tools }), ...(header.workspace === undefined ? {} : { workspace: header.workspace }), }); @@ -161,7 +161,14 @@ function assertFlowHeader(value: unknown, flowName: string): asserts value is Fl at, ); assertOptionalString(value, "identity", at); - assertOptionalString(value, "budget", at); + if (value.budget !== undefined && typeof value.budget !== "string") { + assertHeaderObject(value.budget, `${at}.budget`); + assertKnownKeys(value.budget, ["tokens", "dollars", "wallclock"], `${at}.budget`); + assertOptionalString(value.budget, "wallclock", at); + for (const key of ["tokens", "dollars"]) { + if (value.budget[key] !== undefined && typeof value.budget[key] !== "number") throw new TypeError(`budget_syntax_invalid: ${key} must be a number`); + } + } assertOptionalString(value, "workspace", at); assertOptionalStringArray(value, "use", at); if (value.use !== undefined) { diff --git a/testdata/budget-guarded.flow.yaml b/testdata/budget-guarded.flow.yaml new file mode 100644 index 000000000..360d5dc3c --- /dev/null +++ b/testdata/budget-guarded.flow.yaml @@ -0,0 +1,12 @@ +version: 0.1.0 +name: budget-guarded +budget: + wallclock: 0ms +steps: + - id: measured + type: deterministic + command: sleep 0.01 + - id: guarded + type: deterministic + dependsOn: [measured] + command: printf 'budget should refuse this step' From 3e682c41c89de50524676e13f03b451658e99683 Mon Sep 17 00:00:00 2001 From: kjgbot Date: Fri, 11 Sep 2026 11:59:35 +0200 Subject: [PATCH 2/4] fix(sdk): unpriced models return undefined usage instead of throwing after CLI decode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cursor Bugbot HIGH: "Unlisted models fail after usage decode" — the runtime `pricedUsage` threw `budget_missing_price` after the CLI had already spent tokens, wasting the exact CLI invocation that preflight was meant to prevent. Preflight's `budgetDiagnostics` already refuses declared dollar budgets against unpriced models before any CLI dispatches. This makes the runtime path match that: `pricedUsage` returns `undefined` for unpriced models, callers (`worker`, `llm-worker`) omit `usage` from the step-complete payload rather than sending `dollars: null` (which would break the kernel wire schema). No new kernel field, no wire change. Adds `packages/sdk/tests/model-pricing.test.ts` (8 assertions) covering: - priced/unpriced/undefined-model returns - invalid token counts still journal as `worker_error` - `workerSpend` returns undefined usage for unpriced models without failing the step Session-Id: efeda5df-9b7c-48d4-b2ce-957f5bef0a82 Session-Id: efeda5df-9b7c-48d4-b2ce-957f5bef0a82 Session-Id: efeda5df-9b7c-48d4-b2ce-957f5bef0a82 --- packages/sdk/src/llm-worker.ts | 2 +- packages/sdk/src/model-pricing.ts | 23 +++++++-- packages/sdk/src/worker-spend.ts | 10 +++- packages/sdk/src/worker.ts | 2 +- packages/sdk/tests/model-pricing.test.ts | 62 ++++++++++++++++++++++++ 5 files changed, 93 insertions(+), 6 deletions(-) create mode 100644 packages/sdk/tests/model-pricing.test.ts diff --git a/packages/sdk/src/llm-worker.ts b/packages/sdk/src/llm-worker.ts index 6a287d092..b004b947d 100644 --- a/packages/sdk/src/llm-worker.ts +++ b/packages/sdk/src/llm-worker.ts @@ -75,7 +75,7 @@ export class LlmWorker extends EventEmitter { await this.client.stepComplete(dispatch.run_id, dispatch.step_id, dispatch.attempt, dispatch.idempotency_key, reason, { output, - usage, + ...(usage !== undefined ? { usage } : {}), ...(reason === 'success' ? {} : { trajectory_tail: { error: detail } }), }); } diff --git a/packages/sdk/src/model-pricing.ts b/packages/sdk/src/model-pricing.ts index d828b6fdf..04672fe93 100644 --- a/packages/sdk/src/model-pricing.ts +++ b/packages/sdk/src/model-pricing.ts @@ -6,10 +6,27 @@ export const MODEL_PRICING: Readonly Number.isSafeInteger(n) && n >= 0)) throw new Error('Invalid token usage'); const price = model === undefined || !Object.hasOwn(MODEL_PRICING, model) ? undefined : MODEL_PRICING[model]; - if (model !== undefined && price === undefined && input + output > 0) throw new Error(`budget_missing_price: ${model}`); - const micro = BigInt(input) * BigInt(price?.input ?? 0) + BigInt(output) * BigInt(price?.output ?? 0); + if (price === undefined) return undefined; + const micro = BigInt(input) * BigInt(price.input) + BigInt(output) * BigInt(price.output); return { tokens_in: input, tokens_out: output, dollars: `${micro / 1_000_000n}.${String(micro % 1_000_000n).padStart(6, '0')}` }; } diff --git a/packages/sdk/src/worker-spend.ts b/packages/sdk/src/worker-spend.ts index b6375da63..9aae3d954 100644 --- a/packages/sdk/src/worker-spend.ts +++ b/packages/sdk/src/worker-spend.ts @@ -1,7 +1,15 @@ import { pricedUsage } from './model-pricing.js'; import type { WorkerCliResult } from './worker-cli.js'; -/** Pricing failures are journaled worker errors, never dropped completions. */ +/** + * Attach token/dollar usage to a worker's CLI result. Invalid token counts + * (non-integer or negative) are the one remaining failure mode — those are + * journaled as `worker_error` with the usage projected from clamped counts. + * + * Unpriced models are NOT a failure here: `pricedUsage` returns + * `dollars: null` and preflight (see `budgetDiagnostics`) has already refused + * declared dollar budgets against unpriced models before the CLI dispatched. + */ export function workerSpend(result: WorkerCliResult, model?: string) { try { return { result, usage: pricedUsage(model, result.tokens_input, result.tokens_output) }; } catch (error) { diff --git a/packages/sdk/src/worker.ts b/packages/sdk/src/worker.ts index ff0dad015..1cbb13e3c 100644 --- a/packages/sdk/src/worker.ts +++ b/packages/sdk/src/worker.ts @@ -124,7 +124,7 @@ export class AgentWorker extends EventEmitter { completionReason, { output, - usage, + ...(usage !== undefined ? { usage } : {}), started_pins: dispatch.pins, end_pins: dispatch.pins, }, diff --git a/packages/sdk/tests/model-pricing.test.ts b/packages/sdk/tests/model-pricing.test.ts new file mode 100644 index 000000000..bc0c0182b --- /dev/null +++ b/packages/sdk/tests/model-pricing.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it } from 'vitest'; +import { pricedUsage, hasPricing, MODEL_PRICING } from '../src/model-pricing.js'; +import { workerSpend } from '../src/worker-spend.js'; + +describe('pricedUsage', () => { + it('prices a listed model to microdollar-exact dollars', () => { + const usage = pricedUsage('claude-sonnet-4-6', 1_000_000, 500_000); + expect(usage).toEqual({ tokens_in: 1_000_000, tokens_out: 500_000, dollars: '10.500000' }); + }); + + it('returns undefined for an unpriced model rather than throwing after decode', () => { + // Regression for the Cursor Bugbot HIGH finding "Unlisted models fail + // after usage decode": the runtime previously threw here after the CLI + // had already spent tokens. Preflight now owns the refusal; the runtime + // path is permissive so an unpriced-model step wastes nothing on the + // pricing check itself. + expect(pricedUsage('unlisted-model', 100, 50)).toBeUndefined(); + expect(pricedUsage('unlisted-model', 0, 0)).toBeUndefined(); + }); + + it('returns undefined when no model is declared', () => { + expect(pricedUsage(undefined, 100, 50)).toBeUndefined(); + }); + + it('rejects invalid token counts', () => { + expect(() => pricedUsage('claude-sonnet-4-6', -1, 0)).toThrow(/Invalid token usage/); + expect(() => pricedUsage('claude-sonnet-4-6', 0, 1.5)).toThrow(/Invalid token usage/); + }); + + it('hasPricing agrees with MODEL_PRICING membership', () => { + for (const key of Object.keys(MODEL_PRICING)) expect(hasPricing(key)).toBe(true); + expect(hasPricing('unlisted-model')).toBe(false); + expect(hasPricing(undefined)).toBe(false); + }); +}); + +describe('workerSpend', () => { + const priced = { exit_code: 0, stdout_tail: '', stderr_tail: '', tokens_input: 100, tokens_output: 50 }; + + it('attaches usage for priced models', () => { + const spent = workerSpend(priced, 'claude-sonnet-4-6'); + expect(spent.usage).toEqual({ tokens_in: 100, tokens_out: 50, dollars: '0.001050' }); + expect(spent.result.exit_code).toBe(0); + }); + + it('leaves usage undefined for an unpriced model without failing the step', () => { + // The step still succeeded — an unpriced model is a preflight concern + // when a dollar budget is declared, not a runtime failure per se. + const spent = workerSpend(priced, 'unlisted-model'); + expect(spent.usage).toBeUndefined(); + expect(spent.result.exit_code).toBe(0); + expect(spent.result.stderr_tail).toBe(''); + }); + + it('journals invalid token counts as worker_error, projecting clamped counts', () => { + const bad = { ...priced, tokens_input: -1 }; + const spent = workerSpend(bad, 'claude-sonnet-4-6'); + expect(spent.result.exit_code).toBeNull(); + expect(spent.result.stderr_tail).toMatch(/Invalid token usage/); + expect(spent.usage).toBeUndefined(); + }); +}); From b67e80bc28b908b62503b99f322899d6acb02fef Mon Sep 17 00:00:00 2001 From: kjgbot Date: Fri, 11 Sep 2026 12:42:34 +0200 Subject: [PATCH 3/4] fix(sdk): wrap parseBudget throws as CompileError with kind compileSpec's own JSDoc claims 'Throws CompileError on validation failure,' but a bare BudgetSyntaxError from parseBudget escaped that contract and surfaced as an uncaught exception in callers that only catch CompileError. Preserve the diagnostic classification by adding an optional kind to CompileError and threading budget_syntax_invalid through it; preflight's existing BudgetSyntaxError instanceof branch stays as a fallback for callers that reach parseBudget outside compileSpec. Session-Id: efeda5df-9b7c-48d4-b2ce-957f5bef0a82 Session-Id: efeda5df-9b7c-48d4-b2ce-957f5bef0a82 Session-Id: efeda5df-9b7c-48d4-b2ce-957f5bef0a82 --- packages/sdk/src/compile.ts | 20 ++++++++++++++++++-- packages/sdk/src/preflight.ts | 10 +++++++++- packages/sdk/tests/budget-preflight.test.ts | 9 ++++++++- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/packages/sdk/src/compile.ts b/packages/sdk/src/compile.ts index 701f04107..502f34890 100644 --- a/packages/sdk/src/compile.ts +++ b/packages/sdk/src/compile.ts @@ -43,10 +43,13 @@ import { snapshotJsonValue } from './json-value.js'; export class CompileError extends Error { readonly errors: string[]; - constructor(errors: string[]) { + /** Optional diagnostic kind for callers that classify refusals (e.g. preflight). */ + readonly kind?: string; + constructor(errors: string[], kind?: string) { super('spec compile failed:\n - ' + errors.join('\n - ')); this.name = 'CompileError'; this.errors = errors; + if (kind !== undefined) this.kind = kind; } } @@ -83,7 +86,20 @@ export function compileSpec(spec: unknown): CompiledFlowSpec { ]); } if (snapshot !== null && typeof snapshot === 'object' && !Array.isArray(snapshot) && 'budget' in snapshot && snapshot.budget !== undefined) { - snapshot = { ...snapshot, budget: parseBudget(snapshot.budget) }; + // parseBudget throws BudgetSyntaxError on any malformed header. Without + // this wrap, that throw escaped compileSpec's own CompileError contract, + // so callers (validate, cli/check) that only catch CompileError would + // surface the budget error as an uncaught exception instead of a + // diagnostic. Rewrap as CompileError so it flows through the same + // gate-1 refusal path as every other invalid spec. + try { + snapshot = { ...snapshot, budget: parseBudget(snapshot.budget) }; + } catch (error) { + throw new CompileError( + [`spec.budget: ${error instanceof Error ? error.message : 'budget_syntax_invalid'}`], + 'budget_syntax_invalid', + ); + } } const validation: ValidationResult = validateSpec(snapshot); if (!validation.ok) throw new CompileError(validation.errors); diff --git a/packages/sdk/src/preflight.ts b/packages/sdk/src/preflight.ts index 7620d09f9..9b63e4714 100644 --- a/packages/sdk/src/preflight.ts +++ b/packages/sdk/src/preflight.ts @@ -152,13 +152,21 @@ function preflightSync(flow: unknown, options: PreflightOptions): PreflightResul const errors = error instanceof CompileError ? error.errors : [error instanceof Error ? error.message : 'spec: expected JSON-compatible data']; + // BudgetSyntaxError now flows through CompileError with kind on it; the + // legacy raw-throw instanceof is preserved as a fallback so a caller that + // constructs preflight input through a different path still classifies. + const kind: PreflightDiagnostic['kind'] = error instanceof CompileError && error.kind === 'budget_syntax_invalid' + ? 'budget_syntax_invalid' + : error instanceof BudgetSyntaxError + ? 'budget_syntax_invalid' + : 'invalid_spec'; return { ok: false, gates: [], resolutions: [], diagnostics: [{ severity: 'refusal', - kind: error instanceof BudgetSyntaxError ? 'budget_syntax_invalid' : 'invalid_spec', + kind, message: `Relayflow spec is invalid: ${errors.join('; ')}`, errors, }], diff --git a/packages/sdk/tests/budget-preflight.test.ts b/packages/sdk/tests/budget-preflight.test.ts index aeb72baf7..f6590a959 100644 --- a/packages/sdk/tests/budget-preflight.test.ts +++ b/packages/sdk/tests/budget-preflight.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; import { preflight } from '../src/preflight.js'; -import { compileSpec, toKernelSpec, kernelToAuthoring } from '../src/compile.js'; +import { compileSpec, CompileError, toKernelSpec, kernelToAuthoring } from '../src/compile.js'; import { flow } from '@relayflows/surface'; import { getFlowDefinition } from '@relayflows/surface/runtime'; @@ -50,4 +50,11 @@ describe('budget preflight', () => { it.each(['$1/week', '-$1/run', '$1.0000001/run', {tokens: -1}, {wallclock: 'soon'}, {dollars: Infinity}, {typo: 2}])('refuses malformed budget %j', budget => { expect(preflight(spec(budget), options()).ok).toBe(false); }); + it.each(['$$$', '$1/week', {typo: 2}])('compileSpec wraps parseBudget throws as a CompileError with a budget-scoped message (%j)', budget => { + let caught: unknown; + try { compileSpec(spec(budget)); } catch (error) { caught = error; } + expect(caught).toBeInstanceOf(CompileError); + expect((caught as CompileError).errors[0]).toMatch(/^spec\.budget: /); + expect((caught as CompileError).errors[0]).toContain('budget_syntax_invalid'); + }); }); From ab16f18e5d645a9461ef5d1da6e506dc520074e4 Mon Sep 17 00:00:00 2001 From: kjgbot Date: Fri, 11 Sep 2026 15:56:42 +0200 Subject: [PATCH 4/4] fix(check): whitelist all known header fields (budget, identity, memory, workspace, use) in checkMcpHeader MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit checkMcpHeader only filtered out 'tools' before, so authored flows with any other legitimate header (budget, identity, memory, use, workspace) were rejected as 'unsupported header fields' — including this PR's own budget-authored-live tests. Session-Id: efeda5df-9b7c-48d4-b2ce-957f5bef0a82 --- packages/sdk/src/cli/check-typescript.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/sdk/src/cli/check-typescript.ts b/packages/sdk/src/cli/check-typescript.ts index 356eb42ed..82fc5c66f 100644 --- a/packages/sdk/src/cli/check-typescript.ts +++ b/packages/sdk/src/cli/check-typescript.ts @@ -26,7 +26,8 @@ export async function checkMcpHeader( path: string, ): Promise { const empty = { servers: Object.freeze({}), inventory: Object.freeze({}) }; - const unsupported = Object.keys(definition.header).filter(key => key !== 'tools'); + const KNOWN_HEADER_FIELDS = new Set(['tools', 'budget', 'identity', 'memory', 'workspace', 'use']); + const unsupported = Object.keys(definition.header).filter(key => !KNOWN_HEADER_FIELDS.has(key)); if (definition.header.tools?.relayfile !== undefined) unsupported.push('tools.relayfile'); if (unsupported.length) { return { ...empty, report: inputFailureReport({ kind: 'invalid_spec',