diff --git a/docs/BUDGET.md b/docs/BUDGET.md new file mode 100644 index 000000000..4f2dd9e4d --- /dev/null +++ b/docs/BUDGET.md @@ -0,0 +1,61 @@ +# Budget headers and spend + +`budget` is optional. A flow may declare `"$0.10/run"`, `"$20/day"`, or +`{ tokens: 10000, dollars: 0.10, wallclock: "2m" }`. Objects default to a run +window. Tokens count input plus output; wallclock sums attempt durations from +journaled start to completion. Duration units are `ms`, `s`, `m`, `h`, and `d`. +Limits are non-negative; dollars support up to six decimal places. + +A day is a UTC calendar day within a run. Completed spend resets for admission +at the next UTC day; unrelated runs do not share a global account. Header +syntax errors refuse as `budget_syntax_invalid`. Declared models without a +frozen price refuse as `budget_missing_price`; dollar budgets also require a +model on each worker step. Existing project model allowlist checks still apply. + +Every newly written `step.completed` includes: + +```json +"spend": { + "tokens_input": 1000, + "tokens_output": 200, + "dollars": 0.006, + "wallclock_ms": 25 +} +``` + +The frozen table in `packages/sdk/src/model-pricing.ts` quotes dollars per +million tokens. Workers compute integer microdollars: +`inputTokens * inputPrice + outputTokens * outputPrice`. The existing `budget` +field retains exact decimal dollars; only the journal's `spend.dollars` becomes +a JSON number. Memory-injected costs retain their existing single charge. +Deterministic steps have zero model tokens and dollars, with measured duration. + +The kernel checks accumulated spend before each new attempt. Equality is +permitted. Crossing a limit keeps that completion valid and refuses the next +start with `run.completed.completionReason: "budget_exceeded"`. Running peers +may finish; no new peers start. A final successful step may cross a limit and +still complete its run successfully. Replay reconstructs accounting from the +journal, including retries and prior epochs. + +The internal TypeScript executor currently lowers each authored step to its +own kernel run. For budgeted flows it serializes admission and carries exact +journaled costs into the next run's `budget.prior_spend`. Its generated terminal +marker does not consume the author's step budget. This does not add a durable +TypeScript root or change that runner's existing resume contract. + +Raw Claude/Codex adapters request structured output to extract usage. A custom +wrapper may return an explicit result envelope after its execution handshake: + +```json +{"protocol":"relayflows-agent-cli-v1-result","output":"answer","usage":{"input_tokens":1000,"output_tokens":200}} +``` + +A priced model with missing or malformed usage produces a journaled worker +error. Legacy `maxTokensIn` / `maxTokensOut` / `maxDollars` envelopes keep their +worker-supplied pricing contract, including existing synthetic test models. +New surface headers carry `pricing: "frozen"` through compiled artifacts so +checking a compiled flow preserves the same missing-price refusal. + +`testdata/budget-guarded.flow.yaml` is a local smoke using a zero-duration budget: +its first completed command exceeds the ceiling and its dependent command +never starts. diff --git a/docs/SURFACE.md b/docs/SURFACE.md index 52c77c7df..09812de16 100644 --- a/docs/SURFACE.md +++ b/docs/SURFACE.md @@ -87,7 +87,7 @@ No process runs between events: the handler wakes, executes to its next await, p [the generator notes](../packages/surface/src/helpers/README.md). 4. **`{{prev}}` / return-value chaining.** Output flows downward implicitly; naming steps is for reaching back, not bookkeeping. -5. **Headers are optional escalation.** identity, memory, budget, tools appear only when used. The empty header is the common case. +5. **Headers are optional escalation.** identity, memory, budget, tools appear only when used. The empty header is the common case. [Budget headers and spend](BUDGET.md) specifies parsing, prices, journal attribution, and admission limits. 6. **Agent definitions escalate by composition** — and a reusable agent *is* a flow: ```yaml - agent: Review this diff for security issues. # 1. anonymous diff --git a/evidence/spec-G-budget/PR.md b/evidence/spec-G-budget/PR.md new file mode 100644 index 000000000..77a429cd4 --- /dev/null +++ b/evidence/spec-G-budget/PR.md @@ -0,0 +1,36 @@ +Title: feat(surface,sdk,kernel): budget header + spend attribution (SURFACE §2 rule 5) + +The `budget:` header was declared in surface examples but never enforced. This +change accepts string and object headers, refuses invalid syntax and missing +model prices before execution, and records tokens, dollars, and elapsed attempt +time on every completion. Once completed spend crosses a limit, the kernel +preserves completed work and refuses the next start with `budget_exceeded`. + +Parent issue: the lead will insert the issue link when opening this PR. + +The SDK uses the frozen model-pricing table and integer microdollars; the kernel +keeps exact decimal accounting through replay. UTC day windows, total-token and +wallclock limits are supported. The internal authored executor carries journaled +spend between its existing per-step kernel runs. Legacy explicit envelopes keep +their worker-supplied price contract. See [the budget contract](docs/BUDGET.md). + +Fixture: [budget-guarded.flow.yaml](testdata/budget-guarded.flow.yaml). Its +`measured` command completes; `guarded` never starts. Actual journal excerpt: + +```json +{"entry_type":"step.completed","step_id":"measured","payload":{"completionReason":"success","spend":{"tokens_input":0,"tokens_output":0,"dollars":0,"wallclock_ms":19}}} +{"entry_type":"run.completed","payload":{"completionReason":"budget_exceeded"}} +``` + +Validation commands and complete captured output are in +[evidence/spec-G-budget](evidence/spec-G-budget/README.md). Kernel workspace, +surface tests, and focused SDK tests pass. Plain `npm test` remains blocked by +this host's missing Claude login for the existing live analyzer acceptance test; +that failure has not been skipped or weakened. A timing-sensitive existing +lease-wait test failed in one full run and passed in the subsequent focused run. +The two new exhaustive refusal scenarios were added with lead approval; the +existing assertion is unchanged. + +The lead owns PR creation, CI verification, and bot review because GitHub +credentials on the worker return HTTP 401. Do not mark ready for merge until +those checks and the live analyzer acceptance requirement are satisfied. diff --git a/evidence/spec-G-budget/README.md b/evidence/spec-G-budget/README.md new file mode 100644 index 000000000..3a9c4e939 --- /dev/null +++ b/evidence/spec-G-budget/README.md @@ -0,0 +1,97 @@ +# Spec G verification and handoff + +This branch implements budget headers, frozen SDK pricing, per-completion spend, +and kernel admission limits. The lead approved adding two scenarios to the +existing exhaustive refusal test without changing its assertion, and preserving +legacy explicit-budget synthetic model behavior (Relay message +`224082264437923840`). The same message directs the worker to push and hand off +PR creation because fleet GitHub credentials return HTTP 401. + +## Commands and captured output + +Only trailing whitespace was normalized in the captured text logs for git. + +From `kernel`: + +```sh +PATH=/Users/khaliqgant/.cargo/bin:$PATH cargo test --workspace +``` + +Exit 0. Full literal output: [cargo-test.log](cargo-test.log). New integration +coverage output: + +```text +running 3 tests +test daily_windows_reset_and_exact_limits_do_not_refuse ... ok +test crossing_completion_is_durable_and_next_step_is_refused ... ok +test deterministic_spend_and_wallclock_limit_gate_parallel_batch_starts ... ok +``` + +From `packages/sdk`: + +```sh +PATH=/Users/khaliqgant/.cargo/bin:$PATH npm test +``` + +The full suite is not green on this host. Full output is retained in +[sdk-npm-test.log](sdk-npm-test.log): + +```text + Test Files 1 failed | 53 passed | 1 skipped (55) + Tests 1 failed | 984 passed | 3 skipped (988) +``` + +The real analyzer requires a Claude login; +`claude auth status` returned exit 1; its full output is +[claude-auth-status.log](claude-auth-status.log). + +An [earlier full run](sdk-npm-test-earlier.log) also failed the existing lease-wait timing assertion. The +entire CLI test file subsequently passed in the focused rerun below. Neither +assertion was modified and the analyzer was not skipped with an environment +flag. + +```sh +PATH=/Users/khaliqgant/.cargo/bin:$PATH ./node_modules/.bin/vitest run tests/cli.test.ts tests/budget-preflight.test.ts tests/budget-attribution.test.ts tests/budget-authored-live.test.ts tests/preflight.test.ts +``` + +Final focused output: [sdk-focused.log](sdk-focused.log). + +```text + Test Files 5 passed (5) + Tests 109 passed (109) +``` + +From `packages/surface`, `npm test` could not launch because `bun` is absent. +Its build, test typecheck, and test commands were run directly: + +```sh +npm run build +./node_modules/.bin/tsc -p tsconfig.test.json +./node_modules/.bin/vitest run +``` + +All three exited 0; test output: [surface-test.log](surface-test.log). + +```text + Test Files 1 passed (1) + Tests 7 passed (7) +``` + +## Declarative smoke + +The fixture `testdata/budget-guarded.flow.yaml` was compiled with the SDK's +`compileYaml` and `toKernelSpec` into `/tmp/spec-g-smoke.json`, then run from the +repository root: + +```sh +kernel/target/debug/relayflowd --data-dir /tmp/spec-g-smoke-data run /tmp/spec-g-smoke.json +``` + +Exit 1 is the expected budget refusal. The emitted spec and captured outcome are +[smoke.spec.json](smoke.spec.json) and [smoke.log](smoke.log). The actual SQLite +journal excerpts are [smoke-journal.json](smoke-journal.json): `measured` completed +successfully with 19ms spend; `guarded` never started; the run completed with +`budget_exceeded`. + +PR creation, CI checks and review-bot triage are handed to the lead. This evidence +does not claim required CI passed or that the branch is ready for merge. diff --git a/evidence/spec-G-budget/cargo-test.log b/evidence/spec-G-budget/cargo-test.log new file mode 100644 index 000000000..59c8b6ba4 --- /dev/null +++ b/evidence/spec-G-budget/cargo-test.log @@ -0,0 +1,361 @@ + Compiling relayflowd-core v0.1.0 (/Users/khaliqgant/flows-spec-G-budget/kernel/relayflowd-core) + Compiling relayflowd-journal v0.1.0 (/Users/khaliqgant/flows-spec-G-budget/kernel/relayflowd-journal) + Compiling relayflowd v0.1.0 (/Users/khaliqgant/flows-spec-G-budget/kernel/relayflowd) + Finished `test` profile [unoptimized + debuginfo] target(s) in 5.96s + Running unittests src/lib.rs (target/debug/deps/relayflowd-3287862779f564ca) + +running 40 tests +test engine::remote::worker_failure_detail_tests::a_null_or_blank_output_yields_no_detail ... ok +test engine::remote::worker_failure_detail_tests::a_string_output_is_carried_verbatim_and_trimmed ... ok +test engine::remote::worker_failure_detail_tests::an_output_at_the_boundary_is_not_truncated ... ok +test engine::remote::worker_failure_detail_tests::a_non_string_output_is_rendered_rather_than_dropped ... ok +test engine::remote::worker_failure_detail_tests::truncation_does_not_split_a_multi_byte_char ... ok +test server::client::tests::resume_waits_while_the_heartbeat_renewed_lease_is_live ... ok +test engine::boot_identity_tests::every_engine_in_this_process_shares_one_boot_id ... ok +test server::liveness::tests::sweep_id_buckets_by_the_interval ... ok +test server::channels::tests::unknown_verb_never_falls_through_to_receive ... ok +test exec_det::tests::timeout_has_an_explicit_completion_reason ... ok +test exec_det::tests::captures_deterministic_output ... ok +test engine::wake::claim_guard_tests::a_disarmed_guard_leaves_the_claim_alone ... ok +test server::tests::agent::contract::an_agent_worker_attaching_without_pins_is_refused_at_attach ... ok +test server::liveness::tests::sweep_pass_latches_after_journaling_and_next_bucket_is_empty ... ok +test server::liveness::tests::sweep_pass_healthy_subscription_is_a_noop ... ok +test engine::wake::claim_guard_tests::an_armed_guard_releases_the_claim_when_dropped ... ok +test server::tests::agent::contract::an_oversized_trajectory_tail_is_refused_at_step_complete ... ok +test engine::wake::claim_guard_tests::a_panic_between_claim_and_register_still_releases ... ok +test engine::wake::claim_guard_tests::a_guard_only_releases_its_own_run ... ok +test server::tests::agent::contract::agent_without_a_compatible_worker_parks_without_starting ... ok +test server::tests::hello_enforces_protocol_version ... ok +test server::tests::agent::contract::an_agent_worker_missing_a_declared_surface_parks_the_run_instead_of_erroring ... ok +test server::tests::run_resume_asks_the_registry_instead_of_treating_an_orphan_file_as_a_run ... ok +test server::tests::a_failed_disconnect_journal_append_is_retained_and_retried_not_dropped ... ok +test server::tests::agent::contract::an_llm_completion_claiming_an_effect_fails_closed_with_the_reason_journaled ... ok +test server::tests::agent::pins::reset_worker_reporting_a_revision_other_than_its_pin_fails_closed_as_worker_error ... ok +test server::tests::run_start_fails_closed_on_an_unknown_verification_key ... ok +test socket_path::tests::deep_data_dir_produces_short_socket_path ... ok +test socket_path::tests::different_data_dirs_yield_different_sockets ... ok +test socket_path::tests::relative_and_absolute_data_dirs_agree ... ok +test socket_path::tests::same_data_dir_yields_same_socket ... ok +test server::tests::agent::contract::a_replacement_worker_that_never_reported_the_pinned_surface_is_not_dispatched_to ... ok +test server::tests::run_resume_refuses_a_journal_that_never_recorded_its_run ... ok +test server::tests::run_resume_adopts_a_real_journal_whose_registry_row_is_missing ... ok +test server::tests::run_resume_refuses_a_valid_journal_that_belongs_to_another_run ... ok +test server::tests::agent::pins::consecutive_agent_steps_on_different_surfaces_each_start_from_their_own_pins ... ok +test server::tests::stopped_heartbeats_past_the_deadline_journal_lease_expired_and_release_the_step ... ok +test exec_det::tests::timeout_kills_the_whole_process_group ... ok +test server::tests::agent::pins::a_replacement_worker_at_a_different_revision_is_not_dispatched_the_stale_pins ... ok +test server::tests::an_entry_appended_during_watch_registration_is_delivered_exactly_once ... ok + +test result: ok. 40 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.57s + + Running unittests src/main.rs (target/debug/deps/relayflowd-ae3ceb16e5d803ec) + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/budget_gate.rs (target/debug/deps/budget_gate-48be15f3361f11b0) + +running 3 tests +test daily_windows_reset_and_exact_limits_do_not_refuse ... ok +test crossing_completion_is_durable_and_next_step_is_refused ... ok +test deterministic_spend_and_wallclock_limit_gate_parallel_batch_starts ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.05s + + Running tests/crash_resume.rs (target/debug/deps/crash_resume-4619905d2726c36c) + +running 40 tests +test agent::resume_without_a_worker_parks_immediately_instead_of_timing_out ... ok +test concurrency::cancel_closes_the_lease_and_rejects_a_late_completion ... ok +test concurrency::cancel_and_completion_race_has_one_terminal_fact ... ok +test agent::rung_c_sigkill_after_final_effect_replays_results_without_redispatch ... ok +test channels::channels_reject_foreign_workers_stale_attempts_and_invalid_acknowledgements ... ok +test agent::rung_c_sigkill_between_agent_completion_and_final_effect_memoizes_the_agent ... ok +test agent::rung_c_crash_between_effect_election_and_the_provider_call_performs_it_exactly_once ... ok +test agent::rung_c_reset_sigkill_mid_edit_restores_pins_dedupes_effect_and_explains_attempts ... ok +test concurrency::concurrent_resumes_lease_exactly_one_attempt ... ok +test concurrency::run_start_dispatches_every_independent_lane_before_any_completion ... ok +test concurrency::live_resume_leaves_an_active_lease_running ... ok +test concurrency::server_restart_recovers_every_parallel_lease_without_duplicate_success ... ok +test agent::rung_c_sigkill_boundaries_resume_only_unfinished_steps_via_real_cli ... ok +test llm::serve_plumbs_watch_events_and_replayable_stream_verbs ... ok +test llm::failing_llm_verification_schedules_a_durable_retry_and_succeeds ... ok +test llm::llm_verification_exhaustion_is_a_declared_failure_kind ... ok +test llm::completed_llm_output_is_memoized_when_serve_dies_during_the_next_step ... ok +test llm::sigkill_after_the_final_rung_b_effect_resumes_without_redispatching_llm ... ok +test pin_projection::rejected_completion_cannot_forge_inspect_retry_pins_over_the_real_socket ... ok +test memory::memory_sigkill_after_injection_replays_pack_and_charges_it_once ... ok +test llm::worker_killed_while_holding_a_lease_is_explained_and_released_on_cli_resume ... ok +test llm::sigkill_under_serve_mid_llm_releases_the_lease_and_finishes_via_cli_resume ... ok +test protocol_admission::every_mutating_run_verb_refuses_terminal_before_changing_state ... ok +test parallel_lifecycle::terminal_failure_drains_or_explains_every_live_sibling ... ok +test sigkill_after_cancel_request_resumes_to_one_canceled_fact ... ok +test sigkill_mid_step_replaces_and_explains_the_dead_attempt ... ok +test llm::sigkill_sweep_covers_before_and_between_the_rung_b_steps ... ok +test parallel_lifecycle::overlapping_agent_conflict_survives_server_crash_and_resume ... ok +test sigkill_under_serve_resumes_the_socket_started_run ... ok +test surface_identity::aliases_are_rejected_and_external_ancestors_serialize_over_real_sockets ... ok +test parallel_lifecycle::overlapping_agent_lanes_serialize_while_disjoint_lanes_merge_in_either_order ... ok +test worker_capacity::default_capacity_one_reopens_only_after_durable_completion_or_crash ... ok +test worker_capacity::two_workers_receive_a_deterministic_fair_capacity_bounded_batch ... ok +test placement::declared_placement_keeps_one_source_tree_across_resume ... ok +test workspace_identity::workspace_aliases_are_refused_and_canonical_subtrees_serialize_over_real_sockets ... ok +test channels::channels_sigkill_resume_redelivers_unacked_messages_with_exactly_once_effects ... ok +test sigkill_sweep_covers_every_hello_step_boundary ... ok +test placement::sigkill_before_first_step_preserves_the_submitted_workspace ... ok +test placement::sigkill_mid_step_keeps_the_route_and_source_tree ... ok +test parallel_lifecycle::renewed_parallel_leases_survive_the_original_grant_and_remain_distinct ... ok + +test result: ok. 40 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 37.34s + + Running tests/daemon_lifecycle.rs (target/debug/deps/daemon_lifecycle-e483798b2ec9e14a) + +running 6 tests +test connection_file_is_published_only_after_the_socket_is_live ... ok +test clean_shutdown_removes_advertisement_and_socket ... ok +test deep_data_dir_still_binds ... ok +test sigkill_leaves_a_stale_file_with_a_dead_pid ... ok +test a_second_serve_on_a_served_data_dir_refuses_and_the_first_keeps_serving ... ok +test a_sigkilled_daemons_successor_starts_cleanly ... ok + +test result: ok. 6 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.05s + + Running tests/event_wake.rs (target/debug/deps/event_wake-bc602276d1cebcd0) + +running 3 tests +test two_racing_deliveries_of_one_event_produce_exactly_one_run ... ok +test matching_event_wakes_once_with_fresh_context ... ok +test a_resumed_run_dispatches_the_original_wake_context ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.03s + + Running tests/hn_monitor_integration.rs (target/debug/deps/hn_monitor_integration-6b22693ad979dd80) + +running 1 test +test hn_story_event_wakes_monitor_once_with_story_context ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.04s + + Running tests/input_binding.rs (target/debug/deps/input_binding-4608eb1f1e2c7549) + +running 2 tests +test binding_schema_is_additive_and_fails_closed ... ok +test sigkill_before_consumer_resolves_original_journal_output_without_reexecuting_source ... ok + +test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.09s + + Running tests/invalid_schema_preflight.rs (target/debug/deps/invalid_schema_preflight-2c352b9351d5acf8) + +running 3 tests +test invalid_json_schema_is_refused_before_journal_or_command ... ok +test unbounded_json_schema_is_refused_before_journal_or_command ... ok +test legitimately_recursive_json_schema_still_starts ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 3.65s + + Running tests/memory.rs (target/debug/deps/memory-a74848a8274a290e) + +running 5 tests +test rejected_journal_fact_releases_reservation_and_never_dispatches ... ok +test llm_dispatch_receives_same_pack_after_resume_without_provider ... ok +test replay_and_resume_need_no_provider_and_script_receives_recorded_pack ... ok +test semantic_retry_reuses_memory_without_a_second_charge ... ok +test over_budget_and_provider_errors_fail_without_dispatch_or_charge ... ok + +test result: ok. 5 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.04s + + Running tests/memory_epoch.rs (target/debug/deps/memory_epoch-edee30a47cad81b5) + +running 1 test +test epoch_carries_pack_and_exact_charge_and_refuses_duplicate_injection ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.03s + + Running tests/parallel_driver.rs (target/debug/deps/parallel_driver-74d9d10149c534bd) + +running 4 tests +test stop_after_one_holds_for_an_independent_deterministic_batch ... ok +test pause_before_second_independent_step_holds_the_driver_boundary ... ok +test backpressured_or_mismatched_lane_does_not_drop_a_later_dispatch ... ok +test crash_boundaries_resume_the_real_driver_with_one_effect_per_lane ... ok + +test result: ok. 4 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.07s + + Running tests/placement_pins.rs (target/debug/deps/placement_pins-893a8773fbe94aee) + +running 3 tests +test unsupported_local_pty_is_refused_before_an_earlier_step_can_run ... ok +test default_worker_pins_the_declared_worktree_base_commit_and_refuses_missing_source ... ok +test a_resumed_attempt_keeps_the_original_pin_after_the_worktree_head_moves ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.24s + + Running tests/placement_routing.rs (target/debug/deps/placement_routing-aabf5947f3ca76b4) + +running 3 tests +test a_failed_routing_append_never_starts_or_dispatches_work ... ok +test crash_between_routing_and_start_does_not_redecide ... ok +test worker_retry_consumes_the_original_routing_fact ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s + + Running tests/routing_diagnostics.rs (target/debug/deps/routing_diagnostics-ddddee3a427ee9ae) + +running 2 tests +test duplicate_routes_have_a_distinct_diagnostic_and_leave_the_original_fact_intact ... ok +test malformed_routes_name_the_same_field_at_append_replay_and_epoch_replay ... ok + +test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s + + Running tests/spec_review_routing.rs (target/debug/deps/spec_review_routing-0731f97e90d67407) + +running 4 tests +test attempt_scoped_route_is_rejected_at_append_and_replay ... ok +test malformed_epoch_routes_are_rejected_before_commit ... ok +test epoch_cannot_drop_or_replace_a_durable_route ... ok +test workspace_pin_peels_tags_and_refuses_non_commit_objects ... ok + +test result: ok. 4 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.21s + + Running tests/subscription_liveness.rs (target/debug/deps/subscription_liveness-9506d11f678a9586) + +running 3 tests +test submit_event_upserts_subscription_row_and_sweep_flags_it_stale_after_budget ... ok +test stale_transition_is_journaled_as_subscription_stale_entry_in_the_last_known_run ... ok +test a_fresh_arrival_re_arms_the_latch_and_the_next_silence_can_stale_again ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s + + Running unittests src/lib.rs (target/debug/deps/relayflowd_core-b1fe3b3250e9e7a2) + +running 60 tests +test clock::tests::simulated_clock_is_explicitly_advanced ... ok +test channel::tests::malformed_payloads_and_invalid_new_channel_appends_leave_state_unchanged ... ok +test channel::tests::forged_deliveries_and_acknowledgements_fail_closed ... ok +test journal::tests::memory_journal_assigns_sequences_and_rolls_epochs ... ok +test channel::tests::send_retry_is_stable_and_conflicting_content_is_rejected ... ok +test channel::tests::delivery_replay_and_independent_acknowledged_offsets ... ok +test machine::parallel_tests::machine_starts_every_runnable_step_in_authored_order ... ok +test machine::parallel_tests::every_declared_mutable_surface_participates_in_conflict_selection ... ok +test machine::parallel_tests::external_ancestor_and_descendant_paths_conflict_but_siblings_do_not ... ok +test machine::tests::all_backing_off_steps_return_timers ... ok +test machine::parallel_tests::workspace_ancestor_and_descendant_paths_conflict_but_siblings_do_not ... ok +test machine::parallel_tests::failed_run_drains_open_siblings_before_terminal_entry ... ok +test machine::parallel_tests::parallel_lanes_do_not_cross_the_dependency_barrier_early ... ok +test machine::tests::every_reason_label_matches_its_serialized_form ... ok +test machine::tests::durable_cancel_request_outranks_crash_recovery ... ok +test machine::tests::crashed_attempt_does_not_consume_an_iteration ... ok +test machine::tests::cancel_request_closes_the_active_lease_before_the_terminal_fact ... ok +test machine::parallel_tests::disjoint_agent_lanes_merge_pins_in_either_completion_order ... ok +test machine::parallel_tests::overlapping_agent_surfaces_are_serialized_in_authored_order ... ok +test machine::tests::machine_starts_runnable_step_with_stable_effect_key ... ok +test machine::tests::repeated_cancel_request_is_idempotent ... ok +test machine::tests::manual_recovery_parks_needs_human_and_never_redispatches ... ok +test machine::tests::verification_failure_schedules_a_durable_retry ... ok +test machine::tests::successful_memo_is_never_scheduled_again ... ok +test machine::tests::worker_reported_failure_without_detail_still_records_a_verification ... ok +test machine::parallel_tests::crash_resume_preserves_each_parallel_lease_exactly_once ... ok +test machine::tests::reset_recovery_dispatches_the_original_pinned_revision ... ok +test memory::tests::caps_compare_exact_decimals_and_each_token_dimension ... ok +test machine::tests::inspect_recovery_injects_the_dirty_pin_completion_reason_and_tail ... ok +test retry::tests::jitter_is_repeatable_and_bounded ... ok +test machine::tests::every_failed_run_terminates_with_declared_completion_reasons ... ok +test schema::tests::in_document_uri_references_resolve_to_the_node_they_name ... ok +test spec::tests::a_misspelled_step_level_key_is_a_parse_error ... ok +test schema::tests::refusal_names_the_cycle_it_found ... ok +test spec::tests::a_misspelled_verification_gate_key_is_a_parse_error_not_a_dropped_gate ... ok +test spec::tests::cycles_are_rejected ... ok +test spec::tests::preflight_data_is_fail_closed ... ok +test spec::tests::external_surface_paths_must_have_one_canonical_spelling ... ok +test spec::tests::spec_version_is_semver_and_gated ... ok +test spec::tests::unknown_root_and_nested_fields_are_rejected ... ok +test schema::tests::references_the_bound_leaves_opaque_are_refused_by_the_engine ... ok +test spec::tests::zero_agent_flow_is_valid ... ok +test state::budget::tests::adds_costs_exactly_beyond_machine_decimal_precision ... ok +test state::budget::tests::overflow_and_malformed_cost_leave_total_unchanged ... ok +test spec::tests::workspace_mounts_and_worktrees_must_have_one_canonical_spelling ... ok +test state::tests::budget_decimal_strings_add_without_floats ... ok +test state::tests::a_completion_that_omits_a_surface_does_not_drop_it_from_the_pin_chain ... ok +test state::tests::journal_replays_data_gate_verdict_without_rerunning_completed_code ... ok +test verify::tests::an_unbounded_schema_in_a_journal_fails_its_gate_instead_of_aborting ... ok +test state::tests::end_pin_chain_is_enforced_and_a_broken_chain_is_a_hard_error ... ok +test verify::tests::deterministic_output_requires_successful_exit_and_content ... ok +test spec::tests::the_full_ladder_parses_in_the_one_dialect ... ok +test verify::tests::json_schema_is_a_control_gate ... ok +test schema::tests::a_property_named_ref_is_not_a_reference ... ok +test schema::tests::shared_declarations_and_boolean_schemas_are_validated ... ok +test schema::tests::every_accepted_corpus_schema_is_accepted ... ok +test schema::tests::every_refused_corpus_schema_compiles_but_is_refused_by_the_bound ... ok +test spec::tests::sdk_boundary_rejects_a_10_000_step_cycle_with_a_typed_error ... ok +test spec::tests::sdk_boundary_accepts_a_valid_10_000_step_reverse_chain ... ok +test schema::tests::deeply_nested_schemas_do_not_overflow_the_checker ... ok + +test result: ok. 60 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.60s + + Running tests/spec_parity.rs (target/debug/deps/spec_parity-bbda6cf1e1cf1c19) + +running 9 tests +test placement_requirements_have_identical_canonical_bytes_and_hash ... ok +test the_kernel_parses_the_deterministic_rung_and_stamps_the_same_hash ... ok +test the_kernel_parses_the_event_triggered_spec_and_stamps_the_same_hash ... ok +test step_memory_has_identical_canonical_bytes_and_hash ... ok +test the_kernel_parses_the_rung_c_agent_spec_and_stamps_the_same_hash ... ok +test memory_declaration_acceptance_matches_the_sdk_corpus ... ok +test placement_declaration_acceptance_matches_the_sdk_corpus ... ok +test the_kernel_parses_the_sdk_compiled_spec_and_stamps_the_same_hash ... ok +test the_kernel_parses_the_rung_b_spec_and_stamps_the_same_hash ... ok + +test result: ok. 9 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s + + Running unittests src/lib.rs (target/debug/deps/relayflowd_journal-d13cb7954335385c) + +running 28 tests +test registry::tests::a_same_boot_claim_with_no_run_yet_is_a_duplicate_not_wreckage ... ok +test registry::tests::a_registered_run_dedupes_across_boots ... ok +test registry::tests::a_previous_boots_claim_with_no_run_is_repaired ... ok +test registry::tests::releasing_a_claim_lets_the_same_boot_retry ... ok +test registry::tests::registry_is_a_rebuildable_run_locator ... ok +test registry::tests::a_pre_migration_registry_gains_boot_id_and_its_claims_are_repairable ... ok +test channel::tests::stale_attempts_and_raw_forged_acknowledgements_cannot_change_offsets ... ok +test subscriptions::tests::detect_without_latch_stays_available_for_the_next_sweep ... ok +test registry::tests::releasing_is_scoped_to_the_claiming_run ... ok +test subscriptions::tests::last_run_for_subscription_returns_none_before_first_arrival ... ok +test subscriptions::tests::latch_is_a_no_op_if_a_fresh_event_arrived_between_detect_and_latch ... ok +test subscriptions::tests::last_run_for_subscription_returns_the_lex_greatest_ulid_regardless_of_insertion ... ok +test subscriptions::tests::prune_sweep_claims_deletes_only_rows_older_than_cutoff ... ok +test channel::tests::channels_cross_segment_boundaries_and_terminal_runs_reject_mutations ... ok +test subscriptions::tests::sweep_election_gives_the_first_caller_the_result_and_second_gets_empty ... ok +test subscriptions::tests::sweep_ignores_subscriptions_whose_silence_is_still_within_budget ... ok +test subscriptions::tests::sweep_does_not_re_emit_the_same_stale_row_on_a_later_tick ... ok +test subscriptions::tests::sweep_marks_row_stale_when_silence_exceeds_budget ... ok +test subscriptions::tests::upsert_after_stale_re_arms_and_next_silence_can_re_emit ... ok +test subscriptions::tests::upsert_is_idempotent_across_bumps_and_preserves_event_type_updates ... ok +test tests::an_unconfirmed_election_is_reclaimed_by_the_next_attempt_not_treated_as_done ... ok +test tests::append_is_durable_and_monotonic_after_reopen ... ok +test channel::tests::failed_channel_writes_never_expose_delivery_or_advance_acknowledged_offset ... ok +test tests::failed_commit_is_returned_not_swallowed ... ok +test tests::terminal_run_refuses_every_later_entry_atomically ... ok +test tests::rollover_is_atomic_scaffolding_for_epoch_resume ... ok +test tests::effects_are_deduplicated_at_the_journal_boundary ... ok +test channel::tests::independent_connections_serialize_send_receive_and_acknowledgement ... ok + +test result: ok. 28 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.13s + + Doc-tests relayflowd + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Doc-tests relayflowd_core + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Doc-tests relayflowd_journal + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s diff --git a/evidence/spec-G-budget/claude-auth-status.log b/evidence/spec-G-budget/claude-auth-status.log new file mode 100644 index 000000000..8439cac2a --- /dev/null +++ b/evidence/spec-G-budget/claude-auth-status.log @@ -0,0 +1,8 @@ +{ + "loggedIn": false, + "authMethod": "none", + "apiProvider": "firstParty", + "analyticsDisabled": false, + "projectsDirectory": "/Users/khaliqgant/.claude/projects", + "configDirectory": "/Users/khaliqgant/.claude" +} diff --git a/evidence/spec-G-budget/sdk-focused.log b/evidence/spec-G-budget/sdk-focused.log new file mode 100644 index 000000000..764729802 --- /dev/null +++ b/evidence/spec-G-budget/sdk-focused.log @@ -0,0 +1,23 @@ + + RUN v2.1.9 /Users/khaliqgant/flows-spec-G-budget/packages/sdk + + ✓ tests/budget-attribution.test.ts (4 tests) 3ms + ✓ tests/budget-preflight.test.ts (13 tests) 6ms + ✓ tests/preflight.test.ts (27 tests) 26ms + ✓ tests/budget-authored-live.test.ts (2 tests) 267ms + ✓ tests/cli.test.ts (63 tests) 9479ms + ✓ flows check CLI > binds a checked relative wrapper to the flow directory for worker execution 459ms + ✓ flows check CLI > refuses a typo model before probing or contacting relayflowd 593ms + ✓ flows check CLI > resolves a project CLI path relative to the flows.json that declares it 315ms + ✓ flows check CLI > maps every input refusal path to its declared kind without raw exceptions 799ms + ✓ flows run/resume CLI over the journal protocol > reports a needs_human agent step as parked for human recovery 605ms + ✓ flows run/resume CLI over the journal protocol > classifies a typed hello refusal as a protocol error, not an unreachable daemon 721ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 696ms + ✓ flows run/resume CLI over the journal protocol > bounds a worker wait by its lease and reports what it is waiting for 626ms + ✓ flows run/resume CLI over the journal protocol > resumes a parked run from snapshot step types without reading journal sequence one 532ms + ✓ flows run/resume CLI over the journal protocol > maps only run_not_found resumes to exit 2 1821ms + + Test Files 5 passed (5) + Tests 109 passed (109) + Start at 10:32:46 + Duration 10.07s (transform 473ms, setup 0ms, collect 1.31s, tests 9.78s, environment 0ms, prepare 230ms) diff --git a/evidence/spec-G-budget/sdk-npm-test-earlier.log b/evidence/spec-G-budget/sdk-npm-test-earlier.log new file mode 100644 index 000000000..29b5c6dab --- /dev/null +++ b/evidence/spec-G-budget/sdk-npm-test-earlier.log @@ -0,0 +1,214 @@ + +> @relayflows/sdk@2.0.8 test +> sh scripts/test.sh + + +> @relayflows/sdk@2.0.8 test:prep +> ( cd ../../kernel && sh ../ops/cargo.sh build ) && ( [ ! -d ../../testdata/preflight ] || find ../../testdata/preflight -name '*-cli' -type f -exec chmod +x {} + ) + + Compiling relayflowd-core v0.1.0 (/Users/khaliqgant/flows-spec-G-budget/kernel/relayflowd-core) + Compiling relayflowd-journal v0.1.0 (/Users/khaliqgant/flows-spec-G-budget/kernel/relayflowd-journal) + Compiling relayflowd v0.1.0 (/Users/khaliqgant/flows-spec-G-budget/kernel/relayflowd) + Finished `dev` profile [unoptimized + debuginfo] target(s) in 3.96s + +> @relayflows/sdk@2.0.8 typecheck +> tsc --noEmit && tsc -p tsconfig.type-tests.json + + +> @relayflows/sdk@2.0.8 build +> tsc && node scripts/make-cli-executable.mjs + + +> @relayflows/sdk@2.0.8 typecheck:tests +> tsc -p tsconfig.tests.json + + + RUN v2.1.9 /Users/khaliqgant/flows-spec-G-budget/packages/sdk + +stdout | tests/live-kernel.test.ts +LIVE_KERNEL relayflowd=/Users/khaliqgant/.relayflows-toolchain/target/3804940860/debug/relayflowd +LIVE_KERNEL flows=/Users/khaliqgant/flows-spec-G-budget/packages/sdk/dist/cli.js + + ✓ tests/journal-client.test.ts (14 tests) 70ms + ✓ tests/daemon-lifecycle.test.ts (42 tests) 29ms + ✓ tests/validate.test.ts (68 tests) 23ms + ✓ tests/preflight.test.ts (27 tests) 52ms + ✓ tests/observer-link.test.ts (39 tests) 326ms + ✓ tests/tick-source.test.ts (33 tests) 12ms + ✓ tests/cloud-run.test.ts (47 tests) 119ms + ✓ tests/authored-flow.test.ts (24 tests) 656ms + ✓ tests/verb-field-lint.test.ts (78 tests) 262ms + ✓ tests/gate-contract.test.ts (20 tests) 79ms + ✓ tests/cli-hn-monitor.test.ts (16 tests) 104ms + ✓ tests/authored-flow-lifecycle-executor.test.ts (27 tests) 527ms + ✓ tests/backlog-picker.test.ts (14 tests) 70ms + ✓ tests/tick-runner.test.ts (22 tests) 828ms + ✓ tests/authored-flow-operation.test.ts (23 tests) 324ms + ✓ tests/backlog-picker-flow.test.ts (6 tests) 549ms + ✓ tests/work-package-consumer.test.ts (13 tests) 207ms + ✓ tests/spec-parity.test.ts (31 tests) 187ms + ✓ tests/worker-lease.test.ts (7 tests) 8ms + ✓ tests/typed-output.test.ts (14 tests) 158ms + ✓ tests/model-selection.test.ts (10 tests) 12ms + ✓ tests/relayflowd-path.test.ts (10 tests) 2ms + ✓ tests/json-schema-bound.test.ts (71 tests) 1448ms + ✓ JSON Schema termination bound > walks a deep schema with an explicit stack rather than recursion 1175ms + ✓ tests/yaml-local-agent-live.test.ts (7 tests) 2683ms + ✓ YAML --local-agent through the built CLI and real daemon > runs with the checked step CLI and model and journals done 422ms + ✓ YAML --local-agent through the built CLI and real daemon > runs with the checked named CLI and model and journals done 387ms + ✓ YAML --local-agent through the built CLI and real daemon > runs with the checked flow CLI and model and journals done 448ms + ✓ YAML --local-agent through the built CLI and real daemon > runs with the checked project CLI and model and journals done 429ms + ✓ YAML --local-agent through the built CLI and real daemon > reports the agent process failure 405ms + ✓ tests/local-dev-ux.test.ts (8 tests) 12ms + ✓ tests/dependency-validation.test.ts (6 tests) 273ms + ✓ tests/input-binding.test.ts (12 tests) 121ms + ✓ tests/deterministic-llm.test.ts (5 tests) 38ms + ✓ tests/bin.test.ts (7 tests) 603ms + ✓ tests/hn-poller.test.ts (6 tests) 3ms + ✓ tests/stop-process-group.test.ts (6 tests) 8298ms + ✓ every stop reaches the process group, not just the direct child > exits the run after an execution-timeout stop 976ms + ✓ every stop reaches the process group, not just the direct child > exits the run after a protocol terminate stop 1107ms + ✓ every stop reaches the process group, not just the direct child > kills a SIGTERM-deaf grandchild after a protocol terminate stop 2288ms + ✓ every stop reaches the process group, not just the direct child > kills a SIGTERM-deaf grandchild after an execution-timeout stop 2444ms + ✓ every stop reaches the process group, not just the direct child > holds the loop open long enough for the escalation to run 1172ms + ✓ every stop reaches the process group, not just the direct child > terminate() forces a group that outlives SIGTERM 310ms + ✓ tests/budget-attribution.test.ts (4 tests) 2ms + ✓ tests/direct-input.test.ts (4 tests) 7469ms + ✓ direct .flow.ts input through the built CLI and live runtime > executes inline and file JSON input through relayflowd 3207ms + ✓ direct .flow.ts input through the built CLI and live runtime > refuses missing and malformed input before contacting relayflowd 2587ms + ✓ direct .flow.ts input through the built CLI and live runtime > does not import or execute authored code before daemon availability 874ms + ✓ direct .flow.ts input through the built CLI and live runtime > refuses oversized file input before contacting relayflowd 801ms + ✓ tests/dir-watcher-poller.test.ts (6 tests) 2ms + ✓ tests/work-package-validator.test.ts (7 tests) 3ms + ✓ tests/hello-deterministic.test.ts (5 tests) 12ms + ✓ tests/budget-preflight.test.ts (13 tests) 5ms + ↓ tests/real-cli-adapters.test.ts (3 tests | 3 skipped) + ✓ tests/parse-json-output.test.ts (7 tests) 1ms + ✓ tests/cli-adapter.test.ts (3 tests) 2ms + ✓ tests/journal-client-completion.test.ts (4 tests) 99ms + ✓ tests/direct-run-failure.test.ts (6 tests) 2ms + ✓ tests/budget-authored-live.test.ts (2 tests) 189ms + ✓ tests/daemon-lifecycle-live.test.ts (9 tests) 9560ms + ✓ flows run against a data dir with no daemon (§6 test 7) > cold start spawns exactly one daemon, the run succeeds, and the daemon outlives the CLI 909ms + ✓ flows run against a data dir with no daemon (§6 test 7) > polls, bounded, for a daemon that holds the lock before it binds 1866ms + ✓ flows run against a data dir with no daemon (§6 test 7) > attaches to a serving daemon that has not published a connection file 1079ms + ✓ flows run against a data dir with no daemon (§6 test 7) > a second run attaches to the daemon the first one started, spawning nothing 1091ms + ✓ flows run against a data dir with no daemon (§6 test 7) > detects a stale connection file left by a hard kill and starts a fresh daemon 1748ms + ✓ concurrent invocations against one empty data dir (§6 test 15) > ends with exactly one daemon owning the socket, and both runs succeed 995ms + ✓ refusals from a spawn that cannot produce a daemon > names relayflowd_not_found rather than falling through to PATH 716ms + ✓ refusals from a spawn that cannot produce a daemon > names daemon_start_failed and quotes the daemon log when startup dies 720ms + ✓ refusals from a spawn that cannot produce a daemon > refuses a daemon speaking another protocol version instead of binding over it 436ms + ✓ tests/placement.test.ts (54 tests) 7ms + ✓ tests/memory.test.ts (18 tests) 4ms + ✓ tests/worker-platform.test.ts (1 test) 2ms + ✓ tests/classify-outcome.test.ts (2 tests) 2224ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2068ms + ✓ tests/flow-executor-chain.test.ts (12 tests) 10484ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 1272ms + ✓ flow executor LLM and output-binding chain > fails invalid LLM output before the next step: not JSON 1156ms + ✓ flow executor LLM and output-binding chain > fails invalid LLM output before the next step: {"message":7} 1079ms + ✓ flow executor LLM and output-binding chain > retains tagged-template text output 756ms + ✓ flow executor LLM and output-binding chain > preserves JSON values without promoting them to process wrappers: null 943ms + ✓ flow executor LLM and output-binding chain > preserves JSON values without promoting them to process wrappers: [1,2] 571ms + ✓ flow executor LLM and output-binding chain > preserves JSON values without promoting them to process wrappers: "hello" 622ms + ✓ flow executor LLM and output-binding chain > runs the authored LLM path through the built flows CLI 973ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 654ms + ✓ flow executor LLM and output-binding chain > journals a missing optional field as a failure before the consuming command executes 606ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 1772ms + ✓ tests/cli-progress-wait.test.ts (2 tests) 1662ms + ✓ run starts the wait clock on its first observed lease 657ms + ✓ resume starts the wait clock on its first observed lease 1004ms + ✓ tests/worker-cli-abort.test.ts (2 tests) 2649ms + ✓ stops claude and its process group when lease ownership is lost 1393ms + ✓ stops wrapper.mjs and its process group when lease ownership is lost 1255ms + ❯ tests/cli.test.ts (63 tests | 1 failed) 13468ms + ✓ flows check CLI > binds a checked relative wrapper to the flow directory for worker execution 937ms + ✓ flows check CLI > uses the raw Claude adapter model flag instead of accepting auth status as model proof 512ms + ✓ flows check CLI > uses Codex login status and reports a rejected model as unavailable, not unauthenticated 751ms + ✓ flows check CLI > refuses a nonconforming custom wrapper without calling it an authentication failure 823ms + ✓ flows check CLI > accepts an exact allowlisted named-agent model and probes that model 729ms + ✓ flows check CLI > checks the same named-agent contract from declarative JSON 829ms + ✓ flows check CLI > refuses a typo model before probing or contacting relayflowd 448ms + ✓ flows check CLI > distinguishes an allowlisted but inaccessible model from broken auth 493ms + ✓ flows check CLI > resolves a project CLI path relative to the flows.json that declares it 464ms + ✓ flows check CLI > maps every input refusal path to its declared kind without raw exceptions 983ms + ✓ flows run/resume CLI over the journal protocol > parses run options, submits the kernel dialect, and exits 0 on success 611ms + ✓ flows run/resume CLI over the journal protocol > exits 3 and names the parked llm step 599ms + ✓ flows run/resume CLI over the journal protocol > reports a needs_human agent step as parked for human recovery 592ms + ✓ flows run/resume CLI over the journal protocol > classifies a typed hello refusal as a protocol error, not an unreachable daemon 610ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 672ms + × flows run/resume CLI over the journal protocol > bounds a worker wait by its lease and reports what it is waiting for 641ms + → expected +0 to be 1 // Object.is equality + ✓ flows run/resume CLI over the journal protocol > resumes a parked run from snapshot step types without reading journal sequence one 384ms + ✓ flows run/resume CLI over the journal protocol > maps only run_not_found resumes to exit 2 1764ms + ✓ tests/worker-cli.test.ts (13 tests) 23091ms + ✓ custom wrapper execution identity > refuses a wrapper symlink retarget before delivering private values 897ms + ✓ custom wrapper execution identity > bounds wrapper execution after acknowledgement 710ms + ✓ custom wrapper execution identity > bounds captured wrapper output 762ms + ✓ custom wrapper execution identity > refuses a duplicate execute protocol frame 740ms + ✓ custom wrapper execution bounds are reader-owned > resolves when a conforming wrapper leaks a stdio pipe to a background helper 2348ms + ✓ custom wrapper execution bounds are reader-owned > resolves when the leaked helper inherits stderr only 1926ms + ✓ custom wrapper execution bounds are reader-owned > resolves when a wrapper leaks a stdio pipe and exits before identifying 3255ms + ✓ custom wrapper execution bounds are reader-owned > journals a completionReason at the default bound when a wrapper leaks a stdio pipe 11258ms + ✓ custom wrapper execution bounds are reader-owned > accepts the same over-8KiB payload whether or not it coalesces with the execute token 457ms + ✓ tests/local-agent-live.test.ts (5 tests) 39743ms + ✓ built CLI local agent against a real daemon > dispatches through the wrapper and keeps --json stdout report-shaped 1367ms + ✓ built CLI local agent against a real daemon > runs beyond the initial 30-second lease without a second invocation 35809ms + ✓ built CLI local agent against a real daemon > renders actual agent completion in text output 870ms + ✓ built CLI local agent against a real daemon > returns a failed run when the agent process fails 778ms + ✓ built CLI local agent against a real daemon > refuses a workspace it cannot pin before invoking the agent 918ms +stdout | tests/live-kernel.test.ts > surface resume after a real daemon kill > resumes a three-step run with each successful completion exactly once +LIVE_KERNEL kill -9 pid=90664 run=01M27SEJVGNF1Y8C2SP68CEDSX while step=two state=Running + + ❯ tests/live-kernel.test.ts (30 tests | 1 failed) 53621ms + ✓ built flows CLI against live relayflowd > runs rung (a), parks rung (b), and keeps JSON report-shaped 5226ms + ✓ built flows CLI against live relayflowd > allows a deterministic run to exceed the bounded request timeout 32243ms + ✓ built flows CLI against live relayflowd > follows a live worker dispatch through flows run 1096ms + ✓ built flows CLI against live relayflowd > f.agent lowers to a real agent step and dispatches through a live worker 380ms + ✓ built flows CLI against live relayflowd > f.agent's default flowPath anchors on cwd, not cwd's parent 430ms + ✓ built flows CLI against live relayflowd > can always get a parked run to a late-attaching worker 5570ms + ✓ built flows CLI against live relayflowd > reports a real manual-recovery NeedsHuman state as parked 820ms + ✓ built flows CLI against live relayflowd > AgentWorker passes a declared model to an identified wrapper as RELAYFLOW_MODEL 337ms + × built flows CLI against live relayflowd > hn-monitor analyze-story reaches done through the real Claude analyzer CLI 1404ms + → LIVE_ANALYZER_UNAVAILABLE: "/Users/khaliqgant/flows-spec-G-budget/testdata/preflight/analyze-story-claude-cli auth status" exited 1: analyze-story-claude-cli: "claude -p --model claude-haiku-4-5-20251001" exited 1: — failing because gate-2 acceptance requires the real analyzer to execute. Set RELAYFLOWS_ALLOW_ANALYZER_SKIP=1 only if this run is not gate evidence. + ✓ built flows CLI against live relayflowd > preflights before journaling and names an unreachable socket 1505ms + ✓ built flows CLI against live relayflowd > starts exactly one daemon when two runs race for one empty data dir 967ms + ✓ surface resume after a real daemon kill > resumes a three-step run with each successful completion exactly once 1528ms + +⎯⎯⎯⎯⎯⎯⎯ Failed Tests 2 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/cli.test.ts > flows run/resume CLI over the journal protocol > bounds a worker wait by its lease and reports what it is waiting for +AssertionError: expected +0 to be 1 // Object.is equality + +- Expected ++ Received + +- 1 ++ 0 + + ❯ tests/cli.test.ts:887:18 + 885| ], output.io); + 886| + 887| expect(code).toBe(1); + | ^ + 888| expect(output.stderr.join('\n')).toContain('WAITING [worker_lease]… + 889| expect(output.stderr.join('\n')).toContain(`until ${leaseDeadlineM… + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/2]⎯ + + FAIL tests/live-kernel.test.ts > built flows CLI against live relayflowd > hn-monitor analyze-story reaches done through the real Claude analyzer CLI +Error: LIVE_ANALYZER_UNAVAILABLE: "/Users/khaliqgant/flows-spec-G-budget/testdata/preflight/analyze-story-claude-cli auth status" exited 1: analyze-story-claude-cli: "claude -p --model claude-haiku-4-5-20251001" exited 1: — failing because gate-2 acceptance requires the real analyzer to execute. Set RELAYFLOWS_ALLOW_ANALYZER_SKIP=1 only if this run is not gate evidence. + ❯ tests/live-kernel.test.ts:1186:15 + 1184| const notice = `LIVE_ANALYZER_UNAVAILABLE: ${readiness.detail}`; + 1185| if (process.env['RELAYFLOWS_ALLOW_ANALYZER_SKIP'] !== '1') { + 1186| throw new Error( + | ^ + 1187| `${notice} — failing because gate-2 acceptance requires the … + 1188| + 'Set RELAYFLOWS_ALLOW_ANALYZER_SKIP=1 only if this run is … + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[2/2]⎯ + + Test Files 2 failed | 52 passed | 1 skipped (55) + Tests 2 failed | 983 passed | 3 skipped (988) + Start at 10:29:59 + Duration 54.17s (transform 962ms, setup 0ms, collect 4.39s, tests 182.39s, environment 6ms, prepare 1.78s) diff --git a/evidence/spec-G-budget/sdk-npm-test.log b/evidence/spec-G-budget/sdk-npm-test.log new file mode 100644 index 000000000..d299b7abf --- /dev/null +++ b/evidence/spec-G-budget/sdk-npm-test.log @@ -0,0 +1,193 @@ + +> @relayflows/sdk@2.0.8 test +> sh scripts/test.sh + + +> @relayflows/sdk@2.0.8 test:prep +> ( cd ../../kernel && sh ../ops/cargo.sh build ) && ( [ ! -d ../../testdata/preflight ] || find ../../testdata/preflight -name '*-cli' -type f -exec chmod +x {} + ) + + Finished `dev` profile [unoptimized + debuginfo] target(s) in 0.28s + +> @relayflows/sdk@2.0.8 typecheck +> tsc --noEmit && tsc -p tsconfig.type-tests.json + + +> @relayflows/sdk@2.0.8 build +> tsc && node scripts/make-cli-executable.mjs + + +> @relayflows/sdk@2.0.8 typecheck:tests +> tsc -p tsconfig.tests.json + + + RUN v2.1.9 /Users/khaliqgant/flows-spec-G-budget/packages/sdk + + ✓ tests/journal-client.test.ts (14 tests) 68ms +stdout | tests/live-kernel.test.ts +LIVE_KERNEL relayflowd=/Users/khaliqgant/.relayflows-toolchain/target/3804940860/debug/relayflowd +LIVE_KERNEL flows=/Users/khaliqgant/flows-spec-G-budget/packages/sdk/dist/cli.js + + ✓ tests/validate.test.ts (68 tests) 23ms + ✓ tests/daemon-lifecycle.test.ts (42 tests) 68ms + ✓ tests/preflight.test.ts (27 tests) 48ms + ✓ tests/observer-link.test.ts (39 tests) 259ms + ✓ tests/tick-source.test.ts (33 tests) 64ms + ✓ tests/cloud-run.test.ts (47 tests) 122ms + ✓ tests/authored-flow.test.ts (24 tests) 659ms + ✓ tests/verb-field-lint.test.ts (78 tests) 291ms + ✓ tests/authored-flow-lifecycle-executor.test.ts (27 tests) 506ms + ✓ tests/cli-hn-monitor.test.ts (16 tests) 97ms + ✓ tests/gate-contract.test.ts (20 tests) 163ms + ✓ tests/backlog-picker.test.ts (14 tests) 101ms + ✓ tests/tick-runner.test.ts (22 tests) 943ms + ✓ tests/authored-flow-operation.test.ts (23 tests) 348ms + ✓ tests/backlog-picker-flow.test.ts (6 tests) 768ms + ✓ backlog-picker flow > does not emit a stale entry left by a previous run 373ms + ✓ tests/work-package-consumer.test.ts (13 tests) 267ms + ✓ tests/spec-parity.test.ts (31 tests) 304ms + ✓ tests/worker-lease.test.ts (7 tests) 14ms + ✓ tests/typed-output.test.ts (14 tests) 155ms + ✓ tests/model-selection.test.ts (10 tests) 12ms + ✓ tests/relayflowd-path.test.ts (10 tests) 5ms + ✓ tests/json-schema-bound.test.ts (71 tests) 1983ms + ✓ JSON Schema termination bound > walks a deep schema with an explicit stack rather than recursion 1653ms + ✓ tests/yaml-local-agent-live.test.ts (7 tests) 2794ms + ✓ YAML --local-agent through the built CLI and real daemon > runs with the checked step CLI and model and journals done 533ms + ✓ YAML --local-agent through the built CLI and real daemon > runs with the checked named CLI and model and journals done 476ms + ✓ YAML --local-agent through the built CLI and real daemon > runs with the checked flow CLI and model and journals done 421ms + ✓ YAML --local-agent through the built CLI and real daemon > runs with the checked project CLI and model and journals done 392ms + ✓ YAML --local-agent through the built CLI and real daemon > reports the agent process failure 406ms + ✓ tests/local-dev-ux.test.ts (8 tests) 9ms + ✓ tests/direct-input.test.ts (4 tests) 6554ms + ✓ direct .flow.ts input through the built CLI and live runtime > executes inline and file JSON input through relayflowd 2508ms + ✓ direct .flow.ts input through the built CLI and live runtime > refuses missing and malformed input before contacting relayflowd 2330ms + ✓ direct .flow.ts input through the built CLI and live runtime > does not import or execute authored code before daemon availability 694ms + ✓ direct .flow.ts input through the built CLI and live runtime > refuses oversized file input before contacting relayflowd 1021ms + ✓ tests/input-binding.test.ts (12 tests) 105ms + ✓ tests/dependency-validation.test.ts (6 tests) 297ms + ✓ tests/stop-process-group.test.ts (6 tests) 7894ms + ✓ every stop reaches the process group, not just the direct child > exits the run after an execution-timeout stop 992ms + ✓ every stop reaches the process group, not just the direct child > exits the run after a protocol terminate stop 1143ms + ✓ every stop reaches the process group, not just the direct child > kills a SIGTERM-deaf grandchild after a protocol terminate stop 2228ms + ✓ every stop reaches the process group, not just the direct child > kills a SIGTERM-deaf grandchild after an execution-timeout stop 2055ms + ✓ every stop reaches the process group, not just the direct child > holds the loop open long enough for the escalation to run 1160ms + ✓ every stop reaches the process group, not just the direct child > terminate() forces a group that outlives SIGTERM 315ms + ✓ tests/deterministic-llm.test.ts (5 tests) 36ms + ✓ tests/budget-attribution.test.ts (4 tests) 2ms + ✓ tests/hn-poller.test.ts (6 tests) 3ms + ✓ tests/dir-watcher-poller.test.ts (6 tests) 2ms + ✓ tests/hello-deterministic.test.ts (5 tests) 10ms + ✓ tests/work-package-validator.test.ts (7 tests) 4ms + ✓ tests/budget-preflight.test.ts (13 tests) 8ms + ✓ tests/bin.test.ts (7 tests) 644ms + ↓ tests/real-cli-adapters.test.ts (3 tests | 3 skipped) + ✓ tests/parse-json-output.test.ts (7 tests) 1ms + ✓ tests/cli-adapter.test.ts (3 tests) 2ms + ✓ tests/journal-client-completion.test.ts (4 tests) 98ms + ✓ tests/direct-run-failure.test.ts (6 tests) 2ms + ✓ tests/budget-authored-live.test.ts (2 tests) 187ms + ✓ tests/flow-executor-chain.test.ts (12 tests) 10086ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 1501ms + ✓ flow executor LLM and output-binding chain > fails invalid LLM output before the next step: not JSON 862ms + ✓ flow executor LLM and output-binding chain > fails invalid LLM output before the next step: {"message":7} 1030ms + ✓ flow executor LLM and output-binding chain > retains tagged-template text output 697ms + ✓ flow executor LLM and output-binding chain > preserves JSON values without promoting them to process wrappers: null 882ms + ✓ flow executor LLM and output-binding chain > preserves JSON values without promoting them to process wrappers: [1,2] 721ms + ✓ flow executor LLM and output-binding chain > preserves JSON values without promoting them to process wrappers: "hello" 560ms + ✓ flow executor LLM and output-binding chain > runs the authored LLM path through the built flows CLI 1219ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 659ms + ✓ flow executor LLM and output-binding chain > journals a missing optional field as a failure before the consuming command executes 561ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 1335ms + ✓ tests/classify-outcome.test.ts (2 tests) 2197ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2041ms + ✓ tests/placement.test.ts (54 tests) 7ms + ✓ tests/daemon-lifecycle-live.test.ts (9 tests) 10644ms + ✓ flows run against a data dir with no daemon (§6 test 7) > cold start spawns exactly one daemon, the run succeeds, and the daemon outlives the CLI 1003ms + ✓ flows run against a data dir with no daemon (§6 test 7) > polls, bounded, for a daemon that holds the lock before it binds 1855ms + ✓ flows run against a data dir with no daemon (§6 test 7) > attaches to a serving daemon that has not published a connection file 668ms + ✓ flows run against a data dir with no daemon (§6 test 7) > a second run attaches to the daemon the first one started, spawning nothing 1637ms + ✓ flows run against a data dir with no daemon (§6 test 7) > detects a stale connection file left by a hard kill and starts a fresh daemon 2075ms + ✓ concurrent invocations against one empty data dir (§6 test 15) > ends with exactly one daemon owning the socket, and both runs succeed 979ms + ✓ refusals from a spawn that cannot produce a daemon > names relayflowd_not_found rather than falling through to PATH 796ms + ✓ refusals from a spawn that cannot produce a daemon > names daemon_start_failed and quotes the daemon log when startup dies 853ms + ✓ refusals from a spawn that cannot produce a daemon > refuses a daemon speaking another protocol version instead of binding over it 778ms + ✓ tests/memory.test.ts (18 tests) 5ms + ✓ tests/cli-progress-wait.test.ts (2 tests) 1189ms + ✓ run starts the wait clock on its first observed lease 641ms + ✓ resume starts the wait clock on its first observed lease 547ms + ✓ tests/worker-platform.test.ts (1 test) 2ms + ✓ tests/worker-cli-abort.test.ts (2 tests) 2589ms + ✓ stops claude and its process group when lease ownership is lost 1266ms + ✓ stops wrapper.mjs and its process group when lease ownership is lost 1323ms + ✓ tests/cli.test.ts (63 tests) 14605ms + ✓ flows check CLI > binds a checked relative wrapper to the flow directory for worker execution 944ms + ✓ flows check CLI > uses the raw Claude adapter model flag instead of accepting auth status as model proof 482ms + ✓ flows check CLI > uses Codex login status and reports a rejected model as unavailable, not unauthenticated 936ms + ✓ flows check CLI > refuses a nonconforming custom wrapper without calling it an authentication failure 530ms + ✓ flows check CLI > accepts an exact allowlisted named-agent model and probes that model 519ms + ✓ flows check CLI > checks the same named-agent contract from declarative JSON 516ms + ✓ flows check CLI > refuses a typo model before probing or contacting relayflowd 621ms + ✓ flows check CLI > resolves a project CLI path relative to the flows.json that declares it 617ms + ✓ flows check CLI > maps every input refusal path to its declared kind without raw exceptions 1876ms + ✓ flows run/resume CLI over the journal protocol > parses run options, submits the kernel dialect, and exits 0 on success 719ms + ✓ flows run/resume CLI over the journal protocol > exits 1 and emits the declared completionReason for a failed run 539ms + ✓ flows run/resume CLI over the journal protocol > exits 3 and names the parked llm step 622ms + ✓ flows run/resume CLI over the journal protocol > reports a needs_human agent step as parked for human recovery 552ms + ✓ flows run/resume CLI over the journal protocol > classifies a typed hello refusal as a protocol error, not an unreachable daemon 304ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 571ms + ✓ flows run/resume CLI over the journal protocol > bounds a worker wait by its lease and reports what it is waiting for 597ms + ✓ flows run/resume CLI over the journal protocol > resumes a parked run from snapshot step types without reading journal sequence one 720ms + ✓ flows run/resume CLI over the journal protocol > maps only run_not_found resumes to exit 2 1902ms + ✓ tests/worker-cli.test.ts (13 tests) 23025ms + ✓ custom wrapper execution identity > passes an explicit safe environment at identification and execution 341ms + ✓ custom wrapper execution identity > refuses a wrapper symlink retarget before delivering private values 823ms + ✓ custom wrapper execution identity > bounds wrapper execution after acknowledgement 720ms + ✓ custom wrapper execution identity > bounds captured wrapper output 848ms + ✓ custom wrapper execution identity > refuses a duplicate execute protocol frame 529ms + ✓ custom wrapper execution bounds are reader-owned > resolves when a conforming wrapper leaks a stdio pipe to a background helper 2361ms + ✓ custom wrapper execution bounds are reader-owned > resolves when the leaked helper inherits stderr only 1950ms + ✓ custom wrapper execution bounds are reader-owned > resolves when a wrapper leaks a stdio pipe and exits before identifying 3254ms + ✓ custom wrapper execution bounds are reader-owned > journals a completionReason at the default bound when a wrapper leaks a stdio pipe 11255ms + ✓ custom wrapper execution bounds are reader-owned > accepts the same over-8KiB payload whether or not it coalesces with the execute token 430ms + ✓ tests/local-agent-live.test.ts (5 tests) 39723ms + ✓ built CLI local agent against a real daemon > dispatches through the wrapper and keeps --json stdout report-shaped 1209ms + ✓ built CLI local agent against a real daemon > runs beyond the initial 30-second lease without a second invocation 36039ms + ✓ built CLI local agent against a real daemon > renders actual agent completion in text output 881ms + ✓ built CLI local agent against a real daemon > returns a failed run when the agent process fails 793ms + ✓ built CLI local agent against a real daemon > refuses a workspace it cannot pin before invoking the agent 801ms +stdout | tests/live-kernel.test.ts > surface resume after a real daemon kill > resumes a three-step run with each successful completion exactly once +LIVE_KERNEL kill -9 pid=99452 run=01M27SNDND6GMEMVJVKTEMTSF0 while step=two state=Running + + ❯ tests/live-kernel.test.ts (30 tests | 1 failed) 53714ms + ✓ built flows CLI against live relayflowd > runs rung (a), parks rung (b), and keeps JSON report-shaped 5232ms + ✓ built flows CLI against live relayflowd > allows a deterministic run to exceed the bounded request timeout 32295ms + ✓ built flows CLI against live relayflowd > follows a live worker dispatch through flows run 862ms + ✓ built flows CLI against live relayflowd > f.agent lowers to a real agent step and dispatches through a live worker 304ms + ✓ built flows CLI against live relayflowd > f.agent's default flowPath anchors on cwd, not cwd's parent 372ms + ✓ built flows CLI against live relayflowd > can always get a parked run to a late-attaching worker 5564ms + ✓ built flows CLI against live relayflowd > reports a real manual-recovery NeedsHuman state as parked 823ms + ✓ built flows CLI against live relayflowd > AgentWorker passes a declared model to an identified wrapper as RELAYFLOW_MODEL 407ms + × built flows CLI against live relayflowd > hn-monitor analyze-story reaches done through the real Claude analyzer CLI 1534ms + → LIVE_ANALYZER_UNAVAILABLE: "/Users/khaliqgant/flows-spec-G-budget/testdata/preflight/analyze-story-claude-cli auth status" exited 1: analyze-story-claude-cli: "claude -p --model claude-haiku-4-5-20251001" exited 1: — failing because gate-2 acceptance requires the real analyzer to execute. Set RELAYFLOWS_ALLOW_ANALYZER_SKIP=1 only if this run is not gate evidence. + ✓ built flows CLI against live relayflowd > preflights before journaling and names an unreachable socket 1643ms + ✓ built flows CLI against live relayflowd > starts exactly one daemon when two runs race for one empty data dir 879ms + ✓ surface resume after a real daemon kill > resumes a three-step run with each successful completion exactly once 1669ms + +⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/live-kernel.test.ts > built flows CLI against live relayflowd > hn-monitor analyze-story reaches done through the real Claude analyzer CLI +Error: LIVE_ANALYZER_UNAVAILABLE: "/Users/khaliqgant/flows-spec-G-budget/testdata/preflight/analyze-story-claude-cli auth status" exited 1: analyze-story-claude-cli: "claude -p --model claude-haiku-4-5-20251001" exited 1: — failing because gate-2 acceptance requires the real analyzer to execute. Set RELAYFLOWS_ALLOW_ANALYZER_SKIP=1 only if this run is not gate evidence. + ❯ tests/live-kernel.test.ts:1186:15 + 1184| const notice = `LIVE_ANALYZER_UNAVAILABLE: ${readiness.detail}`; + 1185| if (process.env['RELAYFLOWS_ALLOW_ANALYZER_SKIP'] !== '1') { + 1186| throw new Error( + | ^ + 1187| `${notice} — failing because gate-2 acceptance requires the … + 1188| + 'Set RELAYFLOWS_ALLOW_ANALYZER_SKIP=1 only if this run is … + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/1]⎯ + + Test Files 1 failed | 53 passed | 1 skipped (55) + Tests 1 failed | 984 passed | 3 skipped (988) + Start at 10:33:43 + Duration 54.37s (transform 989ms, setup 0ms, collect 4.98s, tests 183.70s, environment 6ms, prepare 2.02s) diff --git a/evidence/spec-G-budget/smoke-journal.json b/evidence/spec-G-budget/smoke-journal.json new file mode 100644 index 000000000..221eb96ee --- /dev/null +++ b/evidence/spec-G-budget/smoke-journal.json @@ -0,0 +1,48 @@ +[ + { + "entry_type": "step.completed", + "step_id": "measured", + "payload": { + "budget": { + "dollars": "0", + "tokens_in": 0, + "tokens_out": 0 + }, + "completed_by": "kernel", + "completionReason": "success", + "disposition": "step_done", + "effects": [], + "end_pins": null, + "next_attempt_at_ms": null, + "output": { + "exit_code": 0, + "stderr_tail": "", + "stdout_tail": "" + }, + "spend": { + "dollars": 0, + "tokens_input": 0, + "tokens_output": 0, + "wallclock_ms": 19 + }, + "verification": { + "detail": "all gates passed", + "gate": "exit_code", + "verdict": "pass" + } + } + }, + { + "entry_type": "run.completed", + "step_id": null, + "payload": { + "budget_total": { + "dollars": "0", + "tokens_in": 0, + "tokens_out": 0 + }, + "completionReason": "budget_exceeded", + "failed_step_id": null + } + } +] diff --git a/evidence/spec-G-budget/smoke.log b/evidence/spec-G-budget/smoke.log new file mode 100644 index 000000000..f29c767f3 --- /dev/null +++ b/evidence/spec-G-budget/smoke.log @@ -0,0 +1,2 @@ +{"run_id":"01M27SE6KVHAK8G7Y2CHVK0HRF","status":"failed","completion_reason":"budget_exceeded","completed_steps":1} +Error: run 01M27SE6KVHAK8G7Y2CHVK0HRF failed diff --git a/evidence/spec-G-budget/smoke.spec.json b/evidence/spec-G-budget/smoke.spec.json new file mode 100644 index 000000000..2fdf844cf --- /dev/null +++ b/evidence/spec-G-budget/smoke.spec.json @@ -0,0 +1 @@ +{"version":"0.1.0","name":"budget-guarded","steps":[{"id":"measured","depends_on":[],"max_iterations":1,"retry":{"initial_backoff_ms":100,"max_backoff_ms":60000,"multiplier":2,"jitter_percent":20},"verification":{},"type":"deterministic","command":"sleep 0.01"},{"id":"guarded","depends_on":["measured"],"max_iterations":1,"retry":{"initial_backoff_ms":100,"max_backoff_ms":60000,"multiplier":2,"jitter_percent":20},"verification":{},"type":"deterministic","command":"printf 'budget should refuse this step'"}],"budget":{"pricing":"frozen","max_wallclock_ms":0}} diff --git a/evidence/spec-G-budget/surface-test.log b/evidence/spec-G-budget/surface-test.log new file mode 100644 index 000000000..263f20305 --- /dev/null +++ b/evidence/spec-G-budget/surface-test.log @@ -0,0 +1,9 @@ + + RUN v2.1.9 /Users/khaliqgant/flows-spec-G-budget/packages/surface + + ✓ tests/flow.test.ts (7 tests) 3ms + + Test Files 1 passed (1) + Tests 7 passed (7) + Start at 10:30:40 + Duration 228ms (transform 21ms, setup 0ms, collect 21ms, tests 3ms, environment 0ms, prepare 32ms) diff --git a/kernel/relayflowd-core/src/entry.rs b/kernel/relayflowd-core/src/entry.rs index 075a2a218..645f0cf62 100644 --- a/kernel/relayflowd-core/src/entry.rs +++ b/kernel/relayflowd-core/src/entry.rs @@ -133,6 +133,18 @@ pub struct JournalEntry { pub payload: Value, } +pub fn journal_dollars(value: &str) -> Result { + let (whole, fraction) = value.split_once('.').unwrap_or((value, "")); + let whole = whole.trim_start_matches('0'); + let whole = if whole.is_empty() { "0" } else { whole }; + let normalized = if fraction.is_empty() { + whole.to_owned() + } else { + format!("{whole}.{fraction}") + }; + normalized.parse() +} + impl JournalEntry { pub fn new( entry_type: EntryType, @@ -142,6 +154,16 @@ impl JournalEntry { at_ms: i64, payload: T, ) -> Self { + let mut payload = serde_json::to_value(payload).expect("journal payload must serialize"); + if entry_type == EntryType::StepCompleted { + let budget: Budget = + serde_json::from_value(payload["budget"].clone()).unwrap_or_default(); + payload["spend"] = serde_json::json!({ + "tokens_input": budget.tokens_in, "tokens_output": budget.tokens_out, + "dollars": journal_dollars(&budget.dollars).expect("valid journal dollars"), + "wallclock_ms": 0, + }); + } Self { seq: 0, segment_id: 0, @@ -150,7 +172,7 @@ impl JournalEntry { step_id, attempt, at_ms, - payload: serde_json::to_value(payload).expect("journal payload must serialize"), + payload, } } } diff --git a/kernel/relayflowd-core/src/machine.rs b/kernel/relayflowd-core/src/machine.rs index 9c2f8501d..0a0282814 100644 --- a/kernel/relayflowd-core/src/machine.rs +++ b/kernel/relayflowd-core/src/machine.rs @@ -17,6 +17,8 @@ use crate::{ const LEASE_DURATION_MS: i64 = 30_000; +mod budget; +pub use budget::exceeded as budget_exceeded; mod cancel; use cancel::cancel_run_actions; pub use cancel::request_cancel_action; @@ -115,6 +117,17 @@ pub fn next_actions(state: &RunState, now_ms: i64) -> Vec { return complete_run_actions(state, RunCompletionReason::Success, None, now_ms); } + if budget::exceeded(state, now_ms) { + if state + .steps + .values() + .any(|runtime| matches!(runtime.state, StepState::Running { .. })) + { + return Vec::new(); + } + return complete_run_actions(state, RunCompletionReason::BudgetExceeded, None, now_ms); + } + // Wake every retry whose deterministic timer is due before starting work. // Recovery can put several crashed parallel lanes into the same zero-delay // backoff; waking just one would let an already-runnable peer start and park diff --git a/kernel/relayflowd-core/src/machine/budget.rs b/kernel/relayflowd-core/src/machine/budget.rs new file mode 100644 index 000000000..092dcb68b --- /dev/null +++ b/kernel/relayflowd-core/src/machine/budget.rs @@ -0,0 +1,29 @@ +use crate::{Budget, RunState}; +use std::cmp::Ordering; + +/// Completion facts charge the envelope; already-running work remains valid. +pub fn exceeded(state: &RunState, now_ms: i64) -> bool { + let Some(limit) = &state.spec.budget else { + return false; + }; + let empty = Budget::default(); + let (spent, wallclock) = if limit.window.is_some() { + if state.budget_day == Some(now_ms.div_euclid(86_400_000)) { + (&state.daily_budget, state.daily_wallclock_ms) + } else { + (&empty, 0) + } + } else { + (&state.budget, state.wallclock_ms) + }; + limit.max_tokens_in.is_some_and(|n| spent.tokens_in > n) + || limit.max_tokens_out.is_some_and(|n| spent.tokens_out > n) + || limit.max_tokens.is_some_and(|n| { + u128::from(spent.tokens_in) + u128::from(spent.tokens_out) > u128::from(n) + }) + || limit.max_wallclock_ms.is_some_and(|n| wallclock > n) + || limit + .max_dollars + .as_deref() + .is_some_and(|n| crate::memory::decimal_cmp(&spent.dollars, n) == Ordering::Greater) +} diff --git a/kernel/relayflowd-core/src/memory.rs b/kernel/relayflowd-core/src/memory.rs index 6b912297a..06cbf2ea3 100644 --- a/kernel/relayflowd-core/src/memory.rs +++ b/kernel/relayflowd-core/src/memory.rs @@ -32,6 +32,11 @@ pub struct MemoryInjectedPayload { impl MemorySpec { pub fn validate(&self) -> Result<(), String> { + if self.budget.pricing.is_some() || self.budget.prior_spend.is_some() + || self.budget.max_tokens.is_some() || self.budget.max_wallclock_ms.is_some() + || self.budget.window.is_some() { + return Err("memory budgets accept only max_tokens_in, max_tokens_out, max_dollars".into()); + } if self.query.trim().is_empty() { return Err("query must be a non-empty string".into()); } @@ -73,7 +78,7 @@ impl MemorySpec { } } -fn valid_decimal(value: &str) -> bool { +pub fn valid_decimal(value: &str) -> bool { let (whole, fraction) = value .split_once('.') .map_or((value, None), |(w, f)| (w, Some(f))); @@ -83,7 +88,7 @@ fn valid_decimal(value: &str) -> bool { } /// Exact decimal comparison without floating point or fixed-width scaling. -fn decimal_cmp(left: &str, right: &str) -> Ordering { +pub(crate) fn decimal_cmp(left: &str, right: &str) -> Ordering { let parts = |value: &str| { let (whole, fraction) = value.split_once('.').unwrap_or((value, "")); ( diff --git a/kernel/relayflowd-core/src/spec.rs b/kernel/relayflowd-core/src/spec.rs index a2c15d8f3..c4e549e83 100644 --- a/kernel/relayflowd-core/src/spec.rs +++ b/kernel/relayflowd-core/src/spec.rs @@ -74,6 +74,21 @@ impl RunSpec { return Err(SpecError::EmptyCli); } + if let Some(budget) = &self.budget { + if budget + .max_dollars + .as_deref() + .is_some_and(|v| !crate::memory::valid_decimal(v)) + || budget + .prior_spend + .as_ref() + .is_some_and(|p| !crate::memory::valid_decimal(&p.dollars)) + { + return Err(SpecError::Malformed( + "budget dollars must be non-negative decimal strings".into(), + )); + } + } let mut trigger_ids = BTreeSet::new(); for trigger in &self.triggers { if trigger.id.trim().is_empty() @@ -525,6 +540,16 @@ impl TriggerSpec { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct BudgetSpec { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pricing: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub prior_spend: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_tokens: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_wallclock_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub window: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub max_tokens_in: Option, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -533,6 +558,29 @@ pub struct BudgetSpec { pub max_dollars: Option, } +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum BudgetWindow { + Day, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum BudgetPricing { + Frozen, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct PriorSpend { + pub tokens_in: u64, + pub tokens_out: u64, + pub dollars: String, + pub wallclock_ms: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub day: Option, +} + /// v0 verification gates (kernel DESIGN.md §4): `exit_code == 0` is implicit /// for deterministic steps; these two are optional and combinable. Unknown /// keys are a parse error — verification is control flow, and a dropped gate diff --git a/kernel/relayflowd-core/src/state.rs b/kernel/relayflowd-core/src/state.rs index 9575fd854..e14f034a1 100644 --- a/kernel/relayflowd-core/src/state.rs +++ b/kernel/relayflowd-core/src/state.rs @@ -13,10 +13,11 @@ use crate::{ }; mod budget; +#[cfg(test)] +use budget::add_budget; mod memory; mod pins; mod routing; -use budget::add_budget; #[derive(Debug, Clone, PartialEq)] pub enum StepState { @@ -69,6 +70,10 @@ pub struct RunState { pub steps: BTreeMap, pub memo: BTreeMap, pub budget: Budget, + pub wallclock_ms: u64, + pub budget_day: Option, + pub daily_budget: Budget, + pub daily_wallclock_ms: u64, pub completion: Option, /// Durable cancellation intent. Once present, scheduling can only close /// live work and append the terminal canceled fact. @@ -110,12 +115,32 @@ impl RunState { spec, memo: BTreeMap::new(), budget: Budget::default(), + wallclock_ms: 0, + budget_day: None, + daily_budget: Budget::default(), + daily_wallclock_ms: 0, completion: None, cancel_requested: None, current_pins: None, routing: BTreeMap::new(), }; + if let Some(prior) = state + .spec + .budget + .as_ref() + .and_then(|b| b.prior_spend.as_ref()) + { + state.budget = Budget { + tokens_in: prior.tokens_in, + tokens_out: prior.tokens_out, + dollars: prior.dollars.clone(), + }; + state.wallclock_ms = prior.wallclock_ms; + state.budget_day = prior.day; + state.daily_budget = state.budget.clone(); + state.daily_wallclock_ms = prior.wallclock_ms; + } for entry in entries { if entry.run_id != state.run_id { return Err(StateError::WrongRun(entry.run_id.clone())); @@ -251,7 +276,11 @@ impl RunState { fn apply_step_completed(&mut self, entry: &JournalEntry) -> Result<(), StateError> { let payload: StepCompletedPayload = decode(entry)?; - add_budget(&mut self.budget, &payload.budget)?; + self.charge_budget( + &payload.budget, + entry.payload["spend"]["wallclock_ms"].as_u64().unwrap_or(0), + entry.at_ms, + )?; let step_id = entry .step_id .clone() @@ -321,6 +350,9 @@ impl RunState { self.validate_routing(&payload.routing)?; self.routing = payload.routing; self.memo.clear(); + if self.spec.budget.is_some() && self.budget_day.is_some() && self.budget != payload.budget_spent { + return Err(StateError::BudgetSummaryMismatch); + } self.budget = payload.budget_spent; for runtime in self.steps.values_mut() { *runtime = StepRuntime { @@ -464,6 +496,8 @@ pub enum StateError { }, #[error("invalid non-negative decimal dollar amount {0:?}")] InvalidDollars(String), + #[error("epoch budget differs from recorded spend")] + BudgetSummaryMismatch, #[error("budget token total overflow")] BudgetOverflow, } diff --git a/kernel/relayflowd-core/src/state/budget.rs b/kernel/relayflowd-core/src/state/budget.rs index d12f52097..f3e25d4bc 100644 --- a/kernel/relayflowd-core/src/state/budget.rs +++ b/kernel/relayflowd-core/src/state/budget.rs @@ -1,6 +1,6 @@ use crate::entry::Budget; -use super::StateError; +use super::{RunState, StateError}; pub(super) fn add_budget(total: &mut Budget, value: &Budget) -> Result<(), StateError> { let tokens_in = total @@ -133,3 +133,30 @@ mod tests { assert_eq!(total, original); } } + +impl RunState { + pub(super) fn charge_budget( + &mut self, + value: &Budget, + duration: u64, + at_ms: i64, + ) -> Result<(), StateError> { + add_budget(&mut self.budget, value)?; + self.wallclock_ms = self + .wallclock_ms + .checked_add(duration) + .ok_or(StateError::BudgetOverflow)?; + let day = at_ms.div_euclid(86_400_000); + if self.budget_day != Some(day) { + self.budget_day = Some(day); + self.daily_budget = Budget::default(); + self.daily_wallclock_ms = 0; + } + add_budget(&mut self.daily_budget, value)?; + self.daily_wallclock_ms = self + .daily_wallclock_ms + .checked_add(duration) + .ok_or(StateError::BudgetOverflow)?; + Ok(()) + } +} diff --git a/kernel/relayflowd-core/src/state/memory.rs b/kernel/relayflowd-core/src/state/memory.rs index d3334dadf..db1fdce4f 100644 --- a/kernel/relayflowd-core/src/state/memory.rs +++ b/kernel/relayflowd-core/src/state/memory.rs @@ -1,4 +1,4 @@ -use super::{RunState, StateError, budget::add_budget, decode}; +use super::{RunState, StateError, decode}; use crate::{JournalEntry, MemoryInjectedPayload, StepState}; impl RunState { @@ -43,9 +43,7 @@ impl RunState { detail: "memory must be injected once by an active attempt".into(), }); } - let mut total = self.budget.clone(); - add_budget(&mut total, &payload.budget)?; - self.budget = total; + self.charge_budget(&payload.budget, 0, entry.at_ms)?; self.steps.get_mut(step_id).expect("validated step").memory = Some(payload); Ok(()) } diff --git a/kernel/relayflowd/src/engine.rs b/kernel/relayflowd/src/engine.rs index 8b62721cd..a005306c3 100644 --- a/kernel/relayflowd/src/engine.rs +++ b/kernel/relayflowd/src/engine.rs @@ -318,9 +318,15 @@ impl Engine { fn load_state(&self, journal: &SqliteJournal, spec: RunSpec) -> Result { let segment = journal.current_segment().map_err(|error| anyhow!(error))?; - let entries = journal - .scan_segment(segment) - .map_err(|error| anyhow!(error))?; + // Window and elapsed-time counters are reconstructed from completion + // facts across epochs, including facts predating this SDK version. + let entries = if spec.budget.is_some() { + journal.scan_all().map_err(|error| anyhow!(error))? + } else { + journal + .scan_segment(segment) + .map_err(|error| anyhow!(error))? + }; RunState::fold(journal.run_id(), spec, &entries).context("fold run journal") } @@ -332,6 +338,17 @@ impl Engine { self.ensure_journal_mutable(journal)?; let mut entry = entry.clone(); self.stamp_completion(journal, &mut entry)?; + if entry.entry_type == EntryType::StepCompleted { + let start = journal.scan_all()?.into_iter().rev().find(|e| { + e.entry_type == EntryType::StepAttemptStarted + && e.step_id == entry.step_id + && e.attempt == entry.attempt + }); + if let Some(start) = start { + entry.payload["spend"]["wallclock_ms"] = + serde_json::json!(entry.at_ms.saturating_sub(start.at_ms).max(0)); + } + } let persisted = journal.append(&entry).map_err(|error| anyhow!(error))?; if let Some(observer) = &self.observer { observer.appended(&persisted); diff --git a/kernel/relayflowd/src/engine/drive.rs b/kernel/relayflowd/src/engine/drive.rs index 8781005e0..2005b9e05 100644 --- a/kernel/relayflowd/src/engine/drive.rs +++ b/kernel/relayflowd/src/engine/drive.rs @@ -54,6 +54,16 @@ impl Engine { match action { Action::Append(mut entry) => { if entry.entry_type == relayflowd_core::EntryType::StepAttemptStarted { + // A deterministic peer may have completed since this batch + // was elected. Re-fold before admitting the next start. + if spec.budget.is_some() + && relayflowd_core::machine::budget_exceeded( + &self.load_state(&journal, spec.clone())?, + self.clock.now_ms(), + ) + { + break; + } let step_id = entry .step_id .as_deref() diff --git a/kernel/relayflowd/src/engine/remote.rs b/kernel/relayflowd/src/engine/remote.rs index 89051670c..3956b7251 100644 --- a/kernel/relayflowd/src/engine/remote.rs +++ b/kernel/relayflowd/src/engine/remote.rs @@ -38,6 +38,11 @@ impl Engine { step_id: &str, completion: OutOfBandCompletion, ) -> Result { + if !relayflowd_core::memory::valid_decimal(&completion.budget.dollars) + || relayflowd_core::journal_dollars(&completion.budget.dollars).is_err() + { + bail!("step completion usage.dollars must be a non-negative decimal string"); + } let mut journal = self.open_run(run_id)?; let spec = journal.run_spec().context("read run spec")?; let state = self.load_state(&journal, spec.clone())?; @@ -380,7 +385,11 @@ fn worker_failure_detail(output: &Value) -> Option { // and slicing it by byte index would panic on multi-byte input. Some(match trimmed.char_indices().nth(MAX_CHARS) { None => trimmed.to_owned(), - Some((cut, _)) => format!("{}… ({} bytes truncated)", &trimmed[..cut], trimmed.len() - cut), + Some((cut, _)) => format!( + "{}… ({} bytes truncated)", + &trimmed[..cut], + trimmed.len() - cut + ), }) } @@ -433,7 +442,10 @@ mod worker_failure_detail_tests { // 3000 three-byte chars = 9000 bytes. let output = json!("€".repeat(3000)); let detail = worker_failure_detail(&output).expect("detail for a long output"); - assert!(detail.contains('…'), "expected a truncation marker, got {detail:?}"); + assert!( + detail.contains('…'), + "expected a truncation marker, got {detail:?}" + ); // Cut at 2000 CHARS = 6000 bytes, so 3000 bytes remain. assert!( detail.contains("3000 bytes truncated"), diff --git a/kernel/relayflowd/tests/budget_gate.rs b/kernel/relayflowd/tests/budget_gate.rs new file mode 100644 index 000000000..4f810c849 --- /dev/null +++ b/kernel/relayflowd/tests/budget_gate.rs @@ -0,0 +1,172 @@ +use relayflowd::{ + Engine, OutOfBandCompletion, + worker::{DispatchOutcome, JournalObserver, StepDispatch, StepDispatcher}, +}; +use relayflowd_core::{ + Budget, CompletionReason, EntryType, JournalEntry, RunCompletionReason, RunSpec, StepType, +}; +use serde_json::json; +use std::sync::{Arc, Mutex}; + +#[derive(Default)] +struct Worker(Mutex>); +impl JournalObserver for Worker { + fn appended(&self, _: &JournalEntry) {} +} +impl StepDispatcher for Worker { + fn executor(&self, _: StepType) -> Option { + Some("mock".into()) + } + fn available(&self, _: StepType) -> bool { + true + } + fn dispatch(&self, d: StepDispatch) -> anyhow::Result { + self.0.lock().unwrap().push(d); + Ok(DispatchOutcome::Dispatched) + } +} + +#[test] +fn crossing_completion_is_durable_and_next_step_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let worker = Arc::new(Worker::default()); + let engine = Engine::with_runtime(dir.path(), worker.clone(), worker.clone()); + let spec = RunSpec::parse(&json!({"budget":{"max_dollars":"0.001"},"steps":[ + {"id":"first","type":"llm","prompt":"answer"}, + {"id":"second","type":"deterministic","command":"printf must-not-run","depends_on":["first"]} + ]})).unwrap(); + let started = engine.start(spec, "test", None).unwrap(); + let d = worker.0.lock().unwrap()[0].clone(); + let outcome = engine + .complete_out_of_band( + &started.run_id, + "first", + OutOfBandCompletion { + attempt: d.attempt, + idempotency_key: d.idempotency_key, + completion_reason: CompletionReason::Success, + output: json!("answer"), + budget: Budget { + tokens_in: 1000, + tokens_out: 200, + dollars: "0.006000".into(), + }, + completed_by: "mock".into(), + started_pins: None, + end_pins: None, + effects: vec![], + trajectory_tail: None, + }, + ) + .unwrap(); + assert_eq!( + outcome.completion_reason, + Some(RunCompletionReason::BudgetExceeded) + ); + let reopened = Engine::new(dir.path()); + let entries = reopened.journal_entries(&started.run_id, 1, 100).unwrap(); + let completed = entries + .iter() + .find(|e| e.entry_type == EntryType::StepCompleted) + .unwrap(); + assert_eq!(completed.payload["completionReason"], "success"); + assert_eq!(completed.payload["output"], "answer"); + assert_eq!(completed.payload["spend"]["tokens_input"], 1000); + assert_eq!(completed.payload["spend"]["tokens_output"], 200); + assert_eq!(completed.payload["spend"]["dollars"], json!(0.006)); + assert!(completed.payload["spend"]["wallclock_ms"].is_u64()); + assert!( + !entries + .iter() + .any(|e| e.entry_type == EntryType::StepAttemptStarted + && e.step_id.as_deref() == Some("second")) + ); + assert_eq!( + reopened + .resume(&started.run_id, None) + .unwrap() + .completion_reason, + outcome.completion_reason + ); +} + +#[test] +fn deterministic_spend_and_wallclock_limit_gate_parallel_batch_starts() { + let dir = tempfile::tempdir().unwrap(); + let engine = Engine::new(dir.path()); + let spec = RunSpec::parse(&json!({"budget":{"max_wallclock_ms":0},"steps":[ + {"id":"first","type":"deterministic","command":"sleep 0.01"}, + {"id":"second","type":"deterministic","command":"printf must-not-run"} + ]})) + .unwrap(); + let outcome = engine.start(spec, "test", None).unwrap(); + assert_eq!( + outcome.completion_reason, + Some(RunCompletionReason::BudgetExceeded) + ); + let entries = engine.journal_entries(&outcome.run_id, 1, 100).unwrap(); + let completions: Vec<_> = entries + .iter() + .filter(|e| e.entry_type == EntryType::StepCompleted) + .collect(); + assert_eq!(completions.len(), 1); + assert_eq!(completions[0].payload["spend"]["tokens_input"], 0); + assert!( + completions[0].payload["spend"]["wallclock_ms"] + .as_u64() + .unwrap() + > 0 + ); +} + +#[test] +fn daily_windows_reset_and_exact_limits_do_not_refuse() { + use relayflowd_core::{Action, AttemptResult, RunState, completion_actions, next_actions}; + for (window, limit, now, exceeded) in [ + ("day", "0.005", 10, true), + ("day", "0.005", 86_400_000, false), + ("day", "0.006", 10, false), + ] { + let spec = RunSpec::parse( + &json!({"budget":{"max_dollars":limit,"window":window},"steps":[ + {"id":"first","type":"llm","prompt":"answer"}, + {"id":"second","type":"deterministic","command":":","depends_on":["first"]} + ]}), + ) + .unwrap(); + let initial = RunState::fold("run", spec.clone(), &[]).unwrap(); + let mut entries: Vec<_> = next_actions(&initial, 0) + .into_iter() + .filter_map(|a| { + if let Action::Append(e) = a { + Some(e) + } else { + None + } + }) + .collect(); + let mut result = AttemptResult::successful(json!("answer"), "mock"); + result.budget = Budget { + tokens_in: 1000, + tokens_out: 200, + dollars: "0.006".into(), + }; + entries.extend( + completion_actions("run", &spec.steps[0], 1, 0, result, 1) + .into_iter() + .filter_map(|a| { + if let Action::Append(e) = a { + Some(e) + } else { + None + } + }), + ); + let replay = RunState::fold("run", spec, &entries).unwrap(); + assert_eq!( + relayflowd_core::machine::budget_exceeded(&replay, now), + exceeded + ); + assert_eq!(replay.memo["first"], json!("answer")); + } +} diff --git a/packages/sdk/src/authored-budget.ts b/packages/sdk/src/authored-budget.ts new file mode 100644 index 000000000..491a02698 --- /dev/null +++ b/packages/sdk/src/authored-budget.ts @@ -0,0 +1,69 @@ +import { BudgetSyntaxError, parseBudget, toKernelBudget } from './budget.js'; +import { AuthoredFlowExecutionError } from './authored-flow-error.js'; +import type { JournalClient } from './journal-client.js'; +import type { RunOutcome } from './protocol.js'; +import type { KernelBudgetSpec, KernelRunSpec } from './spec.js'; + +/** Serialized admission for the internal authored runner's separate step runs. */ +export class AuthoredBudget { + private readonly limit: KernelBudgetSpec | undefined; + private failed = false; + private tail: Promise = Promise.resolve(); + private charges: { input: bigint; output: bigint; micro: bigint; ms: bigint; day: number }[] = []; + + constructor(header: unknown) { + try { + this.limit = header === undefined ? undefined : toKernelBudget(parseBudget(header)); + } catch (error) { + if (error instanceof BudgetSyntaxError) throw new AuthoredFlowExecutionError('budget_syntax_invalid', error.message); + throw error; + } + } + + async execute(journal: JournalClient, spec: KernelRunSpec, consume: (outcome: RunOutcome) => Promise): Promise { + if (this.limit === undefined) return consume(await journal.runStart(spec)); + const previous = this.tail; + let release!: () => void; + this.tail = new Promise(resolve => { release = resolve; }); + await previous; + try { + if (this.failed) throw new AuthoredFlowExecutionError('step_failed', 'A prior budgeted step did not finish successfully.'); + // Window selection follows journal timestamps, never the SDK host clock. + const day = this.charges.at(-1)?.day; + const total = this.charges.filter(c => this.limit!.window !== 'day' || c.day === day) + .reduce((s, c) => ({ input: s.input + c.input, output: s.output + c.output, micro: s.micro + c.micro, ms: s.ms + c.ms }), + { input: 0n, output: 0n, micro: 0n, ms: 0n }); + const exactNumber = (n: bigint) => { if (n > BigInt(Number.MAX_SAFE_INTEGER)) throw new Error('budget counter overflow'); return Number(n); }; + const outcome = await journal.runStart({ ...spec, budget: { ...this.limit, prior_spend: { + tokens_in: exactNumber(total.input), tokens_out: exactNumber(total.output), + dollars: `${total.micro / 1_000_000n}.${String(total.micro % 1_000_000n).padStart(6, '0')}`, + wallclock_ms: exactNumber(total.ms), ...(this.limit.window === 'day' && day !== undefined ? { day } : {}), + } } }); + try { + if (outcome.completion_reason === 'budget_exceeded') throw new AuthoredFlowExecutionError('step_failed', 'Flow budget exceeded before the next step.', 'budget_exceeded', outcome.run_id); + return await consume(outcome); + } finally { + let seq = 1; + for (;;) { + const { entries } = await journal.journalRead(outcome.run_id, seq); + if (entries.length === 0) break; + for (const raw of entries) { + const e = raw as {seq: number; entry_type: string; at_ms: number; payload: {budget?: {tokens_in: number; tokens_out: number; dollars: string}; spend?: {wallclock_ms: number}}}; + seq = e.seq + 1; + if (!['step.completed', 'memory.injected'].includes(e.entry_type)) continue; + const b = e.payload.budget; + if (b === undefined) throw new Error('journal completion missing budget'); + const [whole, fraction = ''] = b.dollars.split('.'); + if (fraction.length > 6 && /[1-9]/.test(fraction.slice(6))) throw new Error('budget accounting requires microdollar precision'); + this.charges.push({input: BigInt(b.tokens_in), output: BigInt(b.tokens_out), + micro: BigInt(whole!) * 1_000_000n + BigInt(fraction.slice(0, 6).padEnd(6, '0')), + ms: BigInt(e.payload.spend?.wallclock_ms ?? 0), day: Math.floor(e.at_ms / 86_400_000)}); + } + } + } + } catch (error) { + this.failed = true; + throw error; + } finally { release(); } + } +} diff --git a/packages/sdk/src/authored-flow-error.ts b/packages/sdk/src/authored-flow-error.ts index 728a24c46..e3eabc899 100644 --- a/packages/sdk/src/authored-flow-error.ts +++ b/packages/sdk/src/authored-flow-error.ts @@ -6,6 +6,8 @@ import type { export type AuthoredFlowExecutionErrorCode = | 'helper_slack.credential_missing' | 'helper_slack.mount_required' + | 'budget_syntax_invalid' + | 'budget_missing_price' | 'agent_cli_unresolved' | 'agent_parked' | 'llm_cli_unresolved' diff --git a/packages/sdk/src/authored-flow-executor.ts b/packages/sdk/src/authored-flow-executor.ts index 64a5a7005..190f4a851 100644 --- a/packages/sdk/src/authored-flow-executor.ts +++ b/packages/sdk/src/authored-flow-executor.ts @@ -6,6 +6,7 @@ import { snapshotJsonValue } from './json-value.js'; import type { SlackCall } from './slack-writeback.js'; import { checkMcpHeader, McpPreflightError } from './cli/check-typescript.js'; import { buildMcpProxy, runMcpEffect } from './authored-mcp.js'; +import { AuthoredBudget } from './authored-budget.js'; import { authoredWorkerRunner } from './authored-worker-step.js'; import { readSuccessfulOutput, isSurfaceRunCompletionReason } from './authored-step-output.js'; import { @@ -84,7 +85,7 @@ type JournalStepUsesStepCompletionReason = Assert< * * This is deliberately not exported by the SDK package: without a durable * authored root, it is not a resumable public runner. The seam is narrow: an - * empty-header flow may await `f.run`, `f.llm`, and `f.agent` steps and must + * flow with an optional budget may await `f.run`, `f.llm`, and `f.agent` steps and must * finish with `f.done("success")`. Each step and the terminal marker is * a compiled spec submitted through * `JournalClient`; values are read back from `step.completed` journal entries. @@ -134,7 +135,7 @@ export async function executeAuthoredFlow( ...(options.onWait !== undefined ? { onWait: options.onWait } : {}), }; const definition = getDefinition(handle); - const headerFields = Object.keys(definition.header).filter(key => key !== 'tools'); + const headerFields = Object.keys(definition.header).filter(key => key !== 'tools' && key !== 'budget'); if (definition.header.tools && Object.keys(definition.header.tools).some(key => !['slack', 'mcp'].includes(key))) headerFields.push('tools'); if (definition.header.tools?.relayfile !== undefined) headerFields.push('tools.relayfile'); const helperPreflight = checkSlackHelpers(definition); @@ -149,6 +150,7 @@ export async function executeAuthoredFlow( const checkedMcp = await checkMcpHeader(definition, flowPath); if (!checkedMcp.report.ok) throw new McpPreflightError(checkedMcp.report); + const budget = new AuthoredBudget(definition.header.budget); const journalSteps: AuthoredFlowJournalStep[] = []; const authoredSteps: AuthoredFlowOperation[] = []; const lifecycle = new AuthoredFlowLifecycle(); @@ -158,17 +160,18 @@ export async function executeAuthoredFlow( const lowerDeterministic = async ( id: string, command: string, + terminal = false, ): Promise => { const spec = toKernelSpec(compileSpec({ version: SPEC_SCHEMA_VERSION, name: `${definition.name}/${id}`, steps: [{ id, type: 'deterministic', command }], })); - const outcome = await journal.runStart(spec); - return readSuccessfulOutput(journal, outcome, id, journalSteps); + if (terminal) return readSuccessfulOutput(journal, await journal.runStart(spec), id, journalSteps); + return budget.execute(journal, spec, outcome => readSuccessfulOutput(journal, outcome, id, journalSteps)); }; - const worker = authoredWorkerRunner(definition, journal, flowPath, journalSteps, waitOptions, localAgentStream); + const worker = authoredWorkerRunner(definition, journal, flowPath, journalSteps, waitOptions, localAgentStream, budget, definition.header.budget); function llmOperation(strings: TemplateStringsArray, ...values: unknown[]): Step; function llmOperation(prompt: string, options: LlmOptions): Step; @@ -337,7 +340,7 @@ export async function executeAuthoredFlow( lifecycle.close(); } - await lowerDeterministic(`complete-${nextStep}`, ':'); + await lowerDeterministic(`complete-${nextStep}`, ':', true); return Object.freeze({ name: definition.name, completionReason: requestedCompletion, diff --git a/packages/sdk/src/authored-worker-step.ts b/packages/sdk/src/authored-worker-step.ts index 389f449f4..01c965b44 100644 --- a/packages/sdk/src/authored-worker-step.ts +++ b/packages/sdk/src/authored-worker-step.ts @@ -1,3 +1,5 @@ +import type { AuthoredBudget } from './authored-budget.js'; +import { parseBudget } from './budget.js'; import type { AgentOptions, AgentResult, LlmOptions } from '@relayflows/surface'; import { toKernelSpec } from './compile.js'; import { checkAuthoredFlow } from './cli/check.js'; @@ -16,11 +18,11 @@ const WORKSPACE_PERMISSION_ANNOTATION = /:\s*(readonly|readwrite)\s*$/i; export function authoredWorkerRunner( definition: { name: string }, journal: JournalClient, flowPath: string, journalSteps: AuthoredFlowJournalStep[], waitOptions: RunLifecycleOptions, - localAgentStream?: string, + localAgentStream?: string, budget?: AuthoredBudget, headerBudget?: unknown, ) { async function run(step: StepSpec): Promise { const id = step.id; - const authoring: FlowSpec = { version: SPEC_SCHEMA_VERSION, name: `${definition.name}/${id}`, steps: [step] }; + const authoring: FlowSpec = { version: SPEC_SCHEMA_VERSION, name: `${definition.name}/${id}`, steps: [step], ...(headerBudget === undefined ? {} : { budget: parseBudget(headerBudget) }) }; // The kernel never resolves a `cli` on its own — every declarative // `flows run`/`flows check` binds it first via this exact function // (cli/check.ts), searching for the nearest flows.json from `flowPath` @@ -33,14 +35,15 @@ export function authoredWorkerRunner( diagnostic.severity === 'refusal', ); throw new AuthoredFlowExecutionError( - step.type === 'llm' ? 'llm_cli_unresolved' : 'agent_cli_unresolved', + refusal?.kind === 'budget_missing_price' || refusal?.kind === 'budget_syntax_invalid' ? refusal.kind + : step.type === 'llm' ? 'llm_cli_unresolved' : 'agent_cli_unresolved', refusal?.message ?? `flow "${definition.name}" step "${id}": no CLI could be resolved for f.${step.type} ` + `(searched for flows.json from "${flowPath}")`, ); } const spec = toKernelSpec(resolved); - const outcome = await journal.runStart(spec); + const consume = async (outcome: import('./protocol.js').RunOutcome) => { // Reuse the declarative CLI's own wait/classification (cli/run.ts) rather // than a hand-rolled poll: `step.completed` and the run's own terminal // state are appended as two SEPARATE actions (kernel/relayflowd-core/src/machine.rs @@ -75,6 +78,8 @@ export function authoredWorkerRunner( ); } return readCompletedStepOutput(journal, outcome.run_id, id, journalSteps); + }; + return budget === undefined ? consume(await journal.runStart(spec)) : budget.execute(journal, spec, consume); } return { diff --git a/packages/sdk/src/budget-preflight.ts b/packages/sdk/src/budget-preflight.ts new file mode 100644 index 000000000..449f14633 --- /dev/null +++ b/packages/sdk/src/budget-preflight.ts @@ -0,0 +1,31 @@ +import type { CompiledFlowSpec } from './compile.js'; +import type { PreflightRefusal } from './preflight.js'; +import { MODEL_PRICING } from './model-pricing.js'; + +/** Legacy explicit envelopes keep their worker-supplied pricing contract. */ +export function budgetDiagnostics(flow: CompiledFlowSpec): PreflightRefusal[] { + if (flow.budget?.pricing !== 'frozen') return []; + const diagnostics: PreflightRefusal[] = []; + const declared = [ + ...Object.entries(flow.agents ?? {}).map(([agent, d]) => ({ agent, model: d.model })), + ...flow.steps.flatMap(s => s.type !== 'deterministic' && s.model !== undefined + ? [{ stepId: s.id, model: s.model }] : []), + ]; + for (const step of flow.steps) { + if (step.type === 'deterministic' || flow.budget.maxDollars === undefined) continue; + const model = step.model ?? (step.type === 'agent' && step.agent !== undefined + ? flow.agents?.[step.agent]?.model : undefined); + if (model === undefined) diagnostics.push({ + severity: 'refusal', kind: 'budget_missing_price', stepId: step.id, + message: `Step "${step.id}" needs a declared, priced model for its dollar budget.`, + }); + } + for (const declaration of declared) { + if (Object.hasOwn(MODEL_PRICING, declaration.model)) continue; + diagnostics.push({ + severity: 'refusal', kind: 'budget_missing_price', ...declaration, + message: `Model "${declaration.model}" has no frozen price for budget accounting.`, + }); + } + return diagnostics; +} diff --git a/packages/sdk/src/budget.ts b/packages/sdk/src/budget.ts new file mode 100644 index 000000000..264723d95 --- /dev/null +++ b/packages/sdk/src/budget.ts @@ -0,0 +1,49 @@ +import type { BudgetSpec, KernelBudgetSpec } from './spec.js'; + +export type HeaderBudget = string | { tokens?: number; dollars?: number; wallclock?: string }; + +export class BudgetSyntaxError extends Error { + constructor() { super('budget_syntax_invalid: expected $/run, $/day, or { tokens, dollars, wallclock }'); } +} + +/** Normalize surface sugar; existing explicit envelopes remain supported. */ +export function parseBudget(value: unknown): BudgetSpec { + if (typeof value === 'string') { + const match = /^\$(\d+(?:\.\d{1,6})?)\/(run|day)$/.exec(value); + if (!match) throw new BudgetSyntaxError(); + return { pricing: 'frozen', maxDollars: match[1]!, ...(match[2] === 'day' ? { window: 'day' as const } : {}) }; + } + if (typeof value !== 'object' || value === null || Array.isArray(value)) throw new BudgetSyntaxError(); + const b = value as Record; + const legacy = ['maxTokensIn', 'maxTokensOut', 'maxTokens', 'maxDollars', 'maxWallclockMs', 'window', 'pricing']; + if (Object.keys(b).some(k => legacy.includes(k))) { + if (Object.keys(b).some(k => !legacy.includes(k))) throw new BudgetSyntaxError(); + return b as BudgetSpec; + } + if (Object.keys(b).some(k => !['tokens', 'dollars', 'wallclock'].includes(k))) throw new BudgetSyntaxError(); + if (b.tokens !== undefined && (!Number.isSafeInteger(b.tokens) || (b.tokens as number) < 0)) throw new BudgetSyntaxError(); + if (b.dollars !== undefined && (typeof b.dollars !== 'number' || !Number.isFinite(b.dollars) || b.dollars < 0 || !/^\d+(?:\.\d{1,6})?$/.test(String(b.dollars)))) throw new BudgetSyntaxError(); + let ms: number | undefined; + if (b.wallclock !== undefined) { + const match = typeof b.wallclock === 'string' && /^(\d+)(ms|s|m|h|d)$/.exec(b.wallclock); + if (!match) throw new BudgetSyntaxError(); + ms = Number(match[1]) * ({ms:1,s:1000,m:60000,h:3600000,d:86400000}[match[2]!]!); + if (!Number.isSafeInteger(ms)) throw new BudgetSyntaxError(); + } + return { pricing: 'frozen', ...(b.tokens === undefined ? {} : {maxTokens: b.tokens as number}), + ...(b.dollars === undefined ? {} : {maxDollars: String(b.dollars)}), + ...(ms === undefined ? {} : {maxWallclockMs: ms}) }; +} + +/** Lower a normalized envelope to the existing kernel protocol. */ +export function toKernelBudget(budget: BudgetSpec): KernelBudgetSpec { + return { + ...(budget.pricing !== undefined ? { pricing: budget.pricing } : {}), + ...(budget.maxTokens !== undefined ? { max_tokens: budget.maxTokens } : {}), + ...(budget.maxWallclockMs !== undefined ? { max_wallclock_ms: budget.maxWallclockMs } : {}), + ...(budget.window !== undefined ? { window: budget.window } : {}), + ...(budget.maxTokensIn !== undefined ? { max_tokens_in: budget.maxTokensIn } : {}), + ...(budget.maxTokensOut !== undefined ? { max_tokens_out: budget.maxTokensOut } : {}), + ...(budget.maxDollars !== undefined ? { max_dollars: budget.maxDollars } : {}), + }; +} diff --git a/packages/sdk/src/cli/check-typescript.ts b/packages/sdk/src/cli/check-typescript.ts index 356eb42ed..82fc5c66f 100644 --- a/packages/sdk/src/cli/check-typescript.ts +++ b/packages/sdk/src/cli/check-typescript.ts @@ -26,7 +26,8 @@ export async function checkMcpHeader( path: string, ): Promise { const empty = { servers: Object.freeze({}), inventory: Object.freeze({}) }; - const unsupported = Object.keys(definition.header).filter(key => key !== 'tools'); + const KNOWN_HEADER_FIELDS = new Set(['tools', 'budget', 'identity', 'memory', 'workspace', 'use']); + const unsupported = Object.keys(definition.header).filter(key => !KNOWN_HEADER_FIELDS.has(key)); if (definition.header.tools?.relayfile !== undefined) unsupported.push('tools.relayfile'); if (unsupported.length) { return { ...empty, report: inputFailureReport({ kind: 'invalid_spec', diff --git a/packages/sdk/src/cli/direct-run.ts b/packages/sdk/src/cli/direct-run.ts index 1987317d3..d9bc82652 100644 --- a/packages/sdk/src/cli/direct-run.ts +++ b/packages/sdk/src/cli/direct-run.ts @@ -120,6 +120,8 @@ export async function runDirectFlow( || (error instanceof AuthoredFlowExecutionError && (error.code === 'helper_slack.credential_missing' || error.code === 'helper_slack.mount_required' + || error.code === 'budget_syntax_invalid' + || error.code === 'budget_missing_price' || error.code === 'unsupported_header' || error.code === 'agent_cli_unresolved' || error.code === 'llm_cli_unresolved' @@ -128,7 +130,12 @@ export async function runDirectFlow( exitCode: 2, report: { ...fromCheckReport('run', inputFailureReport({ - kind: error instanceof AuthoredFlowExecutionError && (error.code === 'helper_slack.credential_missing' || error.code === 'helper_slack.mount_required') ? error.code : 'invalid_spec', + kind: error instanceof AuthoredFlowExecutionError && ( + error.code === 'helper_slack.credential_missing' + || error.code === 'helper_slack.mount_required' + || error.code === 'budget_syntax_invalid' + || error.code === 'budget_missing_price' + ) ? error.code : 'invalid_spec', message: error.message, }, path)), socketPath, diff --git a/packages/sdk/src/compile.ts b/packages/sdk/src/compile.ts index 1bbf5efb2..502f34890 100644 --- a/packages/sdk/src/compile.ts +++ b/packages/sdk/src/compile.ts @@ -15,6 +15,7 @@ // `tests/spec_parity.rs` pin both sides to the same `testdata/` fixture. import { parse as parseYaml } from 'yaml'; +import { parseBudget, toKernelBudget } from './budget.js'; import { bindingDependencies } from './input-binding.js'; import type { AgentStepSpec, @@ -42,10 +43,13 @@ import { snapshotJsonValue } from './json-value.js'; export class CompileError extends Error { readonly errors: string[]; - constructor(errors: string[]) { + /** Optional diagnostic kind for callers that classify refusals (e.g. preflight). */ + readonly kind?: string; + constructor(errors: string[], kind?: string) { super('spec compile failed:\n - ' + errors.join('\n - ')); this.name = 'CompileError'; this.errors = errors; + if (kind !== undefined) this.kind = kind; } } @@ -53,7 +57,7 @@ export class CompileError extends Error { * Compile a YAML string into a validated authoring `FlowSpec`. * Throws `CompileError` on a YAML parse error or any validation failure. */ -export function compileYaml(yaml: string): FlowSpec { +export function compileYaml(yaml: string): CompiledFlowSpec { const parsed = parseYaml(yaml); if (parsed === null || typeof parsed !== 'object') { throw new CompileError(['YAML: expected a mapping at the top level']); @@ -70,7 +74,9 @@ export function compileYamlToCanonicalJson(yaml: string): string { * Validate a parsed spec object and apply authoring defaults, returning a * normalized `FlowSpec`. Throws `CompileError` on validation failure. */ -export function compileSpec(spec: unknown): FlowSpec { +export type CompiledFlowSpec = Omit & { budget?: import('./spec.js').BudgetSpec }; + +export function compileSpec(spec: unknown): CompiledFlowSpec { let snapshot: unknown; try { snapshot = snapshotJsonValue(spec, 'spec'); @@ -79,15 +85,31 @@ export function compileSpec(spec: unknown): FlowSpec { error instanceof Error ? error.message : 'spec: expected JSON-compatible data', ]); } + if (snapshot !== null && typeof snapshot === 'object' && !Array.isArray(snapshot) && 'budget' in snapshot && snapshot.budget !== undefined) { + // parseBudget throws BudgetSyntaxError on any malformed header. Without + // this wrap, that throw escaped compileSpec's own CompileError contract, + // so callers (validate, cli/check) that only catch CompileError would + // surface the budget error as an uncaught exception instead of a + // diagnostic. Rewrap as CompileError so it flows through the same + // gate-1 refusal path as every other invalid spec. + try { + snapshot = { ...snapshot, budget: parseBudget(snapshot.budget) }; + } catch (error) { + throw new CompileError( + [`spec.budget: ${error instanceof Error ? error.message : 'budget_syntax_invalid'}`], + 'budget_syntax_invalid', + ); + } + } const validation: ValidationResult = validateSpec(snapshot); if (!validation.ok) throw new CompileError(validation.errors); - const input = snapshot as FlowSpec; + const input = snapshot as CompiledFlowSpec; // Preserve named declarations and selectors through authoring normalization. // They are resolved exactly once at the kernel boundary, after public // preflight has validated every declaration with truthful provenance. const steps = input.steps.map(compileStep); - const flow: FlowSpec = { + const flow: CompiledFlowSpec = { version: input.version, ...(input.name !== undefined ? { name: input.name } : {}), ...(input.description !== undefined ? { description: input.description } : {}), @@ -256,15 +278,7 @@ export function toKernelSpec(flow: FlowSpec): KernelRunSpec { // look correct. See ops/reviews/20260903-pr139-repair-0903.md section 10. ...(compiled.triggers?.length ? { triggers: compiled.triggers.map(toKernelTrigger) } : {}), steps: compiled.steps.map((step) => toKernelStep(resolveNamedAgent(step, compiled.agents))), - ...(compiled.budget !== undefined - ? { - budget: { - ...(compiled.budget.maxTokensIn !== undefined ? { max_tokens_in: compiled.budget.maxTokensIn } : {}), - ...(compiled.budget.maxTokensOut !== undefined ? { max_tokens_out: compiled.budget.maxTokensOut } : {}), - ...(compiled.budget.maxDollars !== undefined ? { max_dollars: compiled.budget.maxDollars } : {}), - }, - } - : {}), + ...(compiled.budget !== undefined ? { budget: toKernelBudget(compiled.budget) } : {}), }; } @@ -451,8 +465,12 @@ function kernelMemoryToAuthoring(value: unknown, at: string): unknown { } function kernelBudgetToAuthoring(value: unknown, at: string): unknown { - const budget = requireKernelObject(value, ['max_tokens_in', 'max_tokens_out', 'max_dollars'], at); + const budget = requireKernelObject(value, ['max_tokens_in', 'max_tokens_out', 'max_dollars', 'max_tokens', 'max_wallclock_ms', 'window', 'pricing'], at); return { + ...(budget['pricing'] !== undefined ? { pricing: budget['pricing'] } : {}), + ...(budget['max_tokens'] !== undefined ? { maxTokens: budget['max_tokens'] } : {}), + ...(budget['max_wallclock_ms'] !== undefined ? { maxWallclockMs: budget['max_wallclock_ms'] } : {}), + ...(budget['window'] !== undefined ? { window: budget['window'] } : {}), ...(budget['max_tokens_in'] !== undefined ? { maxTokensIn: budget['max_tokens_in'] } : {}), ...(budget['max_tokens_out'] !== undefined ? { maxTokensOut: budget['max_tokens_out'] } : {}), ...(budget['max_dollars'] !== undefined ? { maxDollars: budget['max_dollars'] } : {}), diff --git a/packages/sdk/src/failure-kinds.ts b/packages/sdk/src/failure-kinds.ts index 4cdafa202..db6f48dfa 100644 --- a/packages/sdk/src/failure-kinds.ts +++ b/packages/sdk/src/failure-kinds.ts @@ -6,6 +6,8 @@ const PREFLIGHT_ENVIRONMENT_FAILURE_KINDS = [ 'helper_slack.mount_required', 'mcp_undeclared_server', 'mcp_unreachable', + 'budget_syntax_invalid', + 'budget_missing_price', 'cli_missing', 'cli_unauthenticated', 'cli_unresolved', diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index 102770910..962df481d 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -45,6 +45,9 @@ export type { VerificationSpec, WorkspaceSurface, } from './spec.js'; +export { MODEL_PRICING } from './model-pricing.js'; +export type { HeaderBudget } from './budget.js'; +export type { StepSpend } from './protocol.js'; export { SPEC_SCHEMA_VERSION } from './spec.js'; export { CloudFlowError, type CloudConnectionOptions } from './cloud-http.js'; diff --git a/packages/sdk/src/llm-worker.ts b/packages/sdk/src/llm-worker.ts index 6d9094f65..b004b947d 100644 --- a/packages/sdk/src/llm-worker.ts +++ b/packages/sdk/src/llm-worker.ts @@ -1,3 +1,5 @@ +import { workerSpend } from './worker-spend.js'; +import type { WorkerCliResult } from './worker-cli.js'; import { EventEmitter } from 'node:events'; import type { JournalClient } from './journal-client.js'; import type { CompletionReason, StepDispatchEvent } from './protocol.js'; @@ -47,10 +49,11 @@ export class LlmWorker extends EventEmitter { const schema = spec.verification?.json_schema; const prompt = schema === undefined ? spec.prompt : `${spec.prompt}\n\nReturn only a JSON value matching this JSON Schema (no Markdown fences):\n${JSON.stringify(schema)}`; - const result = await withWorkerLease(this.client, dispatch, signal => + const completed: WorkerCliResult = await withWorkerLease(this.client, dispatch, signal => typeof spec.cli === 'string' && typeof spec.prompt === 'string' ? runAgentCli(spec.cli, workerInstruction(prompt, dispatch), dispatch.wake_context, spec.model, undefined, signal, 'llm') : Promise.resolve({ exit_code: null, stdout_tail: '', stderr_tail: 'llm step has no declared CLI' })); + const { result, usage } = workerSpend(completed, spec.model); let reason: CompletionReason = result.exit_code === 0 ? 'success' : 'worker_error'; let output: unknown = result.stdout_tail; let detail = result.stderr_tail; @@ -72,6 +75,7 @@ export class LlmWorker extends EventEmitter { await this.client.stepComplete(dispatch.run_id, dispatch.step_id, dispatch.attempt, dispatch.idempotency_key, reason, { output, + ...(usage !== undefined ? { usage } : {}), ...(reason === 'success' ? {} : { trajectory_tail: { error: detail } }), }); } diff --git a/packages/sdk/src/model-pricing.ts b/packages/sdk/src/model-pricing.ts new file mode 100644 index 000000000..04672fe93 --- /dev/null +++ b/packages/sdk/src/model-pricing.ts @@ -0,0 +1,32 @@ +/** Frozen dollars per million tokens. Each rate is an integer microdollar per token. */ +export const MODEL_PRICING: Readonly>> = Object.freeze({ + 'claude-sonnet-4-6': Object.freeze({ input: 3, output: 15 }), + 'claude-opus-4-7': Object.freeze({ input: 15, output: 75 }), + 'codex-medium': Object.freeze({ input: 2, output: 8 }), + 'codex-large': Object.freeze({ input: 5, output: 20 }), +}); + +export function hasPricing(model: string | undefined): boolean { + return model !== undefined && Object.hasOwn(MODEL_PRICING, model); +} + +/** + * Cost accounting for a step's declared model. + * + * Returns `undefined` for unpriced models — callers should omit `usage` + * from the journal payload rather than sending nulls that break the kernel + * wire schema. The refusal for a declared dollar budget against an unpriced + * model is `budgetDiagnostics` at preflight (before any CLI dispatches). + * Throwing here after usage decode would waste the CLI invocation that + * preflight was meant to prevent. + */ +export function pricedUsage(model: string | undefined, input = 0, output = 0): + | { tokens_in: number; tokens_out: number; dollars: string } + | undefined +{ + if (![input, output].every(n => Number.isSafeInteger(n) && n >= 0)) throw new Error('Invalid token usage'); + const price = model === undefined || !Object.hasOwn(MODEL_PRICING, model) ? undefined : MODEL_PRICING[model]; + if (price === undefined) return undefined; + const micro = BigInt(input) * BigInt(price.input) + BigInt(output) * BigInt(price.output); + return { tokens_in: input, tokens_out: output, dollars: `${micro / 1_000_000n}.${String(micro % 1_000_000n).padStart(6, '0')}` }; +} diff --git a/packages/sdk/src/preflight.ts b/packages/sdk/src/preflight.ts index 2e8f74258..9b63e4714 100644 --- a/packages/sdk/src/preflight.ts +++ b/packages/sdk/src/preflight.ts @@ -1,5 +1,7 @@ import type { FlowSpec, StepSpec, TriggerSpec, McpServerConfig } from './spec.js'; import { McpError, openMcpSession, type McpDiagnostic } from './mcp-client.js'; +import { BudgetSyntaxError } from './budget.js'; +import { budgetDiagnostics } from './budget-preflight.js'; import { acceptsAnyOutput, inspectStepGate, type StepGateInspection } from './gate-contract.js'; import { compileSpec, CompileError } from './compile.js'; import type { @@ -112,7 +114,7 @@ export interface PreflightResult { export function preflight(flow: FlowSpec, options: PreflightOptions & { mcpServers: readonly string[] }): Promise; export function preflight(flow: FlowSpec, options: PreflightOptions & { mcpServers?: undefined }): PreflightResult; export function preflight(flow: FlowSpec, options: PreflightOptions): PreflightResult | Promise; -export function preflight(flow: FlowSpec, options: PreflightOptions): PreflightResult | Promise { +export function preflight(flow: unknown, options: PreflightOptions): PreflightResult | Promise { const result = preflightSync(flow, options); if (options.mcpServers === undefined) return result; return probeMcp(result, options); @@ -135,7 +137,7 @@ async function probeMcp(result: PreflightResult, options: PreflightOptions): Pro return { ...result, ok: !result.diagnostics.some(d => d.severity === 'refusal'), mcpTools: Object.freeze(inventory) }; } -function preflightSync(flow: FlowSpec, options: PreflightOptions): PreflightResult { +function preflightSync(flow: unknown, options: PreflightOptions): PreflightResult { // Compile before touching any environment fact. `compileSpec` snapshots raw // input into inert data, validates it against the closed authoring schema, // and lowers `output` sugar into its json_schema gate — so the gate plan @@ -143,20 +145,28 @@ function preflightSync(flow: FlowSpec, options: PreflightOptions): PreflightResu // inspection ever reads a live accessor. The failure is a named refusal // rather than a thrown error (RFC covenant 2), which is the contract main // settled for this boundary. - let compiled: FlowSpec; + let compiled: import('./compile.js').CompiledFlowSpec; try { compiled = compileSpec(flow); } catch (error) { const errors = error instanceof CompileError ? error.errors : [error instanceof Error ? error.message : 'spec: expected JSON-compatible data']; + // BudgetSyntaxError now flows through CompileError with kind on it; the + // legacy raw-throw instanceof is preserved as a fallback so a caller that + // constructs preflight input through a different path still classifies. + const kind: PreflightDiagnostic['kind'] = error instanceof CompileError && error.kind === 'budget_syntax_invalid' + ? 'budget_syntax_invalid' + : error instanceof BudgetSyntaxError + ? 'budget_syntax_invalid' + : 'invalid_spec'; return { ok: false, gates: [], resolutions: [], diagnostics: [{ severity: 'refusal', - kind: 'invalid_spec', + kind, message: `Relayflow spec is invalid: ${errors.join('; ')}`, errors, }], @@ -168,11 +178,12 @@ function preflightSync(flow: FlowSpec, options: PreflightOptions): PreflightResu const cliProbeResults = new Map(); diagnostics.push(...unknownModelDiagnostics(compiled, options)); - for (const server of new Set(options.mcpServers)) { + for (const server of new Set(options.mcpServers ?? [])) { if (options.mcp !== undefined && Object.hasOwn(options.mcp, server)) continue; diagnostics.push({ severity: 'refusal', kind: 'mcp_undeclared_server', server, message: `MCP server "${server}" is not declared in the nearest flows.json mcp map.` }); } + diagnostics.push(...budgetDiagnostics(compiled)); // Resolve the complete flow before touching any environment fact. A later // statically unresolved CLI makes the whole submission impossible, so no // earlier command, provider/model, or trigger probe may run first. diff --git a/packages/sdk/src/protocol.ts b/packages/sdk/src/protocol.ts index e92fd6731..0679b5eb6 100644 --- a/packages/sdk/src/protocol.ts +++ b/packages/sdk/src/protocol.ts @@ -69,6 +69,13 @@ export interface HelloResult { server: string; } +export interface StepSpend { + tokens_input: number; + tokens_output: number; + dollars: number; + wallclock_ms: number; +} + export interface RunStartParams { reuse_from_run_id?: string; /** diff --git a/packages/sdk/src/spec.ts b/packages/sdk/src/spec.ts index 39444b497..895ddeb21 100644 --- a/packages/sdk/src/spec.ts +++ b/packages/sdk/src/spec.ts @@ -100,6 +100,11 @@ export interface PermissionsSpec { * are integers. */ export interface BudgetSpec { + /** Set when normalizing the surface budget header. Legacy explicit envelopes accept worker-supplied prices. */ + pricing?: 'frozen'; + maxTokens?: number; + maxWallclockMs?: number; + window?: 'day'; maxTokensIn?: number; maxTokensOut?: number; /** Decimal string, e.g. "1.50". */ @@ -288,7 +293,7 @@ export interface FlowSpec { /** Declarations checked by preflight; gate 1 never dispatches them. */ triggers?: TriggerSpec[]; steps: StepSpec[]; - budget?: BudgetSpec; + budget?: BudgetSpec | import('./budget.js').HeaderBudget; } /** Current spec schema version emitted by this SDK. */ @@ -370,6 +375,11 @@ export interface KernelAgentStep extends KernelStepCommon { export type KernelStepSpec = KernelDeterministicStep | KernelLlmStep | KernelAgentStep; export interface KernelBudgetSpec { + pricing?: 'frozen'; + prior_spend?: { tokens_in: number; tokens_out: number; dollars: string; wallclock_ms: number; day?: number }; + max_tokens?: number; + max_wallclock_ms?: number; + window?: 'day'; max_tokens_in?: number; max_tokens_out?: number; max_dollars?: string; diff --git a/packages/sdk/src/validate.ts b/packages/sdk/src/validate.ts index a5cb0184a..bce5be7a3 100644 --- a/packages/sdk/src/validate.ts +++ b/packages/sdk/src/validate.ts @@ -69,7 +69,7 @@ function isCanonicalPathSurface(value: unknown): value is string { // unknown keys (AGENTS.md rule 4; RFC covenant 2): a typo'd key like // `depends_on` must be an error naming the nearest valid key, never a // silently discarded field — silently dropping `dependsOn` loses ordering. -const BUDGET_KEYS = ['maxTokensIn', 'maxTokensOut', 'maxDollars'] as const; +const BUDGET_KEYS = ['maxTokensIn', 'maxTokensOut', 'maxDollars', 'maxTokens', 'maxWallclockMs', 'window', 'pricing'] as const; const VERIFICATION_KEYS: Record = { exit_code: ['type', 'expect'], output_contains: ['type', 'value'], @@ -210,8 +210,13 @@ class Validator { this.fail(`${at}: expected an object`); return; } - this.checkKeys(b, BUDGET_KEYS, at); + this.checkKeys(b, at === 'spec.budget' ? BUDGET_KEYS : ['maxTokensIn', 'maxTokensOut', 'maxDollars'], at); const budget = b as BudgetSpec; + for (const key of ['maxTokens', 'maxWallclockMs'] as const) { + if (budget[key] !== undefined && (!Number.isSafeInteger(budget[key]) || budget[key]! < 0)) this.fail(`${at}.${key}: expected a safe non-negative integer`); + } + if (budget.pricing !== undefined && budget.pricing !== 'frozen') this.fail(`${at}.pricing: expected frozen`); + if (budget.window !== undefined && budget.window !== 'day') this.fail(`${at}.window: expected day`); if ( budget.maxTokensIn !== undefined && !isNonNegInt(budget.maxTokensIn) diff --git a/packages/sdk/src/worker-cli.ts b/packages/sdk/src/worker-cli.ts index f72fd2313..f5d7ffb8d 100644 --- a/packages/sdk/src/worker-cli.ts +++ b/packages/sdk/src/worker-cli.ts @@ -1,3 +1,4 @@ +import { decodeProviderResult, decodeWrapperResult, requirePricedUsage } from './worker-usage.js'; import { spawn } from 'node:child_process'; import { childStop, ownsProcessGroup } from './child-stop.js'; import { @@ -23,6 +24,8 @@ export const WAKE_CONTEXT_ENV = 'RELAYFLOW_WAKE_CONTEXT'; export const MODEL_ENV = 'RELAYFLOW_MODEL'; export interface WorkerCliResult { + tokens_input?: number; + tokens_output?: number; exit_code: number | null; stdout_tail: string; stderr_tail: string; @@ -44,7 +47,7 @@ export async function runAgentCli( const kind = cliAdapterKind(cli); if (kind === 'relayflows-wrapper-v1') { - return runWrapperSession( + return requirePricedUsage(decodeWrapperResult(await runWrapperSession( cli, instruction, wakeContext, @@ -52,7 +55,7 @@ export async function runAgentCli( wrapperEnvironment(process.env), wrapperLimits, signal, - ); + )), model); } const env: NodeJS.ProcessEnv = { ...process.env }; @@ -73,7 +76,10 @@ export async function runAgentCli( } if (invocation.modelEnv !== undefined) env[MODEL_ENV] = invocation.modelEnv; - return spawnInvocation(cli, invocation, env, signal); + // Structured provider output carries the authoritative token counts. + const args = [...invocation.args]; + args.splice(args.length - 1, 0, ...(kind === 'claude' ? ['--output-format', 'json'] : ['--json'])); + return requirePricedUsage(decodeProviderResult(await spawnInvocation(cli, { ...invocation, args }, env, signal), kind), model); } function spawnInvocation( diff --git a/packages/sdk/src/worker-spend.ts b/packages/sdk/src/worker-spend.ts new file mode 100644 index 000000000..9aae3d954 --- /dev/null +++ b/packages/sdk/src/worker-spend.ts @@ -0,0 +1,23 @@ +import { pricedUsage } from './model-pricing.js'; +import type { WorkerCliResult } from './worker-cli.js'; + +/** + * Attach token/dollar usage to a worker's CLI result. Invalid token counts + * (non-integer or negative) are the one remaining failure mode — those are + * journaled as `worker_error` with the usage projected from clamped counts. + * + * Unpriced models are NOT a failure here: `pricedUsage` returns + * `dollars: null` and preflight (see `budgetDiagnostics`) has already refused + * declared dollar budgets against unpriced models before the CLI dispatched. + */ +export function workerSpend(result: WorkerCliResult, model?: string) { + try { return { result, usage: pricedUsage(model, result.tokens_input, result.tokens_output) }; } + catch (error) { + return { + result: { ...result, exit_code: null, stderr_tail: error instanceof Error ? error.message : 'Invalid model usage' }, + usage: pricedUsage(undefined, + Number.isSafeInteger(result.tokens_input) && result.tokens_input! >= 0 ? result.tokens_input : 0, + Number.isSafeInteger(result.tokens_output) && result.tokens_output! >= 0 ? result.tokens_output : 0), + }; + } +} diff --git a/packages/sdk/src/worker-usage.ts b/packages/sdk/src/worker-usage.ts new file mode 100644 index 000000000..860e7efea --- /dev/null +++ b/packages/sdk/src/worker-usage.ts @@ -0,0 +1,48 @@ +import type { WorkerCliResult } from './worker-cli.js'; +import { MODEL_PRICING } from './model-pricing.js'; + +type RecordValue = Record; +const record = (value: unknown): value is RecordValue => typeof value === 'object' && value !== null && !Array.isArray(value); +const count = (value: unknown): value is number => typeof value === 'number' && Number.isSafeInteger(value) && value >= 0; + +function invalid(result: WorkerCliResult, detail: string): WorkerCliResult { + return { ...result, exit_code: null, stderr_tail: `${result.stderr_tail}\n${detail}`.trim() }; +} + +function usageResult(result: WorkerCliResult, usage: unknown, output: unknown): WorkerCliResult { + if (output === undefined) return invalid(result, 'Provider completion is missing its output.'); + if (!record(usage) || !count(usage.input_tokens) || !count(usage.output_tokens)) { + return invalid(result, 'Provider completion has invalid token usage.'); + } + return { ...result, stdout_tail: typeof output === 'string' ? output : JSON.stringify(output), + tokens_input: usage.input_tokens, tokens_output: usage.output_tokens }; +} + +/** Provider envelopes are execution metadata, never the authored output. */ +export function decodeProviderResult(result: WorkerCliResult, kind: 'claude' | 'codex'): WorkerCliResult { + const frames: RecordValue[] = []; + for (const line of result.stdout_tail.split('\n')) { + try { const frame: unknown = JSON.parse(line); if (record(frame)) frames.push(frame); } catch { /* Plain text remains output. */ } + } + const terminal = [...frames].reverse().find(f => kind === 'claude' ? f.type === 'result' : f.type === 'turn.completed'); + if (terminal === undefined) return result; + const text = kind === 'claude' ? terminal.result : frames + .flatMap(f => f.type === 'item.completed' && record(f.item) && f.item.type === 'agent_message' && typeof f.item.text === 'string' ? [f.item.text] : []).join('\n'); + return usageResult(result, terminal.usage, text); +} + +/** Optional wrapper result envelope; existing opaque text output stays valid. */ +export function decodeWrapperResult(result: WorkerCliResult): WorkerCliResult { + let value: unknown; + try { value = JSON.parse(result.stdout_tail); } catch { return result; } + if (!record(value) || value.protocol !== 'relayflows-agent-cli-v1-result') return result; + return usageResult(result, value.usage, value.output); +} + +export function requirePricedUsage(result: WorkerCliResult, model?: string): WorkerCliResult { + if (model !== undefined && Object.hasOwn(MODEL_PRICING, model) + && (result.tokens_input === undefined || result.tokens_output === undefined)) { + return invalid(result, 'Priced model completion is missing token usage.'); + } + return result; +} diff --git a/packages/sdk/src/worker.ts b/packages/sdk/src/worker.ts index 67cc5ad15..1cbb13e3c 100644 --- a/packages/sdk/src/worker.ts +++ b/packages/sdk/src/worker.ts @@ -1,3 +1,5 @@ +import { workerSpend } from './worker-spend.js'; +import type { WorkerCliResult } from './worker-cli.js'; import { EventEmitter } from 'node:events'; import type { JournalClient } from './journal-client.js'; import type { Pins, StepDispatchEvent } from './protocol.js'; @@ -92,10 +94,11 @@ export class AgentWorker extends EventEmitter { private async execute(dispatch: StepDispatchEvent): Promise { const spec = dispatch.spec as Partial; - const result = await withWorkerLease(this.client, dispatch, signal => + const completed: WorkerCliResult = await withWorkerLease(this.client, dispatch, signal => typeof spec.cli === 'string' && typeof spec.instruction === 'string' ? runAgentCli(spec.cli, workerInstruction(spec.instruction, dispatch), dispatch.wake_context, spec.model, undefined, signal) : Promise.resolve({ exit_code: null, stdout_tail: '', stderr_tail: 'agent step has no declared CLI' })); + const { result, usage } = workerSpend(completed, spec.model); const completionReason = result.exit_code === 0 ? 'success' : 'worker_error'; // Output shape: if the CLI's stdout parses as JSON, promote THAT @@ -121,6 +124,7 @@ export class AgentWorker extends EventEmitter { completionReason, { output, + ...(usage !== undefined ? { usage } : {}), started_pins: dispatch.pins, end_pins: dispatch.pins, }, diff --git a/packages/sdk/tests/budget-attribution.test.ts b/packages/sdk/tests/budget-attribution.test.ts new file mode 100644 index 000000000..be628d77d --- /dev/null +++ b/packages/sdk/tests/budget-attribution.test.ts @@ -0,0 +1,53 @@ +import { EventEmitter } from 'node:events'; +import { describe, expect, it, vi } from 'vitest'; +import { decodeProviderResult, decodeWrapperResult, requirePricedUsage } from '../src/worker-usage.js'; +import { pricedUsage, MODEL_PRICING } from '../src/model-pricing.js'; +import { AuthoredBudget } from '../src/authored-budget.js'; +import type { JournalClient } from '../src/journal-client.js'; +import type { StepDispatchEvent } from '../src/protocol.js'; +import { LlmWorker } from '../src/llm-worker.js'; + +vi.mock('../src/worker-cli.js', () => ({runAgentCli: vi.fn(async () => ({exit_code: 0, stdout_tail: 'answer', stderr_tail: '', tokens_input: 1000, tokens_output: 200}))})); +vi.mock('../src/worker-lease.js', () => ({withWorkerLease: (_c: unknown, _d: unknown, run: (s: AbortSignal) => unknown) => run(new AbortController().signal)})); + +describe('budget attribution', () => { + it('threads mocked provider tokens through the worker completion', async () => { + const client = Object.assign(new EventEmitter(), {workerAttach: vi.fn(), stepComplete: vi.fn()}); + const worker = new LlmWorker(client as unknown as JournalClient, 'test'); + await worker.attach(); + client.emit('step.dispatch', {run_id:'r', step_id:'s', step_type:'llm', attempt:1, idempotency_key:'k', + spec:{type:'llm', prompt:'hello', cli:'claude', model:'claude-sonnet-4-6'}} as StepDispatchEvent); + await worker.close(); + expect(client.stepComplete.mock.calls[0]?.[5]).toMatchObject({output:'answer', usage:{tokens_in:1000, tokens_out:200, dollars:'0.006000'}}); + }); + it('extracts provider usage while preserving the authored output', () => { + for (const [kind, stdout] of [ + ['claude', JSON.stringify({type:'result', result:'answer', usage:{input_tokens:1000, output_tokens:200}})], + ['codex', [JSON.stringify({type:'item.completed', item:{type:'agent_message', text:'answer'}}), JSON.stringify({type:'turn.completed', usage:{input_tokens:1000, output_tokens:200}})].join('\n')], + ] as const) { + const result = decodeProviderResult({exit_code:0, stdout_tail:stdout, stderr_tail:''}, kind); + expect(result).toMatchObject({stdout_tail:'answer', tokens_input:1000, tokens_output:200}); + expect(pricedUsage('claude-sonnet-4-6', result.tokens_input, result.tokens_output).dollars).toBe('0.006000'); + } + expect(Object.isFrozen(MODEL_PRICING)).toBe(true); + expect(Object.isFrozen(MODEL_PRICING['codex-large'])).toBe(true); + }); + it('accepts explicit wrapper usage and refuses missing or malformed priced usage', () => { + const base = {exit_code: 0, stdout_tail: 'answer', stderr_tail: ''}; + expect(requirePricedUsage(base, 'codex-medium').exit_code).toBeNull(); + expect(decodeWrapperResult({...base, stdout_tail: JSON.stringify({protocol:'relayflows-agent-cli-v1-result', usage:{input_tokens:5, output_tokens:2}})}).exit_code).toBeNull(); + const wrapped = decodeWrapperResult({...base, stdout_tail: JSON.stringify({protocol:'relayflows-agent-cli-v1-result', output:'answer', usage:{input_tokens:5, output_tokens:2}})}); + expect(requirePricedUsage(wrapped, 'codex-medium')).toMatchObject({exit_code:0, stdout_tail:'answer', tokens_input:5, tokens_output:2}); + expect(decodeProviderResult({...base, stdout_tail: JSON.stringify({type:'result', result:'answer', usage:{input_tokens:-1, output_tokens:2}})}, 'claude').exit_code).toBeNull(); + }); + it('carries exact completed spend into the next authored step kernel run', async () => { + const budget = new AuthoredBudget('$0.001/run'); + const client = {runStart: vi.fn(async () => ({run_id:'r', status:'completed', completion_reason:'success', completed_steps:1})), + journalRead: vi.fn(async (_run: string, seq: number) => ({entries: seq === 1 ? [{seq:1, entry_type:'step.completed', at_ms:0, + payload:{budget:{tokens_in:1000, tokens_out:200, dollars:'0.006000'}, spend:{wallclock_ms:25}}}] : []}))}; + const spec = {version:'0.1.0', steps:[]}; + await budget.execute(client as unknown as JournalClient, spec, async () => 'answer'); + await budget.execute(client as unknown as JournalClient, spec, async () => 'answer'); + expect((client.runStart.mock.calls as unknown as [{budget:unknown}][])[1]?.[0].budget).toMatchObject({max_dollars:'0.001', prior_spend:{tokens_in:1000, tokens_out:200, dollars:'0.006000', wallclock_ms:25}}); + }); +}); diff --git a/packages/sdk/tests/budget-authored-live.test.ts b/packages/sdk/tests/budget-authored-live.test.ts new file mode 100644 index 000000000..2edbf10b7 --- /dev/null +++ b/packages/sdk/tests/budget-authored-live.test.ts @@ -0,0 +1,36 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { flow } from '@relayflows/surface'; +import { executeAuthoredFlow } from '../src/authored-flow-executor.js'; +import { chainFixture } from './flow-chain-fixture.js'; + +const cleanup: Array<() => Promise> = []; +afterEach(async () => { for (const close of cleanup.splice(0)) await close(); }); + +describe('authored budgets through the live kernel', () => { + it('retains the first completion and journals refusal of the next authored step', async () => { + const fixture = chainFixture(); + cleanup.push(() => fixture.close()); + const client = await fixture.connect(); + const start = vi.spyOn(client, 'runStart'); + const handle = flow('budgeted', {budget: {wallclock: '0ms'}}, async f => { + await f.run('sleep 0.01'); + await f.run('printf must-not-run'); + f.done('success'); + }); + await expect(executeAuthoredFlow(handle, client)).rejects.toMatchObject({completionReason:'budget_exceeded'}); + expect(start).toHaveBeenCalledTimes(2); + const first = await start.mock.results[0]!.value; + const refused = await start.mock.results[1]!.value; + const entries = (await client.journalRead(first.run_id,1)).entries; + expect(entries).toEqual(expect.arrayContaining([expect.objectContaining({entry_type:'step.completed',payload:expect.objectContaining({completionReason:'success',spend:expect.objectContaining({dollars:0})})})])); + const refusal = (await client.journalRead(refused.run_id,1)).entries; + expect(refusal).toEqual(expect.arrayContaining([expect.objectContaining({entry_type:'run.completed',payload:expect.objectContaining({completionReason:'budget_exceeded'})})])); + expect(refusal).not.toEqual(expect.arrayContaining([expect.objectContaining({entry_type:'step.attempt_started'})])); + }); + it('permits done after the last valid step crosses the limit', async () => { + const fixture = chainFixture(); cleanup.push(() => fixture.close()); + const client = await fixture.connect(); + const handle = flow('last', {budget:{wallclock:'0ms'}}, async f => { await f.run('sleep 0.01'); f.done('success'); }); + expect((await executeAuthoredFlow(handle,client)).completionReason).toBe('success'); + }); +}); diff --git a/packages/sdk/tests/budget-preflight.test.ts b/packages/sdk/tests/budget-preflight.test.ts new file mode 100644 index 000000000..f6590a959 --- /dev/null +++ b/packages/sdk/tests/budget-preflight.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it, vi } from 'vitest'; +import { preflight } from '../src/preflight.js'; +import { compileSpec, CompileError, toKernelSpec, kernelToAuthoring } from '../src/compile.js'; +import { flow } from '@relayflows/surface'; +import { getFlowDefinition } from '@relayflows/surface/runtime'; + +const options = () => ({models: ['claude-sonnet-4-6', 'unknown'], probes: { + cli: vi.fn(() => ({ exists: true, authenticated: true, modelAvailable: true })), + executor: () => true, command: () => true, +}}); +const spec = (budget: unknown, model = 'claude-sonnet-4-6') => ({ version: '0.1.0', budget, + steps: [{ id: 'ask', type: 'llm', cli: 'claude', model, prompt: 'hello' }] }); + +describe('budget preflight', () => { + it('accepts the string header and lowers an exact daily envelope', () => { + expect(preflight(spec('$20/day'), options()).ok).toBe(true); + expect(toKernelSpec(compileSpec(spec('$0.10/run'))).budget).toEqual({ pricing: 'frozen', max_dollars: '0.10' }); + expect(toKernelSpec(compileSpec(spec('$20/day'))).budget).toEqual({ pricing: 'frozen', max_dollars: '20', window: 'day' }); + }); + it('accepts object limits and snapshots the surface header', () => { + const budget = { tokens: 100, dollars: 0.1, wallclock: '2m' }; + const handle = flow('budget', {budget}, async f => { f.done('success'); }); + budget.tokens = 999; + expect(getFlowDefinition(handle).header.budget).toEqual({ tokens: 100, dollars: 0.1, wallclock: '2m' }); + expect(preflight(spec(budget), options()).ok).toBe(true); + expect(toKernelSpec(compileSpec(spec(budget))).budget).toEqual({pricing: 'frozen', max_tokens: 999, max_dollars: '0.1', max_wallclock_ms: 120000}); + }); + it('refuses missing windows before any environment probe', () => { + const o = options(); + const result = preflight(spec('$20'), o); + expect(result.diagnostics.map(d => d.kind)).toEqual(['budget_syntax_invalid']); + expect(o.probes.cli).not.toHaveBeenCalled(); + }); + it('refuses an unpriced declared model before probing', () => { + const o = options(); + const result = preflight(spec('$20/run', 'unknown'), o); + expect(result.diagnostics.map(d => d.kind)).toEqual(['budget_missing_price']); + expect(o.probes.cli).not.toHaveBeenCalled(); + }); + it('retains frozen pricing when checking a compiled artifact', () => { + expect(preflight(kernelToAuthoring(toKernelSpec(compileSpec(spec('$20/run', 'unknown')))), options()).diagnostics) + .toEqual(expect.arrayContaining([expect.objectContaining({kind: 'budget_missing_price'})])); + }); + it('requires a model when declaring a dollar budget', () => { + const input = spec('$20/run'); + const { model, ...step } = input.steps[0]!; + expect(preflight({...input, steps:[step]}, options()).diagnostics) + .toEqual(expect.arrayContaining([expect.objectContaining({kind: 'budget_missing_price'})])); + }); + it.each(['$1/week', '-$1/run', '$1.0000001/run', {tokens: -1}, {wallclock: 'soon'}, {dollars: Infinity}, {typo: 2}])('refuses malformed budget %j', budget => { + expect(preflight(spec(budget), options()).ok).toBe(false); + }); + it.each(['$$$', '$1/week', {typo: 2}])('compileSpec wraps parseBudget throws as a CompileError with a budget-scoped message (%j)', budget => { + let caught: unknown; + try { compileSpec(spec(budget)); } catch (error) { caught = error; } + expect(caught).toBeInstanceOf(CompileError); + expect((caught as CompileError).errors[0]).toMatch(/^spec\.budget: /); + expect((caught as CompileError).errors[0]).toContain('budget_syntax_invalid'); + }); +}); diff --git a/packages/sdk/tests/model-pricing.test.ts b/packages/sdk/tests/model-pricing.test.ts new file mode 100644 index 000000000..bc0c0182b --- /dev/null +++ b/packages/sdk/tests/model-pricing.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it } from 'vitest'; +import { pricedUsage, hasPricing, MODEL_PRICING } from '../src/model-pricing.js'; +import { workerSpend } from '../src/worker-spend.js'; + +describe('pricedUsage', () => { + it('prices a listed model to microdollar-exact dollars', () => { + const usage = pricedUsage('claude-sonnet-4-6', 1_000_000, 500_000); + expect(usage).toEqual({ tokens_in: 1_000_000, tokens_out: 500_000, dollars: '10.500000' }); + }); + + it('returns undefined for an unpriced model rather than throwing after decode', () => { + // Regression for the Cursor Bugbot HIGH finding "Unlisted models fail + // after usage decode": the runtime previously threw here after the CLI + // had already spent tokens. Preflight now owns the refusal; the runtime + // path is permissive so an unpriced-model step wastes nothing on the + // pricing check itself. + expect(pricedUsage('unlisted-model', 100, 50)).toBeUndefined(); + expect(pricedUsage('unlisted-model', 0, 0)).toBeUndefined(); + }); + + it('returns undefined when no model is declared', () => { + expect(pricedUsage(undefined, 100, 50)).toBeUndefined(); + }); + + it('rejects invalid token counts', () => { + expect(() => pricedUsage('claude-sonnet-4-6', -1, 0)).toThrow(/Invalid token usage/); + expect(() => pricedUsage('claude-sonnet-4-6', 0, 1.5)).toThrow(/Invalid token usage/); + }); + + it('hasPricing agrees with MODEL_PRICING membership', () => { + for (const key of Object.keys(MODEL_PRICING)) expect(hasPricing(key)).toBe(true); + expect(hasPricing('unlisted-model')).toBe(false); + expect(hasPricing(undefined)).toBe(false); + }); +}); + +describe('workerSpend', () => { + const priced = { exit_code: 0, stdout_tail: '', stderr_tail: '', tokens_input: 100, tokens_output: 50 }; + + it('attaches usage for priced models', () => { + const spent = workerSpend(priced, 'claude-sonnet-4-6'); + expect(spent.usage).toEqual({ tokens_in: 100, tokens_out: 50, dollars: '0.001050' }); + expect(spent.result.exit_code).toBe(0); + }); + + it('leaves usage undefined for an unpriced model without failing the step', () => { + // The step still succeeded — an unpriced model is a preflight concern + // when a dollar budget is declared, not a runtime failure per se. + const spent = workerSpend(priced, 'unlisted-model'); + expect(spent.usage).toBeUndefined(); + expect(spent.result.exit_code).toBe(0); + expect(spent.result.stderr_tail).toBe(''); + }); + + it('journals invalid token counts as worker_error, projecting clamped counts', () => { + const bad = { ...priced, tokens_input: -1 }; + const spent = workerSpend(bad, 'claude-sonnet-4-6'); + expect(spent.result.exit_code).toBeNull(); + expect(spent.result.stderr_tail).toMatch(/Invalid token usage/); + expect(spent.usage).toBeUndefined(); + }); +}); diff --git a/packages/sdk/tests/preflight.test.ts b/packages/sdk/tests/preflight.test.ts index c01150543..bf26ab44a 100644 --- a/packages/sdk/tests/preflight.test.ts +++ b/packages/sdk/tests/preflight.test.ts @@ -370,6 +370,8 @@ describe('preflight: CLI resolution and refusal predicates', () => { probes: probes(), mcpServers: ['missing-binary'], mcp: { 'missing-binary': { command: '/nonexistent-mcp-test-binary' } }, }), + preflight({ ...flow({ id: 'a', type: 'deterministic', command: 'x' }), budget: '$20' }, { probes: probes() }), + preflight({ ...flow({ id: 'a', type: 'llm', prompt: 'p', cli: 'x', model: 'unpriced' }), budget: '$20/run' }, { models: ['unpriced'], probes: probes() }), preflight({ version: '0.1.0', steps: [{ id: 'a', type: 'deterministic', command: 'x', prompt: 'cross-verb' }], diff --git a/packages/surface/src/flow.ts b/packages/surface/src/flow.ts index d5eb95f77..4013f7358 100644 --- a/packages/surface/src/flow.ts +++ b/packages/surface/src/flow.ts @@ -6,7 +6,7 @@ export interface FlowHeader { use?: string[]; identity?: string; memory?: { script?: boolean; agent?: boolean }; - budget?: string; + budget?: string | { tokens?: number; dollars?: number; wallclock?: string }; tools?: { slack?: boolean; relayfile?: string[]; mcp?: string[] }; workspace?: string; } @@ -17,7 +17,7 @@ export interface ReadonlyFlowHeader { readonly use?: readonly string[]; readonly identity?: string; readonly memory?: Readonly<{ script?: boolean; agent?: boolean }>; - readonly budget?: string; + readonly budget?: string | Readonly<{ tokens?: number; dollars?: number; wallclock?: string }>; readonly tools?: Readonly<{ slack?: boolean; relayfile?: readonly string[]; @@ -146,7 +146,7 @@ function freezeHeader(header: FlowHeader): ReadonlyFlowHeader { ...(header.use === undefined ? {} : { use: Object.freeze([...header.use]) }), ...(header.identity === undefined ? {} : { identity: header.identity }), ...(memory === undefined ? {} : { memory }), - ...(header.budget === undefined ? {} : { budget: header.budget }), + ...(header.budget === undefined ? {} : { budget: typeof header.budget === "string" ? header.budget : Object.freeze({ ...header.budget }) }), ...(tools === undefined ? {} : { tools }), ...(header.workspace === undefined ? {} : { workspace: header.workspace }), }); @@ -161,7 +161,14 @@ function assertFlowHeader(value: unknown, flowName: string): asserts value is Fl at, ); assertOptionalString(value, "identity", at); - assertOptionalString(value, "budget", at); + if (value.budget !== undefined && typeof value.budget !== "string") { + assertHeaderObject(value.budget, `${at}.budget`); + assertKnownKeys(value.budget, ["tokens", "dollars", "wallclock"], `${at}.budget`); + assertOptionalString(value.budget, "wallclock", at); + for (const key of ["tokens", "dollars"]) { + if (value.budget[key] !== undefined && typeof value.budget[key] !== "number") throw new TypeError(`budget_syntax_invalid: ${key} must be a number`); + } + } assertOptionalString(value, "workspace", at); assertOptionalStringArray(value, "use", at); if (value.use !== undefined) { diff --git a/testdata/budget-guarded.flow.yaml b/testdata/budget-guarded.flow.yaml new file mode 100644 index 000000000..360d5dc3c --- /dev/null +++ b/testdata/budget-guarded.flow.yaml @@ -0,0 +1,12 @@ +version: 0.1.0 +name: budget-guarded +budget: + wallclock: 0ms +steps: + - id: measured + type: deterministic + command: sleep 0.01 + - id: guarded + type: deterministic + dependsOn: [measured] + command: printf 'budget should refuse this step'