From 1191c2f2a2f7016b90933350fb818603fec23e4e Mon Sep 17 00:00:00 2001 From: Yasyf Mohamedali Date: Wed, 7 Oct 2026 05:28:09 -0700 Subject: [PATCH 1/2] =?UTF-8?q?decide:=20=E2=9C=A8=20Ask=20typed=20questio?= =?UTF-8?q?ns=20of=20TypeSafe=20Jev=20and=20OpenAI=20Decisions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude-Session-Id: 899e5f7a-9099-4d8f-b96d-79291e63b5b4 --- .../great-docs-local.fragment.yml | 18 + CHANGELOG.md | 18 + README.md | 23 +- .../decide_plan/jev-binary-criteria.json | 43 + .../jev-structured-state-verbatim.json | 117 +++ .../decide_plan/jev-three-questions.json | 93 ++ .../decide_plan/openai-binary-criteria.json | 40 + .../openai-structured-state-compact.json | 124 +++ .../decide_plan/openai-three-questions.json | 112 +++ .../decide_resolve/jev-408-retries.json | 64 ++ .../vectors/decide_resolve/jev-422-fails.json | 65 ++ .../jev-429-retry-after-seconds.json | 64 ++ .../decide_resolve/jev-529-first-backoff.json | 64 ++ .../jev-missing-answer-fails.json | 65 ++ .../jev-recorded-reorders-to-declared.json | 92 ++ .../decide_resolve/no-response-retries.json | 64 ++ .../decide_resolve/openai-401-fails.json | 65 ++ .../openai-429-retry-after-ms-wins.json | 64 ++ .../openai-500-backoff-caps-at-5s.json | 64 ++ ...i-503-http-date-retry-after-backs-off.json | 64 ++ .../openai-misnamed-answer-fails.json | 65 ++ .../decide_resolve/openai-recorded.json | 92 ++ .../openai-refusal-beside-answers.json | 90 ++ .../success-body-not-json-fails.json | 65 ++ great-docs.yml | 18 + rust/conformance-gen/src/cases.rs | 248 ++++++ rust/spawnllm-core/src/decide.rs | 827 ++++++++++++++++++ rust/spawnllm-core/src/lib.rs | 2 + spawnllm/__init__.py | 39 +- spawnllm/cli.py | 15 + spawnllm/decide.py | 514 +++++++++++ tests/test_decide.py | 301 +++++++ 32 files changed, 3595 insertions(+), 4 deletions(-) create mode 100644 conformance/vectors/decide_plan/jev-binary-criteria.json create mode 100644 conformance/vectors/decide_plan/jev-structured-state-verbatim.json create mode 100644 conformance/vectors/decide_plan/jev-three-questions.json create mode 100644 conformance/vectors/decide_plan/openai-binary-criteria.json create mode 100644 conformance/vectors/decide_plan/openai-structured-state-compact.json create mode 100644 conformance/vectors/decide_plan/openai-three-questions.json create mode 100644 conformance/vectors/decide_resolve/jev-408-retries.json create mode 100644 conformance/vectors/decide_resolve/jev-422-fails.json create mode 100644 conformance/vectors/decide_resolve/jev-429-retry-after-seconds.json create mode 100644 conformance/vectors/decide_resolve/jev-529-first-backoff.json create mode 100644 conformance/vectors/decide_resolve/jev-missing-answer-fails.json create mode 100644 conformance/vectors/decide_resolve/jev-recorded-reorders-to-declared.json create mode 100644 conformance/vectors/decide_resolve/no-response-retries.json create mode 100644 conformance/vectors/decide_resolve/openai-401-fails.json create mode 100644 conformance/vectors/decide_resolve/openai-429-retry-after-ms-wins.json create mode 100644 conformance/vectors/decide_resolve/openai-500-backoff-caps-at-5s.json create mode 100644 conformance/vectors/decide_resolve/openai-503-http-date-retry-after-backs-off.json create mode 100644 conformance/vectors/decide_resolve/openai-misnamed-answer-fails.json create mode 100644 conformance/vectors/decide_resolve/openai-recorded.json create mode 100644 conformance/vectors/decide_resolve/openai-refusal-beside-answers.json create mode 100644 conformance/vectors/decide_resolve/success-body-not-json-fails.json create mode 100644 rust/spawnllm-core/src/decide.rs create mode 100644 spawnllm/decide.py create mode 100644 tests/test_decide.py diff --git a/.claude/fragments/great-docs.yml/great-docs-local.fragment.yml b/.claude/fragments/great-docs.yml/great-docs-local.fragment.yml index 5baa122..497565f 100644 --- a/.claude/fragments/great-docs.yml/great-docs-local.fragment.yml +++ b/.claude/fragments/great-docs.yml/great-docs-local.fragment.yml @@ -75,6 +75,24 @@ reference: - ClaudeConfig - CodexConfig - GeminiConfig + - title: Deciding + desc: Typed yes/no, label, and score decisions over TypeSafe Jev and OpenAI Decisions, one question set for either provider. + contents: + - decide.decide + - decide.decide_sync + - Binary + - Label + - Score + - Decision + - BinaryAnswer + - LabelAnswer + - ScoreAnswer + - Refused + - Provider + - DecideError + - DecideKeyMissing + - decide.api_key + - decide.store_key - title: Backends desc: CLI backends (Claude, Codex, and the Gemini family), the in-process Claude Agent SDK backend, the Apple Foundation Models backend over its bundled Swift sidecar, the MLX and OpenAI-endpoint backends, the specialty registry, backend selection, and install/auth status checks. contents: diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ae2024..7db88d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- **`decide` and `decide_sync` ask typed questions of TypeSafe Jev or OpenAI + Decisions.** A `Binary`, `Label`, or `Score` question set runs unchanged + on either provider (`JEV`, pinned to `jev-1.13.0`, or `OPENAI`, on + `gpt-6-luna`), and every answer comes back in declared order: a + `BinaryAnswer` with `p_yes`, a `LabelAnswer` or `ScoreAnswer` with + per-option probabilities and a confidence, or `Refused` when OpenAI + declines one question and answers the rest. The Rust core owns both wire + mappings and the retry policy as the `decide_plan` and `decide_resolve` + ops, pinned by 21 new golden vectors that include one recorded response + from each provider. The policy retries 408, 429, 5xx, 529, and a lost + connection, backing off from 0.5s to a 5s cap and honoring `retry-after` + and `retry-after-ms`, but only while the retry fits inside the caller's + `timeout`. A 401 or 422 raises `DecideError` on the first try. Each + process keeps one keep-alive client, and a warm three-question Jev call + measured 120 to 130 ms end to end from a Mac. Keys come from + `TYPESAFE_API_KEY` or `OPENAI_API_KEY`. On macOS, an unset variable falls + through to the Keychain item that `spawnllm key set jev` (or `openai`) + writes from stdin, read at call time. - **The OpenAI-compatible endpoint backend sets `reasoning_effort`.** A typed `reasoning_effort` (`none`, `minimal`, `low`, `medium`, `high` or `xhigh`) on the endpoint config goes into every `/chat/completions` request body, diff --git a/README.md b/README.md index a7cdc60..fd1349a 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,26 @@ print(call_sync("Reply with just the word: pong", backend=AppleBackend())) Auto-selection tries this backend last and only for `model="small"`; an explicit `backend=AppleBackend()` always reaches it. Session and decoding knobs (`use_case`, `guardrails`, `instructions`, `temperature`, sampling) ride in via `RunSpec(provider_configs={"apple": AppleConfig(...)})`. Structured `extract_sync` works too, nested models included, and schema constraints now bind during decoding: `minimum`/`maximum`, `minItems`/`maxItems`, and string-valued `enum`s are enforced exactly (a non-string `enum` such as `Literal[1, 2]` fails before generation, `extract_sync` raising `BackendCallError`), and a `Field(pattern=...)` constrains the value's shape — length, separators, and character families. Apple's decoder rejects bracket character classes, so the sidecar widens each to the narrowest escape it accepts (`^[A-Z]{3}-\d{4}$` decodes as `\w{3}-\d{4}`), and pydantic stays the exact validator: a value that fits the widened shape but violates your regex raises a plain `ValidationError`. Self-referential models extract cleanly; a mutually recursive pair (A referencing B referencing A) fails cleanly instead, `extract_sync` raising `BackendCallError` and `run` returning an error `Response`. Requires macOS 26+ on Apple Silicon with Apple Intelligence enabled — every other platform gets the pure-Python wheel and reports the backend as not installed. +### Ask a yes/no, label, or score question in about 120 ms + +A classifier that runs on every hook event or every PR comment can't wait seconds for a chat model to write JSON. `decide_sync` asks TypeSafe's Jev or OpenAI Decisions instead. Both return probabilities, not prose, and one question set runs unchanged on either: + +```python +from spawnllm import Binary, Label, decide_sync + +decision = decide_sync( + "Our nightly release has failed for 3 days. Please roll it back now.", + { + "rollback": Binary("Does the message ask to roll back a release?"), + "action": Label("Which release action does it ask for?", {"none": None, "release": None, "rollback": None}), + }, +) +print(decision.answers["rollback"].p_yes) # 0.99 +print(decision.answers["action"].choice) # rollback +``` + +On a warm keep-alive connection from a Mac, a three-question Jev call took 120 to 130 ms end to end. Pass `provider=OPENAI` to ask `gpt-6-luna` instead. Retries cover 408, 429, 5xx, and 529 and stop at `timeout`. Keys come from `TYPESAFE_API_KEY` or `OPENAI_API_KEY`, or on macOS from a Keychain item that `spawnllm key set jev` (or `openai`) writes from stdin. + ### Call the same backends from Go or Rust All three languages run the identical engine: argv planning, output parsing, schema strictification, and retry policy live once in a Rust core — linked natively by the Rust crate, embedded as WASM by the Go module and the Python package — pinned by a shared golden-vector suite and released in lockstep. @@ -119,9 +139,10 @@ Both expose `Call`/`call` and typed `Extract`/`extract` against your existing CL ## More in the docs - **Spec-driven runs** — a literal model id, per-provider flag passthrough, and envelope-aware retry via `RunSpec` — [Running reference](https://yasyf.github.io/spawnllm/reference/#running) +- **Decisions** — every question and answer type, the provider pins, and the retry and key-lookup rules — [Deciding reference](https://yasyf.github.io/spawnllm/reference/#deciding) - **Backend selection** — the priority chain, plus `specialty=` routing (`debugging` and `review` go to Codex, `general` to the Claude Agent SDK backend) — [Backends reference](https://yasyf.github.io/spawnllm/reference/#backends) - **Transport helpers** — `run_cli`, `collect_process`, and `map_concurrent`, the subprocess plumbing shared by every CLI backend — [Transport reference](https://yasyf.github.io/spawnllm/reference/#transport) -- **The CLI** — `spawnllm call`, `status`, and `backends` from any shell — [CLI reference](https://yasyf.github.io/spawnllm/reference/cli/) +- **The CLI** — `spawnllm call`, `status`, `backends`, and `key set` from any shell — [CLI reference](https://yasyf.github.io/spawnllm/reference/cli/) - **MLX internals** — the adapter codec, fuser, and runtime patches behind the local engine — [MLX reference](https://yasyf.github.io/spawnllm/reference/#mlx) Read the [docs](https://yasyf.github.io/spawnllm/) for the full guide and API reference. Licensed under [MIT](https://github.com/yasyf/spawnllm/blob/main/LICENSE). diff --git a/conformance/vectors/decide_plan/jev-binary-criteria.json b/conformance/vectors/decide_plan/jev-binary-criteria.json new file mode 100644 index 0000000..7a8f2f0 --- /dev/null +++ b/conformance/vectors/decide_plan/jev-binary-criteria.json @@ -0,0 +1,43 @@ +{ + "name": "jev-binary-criteria", + "op": "decide_plan", + "input": { + "provider": { + "name": "jev", + "model": "jev-1.13.0" + }, + "api_key": "test-key", + "state": "Help! Our nightly release has been failing for 3 days and customers are blocked. Please roll it back now.", + "questions": [ + { + "type": "binary", + "id": "is_urgent", + "instructions": "Does this convey urgency?", + "yes": "Explicitly time-sensitive", + "no": "No urgency expressed" + } + ] + }, + "expected": { + "kind": "http", + "method": "POST", + "url": "https://api.typesafe.ai/v1/systemone", + "headers": { + "Authorization": "Bearer test-key" + }, + "body": { + "state": "Help! Our nightly release has been failing for 3 days and customers are blocked. Please roll it back now.", + "model": "jev-1.13.0", + "questions": { + "is_urgent": { + "type": "noul", + "instructions": "Does this convey urgency?", + "criteria": { + "true": "Explicitly time-sensitive", + "false": "No urgency expressed" + } + } + } + } + } +} diff --git a/conformance/vectors/decide_plan/jev-structured-state-verbatim.json b/conformance/vectors/decide_plan/jev-structured-state-verbatim.json new file mode 100644 index 0000000..16a3d92 --- /dev/null +++ b/conformance/vectors/decide_plan/jev-structured-state-verbatim.json @@ -0,0 +1,117 @@ +{ + "name": "jev-structured-state-verbatim", + "op": "decide_plan", + "input": { + "provider": { + "name": "jev", + "model": "jev-1.13.0" + }, + "api_key": "test-key", + "state": { + "thread": [ + { + "from": "user", + "text": "ship it" + }, + { + "from": "bot", + "text": "queued", + "attempt": 2 + } + ] + }, + "questions": [ + { + "type": "binary", + "id": "is_rollback_request", + "instructions": "Does the message ask to roll back a release?" + }, + { + "type": "label", + "id": "action", + "instructions": "Which release action does the message ask for?", + "options": [ + { + "value": "none", + "description": "No release action" + }, + { + "value": "release", + "description": "Ship a new release" + }, + { + "value": "rollback", + "description": "Roll back a release" + } + ] + }, + { + "type": "score", + "id": "urgency", + "instructions": "How urgent is the message?", + "levels": [ + { + "label": "Not urgent", + "description": null + }, + { + "label": "Somewhat urgent", + "description": null + }, + { + "label": "Blocking, needs action now", + "description": "customers are blocked" + } + ] + } + ] + }, + "expected": { + "kind": "http", + "method": "POST", + "url": "https://api.typesafe.ai/v1/systemone", + "headers": { + "Authorization": "Bearer test-key" + }, + "body": { + "state": { + "thread": [ + { + "from": "user", + "text": "ship it" + }, + { + "from": "bot", + "text": "queued", + "attempt": 2 + } + ] + }, + "model": "jev-1.13.0", + "questions": { + "is_rollback_request": { + "type": "noul", + "instructions": "Does the message ask to roll back a release?" + }, + "action": { + "type": "choice", + "instructions": "Which release action does the message ask for?", + "criteria": { + "none": "No release action", + "release": "Ship a new release", + "rollback": "Roll back a release" + } + }, + "urgency": { + "type": "score", + "instructions": "How urgent is the message?", + "criteria": [ + "Not urgent", + "Somewhat urgent", + "Blocking, needs action now: customers are blocked" + ] + } + } + } + } +} diff --git a/conformance/vectors/decide_plan/jev-three-questions.json b/conformance/vectors/decide_plan/jev-three-questions.json new file mode 100644 index 0000000..ed919e6 --- /dev/null +++ b/conformance/vectors/decide_plan/jev-three-questions.json @@ -0,0 +1,93 @@ +{ + "name": "jev-three-questions", + "op": "decide_plan", + "input": { + "provider": { + "name": "jev", + "model": "jev-1.13.0" + }, + "api_key": "test-key", + "state": "Help! Our nightly release has been failing for 3 days and customers are blocked. Please roll it back now.", + "questions": [ + { + "type": "binary", + "id": "is_rollback_request", + "instructions": "Does the message ask to roll back a release?" + }, + { + "type": "label", + "id": "action", + "instructions": "Which release action does the message ask for?", + "options": [ + { + "value": "none", + "description": "No release action" + }, + { + "value": "release", + "description": "Ship a new release" + }, + { + "value": "rollback", + "description": "Roll back a release" + } + ] + }, + { + "type": "score", + "id": "urgency", + "instructions": "How urgent is the message?", + "levels": [ + { + "label": "Not urgent", + "description": null + }, + { + "label": "Somewhat urgent", + "description": null + }, + { + "label": "Blocking, needs action now", + "description": "customers are blocked" + } + ] + } + ] + }, + "expected": { + "kind": "http", + "method": "POST", + "url": "https://api.typesafe.ai/v1/systemone", + "headers": { + "Authorization": "Bearer test-key" + }, + "body": { + "state": "Help! Our nightly release has been failing for 3 days and customers are blocked. Please roll it back now.", + "model": "jev-1.13.0", + "questions": { + "is_rollback_request": { + "type": "noul", + "instructions": "Does the message ask to roll back a release?" + }, + "action": { + "type": "choice", + "instructions": "Which release action does the message ask for?", + "criteria": { + "none": "No release action", + "release": "Ship a new release", + "rollback": "Roll back a release" + } + }, + "urgency": { + "type": "score", + "instructions": "How urgent is the message?", + "criteria": [ + "Not urgent", + "Somewhat urgent", + "Blocking, needs action now: customers are blocked" + ] + } + } + } + } +} diff --git a/conformance/vectors/decide_plan/openai-binary-criteria.json b/conformance/vectors/decide_plan/openai-binary-criteria.json new file mode 100644 index 0000000..faeac83 --- /dev/null +++ b/conformance/vectors/decide_plan/openai-binary-criteria.json @@ -0,0 +1,40 @@ +{ + "name": "openai-binary-criteria", + "op": "decide_plan", + "input": { + "provider": { + "name": "openai", + "model": "gpt-6-luna" + }, + "api_key": "test-key", + "state": "Help! Our nightly release has been failing for 3 days and customers are blocked. Please roll it back now.", + "questions": [ + { + "type": "binary", + "id": "is_urgent", + "instructions": "Does this convey urgency?", + "yes": "Explicitly time-sensitive", + "no": "No urgency expressed" + } + ] + }, + "expected": { + "kind": "http", + "method": "POST", + "url": "https://api.openai.com/v1/decisions", + "headers": { + "Authorization": "Bearer test-key" + }, + "body": { + "model": "gpt-6-luna", + "input": "Help! Our nightly release has been failing for 3 days and customers are blocked. Please roll it back now.", + "questions": [ + { + "type": "predicate", + "name": "is_urgent", + "instructions": "Does this convey urgency?\nYes means: Explicitly time-sensitive\nNo means: No urgency expressed" + } + ] + } + } +} diff --git a/conformance/vectors/decide_plan/openai-structured-state-compact.json b/conformance/vectors/decide_plan/openai-structured-state-compact.json new file mode 100644 index 0000000..7c3f917 --- /dev/null +++ b/conformance/vectors/decide_plan/openai-structured-state-compact.json @@ -0,0 +1,124 @@ +{ + "name": "openai-structured-state-compact", + "op": "decide_plan", + "input": { + "provider": { + "name": "openai", + "model": "gpt-6-luna" + }, + "api_key": "test-key", + "state": { + "thread": [ + { + "from": "user", + "text": "ship it" + }, + { + "from": "bot", + "text": "queued", + "attempt": 2 + } + ] + }, + "questions": [ + { + "type": "binary", + "id": "is_rollback_request", + "instructions": "Does the message ask to roll back a release?" + }, + { + "type": "label", + "id": "action", + "instructions": "Which release action does the message ask for?", + "options": [ + { + "value": "none", + "description": "No release action" + }, + { + "value": "release", + "description": "Ship a new release" + }, + { + "value": "rollback", + "description": "Roll back a release" + } + ] + }, + { + "type": "score", + "id": "urgency", + "instructions": "How urgent is the message?", + "levels": [ + { + "label": "Not urgent", + "description": null + }, + { + "label": "Somewhat urgent", + "description": null + }, + { + "label": "Blocking, needs action now", + "description": "customers are blocked" + } + ] + } + ] + }, + "expected": { + "kind": "http", + "method": "POST", + "url": "https://api.openai.com/v1/decisions", + "headers": { + "Authorization": "Bearer test-key" + }, + "body": { + "model": "gpt-6-luna", + "input": "{\"thread\":[{\"from\":\"user\",\"text\":\"ship it\"},{\"from\":\"bot\",\"text\":\"queued\",\"attempt\":2}]}", + "questions": [ + { + "type": "predicate", + "name": "is_rollback_request", + "instructions": "Does the message ask to roll back a release?" + }, + { + "type": "choice", + "name": "action", + "instructions": "Which release action does the message ask for?", + "choices": [ + { + "value": "none", + "description": "No release action" + }, + { + "value": "release", + "description": "Ship a new release" + }, + { + "value": "rollback", + "description": "Roll back a release" + } + ] + }, + { + "type": "score", + "name": "urgency", + "instructions": "How urgent is the message?", + "levels": [ + { + "label": "Not urgent" + }, + { + "label": "Somewhat urgent" + }, + { + "label": "Blocking, needs action now", + "description": "customers are blocked" + } + ] + } + ] + } + } +} diff --git a/conformance/vectors/decide_plan/openai-three-questions.json b/conformance/vectors/decide_plan/openai-three-questions.json new file mode 100644 index 0000000..508803f --- /dev/null +++ b/conformance/vectors/decide_plan/openai-three-questions.json @@ -0,0 +1,112 @@ +{ + "name": "openai-three-questions", + "op": "decide_plan", + "input": { + "provider": { + "name": "openai", + "model": "gpt-6-luna" + }, + "api_key": "test-key", + "state": "Help! Our nightly release has been failing for 3 days and customers are blocked. Please roll it back now.", + "questions": [ + { + "type": "binary", + "id": "is_rollback_request", + "instructions": "Does the message ask to roll back a release?" + }, + { + "type": "label", + "id": "action", + "instructions": "Which release action does the message ask for?", + "options": [ + { + "value": "none", + "description": "No release action" + }, + { + "value": "release", + "description": "Ship a new release" + }, + { + "value": "rollback", + "description": "Roll back a release" + } + ] + }, + { + "type": "score", + "id": "urgency", + "instructions": "How urgent is the message?", + "levels": [ + { + "label": "Not urgent", + "description": null + }, + { + "label": "Somewhat urgent", + "description": null + }, + { + "label": "Blocking, needs action now", + "description": "customers are blocked" + } + ] + } + ] + }, + "expected": { + "kind": "http", + "method": "POST", + "url": "https://api.openai.com/v1/decisions", + "headers": { + "Authorization": "Bearer test-key" + }, + "body": { + "model": "gpt-6-luna", + "input": "Help! Our nightly release has been failing for 3 days and customers are blocked. Please roll it back now.", + "questions": [ + { + "type": "predicate", + "name": "is_rollback_request", + "instructions": "Does the message ask to roll back a release?" + }, + { + "type": "choice", + "name": "action", + "instructions": "Which release action does the message ask for?", + "choices": [ + { + "value": "none", + "description": "No release action" + }, + { + "value": "release", + "description": "Ship a new release" + }, + { + "value": "rollback", + "description": "Roll back a release" + } + ] + }, + { + "type": "score", + "name": "urgency", + "instructions": "How urgent is the message?", + "levels": [ + { + "label": "Not urgent" + }, + { + "label": "Somewhat urgent" + }, + { + "label": "Blocking, needs action now", + "description": "customers are blocked" + } + ] + } + ] + } + } +} diff --git a/conformance/vectors/decide_resolve/jev-408-retries.json b/conformance/vectors/decide_resolve/jev-408-retries.json new file mode 100644 index 0000000..9bbf4ba --- /dev/null +++ b/conformance/vectors/decide_resolve/jev-408-retries.json @@ -0,0 +1,64 @@ +{ + "name": "jev-408-retries", + "op": "decide_resolve", + "input": { + "provider": { + "name": "jev", + "model": "m" + }, + "questions": [ + { + "type": "binary", + "id": "is_rollback_request", + "instructions": "Does the message ask to roll back a release?" + }, + { + "type": "label", + "id": "action", + "instructions": "Which release action does the message ask for?", + "options": [ + { + "value": "none", + "description": "No release action" + }, + { + "value": "release", + "description": "Ship a new release" + }, + { + "value": "rollback", + "description": "Roll back a release" + } + ] + }, + { + "type": "score", + "id": "urgency", + "instructions": "How urgent is the message?", + "levels": [ + { + "label": "Not urgent", + "description": null + }, + { + "label": "Somewhat urgent", + "description": null + }, + { + "label": "Blocking, needs action now", + "description": "customers are blocked" + } + ] + } + ], + "status": 408, + "body": "", + "retry_after": null, + "retry_after_ms": null, + "attempt": 1 + }, + "expected": { + "kind": "retry", + "sleep_s": 1.0 + } +} diff --git a/conformance/vectors/decide_resolve/jev-422-fails.json b/conformance/vectors/decide_resolve/jev-422-fails.json new file mode 100644 index 0000000..1fa0c93 --- /dev/null +++ b/conformance/vectors/decide_resolve/jev-422-fails.json @@ -0,0 +1,65 @@ +{ + "name": "jev-422-fails", + "op": "decide_resolve", + "input": { + "provider": { + "name": "jev", + "model": "m" + }, + "questions": [ + { + "type": "binary", + "id": "is_rollback_request", + "instructions": "Does the message ask to roll back a release?" + }, + { + "type": "label", + "id": "action", + "instructions": "Which release action does the message ask for?", + "options": [ + { + "value": "none", + "description": "No release action" + }, + { + "value": "release", + "description": "Ship a new release" + }, + { + "value": "rollback", + "description": "Roll back a release" + } + ] + }, + { + "type": "score", + "id": "urgency", + "instructions": "How urgent is the message?", + "levels": [ + { + "label": "Not urgent", + "description": null + }, + { + "label": "Somewhat urgent", + "description": null + }, + { + "label": "Blocking, needs action now", + "description": "customers are blocked" + } + ] + } + ], + "status": 422, + "body": "{\"detail\":[{\"loc\":[\"body\",\"questions\",\"urgency\",\"criteria\"],\"msg\":\"too short\"}]}", + "retry_after": null, + "retry_after_ms": null, + "attempt": 0 + }, + "expected": { + "kind": "failed", + "status": 422, + "message": "{\"detail\":[{\"loc\":[\"body\",\"questions\",\"urgency\",\"criteria\"],\"msg\":\"too short\"}]}" + } +} diff --git a/conformance/vectors/decide_resolve/jev-429-retry-after-seconds.json b/conformance/vectors/decide_resolve/jev-429-retry-after-seconds.json new file mode 100644 index 0000000..4aa29fc --- /dev/null +++ b/conformance/vectors/decide_resolve/jev-429-retry-after-seconds.json @@ -0,0 +1,64 @@ +{ + "name": "jev-429-retry-after-seconds", + "op": "decide_resolve", + "input": { + "provider": { + "name": "jev", + "model": "m" + }, + "questions": [ + { + "type": "binary", + "id": "is_rollback_request", + "instructions": "Does the message ask to roll back a release?" + }, + { + "type": "label", + "id": "action", + "instructions": "Which release action does the message ask for?", + "options": [ + { + "value": "none", + "description": "No release action" + }, + { + "value": "release", + "description": "Ship a new release" + }, + { + "value": "rollback", + "description": "Roll back a release" + } + ] + }, + { + "type": "score", + "id": "urgency", + "instructions": "How urgent is the message?", + "levels": [ + { + "label": "Not urgent", + "description": null + }, + { + "label": "Somewhat urgent", + "description": null + }, + { + "label": "Blocking, needs action now", + "description": "customers are blocked" + } + ] + } + ], + "status": 429, + "body": "", + "retry_after": "2", + "retry_after_ms": null, + "attempt": 0 + }, + "expected": { + "kind": "retry", + "sleep_s": 2.0 + } +} diff --git a/conformance/vectors/decide_resolve/jev-529-first-backoff.json b/conformance/vectors/decide_resolve/jev-529-first-backoff.json new file mode 100644 index 0000000..c264826 --- /dev/null +++ b/conformance/vectors/decide_resolve/jev-529-first-backoff.json @@ -0,0 +1,64 @@ +{ + "name": "jev-529-first-backoff", + "op": "decide_resolve", + "input": { + "provider": { + "name": "jev", + "model": "m" + }, + "questions": [ + { + "type": "binary", + "id": "is_rollback_request", + "instructions": "Does the message ask to roll back a release?" + }, + { + "type": "label", + "id": "action", + "instructions": "Which release action does the message ask for?", + "options": [ + { + "value": "none", + "description": "No release action" + }, + { + "value": "release", + "description": "Ship a new release" + }, + { + "value": "rollback", + "description": "Roll back a release" + } + ] + }, + { + "type": "score", + "id": "urgency", + "instructions": "How urgent is the message?", + "levels": [ + { + "label": "Not urgent", + "description": null + }, + { + "label": "Somewhat urgent", + "description": null + }, + { + "label": "Blocking, needs action now", + "description": "customers are blocked" + } + ] + } + ], + "status": 529, + "body": "overloaded", + "retry_after": null, + "retry_after_ms": null, + "attempt": 0 + }, + "expected": { + "kind": "retry", + "sleep_s": 0.5 + } +} diff --git a/conformance/vectors/decide_resolve/jev-missing-answer-fails.json b/conformance/vectors/decide_resolve/jev-missing-answer-fails.json new file mode 100644 index 0000000..d96f6b2 --- /dev/null +++ b/conformance/vectors/decide_resolve/jev-missing-answer-fails.json @@ -0,0 +1,65 @@ +{ + "name": "jev-missing-answer-fails", + "op": "decide_resolve", + "input": { + "provider": { + "name": "jev", + "model": "m" + }, + "questions": [ + { + "type": "binary", + "id": "is_rollback_request", + "instructions": "Does the message ask to roll back a release?" + }, + { + "type": "label", + "id": "action", + "instructions": "Which release action does the message ask for?", + "options": [ + { + "value": "none", + "description": "No release action" + }, + { + "value": "release", + "description": "Ship a new release" + }, + { + "value": "rollback", + "description": "Roll back a release" + } + ] + }, + { + "type": "score", + "id": "urgency", + "instructions": "How urgent is the message?", + "levels": [ + { + "label": "Not urgent", + "description": null + }, + { + "label": "Somewhat urgent", + "description": null + }, + { + "label": "Blocking, needs action now", + "description": "customers are blocked" + } + ] + } + ], + "status": 200, + "body": "{\"model\":\"jev-1.13.0\",\"answers\":{},\"usage\":{\"input_tokens\":1}}", + "retry_after": null, + "retry_after_ms": null, + "attempt": 0 + }, + "expected": { + "kind": "failed", + "status": 200, + "message": "no answer for is_rollback_request" + } +} diff --git a/conformance/vectors/decide_resolve/jev-recorded-reorders-to-declared.json b/conformance/vectors/decide_resolve/jev-recorded-reorders-to-declared.json new file mode 100644 index 0000000..384868f --- /dev/null +++ b/conformance/vectors/decide_resolve/jev-recorded-reorders-to-declared.json @@ -0,0 +1,92 @@ +{ + "name": "jev-recorded-reorders-to-declared", + "op": "decide_resolve", + "input": { + "provider": { + "name": "jev", + "model": "m" + }, + "questions": [ + { + "type": "binary", + "id": "is_rollback_request", + "instructions": "Does the message ask to roll back a release?" + }, + { + "type": "label", + "id": "action", + "instructions": "Which release action does the message ask for?", + "options": [ + { + "value": "none", + "description": "No release action" + }, + { + "value": "release", + "description": "Ship a new release" + }, + { + "value": "rollback", + "description": "Roll back a release" + } + ] + }, + { + "type": "score", + "id": "urgency", + "instructions": "How urgent is the message?", + "levels": [ + { + "label": "Not urgent", + "description": null + }, + { + "label": "Somewhat urgent", + "description": null + }, + { + "label": "Blocking, needs action now", + "description": "customers are blocked" + } + ] + } + ], + "status": 200, + "body": "{\"model\":\"jev-1.13.0\",\"answers\":{\"is_rollback_request\":{\"type\":\"noul\",\"noul\":0.99},\"action\":{\"type\":\"choice\",\"choice\":\"rollback\",\"confidence\":1.0,\"probabilities\":{\"release\":0.0,\"rollback\":1.0,\"none\":0.0}},\"urgency\":{\"type\":\"score\",\"score\":2.0,\"confidence\":1.0,\"legend\":{\"0\":\"Not urgent\",\"1\":\"Somewhat urgent\",\"2\":\"Blocking, needs action now\"},\"probabilities\":{\"0\":0.0,\"1\":0.0,\"2\":1.0}}},\"usage\":{\"input_tokens\":414,\"output_tokens\":73}}", + "retry_after": null, + "retry_after_ms": null, + "attempt": 0 + }, + "expected": { + "kind": "decision", + "model": "jev-1.13.0", + "input_tokens": 414, + "answers": { + "is_rollback_request": { + "type": "binary", + "p_yes": 0.99, + "confidence": 0.98 + }, + "action": { + "type": "label", + "choice": "rollback", + "probabilities": [ + 0.0, + 0.0, + 1.0 + ], + "confidence": 1.0 + }, + "urgency": { + "type": "score", + "score": 2.0, + "probabilities": [ + 0.0, + 0.0, + 1.0 + ], + "confidence": 1.0 + } + } + } +} diff --git a/conformance/vectors/decide_resolve/no-response-retries.json b/conformance/vectors/decide_resolve/no-response-retries.json new file mode 100644 index 0000000..0666783 --- /dev/null +++ b/conformance/vectors/decide_resolve/no-response-retries.json @@ -0,0 +1,64 @@ +{ + "name": "no-response-retries", + "op": "decide_resolve", + "input": { + "provider": { + "name": "jev", + "model": "m" + }, + "questions": [ + { + "type": "binary", + "id": "is_rollback_request", + "instructions": "Does the message ask to roll back a release?" + }, + { + "type": "label", + "id": "action", + "instructions": "Which release action does the message ask for?", + "options": [ + { + "value": "none", + "description": "No release action" + }, + { + "value": "release", + "description": "Ship a new release" + }, + { + "value": "rollback", + "description": "Roll back a release" + } + ] + }, + { + "type": "score", + "id": "urgency", + "instructions": "How urgent is the message?", + "levels": [ + { + "label": "Not urgent", + "description": null + }, + { + "label": "Somewhat urgent", + "description": null + }, + { + "label": "Blocking, needs action now", + "description": "customers are blocked" + } + ] + } + ], + "status": null, + "body": "", + "retry_after": null, + "retry_after_ms": null, + "attempt": 0 + }, + "expected": { + "kind": "retry", + "sleep_s": 0.5 + } +} diff --git a/conformance/vectors/decide_resolve/openai-401-fails.json b/conformance/vectors/decide_resolve/openai-401-fails.json new file mode 100644 index 0000000..558e7df --- /dev/null +++ b/conformance/vectors/decide_resolve/openai-401-fails.json @@ -0,0 +1,65 @@ +{ + "name": "openai-401-fails", + "op": "decide_resolve", + "input": { + "provider": { + "name": "openai", + "model": "m" + }, + "questions": [ + { + "type": "binary", + "id": "is_rollback_request", + "instructions": "Does the message ask to roll back a release?" + }, + { + "type": "label", + "id": "action", + "instructions": "Which release action does the message ask for?", + "options": [ + { + "value": "none", + "description": "No release action" + }, + { + "value": "release", + "description": "Ship a new release" + }, + { + "value": "rollback", + "description": "Roll back a release" + } + ] + }, + { + "type": "score", + "id": "urgency", + "instructions": "How urgent is the message?", + "levels": [ + { + "label": "Not urgent", + "description": null + }, + { + "label": "Somewhat urgent", + "description": null + }, + { + "label": "Blocking, needs action now", + "description": "customers are blocked" + } + ] + } + ], + "status": 401, + "body": "{\"error\":{\"message\":\"Incorrect API key provided\",\"type\":\"invalid_request_error\"}}", + "retry_after": null, + "retry_after_ms": null, + "attempt": 0 + }, + "expected": { + "kind": "failed", + "status": 401, + "message": "{\"error\":{\"message\":\"Incorrect API key provided\",\"type\":\"invalid_request_error\"}}" + } +} diff --git a/conformance/vectors/decide_resolve/openai-429-retry-after-ms-wins.json b/conformance/vectors/decide_resolve/openai-429-retry-after-ms-wins.json new file mode 100644 index 0000000..cab92ad --- /dev/null +++ b/conformance/vectors/decide_resolve/openai-429-retry-after-ms-wins.json @@ -0,0 +1,64 @@ +{ + "name": "openai-429-retry-after-ms-wins", + "op": "decide_resolve", + "input": { + "provider": { + "name": "openai", + "model": "m" + }, + "questions": [ + { + "type": "binary", + "id": "is_rollback_request", + "instructions": "Does the message ask to roll back a release?" + }, + { + "type": "label", + "id": "action", + "instructions": "Which release action does the message ask for?", + "options": [ + { + "value": "none", + "description": "No release action" + }, + { + "value": "release", + "description": "Ship a new release" + }, + { + "value": "rollback", + "description": "Roll back a release" + } + ] + }, + { + "type": "score", + "id": "urgency", + "instructions": "How urgent is the message?", + "levels": [ + { + "label": "Not urgent", + "description": null + }, + { + "label": "Somewhat urgent", + "description": null + }, + { + "label": "Blocking, needs action now", + "description": "customers are blocked" + } + ] + } + ], + "status": 429, + "body": "", + "retry_after": "2", + "retry_after_ms": "250", + "attempt": 1 + }, + "expected": { + "kind": "retry", + "sleep_s": 0.25 + } +} diff --git a/conformance/vectors/decide_resolve/openai-500-backoff-caps-at-5s.json b/conformance/vectors/decide_resolve/openai-500-backoff-caps-at-5s.json new file mode 100644 index 0000000..4fa2259 --- /dev/null +++ b/conformance/vectors/decide_resolve/openai-500-backoff-caps-at-5s.json @@ -0,0 +1,64 @@ +{ + "name": "openai-500-backoff-caps-at-5s", + "op": "decide_resolve", + "input": { + "provider": { + "name": "openai", + "model": "m" + }, + "questions": [ + { + "type": "binary", + "id": "is_rollback_request", + "instructions": "Does the message ask to roll back a release?" + }, + { + "type": "label", + "id": "action", + "instructions": "Which release action does the message ask for?", + "options": [ + { + "value": "none", + "description": "No release action" + }, + { + "value": "release", + "description": "Ship a new release" + }, + { + "value": "rollback", + "description": "Roll back a release" + } + ] + }, + { + "type": "score", + "id": "urgency", + "instructions": "How urgent is the message?", + "levels": [ + { + "label": "Not urgent", + "description": null + }, + { + "label": "Somewhat urgent", + "description": null + }, + { + "label": "Blocking, needs action now", + "description": "customers are blocked" + } + ] + } + ], + "status": 500, + "body": "", + "retry_after": null, + "retry_after_ms": null, + "attempt": 6 + }, + "expected": { + "kind": "retry", + "sleep_s": 5.0 + } +} diff --git a/conformance/vectors/decide_resolve/openai-503-http-date-retry-after-backs-off.json b/conformance/vectors/decide_resolve/openai-503-http-date-retry-after-backs-off.json new file mode 100644 index 0000000..335045e --- /dev/null +++ b/conformance/vectors/decide_resolve/openai-503-http-date-retry-after-backs-off.json @@ -0,0 +1,64 @@ +{ + "name": "openai-503-http-date-retry-after-backs-off", + "op": "decide_resolve", + "input": { + "provider": { + "name": "openai", + "model": "m" + }, + "questions": [ + { + "type": "binary", + "id": "is_rollback_request", + "instructions": "Does the message ask to roll back a release?" + }, + { + "type": "label", + "id": "action", + "instructions": "Which release action does the message ask for?", + "options": [ + { + "value": "none", + "description": "No release action" + }, + { + "value": "release", + "description": "Ship a new release" + }, + { + "value": "rollback", + "description": "Roll back a release" + } + ] + }, + { + "type": "score", + "id": "urgency", + "instructions": "How urgent is the message?", + "levels": [ + { + "label": "Not urgent", + "description": null + }, + { + "label": "Somewhat urgent", + "description": null + }, + { + "label": "Blocking, needs action now", + "description": "customers are blocked" + } + ] + } + ], + "status": 503, + "body": "", + "retry_after": "Wed, 21 Oct 2026 07:28:00 GMT", + "retry_after_ms": null, + "attempt": 2 + }, + "expected": { + "kind": "retry", + "sleep_s": 2.0 + } +} diff --git a/conformance/vectors/decide_resolve/openai-misnamed-answer-fails.json b/conformance/vectors/decide_resolve/openai-misnamed-answer-fails.json new file mode 100644 index 0000000..7f1340d --- /dev/null +++ b/conformance/vectors/decide_resolve/openai-misnamed-answer-fails.json @@ -0,0 +1,65 @@ +{ + "name": "openai-misnamed-answer-fails", + "op": "decide_resolve", + "input": { + "provider": { + "name": "openai", + "model": "m" + }, + "questions": [ + { + "type": "binary", + "id": "is_rollback_request", + "instructions": "Does the message ask to roll back a release?" + }, + { + "type": "label", + "id": "action", + "instructions": "Which release action does the message ask for?", + "options": [ + { + "value": "none", + "description": "No release action" + }, + { + "value": "release", + "description": "Ship a new release" + }, + { + "value": "rollback", + "description": "Roll back a release" + } + ] + }, + { + "type": "score", + "id": "urgency", + "instructions": "How urgent is the message?", + "levels": [ + { + "label": "Not urgent", + "description": null + }, + { + "label": "Somewhat urgent", + "description": null + }, + { + "label": "Blocking, needs action now", + "description": "customers are blocked" + } + ] + } + ], + "status": 200, + "body": "{\"model\":\"gpt-6-luna\",\"answers\":[{\"type\":\"refusal\",\"name\":\"urgency\"},{\"type\":\"refusal\"},{\"type\":\"refusal\"}],\"usage\":{\"input_tokens\":1}}", + "retry_after": null, + "retry_after_ms": null, + "attempt": 0 + }, + "expected": { + "kind": "failed", + "status": 200, + "message": "answer urgency arrived where is_rollback_request was asked" + } +} diff --git a/conformance/vectors/decide_resolve/openai-recorded.json b/conformance/vectors/decide_resolve/openai-recorded.json new file mode 100644 index 0000000..5d8acd2 --- /dev/null +++ b/conformance/vectors/decide_resolve/openai-recorded.json @@ -0,0 +1,92 @@ +{ + "name": "openai-recorded", + "op": "decide_resolve", + "input": { + "provider": { + "name": "openai", + "model": "m" + }, + "questions": [ + { + "type": "binary", + "id": "is_rollback_request", + "instructions": "Does the message ask to roll back a release?" + }, + { + "type": "label", + "id": "action", + "instructions": "Which release action does the message ask for?", + "options": [ + { + "value": "none", + "description": "No release action" + }, + { + "value": "release", + "description": "Ship a new release" + }, + { + "value": "rollback", + "description": "Roll back a release" + } + ] + }, + { + "type": "score", + "id": "urgency", + "instructions": "How urgent is the message?", + "levels": [ + { + "label": "Not urgent", + "description": null + }, + { + "label": "Somewhat urgent", + "description": null + }, + { + "label": "Blocking, needs action now", + "description": "customers are blocked" + } + ] + } + ], + "status": 200, + "body": "{\"model\":\"gpt-6-luna\",\"answers\":[{\"type\":\"predicate\",\"name\":\"is_rollback_request\",\"probability\":1.0},{\"type\":\"choice\",\"name\":\"action\",\"choice\":\"rollback\",\"probabilities\":[{\"value\":\"none\",\"probability\":0.0},{\"value\":\"release\",\"probability\":0.0},{\"value\":\"rollback\",\"probability\":1.0}],\"confidence\":1.0},{\"type\":\"score\",\"name\":\"urgency\",\"score\":2.0,\"probabilities\":[{\"value\":0,\"label\":\"Not urgent\",\"probability\":0.0},{\"value\":1,\"label\":\"Somewhat urgent\",\"probability\":0.0},{\"value\":2,\"label\":\"Blocking, needs action now\",\"probability\":1.0}],\"confidence\":1.0}],\"usage\":{\"input_tokens\":419,\"input_tokens_details\":{\"cached_tokens\":0,\"cache_write_tokens\":0},\"output_tokens\":0,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":419}}", + "retry_after": null, + "retry_after_ms": null, + "attempt": 0 + }, + "expected": { + "kind": "decision", + "model": "gpt-6-luna", + "input_tokens": 419, + "answers": { + "is_rollback_request": { + "type": "binary", + "p_yes": 1.0, + "confidence": 1.0 + }, + "action": { + "type": "label", + "choice": "rollback", + "probabilities": [ + 0.0, + 0.0, + 1.0 + ], + "confidence": 1.0 + }, + "urgency": { + "type": "score", + "score": 2.0, + "probabilities": [ + 0.0, + 0.0, + 1.0 + ], + "confidence": 1.0 + } + } + } +} diff --git a/conformance/vectors/decide_resolve/openai-refusal-beside-answers.json b/conformance/vectors/decide_resolve/openai-refusal-beside-answers.json new file mode 100644 index 0000000..a91f923 --- /dev/null +++ b/conformance/vectors/decide_resolve/openai-refusal-beside-answers.json @@ -0,0 +1,90 @@ +{ + "name": "openai-refusal-beside-answers", + "op": "decide_resolve", + "input": { + "provider": { + "name": "openai", + "model": "m" + }, + "questions": [ + { + "type": "binary", + "id": "is_rollback_request", + "instructions": "Does the message ask to roll back a release?" + }, + { + "type": "label", + "id": "action", + "instructions": "Which release action does the message ask for?", + "options": [ + { + "value": "none", + "description": "No release action" + }, + { + "value": "release", + "description": "Ship a new release" + }, + { + "value": "rollback", + "description": "Roll back a release" + } + ] + }, + { + "type": "score", + "id": "urgency", + "instructions": "How urgent is the message?", + "levels": [ + { + "label": "Not urgent", + "description": null + }, + { + "label": "Somewhat urgent", + "description": null + }, + { + "label": "Blocking, needs action now", + "description": "customers are blocked" + } + ] + } + ], + "status": 200, + "body": "{\"model\":\"gpt-6-luna\",\"answers\":[{\"type\":\"refusal\",\"name\":\"is_rollback_request\"},{\"type\":\"choice\",\"name\":\"action\",\"choice\":\"none\",\"probabilities\":[{\"value\":\"rollback\",\"probability\":0.1},{\"value\":\"none\",\"probability\":0.8},{\"value\":\"release\",\"probability\":0.1}],\"confidence\":0.7},{\"type\":\"score\",\"name\":\"urgency\",\"score\":0.9,\"probabilities\":[{\"value\":2,\"label\":\"Blocking, needs action now\",\"probability\":0.2},{\"value\":0,\"label\":\"Not urgent\",\"probability\":0.3},{\"value\":1,\"label\":\"Somewhat urgent\",\"probability\":0.5}],\"confidence\":0.2}],\"usage\":{\"input_tokens\":419}}", + "retry_after": null, + "retry_after_ms": null, + "attempt": 0 + }, + "expected": { + "kind": "decision", + "model": "gpt-6-luna", + "input_tokens": 419, + "answers": { + "is_rollback_request": { + "type": "refused" + }, + "action": { + "type": "label", + "choice": "none", + "probabilities": [ + 0.8, + 0.1, + 0.1 + ], + "confidence": 0.7 + }, + "urgency": { + "type": "score", + "score": 0.9, + "probabilities": [ + 0.3, + 0.5, + 0.2 + ], + "confidence": 0.2 + } + } + } +} diff --git a/conformance/vectors/decide_resolve/success-body-not-json-fails.json b/conformance/vectors/decide_resolve/success-body-not-json-fails.json new file mode 100644 index 0000000..6bd9a7e --- /dev/null +++ b/conformance/vectors/decide_resolve/success-body-not-json-fails.json @@ -0,0 +1,65 @@ +{ + "name": "success-body-not-json-fails", + "op": "decide_resolve", + "input": { + "provider": { + "name": "jev", + "model": "m" + }, + "questions": [ + { + "type": "binary", + "id": "is_rollback_request", + "instructions": "Does the message ask to roll back a release?" + }, + { + "type": "label", + "id": "action", + "instructions": "Which release action does the message ask for?", + "options": [ + { + "value": "none", + "description": "No release action" + }, + { + "value": "release", + "description": "Ship a new release" + }, + { + "value": "rollback", + "description": "Roll back a release" + } + ] + }, + { + "type": "score", + "id": "urgency", + "instructions": "How urgent is the message?", + "levels": [ + { + "label": "Not urgent", + "description": null + }, + { + "label": "Somewhat urgent", + "description": null + }, + { + "label": "Blocking, needs action now", + "description": "customers are blocked" + } + ] + } + ], + "status": 200, + "body": "", + "retry_after": null, + "retry_after_ms": null, + "attempt": 0 + }, + "expected": { + "kind": "failed", + "status": 200, + "message": "the decision response is malformed: expected value at line 1 column 1" + } +} diff --git a/great-docs.yml b/great-docs.yml index 46a3b3c..d4bf23b 100644 --- a/great-docs.yml +++ b/great-docs.yml @@ -76,6 +76,24 @@ reference: - ClaudeConfig - CodexConfig - GeminiConfig + - title: Deciding + desc: Typed yes/no, label, and score decisions over TypeSafe Jev and OpenAI Decisions, one question set for either provider. + contents: + - decide.decide + - decide.decide_sync + - Binary + - Label + - Score + - Decision + - BinaryAnswer + - LabelAnswer + - ScoreAnswer + - Refused + - Provider + - DecideError + - DecideKeyMissing + - decide.api_key + - decide.store_key - title: Backends desc: CLI backends (Claude, Codex, and the Gemini family), the in-process Claude Agent SDK backend, the Apple Foundation Models backend over its bundled Swift sidecar, the MLX and OpenAI-endpoint backends, the specialty registry, backend selection, and install/auth status checks. contents: diff --git a/rust/conformance-gen/src/cases.rs b/rust/conformance-gen/src/cases.rs index 41bb3dd..834a362 100644 --- a/rust/conformance-gen/src/cases.rs +++ b/rust/conformance-gen/src/cases.rs @@ -1227,6 +1227,252 @@ fn iso_seed_cases() -> Vec { ] } +fn decide_questions() -> Value { + json!([ + {"type": "binary", "id": "is_rollback_request", "instructions": "Does the message ask to roll back a release?"}, + {"type": "label", "id": "action", "instructions": "Which release action does the message ask for?", "options": [ + {"value": "none", "description": "No release action"}, + {"value": "release", "description": "Ship a new release"}, + {"value": "rollback", "description": "Roll back a release"}, + ]}, + {"type": "score", "id": "urgency", "instructions": "How urgent is the message?", "levels": [ + {"label": "Not urgent", "description": null}, + {"label": "Somewhat urgent", "description": null}, + {"label": "Blocking, needs action now", "description": "customers are blocked"}, + ]}, + ]) +} + +const DECIDE_STATE: &str = "Help! Our nightly release has been failing for 3 days and customers are blocked. Please roll it back now."; + +fn decide_plan_case( + name: &str, + provider: &str, + model: &str, + state: Value, + questions: Value, +) -> Case { + Case { + op: "decide_plan", + name: name.to_owned(), + input: json!({ + "provider": {"name": provider, "model": model}, + "api_key": "test-key", + "state": state, + "questions": questions, + }), + } +} + +fn decide_plan_cases() -> Vec { + let binary_with_criteria = json!([{ + "type": "binary", + "id": "is_urgent", + "instructions": "Does this convey urgency?", + "yes": "Explicitly time-sensitive", + "no": "No urgency expressed", + }]); + let structured = json!({"thread": [{"from": "user", "text": "ship it"}, {"from": "bot", "text": "queued", "attempt": 2}]}); + vec![ + decide_plan_case( + "jev-three-questions", + "jev", + "jev-1.13.0", + json!(DECIDE_STATE), + decide_questions(), + ), + decide_plan_case( + "openai-three-questions", + "openai", + "gpt-6-luna", + json!(DECIDE_STATE), + decide_questions(), + ), + decide_plan_case( + "jev-binary-criteria", + "jev", + "jev-1.13.0", + json!(DECIDE_STATE), + binary_with_criteria.clone(), + ), + decide_plan_case( + "openai-binary-criteria", + "openai", + "gpt-6-luna", + json!(DECIDE_STATE), + binary_with_criteria, + ), + decide_plan_case( + "jev-structured-state-verbatim", + "jev", + "jev-1.13.0", + structured.clone(), + decide_questions(), + ), + decide_plan_case( + "openai-structured-state-compact", + "openai", + "gpt-6-luna", + structured, + decide_questions(), + ), + ] +} + +fn decide_resolve_case( + name: &str, + provider: &str, + status: Option, + body: &str, + retry_after: Option<&str>, + retry_after_ms: Option<&str>, + attempt: u32, +) -> Case { + Case { + op: "decide_resolve", + name: name.to_owned(), + input: json!({ + "provider": {"name": provider, "model": "m"}, + "questions": decide_questions(), + "status": status, + "body": body, + "retry_after": retry_after, + "retry_after_ms": retry_after_ms, + "attempt": attempt, + }), + } +} + +const JEV_RECORDED: &str = r#"{"model":"jev-1.13.0","answers":{"is_rollback_request":{"type":"noul","noul":0.99},"action":{"type":"choice","choice":"rollback","confidence":1.0,"probabilities":{"release":0.0,"rollback":1.0,"none":0.0}},"urgency":{"type":"score","score":2.0,"confidence":1.0,"legend":{"0":"Not urgent","1":"Somewhat urgent","2":"Blocking, needs action now"},"probabilities":{"0":0.0,"1":0.0,"2":1.0}}},"usage":{"input_tokens":414,"output_tokens":73}}"#; + +const OPENAI_RECORDED: &str = r#"{"model":"gpt-6-luna","answers":[{"type":"predicate","name":"is_rollback_request","probability":1.0},{"type":"choice","name":"action","choice":"rollback","probabilities":[{"value":"none","probability":0.0},{"value":"release","probability":0.0},{"value":"rollback","probability":1.0}],"confidence":1.0},{"type":"score","name":"urgency","score":2.0,"probabilities":[{"value":0,"label":"Not urgent","probability":0.0},{"value":1,"label":"Somewhat urgent","probability":0.0},{"value":2,"label":"Blocking, needs action now","probability":1.0}],"confidence":1.0}],"usage":{"input_tokens":419,"input_tokens_details":{"cached_tokens":0,"cache_write_tokens":0},"output_tokens":0,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":419}}"#; + +const OPENAI_REFUSAL: &str = r#"{"model":"gpt-6-luna","answers":[{"type":"refusal","name":"is_rollback_request"},{"type":"choice","name":"action","choice":"none","probabilities":[{"value":"rollback","probability":0.1},{"value":"none","probability":0.8},{"value":"release","probability":0.1}],"confidence":0.7},{"type":"score","name":"urgency","score":0.9,"probabilities":[{"value":2,"label":"Blocking, needs action now","probability":0.2},{"value":0,"label":"Not urgent","probability":0.3},{"value":1,"label":"Somewhat urgent","probability":0.5}],"confidence":0.2}],"usage":{"input_tokens":419}}"#; + +fn decide_resolve_cases() -> Vec { + vec![ + decide_resolve_case( + "jev-recorded-reorders-to-declared", + "jev", + Some(200), + JEV_RECORDED, + None, + None, + 0, + ), + decide_resolve_case( + "openai-recorded", + "openai", + Some(200), + OPENAI_RECORDED, + None, + None, + 0, + ), + decide_resolve_case( + "openai-refusal-beside-answers", + "openai", + Some(200), + OPENAI_REFUSAL, + None, + None, + 0, + ), + decide_resolve_case( + "openai-misnamed-answer-fails", + "openai", + Some(200), + r#"{"model":"gpt-6-luna","answers":[{"type":"refusal","name":"urgency"},{"type":"refusal"},{"type":"refusal"}],"usage":{"input_tokens":1}}"#, + None, + None, + 0, + ), + decide_resolve_case( + "jev-missing-answer-fails", + "jev", + Some(200), + r#"{"model":"jev-1.13.0","answers":{},"usage":{"input_tokens":1}}"#, + None, + None, + 0, + ), + decide_resolve_case( + "success-body-not-json-fails", + "jev", + Some(200), + "", + None, + None, + 0, + ), + decide_resolve_case( + "jev-529-first-backoff", + "jev", + Some(529), + "overloaded", + None, + None, + 0, + ), + decide_resolve_case( + "jev-429-retry-after-seconds", + "jev", + Some(429), + "", + Some("2"), + None, + 0, + ), + decide_resolve_case( + "openai-429-retry-after-ms-wins", + "openai", + Some(429), + "", + Some("2"), + Some("250"), + 1, + ), + decide_resolve_case( + "openai-503-http-date-retry-after-backs-off", + "openai", + Some(503), + "", + Some("Wed, 21 Oct 2026 07:28:00 GMT"), + None, + 2, + ), + decide_resolve_case( + "openai-500-backoff-caps-at-5s", + "openai", + Some(500), + "", + None, + None, + 6, + ), + decide_resolve_case("jev-408-retries", "jev", Some(408), "", None, None, 1), + decide_resolve_case("no-response-retries", "jev", None, "", None, None, 0), + decide_resolve_case( + "openai-401-fails", + "openai", + Some(401), + r#"{"error":{"message":"Incorrect API key provided","type":"invalid_request_error"}}"#, + None, + None, + 0, + ), + decide_resolve_case( + "jev-422-fails", + "jev", + Some(422), + r#"{"detail":[{"loc":["body","questions","urgency","criteria"],"msg":"too short"}]}"#, + None, + None, + 0, + ), + ] +} + pub fn all_cases() -> Vec { let mut cases = plan_cases(); cases.extend(resolve_cases()); @@ -1237,6 +1483,8 @@ pub fn all_cases() -> Vec { cases.extend(capabilities_cases()); cases.extend(iso_sources_cases()); cases.extend(iso_seed_cases()); + cases.extend(decide_plan_cases()); + cases.extend(decide_resolve_cases()); cases } diff --git a/rust/spawnllm-core/src/decide.rs b/rust/spawnllm-core/src/decide.rs new file mode 100644 index 0000000..7d1ac9f --- /dev/null +++ b/rust/spawnllm-core/src/decide.rs @@ -0,0 +1,827 @@ +use std::collections::{BTreeMap, HashMap, HashSet}; + +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value, json}; + +use crate::wire::{HttpPlan, InvocationPlan}; +use crate::{OpError, OpResult, from_input, unimplemented}; + +const JEV_URL: &str = "https://api.typesafe.ai/v1/systemone"; +const OPENAI_URL: &str = "https://api.openai.com/v1/decisions"; +const MAX_OPTIONS: usize = 255; +const MAX_LEVELS: usize = 10; +const BACKOFF_BASE_S: f64 = 0.5; +const BACKOFF_CAP_S: f64 = 5.0; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "snake_case")] +enum Vendor { + Jev, + Openai, +} + +#[derive(Debug, Clone, Deserialize)] +struct Provider { + name: Vendor, + model: String, +} + +#[derive(Debug, Clone, Deserialize)] +struct Choice { + value: String, + description: Option, +} + +#[derive(Debug, Clone, Deserialize)] +struct Level { + label: String, + description: Option, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +enum Question { + Binary { + id: String, + instructions: String, + yes: Option, + no: Option, + }, + Label { + id: String, + instructions: String, + options: Vec, + }, + Score { + id: String, + instructions: String, + levels: Vec, + }, +} + +impl Question { + fn id(&self) -> &str { + match self { + Self::Binary { id, .. } | Self::Label { id, .. } | Self::Score { id, .. } => id, + } + } +} + +#[derive(Debug, Clone, Deserialize)] +struct PlanInput { + provider: Provider, + api_key: String, + state: Value, + questions: Vec, +} + +#[derive(Debug, Clone, Deserialize)] +struct ResolveInput { + provider: Provider, + questions: Vec, + status: Option, + body: String, + retry_after: Option, + retry_after_ms: Option, + attempt: u32, +} + +#[derive(Debug, Clone, PartialEq, Serialize)] +#[serde(tag = "type", rename_all = "snake_case")] +enum Answer { + Binary { + p_yes: f64, + confidence: f64, + }, + Label { + choice: String, + probabilities: Vec, + confidence: f64, + }, + Score { + score: f64, + probabilities: Vec, + confidence: f64, + }, + Refused, +} + +#[derive(Debug, Clone, PartialEq, Serialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +enum Outcome { + Decision { + model: String, + input_tokens: u64, + answers: Map, + }, + Retry { + sleep_s: f64, + }, + Failed { + status: Option, + message: String, + }, +} + +#[derive(Deserialize)] +struct Usage { + input_tokens: u64, +} + +#[derive(Deserialize)] +struct Envelope { + model: String, + answers: Value, + usage: Usage, +} + +#[derive(Deserialize)] +struct Noul { + noul: f64, +} + +#[derive(Deserialize)] +struct Predicate { + probability: f64, +} + +#[derive(Deserialize)] +struct Distribution

{ + probabilities: P, + confidence: f64, +} + +#[derive(Deserialize)] +struct Chosen { + choice: String, +} + +#[derive(Deserialize)] +struct Scored { + score: f64, +} + +#[derive(Deserialize)] +struct Weighted { + value: V, + probability: f64, +} + +fn validate(state: &Value, questions: &[Question]) -> Result<(), &'static str> { + if !matches!(state, Value::String(_) | Value::Object(_) | Value::Array(_)) { + return Err("decide state must be a string, an object, or an array"); + } + if questions.is_empty() { + return Err("decide needs at least one question"); + } + let mut ids = HashSet::new(); + if !questions.iter().all(|question| ids.insert(question.id())) { + return Err("decide question ids must be unique"); + } + questions.iter().try_for_each(|question| match question { + Question::Label { options, .. } if !(2..=MAX_OPTIONS).contains(&options.len()) => { + Err("a label question takes 2 to 255 options") + } + Question::Label { options, .. } + if options + .iter() + .map(|option| &option.value) + .collect::>() + .len() + != options.len() => + { + Err("a label question's option values must be unique") + } + Question::Score { levels, .. } if !(2..=MAX_LEVELS).contains(&levels.len()) => { + Err("a score question takes 2 to 10 levels") + } + _ => Ok(()), + }) +} + +fn jev_question(question: &Question) -> Value { + match question { + Question::Binary { + instructions, + yes, + no, + .. + } => { + let criteria: Map = [("true", yes), ("false", no)] + .into_iter() + .filter_map(|(key, text)| text.as_ref().map(|text| (key.to_owned(), json!(text)))) + .collect(); + let mut body = json!({"type": "noul", "instructions": instructions}); + if !criteria.is_empty() { + body["criteria"] = Value::Object(criteria); + } + body + } + Question::Label { + instructions, + options, + .. + } => json!({ + "type": "choice", + "instructions": instructions, + "criteria": options + .iter() + .map(|option| (option.value.clone(), json!(option.description))) + .collect::>(), + }), + Question::Score { + instructions, + levels, + .. + } => json!({ + "type": "score", + "instructions": instructions, + "criteria": levels + .iter() + .map(|level| match &level.description { + Some(description) => format!("{}: {description}", level.label), + None => level.label.clone(), + }) + .collect::>(), + }), + } +} + +fn described(key: &str, name: &str, description: Option<&str>) -> Value { + let mut entry = Map::from_iter([(key.to_owned(), json!(name))]); + if let Some(description) = description { + entry.insert("description".to_owned(), json!(description)); + } + Value::Object(entry) +} + +fn predicate_instructions(instructions: &str, yes: Option<&str>, no: Option<&str>) -> String { + [ + Some(instructions.to_owned()), + yes.map(|yes| format!("Yes means: {yes}")), + no.map(|no| format!("No means: {no}")), + ] + .into_iter() + .flatten() + .collect::>() + .join("\n") +} + +fn openai_question(question: &Question) -> Value { + match question { + Question::Binary { + id, + instructions, + yes, + no, + } => json!({ + "type": "predicate", + "name": id, + "instructions": predicate_instructions(instructions, yes.as_deref(), no.as_deref()), + }), + Question::Label { + id, + instructions, + options, + } => json!({ + "type": "choice", + "name": id, + "instructions": instructions, + "choices": options + .iter() + .map(|option| described("value", &option.value, option.description.as_deref())) + .collect::>(), + }), + Question::Score { + id, + instructions, + levels, + } => json!({ + "type": "score", + "name": id, + "instructions": instructions, + "levels": levels + .iter() + .map(|level| described("label", &level.label, level.description.as_deref())) + .collect::>(), + }), + } +} + +fn plan(input: &PlanInput) -> InvocationPlan { + let (url, body) = match input.provider.name { + Vendor::Jev => ( + JEV_URL, + Map::from_iter([ + ("state".to_owned(), input.state.clone()), + ("model".to_owned(), json!(input.provider.model)), + ( + "questions".to_owned(), + Value::Object( + input + .questions + .iter() + .map(|question| (question.id().to_owned(), jev_question(question))) + .collect(), + ), + ), + ]), + ), + Vendor::Openai => ( + OPENAI_URL, + Map::from_iter([ + ("model".to_owned(), json!(input.provider.model)), + ( + "input".to_owned(), + json!(match &input.state { + Value::String(text) => text.clone(), + structured => structured.to_string(), + }), + ), + ( + "questions".to_owned(), + Value::Array(input.questions.iter().map(openai_question).collect()), + ), + ]), + ), + }; + InvocationPlan::Http(HttpPlan { + method: "POST".to_owned(), + url: url.to_owned(), + headers: BTreeMap::from([( + "Authorization".to_owned(), + format!("Bearer {}", input.api_key), + )]), + body, + }) +} + +fn parse(id: &str, answer: &Value) -> Result { + serde_json::from_value(answer.clone()) + .map_err(|error| format!("answer {id} is malformed: {error}")) +} + +fn ordered( + id: &str, + probabilities: &HashMap, + keys: impl Iterator, +) -> Result, String> { + keys.map(|key| { + probabilities + .get(&key) + .copied() + .ok_or_else(|| format!("answer {id} has no probability for {key:?}")) + }) + .collect() +} + +fn binary(p_yes: f64) -> Answer { + Answer::Binary { + p_yes, + confidence: (2.0 * p_yes - 1.0).abs(), + } +} + +fn option_keys(options: &[Choice]) -> impl Iterator { + options.iter().map(|option| option.value.clone()) +} + +fn level_keys(levels: &[Level]) -> impl Iterator { + (0..levels.len()).map(|index| index.to_string()) +} + +fn answer_type(answer: &Value) -> Option<&str> { + answer.get("type").and_then(Value::as_str) +} + +fn jev_answer(question: &Question, answer: &Value) -> Result { + let id = question.id(); + match (question, answer_type(answer)) { + (Question::Binary { .. }, Some("noul")) => Ok(binary(parse::(id, answer)?.noul)), + (Question::Label { options, .. }, Some("choice")) => { + let spread = parse::>>(id, answer)?; + Ok(Answer::Label { + choice: parse::(id, answer)?.choice, + probabilities: ordered(id, &spread.probabilities, option_keys(options))?, + confidence: spread.confidence, + }) + } + (Question::Score { levels, .. }, Some("score")) => { + let spread = parse::>>(id, answer)?; + Ok(Answer::Score { + score: parse::(id, answer)?.score, + probabilities: ordered(id, &spread.probabilities, level_keys(levels))?, + confidence: spread.confidence, + }) + } + (_, kind) => Err(format!("answer {id} came back as {kind:?}")), + } +} + +fn openai_answer(question: &Question, answer: &Value) -> Result { + let id = question.id(); + if let Some(name) = answer.get("name").and_then(Value::as_str) + && name != id + { + return Err(format!("answer {name} arrived where {id} was asked")); + } + match (question, answer_type(answer)) { + (_, Some("refusal")) => Ok(Answer::Refused), + (Question::Binary { .. }, Some("predicate")) => { + Ok(binary(parse::(id, answer)?.probability)) + } + (Question::Label { options, .. }, Some("choice")) => { + let spread = parse::>>>(id, answer)?; + let probabilities = spread + .probabilities + .into_iter() + .map(|weighted| (weighted.value, weighted.probability)) + .collect(); + Ok(Answer::Label { + choice: parse::(id, answer)?.choice, + probabilities: ordered(id, &probabilities, option_keys(options))?, + confidence: spread.confidence, + }) + } + (Question::Score { levels, .. }, Some("score")) => { + let spread = parse::>>>(id, answer)?; + let probabilities = spread + .probabilities + .into_iter() + .map(|weighted| (weighted.value.to_string(), weighted.probability)) + .collect(); + Ok(Answer::Score { + score: parse::(id, answer)?.score, + probabilities: ordered(id, &probabilities, level_keys(levels))?, + confidence: spread.confidence, + }) + } + (_, kind) => Err(format!("answer {id} came back as {kind:?}")), + } +} + +fn answers(input: &ResolveInput) -> Result { + let envelope: Envelope = serde_json::from_str(&input.body) + .map_err(|error| format!("the decision response is malformed: {error}"))?; + let answers: Vec = match (input.provider.name, &envelope.answers) { + (Vendor::Jev, Value::Object(by_id)) => input + .questions + .iter() + .map(|question| { + by_id + .get(question.id()) + .ok_or_else(|| format!("no answer for {}", question.id())) + .and_then(|answer| jev_answer(question, answer)) + }) + .collect::>()?, + (Vendor::Openai, Value::Array(in_order)) if in_order.len() == input.questions.len() => { + input + .questions + .iter() + .zip(in_order) + .map(|(question, answer)| openai_answer(question, answer)) + .collect::>()? + } + _ => return Err("the decision response's answers do not match the questions".to_owned()), + }; + Ok(Outcome::Decision { + model: envelope.model, + input_tokens: envelope.usage.input_tokens, + answers: input + .questions + .iter() + .zip(answers) + .map(|(question, answer)| { + ( + question.id().to_owned(), + serde_json::to_value(answer).expect("answers always serialize"), + ) + }) + .collect(), + }) +} + +fn seconds(header: Option<&str>) -> Option { + header + .and_then(|value| value.trim().parse::().ok()) + .filter(|value| value.is_finite() && *value >= 0.0) +} + +fn backoff(attempt: u32) -> f64 { + (BACKOFF_BASE_S * 2f64.powi(attempt.min(16) as i32)).min(BACKOFF_CAP_S) +} + +fn resolve(input: &ResolveInput) -> Outcome { + match input.status { + Some(200) => answers(input).unwrap_or_else(|message| Outcome::Failed { + status: Some(200), + message, + }), + None | Some(408 | 429 | 500..=599) => Outcome::Retry { + sleep_s: seconds(input.retry_after_ms.as_deref()) + .map(|ms| ms / 1000.0) + .or_else(|| seconds(input.retry_after.as_deref())) + .unwrap_or_else(|| backoff(input.attempt)), + }, + Some(status) => Outcome::Failed { + status: Some(status), + message: input.body.clone(), + }, + } +} + +pub(crate) fn dispatch(op: &str, input: Value) -> OpResult { + match op { + "decide_plan" => { + let input = from_input::(input)?; + validate(&input.state, &input.questions).map_err(OpError::invalid_spec)?; + serde_json::to_value(plan(&input)).map_err(OpError::internal) + } + "decide_resolve" => serde_json::to_value(resolve(&from_input::(input)?)) + .map_err(OpError::internal), + other => Err(unimplemented(other)), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn questions() -> Value { + json!([ + {"type": "binary", "id": "is_rollback_request", "instructions": "Does the message ask to roll back a release?"}, + {"type": "label", "id": "action", "instructions": "Which release action does the message ask for?", "options": [ + {"value": "none", "description": "No release action"}, + {"value": "release", "description": "Ship a new release"}, + {"value": "rollback", "description": "Roll back a release"}, + ]}, + {"type": "score", "id": "urgency", "instructions": "How urgent is the message?", "levels": [ + {"label": "Not urgent"}, + {"label": "Somewhat urgent"}, + {"label": "Blocking, needs action now"}, + ]}, + ]) + } + + fn plan_for(name: &str, model: &str, state: Value, questions: Value) -> Value { + dispatch( + "decide_plan", + json!({"provider": {"name": name, "model": model}, "api_key": "k", "state": state, "questions": questions}), + ) + .unwrap_or_else(|error| panic!("{}: {}", error.kind, error.msg)) + } + + fn resolve_for(name: &str, status: Option, body: &str, attempt: u32) -> Value { + dispatch( + "decide_resolve", + json!({ + "provider": {"name": name, "model": "m"}, + "questions": questions(), + "status": status, + "body": body, + "retry_after": null, + "retry_after_ms": null, + "attempt": attempt, + }), + ) + .unwrap_or_else(|error| panic!("{}: {}", error.kind, error.msg)) + } + + fn rejection(state: Value, questions: Value) -> String { + match dispatch( + "decide_plan", + json!({"provider": {"name": "jev", "model": "m"}, "api_key": "k", "state": state, "questions": questions}), + ) { + Err(error) => { + assert_eq!(error.kind, "invalid_spec"); + error.msg + } + Ok(plan) => panic!("expected a rejection, planned {plan}"), + } + } + + #[test] + fn jev_plan_keys_questions_by_id_in_declared_order() { + let plan = plan_for("jev", "jev-1.13.0", json!("roll it back"), questions()); + assert_eq!(plan["url"], JEV_URL); + assert_eq!(plan["headers"], json!({"Authorization": "Bearer k"})); + assert_eq!( + plan["body"]["questions"]["action"]["criteria"] + .as_object() + .unwrap() + .keys() + .collect::>(), + ["none", "release", "rollback"] + ); + assert_eq!( + plan["body"]["questions"]["urgency"]["criteria"], + json!([ + "Not urgent", + "Somewhat urgent", + "Blocking, needs action now" + ]) + ); + assert!( + plan["body"]["questions"]["is_rollback_request"] + .get("criteria") + .is_none() + ); + } + + #[test] + fn jev_binary_sends_only_the_criteria_it_was_given() { + let plan = plan_for( + "jev", + "m", + json!("x"), + json!([{"type": "binary", "id": "b", "instructions": "Is it?", "yes": "it is"}]), + ); + assert_eq!( + plan["body"]["questions"]["b"], + json!({"type": "noul", "instructions": "Is it?", "criteria": {"true": "it is"}}) + ); + } + + #[test] + fn openai_plan_folds_binary_criteria_into_instructions_and_compacts_structured_state() { + let plan = plan_for( + "openai", + "gpt-6-luna", + json!({"b": 1, "a": [true, null]}), + json!([{"type": "binary", "id": "b", "instructions": "Is it?", "yes": "it is", "no": "it is not"}]), + ); + assert_eq!(plan["url"], OPENAI_URL); + assert_eq!(plan["body"]["input"], r#"{"b":1,"a":[true,null]}"#); + assert_eq!( + plan["body"]["questions"], + json!([{"type": "predicate", "name": "b", "instructions": "Is it?\nYes means: it is\nNo means: it is not"}]) + ); + } + + #[test] + fn plan_rejects_invalid_specs() { + assert_eq!( + rejection(json!(3), questions()), + "decide state must be a string, an object, or an array" + ); + assert_eq!( + rejection(json!("x"), json!([])), + "decide needs at least one question" + ); + assert_eq!( + rejection( + json!("x"), + json!([ + {"type": "binary", "id": "a", "instructions": "?"}, + {"type": "binary", "id": "a", "instructions": "?"}, + ]) + ), + "decide question ids must be unique" + ); + assert_eq!( + rejection( + json!("x"), + json!([{"type": "label", "id": "a", "instructions": "?", "options": [{"value": "only"}]}]) + ), + "a label question takes 2 to 255 options" + ); + assert_eq!( + rejection( + json!("x"), + json!([{"type": "label", "id": "a", "instructions": "?", "options": [{"value": "x"}, {"value": "x"}]}]) + ), + "a label question's option values must be unique" + ); + let eleven: Vec = (0..11) + .map(|level| json!({"label": level.to_string()})) + .collect(); + assert_eq!( + rejection( + json!("x"), + json!([{"type": "score", "id": "a", "instructions": "?", "levels": eleven}]) + ), + "a score question takes 2 to 10 levels" + ); + } + + #[test] + fn jev_answers_reorder_to_declared_order() { + let body = json!({ + "model": "jev-1.13.0", + "answers": { + "urgency": {"type": "score", "score": 1.5, "confidence": 0.5, "legend": {"0": "a", "1": "b", "2": "c"}, "probabilities": {"2": 0.5, "0": 0.0, "1": 0.5}}, + "action": {"type": "choice", "choice": "rollback", "confidence": 0.7, "probabilities": {"release": 0.1, "rollback": 0.8, "none": 0.1}}, + "is_rollback_request": {"type": "noul", "noul": 0.25}, + }, + "usage": {"input_tokens": 414, "output_tokens": 73}, + }); + let outcome = resolve_for("jev", Some(200), &body.to_string(), 0); + assert_eq!( + outcome, + json!({ + "kind": "decision", + "model": "jev-1.13.0", + "input_tokens": 414, + "answers": { + "is_rollback_request": {"type": "binary", "p_yes": 0.25, "confidence": 0.5}, + "action": {"type": "label", "choice": "rollback", "probabilities": [0.1, 0.1, 0.8], "confidence": 0.7}, + "urgency": {"type": "score", "score": 1.5, "probabilities": [0.0, 0.5, 0.5], "confidence": 0.5}, + }, + }) + ); + } + + #[test] + fn openai_refusal_maps_to_refused_while_others_answer() { + let body = json!({ + "model": "gpt-6-luna", + "answers": [ + {"type": "refusal", "name": "is_rollback_request"}, + {"type": "choice", "name": "action", "choice": "none", "confidence": 1.0, "probabilities": [ + {"value": "rollback", "probability": 0.0}, {"value": "none", "probability": 1.0}, {"value": "release", "probability": 0.0}, + ]}, + {"type": "score", "name": "urgency", "score": 0.0, "confidence": 1.0, "probabilities": [ + {"value": 1, "label": "Somewhat urgent", "probability": 0.0}, {"value": 0, "label": "Not urgent", "probability": 1.0}, {"value": 2, "label": "Blocking, needs action now", "probability": 0.0}, + ]}, + ], + "usage": {"input_tokens": 419}, + }); + let outcome = resolve_for("openai", Some(200), &body.to_string(), 0); + assert_eq!( + outcome["answers"]["is_rollback_request"], + json!({"type": "refused"}) + ); + assert_eq!( + outcome["answers"]["action"]["probabilities"], + json!([1.0, 0.0, 0.0]) + ); + assert_eq!( + outcome["answers"]["urgency"]["probabilities"], + json!([1.0, 0.0, 0.0]) + ); + } + + #[test] + fn mismatched_success_bodies_fail_instead_of_guessing() { + let misnamed = json!({ + "model": "gpt-6-luna", + "answers": [{"type": "refusal", "name": "urgency"}, {"type": "refusal"}, {"type": "refusal"}], + "usage": {"input_tokens": 1}, + }); + assert_eq!( + resolve_for("openai", Some(200), &misnamed.to_string(), 0), + json!({"kind": "failed", "status": 200, "message": "answer urgency arrived where is_rollback_request was asked"}) + ); + let short = json!({"model": "gpt-6-luna", "answers": [], "usage": {"input_tokens": 1}}); + assert_eq!( + resolve_for("openai", Some(200), &short.to_string(), 0)["message"], + "the decision response's answers do not match the questions" + ); + let missing = json!({"model": "jev-1.13.0", "answers": {}, "usage": {"input_tokens": 1}}); + assert_eq!( + resolve_for("jev", Some(200), &missing.to_string(), 0)["message"], + "no answer for is_rollback_request" + ); + } + + #[test] + fn transient_statuses_back_off_to_the_cap_and_others_fail() { + assert_eq!( + resolve_for("jev", Some(529), "overloaded", 0), + json!({"kind": "retry", "sleep_s": 0.5}) + ); + assert_eq!(resolve_for("jev", Some(503), "", 2)["sleep_s"], json!(2.0)); + assert_eq!(resolve_for("jev", None, "", 9)["sleep_s"], json!(5.0)); + assert_eq!(resolve_for("jev", Some(408), "", 1)["sleep_s"], json!(1.0)); + assert_eq!( + resolve_for("openai", Some(401), "bad key", 0), + json!({"kind": "failed", "status": 401, "message": "bad key"}) + ); + } + + #[test] + fn retry_after_headers_override_the_backoff() { + let outcome = |retry_after: Value, retry_after_ms: Value| { + dispatch( + "decide_resolve", + json!({ + "provider": {"name": "openai", "model": "m"}, + "questions": questions(), + "status": 429, + "body": "", + "retry_after": retry_after, + "retry_after_ms": retry_after_ms, + "attempt": 0, + }), + ) + .unwrap_or_else(|error| panic!("{}", error.msg)) + }; + assert_eq!(outcome(json!("3"), json!("250"))["sleep_s"], json!(0.25)); + assert_eq!(outcome(json!("3"), Value::Null)["sleep_s"], json!(3.0)); + assert_eq!( + outcome(json!("Wed, 21 Oct 2026 07:28:00 GMT"), Value::Null)["sleep_s"], + json!(0.5) + ); + } +} diff --git a/rust/spawnllm-core/src/lib.rs b/rust/spawnllm-core/src/lib.rs index de09afb..3a075d8 100644 --- a/rust/spawnllm-core/src/lib.rs +++ b/rust/spawnllm-core/src/lib.rs @@ -1,4 +1,5 @@ mod auth; +mod decide; mod extract; mod isolate; mod plan; @@ -96,6 +97,7 @@ fn run(op: &str, input: Value) -> OpResult { "strict_schema" => schema::dispatch(input), "auth_probes" => probe::dispatch(input), "claude_isolation_sources" | "claude_isolation_seed" => isolate::dispatch(op, input), + "decide_plan" | "decide_resolve" => decide::dispatch(op, input), other => Err(OpError { kind: "unknown_op", msg: other.to_string(), diff --git a/spawnllm/__init__.py b/spawnllm/__init__.py index 7c8a85f..5cba72e 100644 --- a/spawnllm/__init__.py +++ b/spawnllm/__init__.py @@ -2,9 +2,10 @@ The top-level namespace exposes the three primitives — `run`/`call`/`extract` and their `_sync` companions — over a `Backend` family that fully encapsulates -execution and returns one shared `Response`. The MLX engine lives under -`spawnllm.mlx`, whose imports are lazy so that `import spawnllm` never pulls -`mlx_lm`/`zstandard`. +execution and returns one shared `Response`, plus `decide`/`decide_sync` for +typed yes/no, label, and score decisions over TypeSafe Jev and OpenAI +Decisions. The MLX engine lives under `spawnllm.mlx`, whose imports are lazy so +that `import spawnllm` never pulls `mlx_lm`/`zstandard`. """ from __future__ import annotations @@ -30,6 +31,23 @@ select_backend, ) from spawnllm.call import call, call_sync +from spawnllm.decide import ( + JEV, + OPENAI, + Binary, + BinaryAnswer, + DecideError, + DecideKeyMissing, + Decision, + Label, + LabelAnswer, + Provider, + Refused, + Score, + ScoreAnswer, + decide, + decide_sync, +) from spawnllm.extract import extract, extract_sync from spawnllm.response import DiscardedAttempt, Error, Output, Response, Result from spawnllm.run import run, run_sync @@ -37,6 +55,8 @@ from spawnllm.types import ProviderName, TModel, TReasoningEffort, TSpecialty __all__ = [ + "JEV", + "OPENAI", "AntigravityCliBackend", "AppleBackend", "AppleConfig", @@ -46,30 +66,43 @@ "BackendReady", "BackendStatus", "BackendUnavailable", + "Binary", + "BinaryAnswer", "ClaudeCliBackend", "ClaudeConfig", "ClaudeSdkBackend", "CliBackend", "CodexCliBackend", "CodexConfig", + "DecideError", + "DecideKeyMissing", + "Decision", "DiscardedAttempt", "Error", "GeminiCliBackend", "GeminiConfig", + "Label", + "LabelAnswer", "LlmBackend", "LlmBackends", "MlxBackend", "OpenAiEndpointBackend", "Output", + "Provider", "ProviderName", + "Refused", "Response", "Result", "RunSpec", + "Score", + "ScoreAnswer", "TModel", "TReasoningEffort", "TSpecialty", "call", "call_sync", + "decide", + "decide_sync", "extract", "extract_sync", "run", diff --git a/spawnllm/cli.py b/spawnllm/cli.py index e77d8e6..3aa17f4 100644 --- a/spawnllm/cli.py +++ b/spawnllm/cli.py @@ -15,8 +15,10 @@ ) from spawnllm.backends.registry import BACKENDS_BY_NAME, PRIORITY, select_backend from spawnllm.call import DEFAULT_MODEL, call_sync +from spawnllm.decide import JEV, OPENAI, Provider, store_key BACKEND_NAMES = tuple(BACKENDS_BY_NAME) +DECIDE_PROVIDERS: dict[str, Provider] = {provider.name: provider for provider in (JEV, OPENAI)} @click.group() @@ -69,3 +71,16 @@ def status() -> None: except BackendUnavailable: click.echo("selected: none available") click.echo(f"core: {(v := _core.version())['core_version']}@{v['source_hash'][:12]}") + + +@main.group() +def key() -> None: + """Manage the decision providers' API keys in the macOS Keychain.""" + + +@key.command("set") +@click.argument("provider", type=click.Choice(list(DECIDE_PROVIDERS))) +def key_set(provider: str) -> None: + """Store PROVIDER's API key, read from stdin, in the macOS Keychain, where `decide` reads it.""" + store_key(DECIDE_PROVIDERS[provider], sys.stdin.read().strip()) + click.echo(f"stored the {provider} key in the Keychain item spawnllm-{provider}-api-key") diff --git a/spawnllm/decide.py b/spawnllm/decide.py new file mode 100644 index 0000000..1e89869 --- /dev/null +++ b/spawnllm/decide.py @@ -0,0 +1,514 @@ +"""Typed yes/no, label, and score decisions over TypeSafe Jev and OpenAI Decisions. + +The Rust core maps one question set onto either provider's wire format and resolves the +answer, retry, or failure each response means; this module only moves bytes over one +keep-alive HTTP client per process and keeps every attempt inside the caller's deadline. +""" + +from __future__ import annotations + +import asyncio +import os +import re +import subprocess +import sys +import threading +import time +import weakref +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import Any, Literal + +import httpx + +from spawnllm import _core + +__all__ = [ + "JEV", + "OPENAI", + "Binary", + "BinaryAnswer", + "DecideError", + "DecideKeyMissing", + "Decision", + "Label", + "LabelAnswer", + "Provider", + "Refused", + "Score", + "ScoreAnswer", + "TDecideProvider", + "api_key", + "decide", + "decide_sync", + "store_key", +] + +TDecideProvider = Literal["jev", "openai"] +"""A decision provider: TypeSafe's Jev (`jev`) or OpenAI Decisions (`openai`).""" + +KEY_ENV: dict[TDecideProvider, str] = {"jev": "TYPESAFE_API_KEY", "openai": "OPENAI_API_KEY"} +KEY_PATTERN = re.compile(r"[A-Za-z0-9._-]+") +DEFAULT_TIMEOUT = 10.0 + +CLIENT_LOCK = threading.Lock() +CLIENT: httpx.Client | None = None +ASYNC_CLIENTS: weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, httpx.AsyncClient] = weakref.WeakKeyDictionary() + + +@dataclass(frozen=True, slots=True) +class Provider: + """A decision provider and the model it runs. + + Attributes: + name: `jev` posts to TypeSafe's System One endpoint, `openai` to OpenAI's `/v1/decisions`. + model: The literal model id sent with every request; pin a versioned id once a + threshold is tuned against it. + + Example: + >>> Provider("jev", "jev-1.13.0") + """ + + name: TDecideProvider + model: str + + +JEV = Provider("jev", "jev-1.13.0") +"""TypeSafe Jev pinned to `jev-1.13.0`, the model `jev-latest` resolves to today.""" + +OPENAI = Provider("openai", "gpt-6-luna") +"""OpenAI Decisions on `gpt-6-luna`, its only model.""" + + +@dataclass(frozen=True, slots=True) +class Binary: + """A yes/no question; its answer is the probability of yes. + + Attributes: + instructions: The question, about one observable fact. + yes: What a yes means, when the question alone leaves it open. + no: What a no means, when the question alone leaves it open. + + Example: + >>> Binary("Does the message ask to roll back a release?") + """ + + instructions: str + yes: str | None = None + no: str | None = None + + +@dataclass(frozen=True, slots=True) +class Label: + """Pick one option from 2 to 255; put the fail-safe option first, since Jev leans toward it. + + Attributes: + instructions: What to decide. + options: Each option value mapped to its description, or `None`, in declared order. + + Example: + >>> Label("Which release action does the message ask for?", {"none": None, "rollback": "Undo a release"}) + """ + + instructions: str + options: Mapping[str, str | None] + + +@dataclass(frozen=True, slots=True) +class Score: + """Rate along 2 to 10 ordered levels, lowest first; the score is the probability-weighted level index. + + Attributes: + instructions: What to rate. + levels: Each level label mapped to its description, or `None`, lowest first. + + Example: + >>> Score("How urgent is the message?", {"Not urgent": None, "Somewhat urgent": None, "Blocking": None}) + """ + + instructions: str + levels: Mapping[str, str | None] + + +type Question = Binary | Label | Score + + +@dataclass(frozen=True, slots=True) +class BinaryAnswer: + """The answer to a `Binary` question. + + Attributes: + p_yes: Probability the answer is yes. + confidence: `abs(2 * p_yes - 1)`, 0 at a coin flip and 1 at certainty. + """ + + p_yes: float + confidence: float + + +@dataclass(frozen=True, slots=True) +class LabelAnswer: + """The answer to a `Label` question. + + Attributes: + choice: The most likely option value. + probabilities: Every option value mapped to its probability, in declared order. + confidence: How far the distribution sits from uniform, 0 to 1. + """ + + choice: str + probabilities: Mapping[str, float] + confidence: float + + +@dataclass(frozen=True, slots=True) +class ScoreAnswer: + """The answer to a `Score` question. + + Attributes: + score: The probability-weighted level index; it can land between levels. + probabilities: Every level label mapped to its probability, lowest first. + confidence: How concentrated the distribution is around its top level, 0 to 1. + """ + + score: float + probabilities: Mapping[str, float] + confidence: float + + +@dataclass(frozen=True, slots=True) +class Refused: + """A question the provider declined to answer while it answered the others.""" + + +type Answer = BinaryAnswer | LabelAnswer | ScoreAnswer | Refused + + +@dataclass(frozen=True, slots=True) +class Decision: + """One provider response: an answer per question id, in the order asked. + + Attributes: + answers: Each question id mapped to its answer. + model: The model that answered, as the provider reports it. + input_tokens: Billed input tokens. + latency_ms: Wall time of the attempt that answered, request to parsed response. + """ + + answers: Mapping[str, Answer] + model: str + input_tokens: int + latency_ms: float + + +class DecideError(Exception): + """The provider rejected a decision request, or answered one with a body that matches no question. + + Attributes: + status: The HTTP status of the failed response. + message: The provider's error body, or what about its answer failed to match. + """ + + def __init__(self, status: int, message: str) -> None: + super().__init__(f"decision failed with HTTP {status}: {message}") + self.status = status + self.message = message + + +class DecideKeyMissing(LookupError): + """No API key for the provider: none was passed, its variable is unset, and the Keychain holds none.""" + + +def keychain_item(name: TDecideProvider) -> tuple[str, str]: + return f"spawnllm-{name}-api-key", name + + +def api_key(provider: Provider) -> str: + """Return the provider's API key from its environment variable, else from the macOS Keychain. + + Jev reads `TYPESAFE_API_KEY` and OpenAI reads `OPENAI_API_KEY`; on macOS an unset variable falls + through to the Keychain item `store_key` wrote, read when the call is made. + + Raises: + DecideKeyMissing: When neither source holds a key. + """ + if key := os.environ.get(KEY_ENV[provider.name]): + return key + service, account = keychain_item(provider.name) + if ( + sys.platform == "darwin" + and ( + found := subprocess.run( + ["security", "find-generic-password", "-s", service, "-a", account, "-w"], + capture_output=True, + text=True, + check=False, + ) + ).returncode + == 0 + ): + return found.stdout.strip() + raise DecideKeyMissing( + f"no {provider.name} API key: set {KEY_ENV[provider.name]} or run `spawnllm key set {provider.name}`" + ) + + +def store_key(provider: Provider, key: str) -> None: + """Store `key` as the provider's macOS Keychain item, replacing any earlier one. + + The key reaches `security` over stdin, never its argv, so no process listing shows it. + + Raises: + ValueError: When the key holds characters outside `[A-Za-z0-9._-]`. + subprocess.CalledProcessError: When `security` refuses the write. + """ + if not KEY_PATTERN.fullmatch(key): + raise ValueError(f"a {provider.name} API key holds only letters, digits, '.', '_', and '-'") + service, account = keychain_item(provider.name) + subprocess.run( + ["security", "-i"], + input=f"add-generic-password -U -s {service} -a {account} -w {key}\n", + capture_output=True, + text=True, + check=True, + ) + + +def sync_client() -> httpx.Client: + global CLIENT + with CLIENT_LOCK: + if CLIENT is None: + CLIENT = httpx.Client() + return CLIENT + + +def async_client() -> httpx.AsyncClient: + loop = asyncio.get_running_loop() + if (client := ASYNC_CLIENTS.get(loop)) is None: + client = ASYNC_CLIENTS[loop] = httpx.AsyncClient() + return client + + +def fork_reset() -> None: + global CLIENT, CLIENT_LOCK + CLIENT_LOCK = threading.Lock() + CLIENT = None + ASYNC_CLIENTS.clear() + + +os.register_at_fork(after_in_child=fork_reset) + + +def wire_question(id: str, question: Question) -> dict[str, Any]: + match question: + case Binary(instructions=instructions, yes=yes, no=no): + return {"type": "binary", "id": id, "instructions": instructions, "yes": yes, "no": no} + case Label(instructions=instructions, options=options): + return { + "type": "label", + "id": id, + "instructions": instructions, + "options": [{"value": value, "description": text} for value, text in options.items()], + } + case Score(instructions=instructions, levels=levels): + return { + "type": "score", + "id": id, + "instructions": instructions, + "levels": [{"label": label, "description": text} for label, text in levels.items()], + } + + +def keys(question: Question) -> Sequence[str]: + match question: + case Label(options=options): + return tuple(options) + case Score(levels=levels): + return tuple(levels) + case Binary(): + return () + + +def answer(question: Question, wire: dict[str, Any]) -> Answer: + match wire: + case {"type": "binary", "p_yes": p_yes, "confidence": confidence}: + return BinaryAnswer(p_yes, confidence) + case {"type": "label", "choice": choice, "probabilities": probabilities, "confidence": confidence}: + return LabelAnswer(choice, dict(zip(keys(question), probabilities, strict=True)), confidence) + case {"type": "score", "score": score, "probabilities": probabilities, "confidence": confidence}: + return ScoreAnswer(score, dict(zip(keys(question), probabilities, strict=True)), confidence) + case {"type": "refused"}: + return Refused() + raise ValueError(f"the core resolved an unknown answer: {wire}") + + +@dataclass(frozen=True, slots=True) +class Call: + provider: Provider + questions: Mapping[str, Question] + wired: list[dict[str, Any]] + request: dict[str, Any] + deadline: float + timeout: float + + @classmethod + def plan( + cls, + state: str | Mapping[str, Any] | Sequence[Any], + questions: Mapping[str, Question], + provider: Provider, + timeout: float, + key: str | None, + ) -> Call: + wired = [wire_question(id, question) for id, question in questions.items()] + planned = { + "provider": {"name": provider.name, "model": provider.model}, + "api_key": key or api_key(provider), + "state": state, + "questions": wired, + } + try: + request = _core.dispatch("decide_plan", planned) + except _core.CoreError as error: + raise ValueError(error.msg) from error + return cls(provider, questions, wired, request, time.monotonic() + timeout, timeout) + + def remaining(self) -> float: + if (left := self.deadline - time.monotonic()) <= 0: + raise TimeoutError(f"no {self.provider.name} decision within {self.timeout:g}s") + return left + + def outcome(self, response: httpx.Response | None, attempt: int, started: float) -> Decision | float: + resolved = _core.dispatch( + "decide_resolve", + { + "provider": {"name": self.provider.name, "model": self.provider.model}, + "questions": self.wired, + "status": None if response is None else response.status_code, + "body": "" if response is None else response.text, + "retry_after": None if response is None else response.headers.get("retry-after"), + "retry_after_ms": None if response is None else response.headers.get("retry-after-ms"), + "attempt": attempt, + }, + ) + match resolved: + case {"kind": "decision", "model": model, "input_tokens": tokens, "answers": answers}: + return Decision( + {id: answer(self.questions[id], wire) for id, wire in answers.items()}, + model, + tokens, + (time.monotonic() - started) * 1000, + ) + case {"kind": "retry", "sleep_s": sleep_s} if time.monotonic() + sleep_s < self.deadline: + return sleep_s + case {"kind": "retry"}: + last = "no response" if response is None else f"HTTP {response.status_code}" + raise TimeoutError( + f"no {self.provider.name} decision within {self.timeout:g}s; the last try got {last}" + ) + case {"kind": "failed", "status": status, "message": message}: + raise DecideError(status, message) + raise ValueError(f"the core resolved an unknown outcome: {resolved}") + + +def decide_sync( + state: str | Mapping[str, Any] | Sequence[Any], + questions: Mapping[str, Question], + *, + provider: Provider = JEV, + timeout: float = DEFAULT_TIMEOUT, + api_key: str | None = None, +) -> Decision: + """Ask every question about `state` in one request and return the typed answers. + + Keep each question to one observable fact and combine the answers in code; put counting, + dates, and arithmetic in code too. Retries 408, 429, 5xx, 529, and a lost connection with + backoff from 0.5s doubling to 5s, honoring `retry-after`, and only while the retry still fits + inside `timeout`. + + Args: + state: The text to judge, or JSON data; Jev reads structure natively and OpenAI receives + it as compact JSON text. + questions: Each question id mapped to its question, in the order to ask them. + provider: `JEV` or `OPENAI`, or a `Provider` pinning another model. + timeout: Seconds for the whole call, retries included. + api_key: The provider key; `None` reads it through `api_key`. + + Returns: + The decision, with a `Refused` answer for any question the provider declined. + + Raises: + TimeoutError: When no answer arrives within `timeout`. + DecideError: When the provider rejects the request. + DecideKeyMissing: When no key is passed or found. + ValueError: When the question set breaks a limit both providers share, such as a label + with fewer than 2 options or a score with more than 10 levels. + + Example: + >>> decision = decide_sync("Please roll back last night's release.", { + ... "rollback": Binary("Does the message ask to roll back a release?"), + ... }) + >>> decision.answers["rollback"].p_yes > 0.5 + True + """ + call = Call.plan(state, questions, provider, timeout, api_key) + tries = 0 + while True: + started = time.monotonic() + try: + response = sync_client().post( + call.request["url"], + headers=call.request["headers"], + json=call.request["body"], + timeout=call.remaining(), + ) + except httpx.TimeoutException as exc: + raise TimeoutError(f"no {provider.name} decision within {timeout:g}s") from exc + except httpx.TransportError: + response = None + match call.outcome(response, tries, started): + case Decision() as decision: + return decision + case float() as sleep_s: + time.sleep(sleep_s) + tries += 1 + + +async def decide( + state: str | Mapping[str, Any] | Sequence[Any], + questions: Mapping[str, Question], + *, + provider: Provider = JEV, + timeout: float = DEFAULT_TIMEOUT, + api_key: str | None = None, +) -> Decision: + """Ask every question about `state` in one request, asynchronously; `decide_sync` documents the contract. + + Each event loop keeps one keep-alive client. + + Example: + >>> decision = await decide({"title": "Revert the release"}, { + ... "kind": Label("What does the title ask for?", {"other": None, "rollback": None}), + ... }, provider=OPENAI) + """ + call = Call.plan(state, questions, provider, timeout, api_key) + tries = 0 + while True: + started = time.monotonic() + try: + response = await async_client().post( + call.request["url"], + headers=call.request["headers"], + json=call.request["body"], + timeout=call.remaining(), + ) + except httpx.TimeoutException as exc: + raise TimeoutError(f"no {provider.name} decision within {timeout:g}s") from exc + except httpx.TransportError: + response = None + match call.outcome(response, tries, started): + case Decision() as decision: + return decision + case float() as sleep_s: + await asyncio.sleep(sleep_s) + tries += 1 diff --git a/tests/test_decide.py b/tests/test_decide.py new file mode 100644 index 0000000..4cec108 --- /dev/null +++ b/tests/test_decide.py @@ -0,0 +1,301 @@ +from __future__ import annotations + +import importlib +import json +import subprocess +from typing import TYPE_CHECKING, Any + +import httpx +import pytest +from click.testing import CliRunner + +from spawnllm import ( + JEV, + OPENAI, + Binary, + BinaryAnswer, + DecideError, + DecideKeyMissing, + Label, + LabelAnswer, + Refused, + Score, + ScoreAnswer, + decide, + decide_sync, +) +from spawnllm.cli import main + +if TYPE_CHECKING: + from collections.abc import Callable + +module = importlib.import_module("spawnllm.decide") + +STATE = "Help! Our nightly release has been failing for 3 days and customers are blocked. Please roll it back now." +QUESTIONS = { + "is_rollback_request": Binary("Does the message ask to roll back a release?"), + "action": Label( + "Which release action does the message ask for?", + {"none": "No release action", "release": "Ship a new release", "rollback": "Roll back a release"}, + ), + "urgency": Score( + "How urgent is the message?", {"Not urgent": None, "Somewhat urgent": None, "Blocking, needs action now": None} + ), +} +JEV_RECORDED = { + "model": "jev-1.13.0", + "answers": { + "is_rollback_request": {"type": "noul", "noul": 0.99}, + "action": { + "type": "choice", + "choice": "rollback", + "confidence": 1.0, + "probabilities": {"release": 0.0, "rollback": 1.0, "none": 0.0}, + }, + "urgency": { + "type": "score", + "score": 2.0, + "confidence": 1.0, + "legend": {"0": "Not urgent", "1": "Somewhat urgent", "2": "Blocking, needs action now"}, + "probabilities": {"0": 0.0, "1": 0.0, "2": 1.0}, + }, + }, + "usage": {"input_tokens": 414, "output_tokens": 73}, +} +OPENAI_REFUSAL = { + "model": "gpt-6-luna", + "answers": [ + {"type": "refusal", "name": "is_rollback_request"}, + { + "type": "choice", + "name": "action", + "choice": "none", + "confidence": 0.7, + "probabilities": [ + {"value": "rollback", "probability": 0.1}, + {"value": "none", "probability": 0.8}, + {"value": "release", "probability": 0.1}, + ], + }, + { + "type": "score", + "name": "urgency", + "score": 0.9, + "confidence": 0.2, + "probabilities": [ + {"value": 2, "label": "Blocking, needs action now", "probability": 0.2}, + {"value": 0, "label": "Not urgent", "probability": 0.3}, + {"value": 1, "label": "Somewhat urgent", "probability": 0.5}, + ], + }, + ], + "usage": {"input_tokens": 419}, +} + +type Reply = httpx.Response | httpx.TransportError + + +def serve(monkeypatch: pytest.MonkeyPatch, *replies: Reply) -> list[httpx.Request]: + seen: list[httpx.Request] = [] + pending = list(replies) + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(request) + match pending.pop(0): + case httpx.TransportError() as error: + raise error + case httpx.Response() as response: + return response + + transport = httpx.MockTransport(handler) + monkeypatch.setattr(module, "sync_client", lambda: httpx.Client(transport=transport)) + monkeypatch.setattr(module, "async_client", lambda: httpx.AsyncClient(transport=transport)) + return seen + + +def sleeps(monkeypatch: pytest.MonkeyPatch) -> list[float]: + slept: list[float] = [] + monkeypatch.setattr(module.time, "sleep", slept.append) + return slept + + +def test_jev_round_trip_sends_the_planned_request_and_orders_answers_as_declared( + monkeypatch: pytest.MonkeyPatch, +) -> None: + seen = serve(monkeypatch, httpx.Response(200, json=JEV_RECORDED)) + + decision = decide_sync(STATE, QUESTIONS, api_key="test-key") + + assert str(seen[0].url) == "https://api.typesafe.ai/v1/systemone" + assert seen[0].headers["authorization"] == "Bearer test-key" + body = json.loads(seen[0].content) + assert body["model"] == "jev-1.13.0" + assert list(body["questions"]["action"]["criteria"]) == ["none", "release", "rollback"] + assert decision.model == "jev-1.13.0" + assert decision.input_tokens == 414 + assert list(decision.answers) == ["is_rollback_request", "action", "urgency"] + assert decision.answers["is_rollback_request"] == BinaryAnswer(p_yes=0.99, confidence=0.98) + assert decision.answers["action"] == LabelAnswer( + choice="rollback", probabilities={"none": 0.0, "release": 0.0, "rollback": 1.0}, confidence=1.0 + ) + match decision.answers["action"]: + case LabelAnswer(probabilities=probabilities): + assert list(probabilities) == ["none", "release", "rollback"] + assert decision.answers["urgency"] == ScoreAnswer( + score=2.0, + probabilities={"Not urgent": 0.0, "Somewhat urgent": 0.0, "Blocking, needs action now": 1.0}, + confidence=1.0, + ) + assert decision.latency_ms > 0 + + +def test_openai_refusal_comes_back_beside_the_other_answers(monkeypatch: pytest.MonkeyPatch) -> None: + seen = serve(monkeypatch, httpx.Response(200, json=OPENAI_REFUSAL)) + + decision = decide_sync({"message": STATE}, QUESTIONS, provider=OPENAI, api_key="test-key") + + body = json.loads(seen[0].content) + assert str(seen[0].url) == "https://api.openai.com/v1/decisions" + assert body["input"] == json.dumps({"message": STATE}, separators=(",", ":")) + assert [question["name"] for question in body["questions"]] == ["is_rollback_request", "action", "urgency"] + assert decision.answers["is_rollback_request"] == Refused() + assert decision.answers["action"] == LabelAnswer( + choice="none", probabilities={"none": 0.8, "release": 0.1, "rollback": 0.1}, confidence=0.7 + ) + assert decision.answers["urgency"] == ScoreAnswer( + score=0.9, + probabilities={"Not urgent": 0.3, "Somewhat urgent": 0.5, "Blocking, needs action now": 0.2}, + confidence=0.2, + ) + + +@pytest.mark.parametrize( + ("first", "slept"), + [ + pytest.param(httpx.Response(529, text="overloaded"), [0.5], id="529-backs-off"), + pytest.param(httpx.Response(429, headers={"retry-after-ms": "250"}), [0.25], id="429-honors-retry-after-ms"), + pytest.param(httpx.ConnectError("reset"), [0.5], id="lost-connection-retries"), + ], +) +def test_transient_failures_retry_inside_the_deadline( + monkeypatch: pytest.MonkeyPatch, first: Reply, slept: list[float] +) -> None: + serve(monkeypatch, first, httpx.Response(200, json=JEV_RECORDED)) + recorded = sleeps(monkeypatch) + + decision = decide_sync(STATE, QUESTIONS, api_key="test-key") + + assert recorded == slept + assert decision.answers["action"] == LabelAnswer( + choice="rollback", probabilities={"none": 0.0, "release": 0.0, "rollback": 1.0}, confidence=1.0 + ) + + +def test_a_retry_that_would_outlast_the_deadline_raises_timeout(monkeypatch: pytest.MonkeyPatch) -> None: + serve(monkeypatch, httpx.Response(429, headers={"retry-after": "30"})) + recorded = sleeps(monkeypatch) + + with pytest.raises(TimeoutError, match="the last try got HTTP 429"): + decide_sync(STATE, QUESTIONS, api_key="test-key", timeout=2.0) + assert recorded == [] + + +def test_a_rejected_request_raises_decide_error_without_retrying(monkeypatch: pytest.MonkeyPatch) -> None: + seen = serve(monkeypatch, httpx.Response(401, text="Incorrect API key provided")) + + with pytest.raises(DecideError) as raised: + decide_sync(STATE, QUESTIONS, provider=OPENAI, api_key="bad") + + assert (raised.value.status, raised.value.message) == (401, "Incorrect API key provided") + assert len(seen) == 1 + + +def test_an_invalid_question_set_fails_before_any_request(monkeypatch: pytest.MonkeyPatch) -> None: + seen = serve(monkeypatch) + + with pytest.raises(ValueError, match="a label question takes 2 to 255 options"): + decide_sync(STATE, {"only": Label("Which?", {"one": None})}, api_key="test-key") + assert seen == [] + + +async def test_async_decide_shares_the_contract(monkeypatch: pytest.MonkeyPatch) -> None: + serve(monkeypatch, httpx.Response(503), httpx.Response(200, json=JEV_RECORDED)) + slept: list[float] = [] + + async def record(seconds: float) -> None: + slept.append(seconds) + + monkeypatch.setattr(module.asyncio, "sleep", record) + + decision = await decide(STATE, QUESTIONS, api_key="test-key") + + assert slept == [0.5] + assert decision.answers["is_rollback_request"] == BinaryAnswer(p_yes=0.99, confidence=0.98) + + +def security(monkeypatch: pytest.MonkeyPatch, returncode: int, stdout: str = "") -> list[dict[str, Any]]: + calls: list[dict[str, Any]] = [] + real: Callable[..., subprocess.CompletedProcess[str]] = subprocess.run + + def fake(argv: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]: + if argv[0] != "security": + return real(argv, **kwargs) + calls.append({"argv": argv} | kwargs) + return subprocess.CompletedProcess(argv, returncode, stdout, "") + + monkeypatch.setattr(module.subprocess, "run", fake) + return calls + + +def test_the_environment_variable_outranks_the_keychain(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("TYPESAFE_API_KEY", "from-env") + calls = security(monkeypatch, 0, "from-keychain\n") + + assert module.api_key(JEV) == "from-env" + assert calls == [] + + +def test_the_keychain_answers_when_the_variable_is_unset(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.setattr(module.sys, "platform", "darwin") + calls = security(monkeypatch, 0, "from-keychain\n") + + assert module.api_key(OPENAI) == "from-keychain" + assert calls[0]["argv"] == [ + "security", + "find-generic-password", + "-s", + "spawnllm-openai-api-key", + "-a", + "openai", + "-w", + ] + + +def test_no_key_anywhere_raises(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("TYPESAFE_API_KEY", raising=False) + monkeypatch.setattr(module.sys, "platform", "darwin") + security(monkeypatch, 44) + + with pytest.raises(DecideKeyMissing, match="spawnllm key set jev"): + module.api_key(JEV) + + +def test_key_set_writes_the_keychain_over_stdin_only(monkeypatch: pytest.MonkeyPatch) -> None: + calls = security(monkeypatch, 0) + + result = CliRunner().invoke(main, ["key", "set", "jev"], input="ts_live-AbC.123\n") + + assert result.exit_code == 0, result.output + assert calls[0]["argv"] == ["security", "-i"] + assert calls[0]["input"] == "add-generic-password -U -s spawnllm-jev-api-key -a jev -w ts_live-AbC.123\n" + assert all("ts_live" not in arg for arg in calls[0]["argv"]) + + +def test_key_set_refuses_a_key_that_could_break_the_security_command(monkeypatch: pytest.MonkeyPatch) -> None: + calls = security(monkeypatch, 0) + + result = CliRunner().invoke(main, ["key", "set", "openai"], input='sk-x" -A\n') + + assert isinstance(result.exception, ValueError) + assert calls == [] From 5803bc5b5d89bd484f2ae2afee9ad8f6ff5e9d96 Mon Sep 17 00:00:00 2001 From: Yasyf Mohamedali Date: Wed, 7 Oct 2026 05:38:09 -0700 Subject: [PATCH 2/2] =?UTF-8?q?decide:=20=F0=9F=90=9B=20Hold=20the=20whole?= =?UTF-8?q?=20call,=20Keychain=20read=20included,=20to=20its=20timeout?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude-Session-Id: 899e5f7a-9099-4d8f-b96d-79291e63b5b4 --- great-docs.yml | 18 ------------- spawnllm/decide.py | 63 +++++++++++++------------------------------- tests/test_decide.py | 36 +++++++++++++++++++------ 3 files changed, 47 insertions(+), 70 deletions(-) diff --git a/great-docs.yml b/great-docs.yml index d4bf23b..46a3b3c 100644 --- a/great-docs.yml +++ b/great-docs.yml @@ -76,24 +76,6 @@ reference: - ClaudeConfig - CodexConfig - GeminiConfig - - title: Deciding - desc: Typed yes/no, label, and score decisions over TypeSafe Jev and OpenAI Decisions, one question set for either provider. - contents: - - decide.decide - - decide.decide_sync - - Binary - - Label - - Score - - Decision - - BinaryAnswer - - LabelAnswer - - ScoreAnswer - - Refused - - Provider - - DecideError - - DecideKeyMissing - - decide.api_key - - decide.store_key - title: Backends desc: CLI backends (Claude, Codex, and the Gemini family), the in-process Claude Agent SDK backend, the Apple Foundation Models backend over its bundled Swift sidecar, the MLX and OpenAI-endpoint backends, the specialty registry, backend selection, and install/auth status checks. contents: diff --git a/spawnllm/decide.py b/spawnllm/decide.py index 1e89869..b521ba3 100644 --- a/spawnllm/decide.py +++ b/spawnllm/decide.py @@ -14,7 +14,6 @@ import sys import threading import time -import weakref from collections.abc import Mapping, Sequence from dataclasses import dataclass from typing import Any, Literal @@ -53,7 +52,6 @@ CLIENT_LOCK = threading.Lock() CLIENT: httpx.Client | None = None -ASYNC_CLIENTS: weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, httpx.AsyncClient] = weakref.WeakKeyDictionary() @dataclass(frozen=True, slots=True) @@ -223,7 +221,7 @@ def keychain_item(name: TDecideProvider) -> tuple[str, str]: return f"spawnllm-{name}-api-key", name -def api_key(provider: Provider) -> str: +def api_key(provider: Provider, *, timeout: float = DEFAULT_TIMEOUT) -> str: """Return the provider's API key from its environment variable, else from the macOS Keychain. Jev reads `TYPESAFE_API_KEY` and OpenAI reads `OPENAI_API_KEY`; on macOS an unset variable falls @@ -231,23 +229,24 @@ def api_key(provider: Provider) -> str: Raises: DecideKeyMissing: When neither source holds a key. + TimeoutError: When the Keychain read outlasts `timeout`. """ if key := os.environ.get(KEY_ENV[provider.name]): return key service, account = keychain_item(provider.name) - if ( - sys.platform == "darwin" - and ( - found := subprocess.run( + if sys.platform == "darwin": + try: + found = subprocess.run( ["security", "find-generic-password", "-s", service, "-a", account, "-w"], capture_output=True, text=True, check=False, + timeout=timeout, ) - ).returncode - == 0 - ): - return found.stdout.strip() + except subprocess.TimeoutExpired as exc: + raise TimeoutError(f"the Keychain read for the {provider.name} key outlasted {timeout:g}s") from exc + if found.returncode == 0: + return found.stdout.strip() raise DecideKeyMissing( f"no {provider.name} API key: set {KEY_ENV[provider.name]} or run `spawnllm key set {provider.name}`" ) @@ -282,18 +281,10 @@ def sync_client() -> httpx.Client: return CLIENT -def async_client() -> httpx.AsyncClient: - loop = asyncio.get_running_loop() - if (client := ASYNC_CLIENTS.get(loop)) is None: - client = ASYNC_CLIENTS[loop] = httpx.AsyncClient() - return client - - def fork_reset() -> None: global CLIENT, CLIENT_LOCK CLIENT_LOCK = threading.Lock() CLIENT = None - ASYNC_CLIENTS.clear() os.register_at_fork(after_in_child=fork_reset) @@ -360,10 +351,11 @@ def plan( timeout: float, key: str | None, ) -> Call: + deadline = time.monotonic() + timeout wired = [wire_question(id, question) for id, question in questions.items()] planned = { "provider": {"name": provider.name, "model": provider.model}, - "api_key": key or api_key(provider), + "api_key": key or api_key(provider, timeout=timeout), "state": state, "questions": wired, } @@ -371,7 +363,7 @@ def plan( request = _core.dispatch("decide_plan", planned) except _core.CoreError as error: raise ValueError(error.msg) from error - return cls(provider, questions, wired, request, time.monotonic() + timeout, timeout) + return cls(provider, questions, wired, request, deadline, timeout) def remaining(self) -> float: if (left := self.deadline - time.monotonic()) <= 0: @@ -424,7 +416,8 @@ def decide_sync( Keep each question to one observable fact and combine the answers in code; put counting, dates, and arithmetic in code too. Retries 408, 429, 5xx, 529, and a lost connection with backoff from 0.5s doubling to 5s, honoring `retry-after`, and only while the retry still fits - inside `timeout`. + inside `timeout`. The Keychain read counts against `timeout` too, and an answer that arrives + after it raises `TimeoutError` rather than returning late. Args: state: The text to judge, or JSON data; Jev reads structure natively and OpenAI receives @@ -466,6 +459,7 @@ def decide_sync( raise TimeoutError(f"no {provider.name} decision within {timeout:g}s") from exc except httpx.TransportError: response = None + call.remaining() match call.outcome(response, tries, started): case Decision() as decision: return decision @@ -484,31 +478,12 @@ async def decide( ) -> Decision: """Ask every question about `state` in one request, asynchronously; `decide_sync` documents the contract. - Each event loop keeps one keep-alive client. + The call runs `decide_sync` on a worker thread, so it shares the process's keep-alive client and + holds no connection to the event loop. Example: >>> decision = await decide({"title": "Revert the release"}, { ... "kind": Label("What does the title ask for?", {"other": None, "rollback": None}), ... }, provider=OPENAI) """ - call = Call.plan(state, questions, provider, timeout, api_key) - tries = 0 - while True: - started = time.monotonic() - try: - response = await async_client().post( - call.request["url"], - headers=call.request["headers"], - json=call.request["body"], - timeout=call.remaining(), - ) - except httpx.TimeoutException as exc: - raise TimeoutError(f"no {provider.name} decision within {timeout:g}s") from exc - except httpx.TransportError: - response = None - match call.outcome(response, tries, started): - case Decision() as decision: - return decision - case float() as sleep_s: - await asyncio.sleep(sleep_s) - tries += 1 + return await asyncio.to_thread(decide_sync, state, questions, provider=provider, timeout=timeout, api_key=api_key) diff --git a/tests/test_decide.py b/tests/test_decide.py index 4cec108..f92f7a2 100644 --- a/tests/test_decide.py +++ b/tests/test_decide.py @@ -3,6 +3,7 @@ import importlib import json import subprocess +import time from typing import TYPE_CHECKING, Any import httpx @@ -109,7 +110,6 @@ def handler(request: httpx.Request) -> httpx.Response: transport = httpx.MockTransport(handler) monkeypatch.setattr(module, "sync_client", lambda: httpx.Client(transport=transport)) - monkeypatch.setattr(module, "async_client", lambda: httpx.AsyncClient(transport=transport)) return seen @@ -220,19 +220,39 @@ def test_an_invalid_question_set_fails_before_any_request(monkeypatch: pytest.Mo async def test_async_decide_shares_the_contract(monkeypatch: pytest.MonkeyPatch) -> None: serve(monkeypatch, httpx.Response(503), httpx.Response(200, json=JEV_RECORDED)) - slept: list[float] = [] - - async def record(seconds: float) -> None: - slept.append(seconds) - - monkeypatch.setattr(module.asyncio, "sleep", record) + recorded = sleeps(monkeypatch) decision = await decide(STATE, QUESTIONS, api_key="test-key") - assert slept == [0.5] + assert recorded == [0.5] assert decision.answers["is_rollback_request"] == BinaryAnswer(p_yes=0.99, confidence=0.98) +def test_an_answer_that_arrives_after_the_deadline_raises_timeout(monkeypatch: pytest.MonkeyPatch) -> None: + def slow(request: httpx.Request) -> httpx.Response: + time.sleep(0.15) + return httpx.Response(200, json=JEV_RECORDED) + + transport = httpx.MockTransport(slow) + monkeypatch.setattr(module, "sync_client", lambda: httpx.Client(transport=transport)) + + with pytest.raises(TimeoutError, match="no jev decision within 0.05s"): + decide_sync(STATE, QUESTIONS, api_key="test-key", timeout=0.05) + + +def test_a_stalled_keychain_read_counts_against_the_timeout(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("TYPESAFE_API_KEY", raising=False) + monkeypatch.setattr(module.sys, "platform", "darwin") + + def stalled(argv: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]: + raise subprocess.TimeoutExpired(argv, kwargs["timeout"]) + + monkeypatch.setattr(module.subprocess, "run", stalled) + + with pytest.raises(TimeoutError, match="outlasted 0.5s"): + decide_sync(STATE, QUESTIONS, timeout=0.5) + + def security(monkeypatch: pytest.MonkeyPatch, returncode: int, stdout: str = "") -> list[dict[str, Any]]: calls: list[dict[str, Any]] = [] real: Callable[..., subprocess.CompletedProcess[str]] = subprocess.run