From e63d093896efd9688ff20b53f0756759a82523b2 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 23 Sep 2026 11:31:24 +0000 Subject: [PATCH] Add a local label filter that can skip two closed decisions. An ollama model can score single-letter labels and skip a presence review on an extreme silent, or route a no-cue utterance without the advisor completion. Both margins default to off, and the filter cannot write a note, seeds, or a halt. Co-authored-by: Tyler Reckart --- CHANGELOG.md | 9 ++ CMakeLists.txt | 32 +++++ ROADMAP.md | 1 + docs/api/intent.md | 3 +- docs/cli/environment.md | 3 + docs/concepts/intent.md | 4 + docs/concepts/presence.md | 6 + docs/concepts/sse-events.md | 4 +- include/api_client.h | 5 + include/intent.h | 21 +++- include/label_score.h | 123 ++++++++++++++++++ include/orchestrator.h | 9 ++ include/presence.h | 14 +++ src/api_client.cpp | 43 +++++++ src/api_server.cpp | 29 ++++- src/intent.cpp | 53 +++++++- src/label_score.cpp | 241 ++++++++++++++++++++++++++++++++++++ src/orchestrator.cpp | 82 ++++++++++-- src/presence.cpp | 60 +++++++++ tests/test_api_client.cpp | 26 ++++ tests/test_intent.cpp | 230 ++++++++++++++++++++++++++++++++++ tests/test_label_score.cpp | 159 ++++++++++++++++++++++++ tests/test_presence.cpp | 35 ++++++ 23 files changed, 1173 insertions(+), 19 deletions(-) create mode 100644 include/label_score.h create mode 100644 src/label_score.cpp create mode 100644 tests/test_label_score.cpp diff --git a/CHANGELOG.md b/CHANGELOG.md index 73c0878d..0cf701fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ loosely while pre-1.0 (breaking changes can land on minor bumps). ## [Unreleased] +- **Local label filter for presence silence and no-cue intent.** + With `ARBITER_DECISION_MODEL` set to an `ollama/…` id, a closed-set + next-token distribution can skip a presence review on an extreme + `silent`, or route a no-cue utterance without the advisor completion. + Both margins default to off. The filter cannot write a note, a brief + beyond the canned route line, or a halt. See + [Presence](docs/concepts/presence.md#silence-filter) and + [Intent](docs/concepts/intent.md#hybrid-classify). + ## [0.13.11] — 2026-09-21 - **Fleet dashboard pane (Phase C1).** The TUI consumes fleet SSE (`stream_id` + diff --git a/CMakeLists.txt b/CMakeLists.txt index a1cf8fdc..4a516f98 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -195,6 +195,7 @@ set(ARBITER_SOURCES src/constitution.cpp src/orchestrator.cpp src/intent.cpp + src/label_score.cpp src/reconcile.cpp src/advisor_gate.cpp src/advisor.cpp @@ -560,6 +561,7 @@ if(INDEX_BUILD_TESTS) add_executable(unit_api_client tests/test_api_client.cpp src/api_client.cpp + src/label_score.cpp src/circuit_breaker.cpp src/metrics.cpp src/json.cpp @@ -582,6 +584,7 @@ if(INDEX_BUILD_TESTS) add_executable(unit_api_client_pool tests/test_api_client_pool.cpp src/api_client.cpp + src/label_score.cpp src/circuit_breaker.cpp src/metrics.cpp src/json.cpp @@ -603,6 +606,7 @@ if(INDEX_BUILD_TESTS) add_executable(unit_api_client_pool_cancel tests/test_api_client_pool_cancel.cpp src/api_client.cpp + src/label_score.cpp src/circuit_breaker.cpp src/metrics.cpp src/json.cpp @@ -846,6 +850,7 @@ if(INDEX_BUILD_TESTS) src/atomic_file.cpp src/json.cpp src/api_client.cpp + src/label_score.cpp src/circuit_breaker.cpp src/metrics.cpp src/tui/opentui/pane_input_editor.cpp @@ -969,6 +974,7 @@ if(INDEX_BUILD_TESTS) src/workspace_root.cpp src/json.cpp src/api_client.cpp + src/label_score.cpp src/circuit_breaker.cpp src/metrics.cpp src/styled_text.cpp @@ -1015,6 +1021,7 @@ if(INDEX_BUILD_TESTS) src/workspace_root.cpp src/json.cpp src/api_client.cpp + src/label_score.cpp src/circuit_breaker.cpp src/metrics.cpp src/styled_text.cpp @@ -1048,6 +1055,7 @@ if(INDEX_BUILD_TESTS) src/workspace_root.cpp src/json.cpp src/api_client.cpp + src/label_score.cpp src/circuit_breaker.cpp src/metrics.cpp src/styled_text.cpp @@ -1086,6 +1094,7 @@ if(INDEX_BUILD_TESTS) src/workspace_root.cpp src/json.cpp src/api_client.cpp + src/label_score.cpp src/circuit_breaker.cpp src/metrics.cpp src/styled_text.cpp @@ -1390,6 +1399,7 @@ if(INDEX_BUILD_TESTS) src/event_routing.cpp src/constitution.cpp src/api_client.cpp + src/label_score.cpp src/circuit_breaker.cpp src/metrics.cpp src/json.cpp @@ -1614,6 +1624,7 @@ if(INDEX_BUILD_TESTS) src/a2a/sse_reader.cpp src/constitution.cpp src/api_client.cpp + src/label_score.cpp src/circuit_breaker.cpp src/metrics.cpp src/json.cpp @@ -1655,6 +1666,7 @@ if(INDEX_BUILD_TESTS) tests/test_constitution.cpp src/constitution.cpp src/api_client.cpp + src/label_score.cpp src/circuit_breaker.cpp src/metrics.cpp src/json.cpp @@ -1680,6 +1692,7 @@ if(INDEX_BUILD_TESTS) src/model_context.cpp src/constitution.cpp src/api_client.cpp + src/label_score.cpp src/circuit_breaker.cpp src/metrics.cpp src/json.cpp @@ -1707,6 +1720,7 @@ if(INDEX_BUILD_TESTS) src/model_context.cpp src/constitution.cpp src/api_client.cpp + src/label_score.cpp src/circuit_breaker.cpp src/metrics.cpp src/json.cpp @@ -1731,6 +1745,7 @@ if(INDEX_BUILD_TESTS) src/model_catalog.cpp src/model_context.cpp src/api_client.cpp + src/label_score.cpp src/circuit_breaker.cpp src/metrics.cpp src/json.cpp @@ -1772,6 +1787,7 @@ if(INDEX_BUILD_TESTS) src/advisor_gate.cpp src/advisor.cpp src/api_client.cpp + src/label_score.cpp src/circuit_breaker.cpp src/metrics.cpp src/json.cpp @@ -1794,6 +1810,7 @@ if(INDEX_BUILD_TESTS) tests/test_presence.cpp src/presence.cpp src/api_client.cpp + src/label_score.cpp src/circuit_breaker.cpp src/metrics.cpp src/json.cpp @@ -1815,6 +1832,8 @@ if(INDEX_BUILD_TESTS) add_executable(unit_intent tests/test_intent.cpp src/intent.cpp + src/label_score.cpp + src/json.cpp ) target_include_directories(unit_intent PRIVATE ${CMAKE_SOURCE_DIR}/include @@ -1822,6 +1841,19 @@ if(INDEX_BUILD_TESTS) ) add_test(NAME unit_intent COMMAND unit_intent) + # Closed-set label distribution: renormalize, missing-label fail-open, + # OpenAI logprob body parse, margin floors. No provider client. + add_executable(unit_label_score + tests/test_label_score.cpp + src/label_score.cpp + src/json.cpp + ) + target_include_directories(unit_label_score PRIVATE + ${CMAKE_SOURCE_DIR}/include + ${CMAKE_SOURCE_DIR}/third_party/doctest + ) + add_test(NAME unit_label_score COMMAND unit_label_score) + # Intent reconcile: admit, contract compile, observe ΔS, JIT ensure # waves, verification mandate, implement stub, snapshot rollback. # No provider client. diff --git a/ROADMAP.md b/ROADMAP.md index c7b52094..ce71a1d4 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -66,6 +66,7 @@ calendar commitments. - [ ] **Delegation policies-** Consitutions declare max depth, allowed callees, budget caps (tokens/$) - [ ] ~~**Workflow recipes-** Checked-in “crews” (JSON): ordered/parallel graphs of agents + shared todo board~~ - [ ] **Advisor policy packs-** Reusable gate profiles (strict / coding / research) +- [ ] **Decision-filter measurement-** Refit `ARBITER_PRESENCE_SILENCE_MARGIN` and `ARBITER_INTENT_ROUTE_MARGIN` from `filter_*` / `decision_*` traces. The advisor gate stays off this path until those floors exist. **Acceptance criteria:** - [x] A user can watch and steer a 5-agent job. diff --git a/docs/api/intent.md b/docs/api/intent.md index 14718630..3f711a24 100644 --- a/docs/api/intent.md +++ b/docs/api/intent.md @@ -55,7 +55,8 @@ curl http://127.0.0.1:8080/v1/intent \ | Field | Meaning | |-------|---------| | `kind` | Closed taxonomy: `research` \| `review` \| `write` \| `ops` \| `frontend` \| `backend` \| `plan` \| `market` \| `social` \| `multi` \| `unknown`. | -| `source` | `heuristic` \| `llm` \| `explicit` \| `event` \| `none`. | +| `source` | `heuristic` \| `llm` \| `decision` \| `explicit` \| `event` \| `none`. `decision` is a peaked local label distribution on a no-cue utterance. It carries the canned brief and no seeds. | +| `decision_peak`, `decision_margin`, `decision_label` | Present when the local filter ran, including when it fell through to the LLM. Omitted when the filter is off or the heuristic already routed. | | `target_agent` | Suggested specialist, or `""` if index should handle it. | | `applied` | True when `apply_routing` would rewrite an `index` ingress to `target_agent` **and** that id is in the roster used for this request. This endpoint never dispatches. [`POST /v1/orchestrate`](orchestrate.md) additionally requires the agent to be loaded on the request orch (API: tenant catalog; TUI: `agents_dir`). | | `todo_seeds` / `plan_seeds` | Optional decomposition hints. Not persisted. | diff --git a/docs/cli/environment.md b/docs/cli/environment.md index 9a07b760..3b065889 100644 --- a/docs/cli/environment.md +++ b/docs/cli/environment.md @@ -37,6 +37,9 @@ See [`docs/cli/connect.md`](connect.md). Local provider keys (`OPENROUTER_API_KE | `ARBITER_CIRCUIT_COOLDOWN_SECONDS` | Cooldown in Open before a half-open probe. Default `30` (clamped ≥0). | | `ARBITER_INTENT_LLM_RATE_PER_MIN` | Extra per-tenant token bucket for `POST /v1/intent` hybrid/llm classify. Default `20`. `0` disables. Independent of `ARBITER_TENANT_RATE_PER_MIN`. See [Operations → Per-tenant rate / concurrency limiting](../concepts/operations.md#per-tenant-rate--concurrency-limiting). | | `ARBITER_INTENT_LLM_RATE_BURST` | Burst for the intent LLM bucket. Default `5`. | +| `ARBITER_DECISION_MODEL` | Local `ollama/…` id for the closed-set label filter. Unset, empty, or any non-`ollama/` id leaves the filter off. Hosted models are not called. | +| `ARBITER_PRESENCE_SILENCE_MARGIN` | Margin (peak minus runner-up) required before an extreme `silent` skips a presence review. `0` or unset disables that skip. Clamped to `[0, 1]`. A skip also requires peak ≥ 0.9. See [Presence](../concepts/presence.md#silence-filter). | +| `ARBITER_INTENT_ROUTE_MARGIN` | Margin required before a no-cue utterance routes from the label filter without the advisor completion. `0` or unset disables that route. Clamped to `[0, 1]`. A route also requires peak ≥ 0.9. See [Intent](../concepts/intent.md#hybrid-classify). | ## TUI session durability diff --git a/docs/concepts/intent.md b/docs/concepts/intent.md index 6bddaa8d..190cedb3 100644 --- a/docs/concepts/intent.md +++ b/docs/concepts/intent.md @@ -19,10 +19,14 @@ The executor still speaks [writs](writ.md). Intent never becomes a writ. ```text explicit non-index agent → source=explicit (no reroute) heuristic unique match → source=heuristic, maybe reroute +else if no cue and a local label distribution is peaked + → source=decision, canned brief, no seeds else if mode=hybrid|llm → one LLM call, tagged reply else → fall through to index ``` +The decision step is off unless `ARBITER_DECISION_MODEL` is an `ollama/…` id and `ARBITER_INTENT_ROUTE_MARGIN` is greater than 0. It runs only when the heuristic kind is `unknown` (no cue hit). A multi-cue result, a confident cue, and `mode=heuristic` never consult it. The model scores single-letter labels, including `unknown`, and the runtime renormalizes over those letters alone. A route is taken only when the peak is at least 0.9, the margin (peak minus the runner-up) clears the env floor, the winner is a real kind, and that kind maps to an agent on the roster. The brief is the canned `Route to ` line. Todo and phase seeds still come only from the tagged completion, which runs for every other case. A missing label in the provider's top logprobs, a flat distribution, or a transport error leaves routing on the existing path. The margin is a local gate to refit from `decision_peak` / `decision_margin` / `decision_label` on the intent event. It is not a calibrated probability. See [Environment](../cli/environment.md). + Closed `kind` values: `research`, `review`, `write`, `ops`, `frontend`, `backend`, `plan`, `market`, `social`, `multi`, `unknown`. They map onto the personal starter callsigns under `agents/*.json` (`scout`, `vera`, `quill`, `forge`, `loom`, `nexus`, `compass`, `beacon`, `echo`). `multi` means "needs decomposition"; seed slots may be filled. LLM replies use a tagged grammar (same idea as the [advisor gate](advisor.md)): diff --git a/docs/concepts/presence.md b/docs/concepts/presence.md index e8f7d578..7cdaf125 100644 --- a/docs/concepts/presence.md +++ b/docs/concepts/presence.md @@ -67,6 +67,12 @@ The review is history-less — one snapshot in, one signal out — matching the `SILENT` is the default. Surrounding prose is tolerated. Missing ``, an unknown token, or `CONTEXT` without `` is **malformed Silent** (fail-open). Presence cannot `HALT` or `REDIRECT` — that remains the advisor's job. +## Silence filter + +When `ARBITER_DECISION_MODEL` is an `ollama/…` id and `ARBITER_PRESENCE_SILENCE_MARGIN` is greater than 0, each checkpoint scores a short snapshot before `run_presence_review`. The snapshot is the first line of the task (240 bytes) plus tool names. Arguments, result bodies, recent assistant text, and watcher rules are not in it. + +The labels are `silent` and `context`. An extreme `silent` — peak at least 0.9 and margin at least the env floor — skips the review and emits `presence` with `kind=silent`, `detail=filter`, plus `filter_peak`, `filter_margin`, and `filter_label`. A peaked `context`, a flat distribution, a missing label, or a transport error still runs the review, which is the only place a note can be written. The filter cannot inject `[PRESENCE: …]`. The margin is a local gate to refit from those fields. It is not a calibrated probability. See [Environment](../cli/environment.md). + ## Runtime control flow Where presence fires inside the dispatch loop: diff --git a/docs/concepts/sse-events.md b/docs/concepts/sse-events.md index 329fca14..a08638e5 100644 --- a/docs/concepts/sse-events.md +++ b/docs/concepts/sse-events.md @@ -7,7 +7,7 @@ Every event on the `/v1/orchestrate` stream has an `event:` line and a `data:` l | Event | When | Fields | |-------|------|--------| | `request_received` | Exactly once, first event on the stream. | `agent`, `tenant`, `tenant_id`, `message` (first 200 chars, ellipsis added if truncated), `channel?` (`"voice"` when the request asked for spoken output). | -| `intent` | Pre-dispatch classify/route on **fresh** ingress (`original_query` omitted) when the ingress agent's `intent.mode` is not `off`. After `request_received`, before `stream_start`. See [Intent](intent.md). | `kind`, `confidence`, `source`, `target_agent`, `applied`, `requested_agent`, `applied_agent`, `brief?`, `todo_seed_count`, `plan_seed_count`, `llm_used?`, `malformed?`. | +| `intent` | Pre-dispatch classify/route on **fresh** ingress (`original_query` omitted) when the ingress agent's `intent.mode` is not `off`. After `request_received`, before `stream_start`. See [Intent](intent.md). | `kind`, `confidence`, `source`, `target_agent`, `applied`, `requested_agent`, `applied_agent`, `brief?`, `todo_seed_count`, `plan_seed_count`, `llm_used?`, `malformed?`, `decision_peak?`, `decision_margin?`, `decision_label?`. | | `reconcile.progress` | Phase note on [`POST /v1/reconcile`](../api/reconcile.md). | `request_id`, `phase`, `detail?`. | | `reconcile.delta` | Contract observation: residual vs held clauses. Emitted after the initial observe and after every ensure wave. | `residual`, `held`, `empty`, `wave`. | | `agent.spawned` | JIT ephemeral clone created for a ΔS cover (`mode=ensure` waves). | `agent`, `clone_id`, `clauses`. | @@ -23,7 +23,7 @@ Every event on the `/v1/orchestrate` stream has an `event:` line and a `data:` l | `sub_agent_response` | After a delegated turn completes (depth > 0). The full turn body in one payload — useful for consumers that don't want to reconstruct from deltas. | `agent`, `stream_id`, `depth`, `content`. | | `token_usage` | After each turn completes. | `agent`, `stream_id`, `depth`, `model`, `input_tokens`, `output_tokens`, `cache_read_tokens?`, `cache_create_tokens?`. | | `advisor` | Every advisor interaction. Independent of `tool_call` — fires for the runtime's gate decisions (which the executor never invokes directly) and for the executor's own `/advise` consults. See [Advisor concept](advisor.md). | `agent`, `stream_id`, `kind`, `detail?`, `preview?`, `malformed?`. | -| `presence` | Always-on watcher consult after a peer's tool batch. Fires for `SILENT` and `CONTEXT`. See [Presence](presence.md). | `watcher`, `agent` (the working agent), `stream_id`, `kind` (`silent` \| `context`), `detail?`, `malformed?`. | +| `presence` | Always-on watcher consult after a peer's tool batch. Fires for `SILENT` and `CONTEXT`. See [Presence](presence.md). | `watcher`, `agent` (the working agent), `stream_id`, `kind` (`silent` \| `context`), `detail?`, `malformed?`, `filter_action?`, `filter_peak?`, `filter_margin?`, `filter_label?`. | | `escalation` | Out-of-band advisor halt. Fires before the corresponding `stream_end` (which arrives with `ok: false`). Only fires at the originating depth — sub-agent halts bubble up via the parent's response, not via duplicate escalation events. | `agent`, `stream_id`, `reason`. | | `stream_end` | Closes each turn. Any remaining partial line is flushed before this fires, so no `text` events arrive with this `stream_id` after. | `agent`, `stream_id`, `ok`. | | `error` | Recoverable errors during the request (transient upstream issue). The stream continues or terminates depending on severity. | `message`, plus optional context fields (e.g. `reason`). | diff --git a/include/api_client.h b/include/api_client.h index 674fc7c8..3c86d4b5 100644 --- a/include/api_client.h +++ b/include/api_client.h @@ -132,6 +132,11 @@ struct ApiRequest { int max_tokens = 1024; double temperature = 0.3; bool include_temperature = true; + // Decision-filter only. Ollama's OpenAI-compatible endpoint. Left + // false on every generative call so older Ollama builds never see + // the fields. + bool logprobs = false; + int top_logprobs = 0; }; struct ApiResponse { diff --git a/include/intent.h b/include/intent.h index c7522a9e..95a698f0 100644 --- a/include/intent.h +++ b/include/intent.h @@ -16,6 +16,8 @@ // Distinct from Constitution::MemoryConfig::intent_routing, which only // boosts /mem search types. +#include "label_score.h" + #include #include #include @@ -65,13 +67,20 @@ struct Intent { // Closed taxonomy aligned with starter agents, plus multi/unknown. std::string kind; // research|review|write|ops|frontend|backend|plan|market|social|multi|unknown double confidence = 0; // 0..1 - std::string source; // heuristic|llm|explicit|event|none + std::string source; // heuristic|llm|decision|explicit|event|none std::string target_agent; // empty = do not reroute std::string brief; std::vector todo_seeds; std::vector plan_seeds; bool llm_used = false; bool malformed = false; + // Set when the local label scorer ran on a no-cue utterance. peak and + // margin are the renormalized distribution, even when the route was + // not taken and source stayed heuristic/llm. + bool decision_consulted = false; + double decision_peak = 0; + double decision_margin = 0; + std::string decision_label; }; struct IntentInput { @@ -105,11 +114,15 @@ bool intent_should_apply(const IntentConfig& cfg, bool fresh_ingress = true); // Hybrid classify: explicit specialist short-circuits; heuristic when -// confident; LLM when mode is hybrid/llm and heuristic is below threshold. -// llm may be null — hybrid then degrades to heuristic-only. +// confident; a peaked local label distribution when the heuristic found +// no cue and `decision_margin` > 0; otherwise the LLM. llm and scorer +// may be null — hybrid then degrades to heuristic-only. decision_margin +// <= 0 leaves the scorer unconsulted. Intent resolve_intent(const IntentInput& in, const IntentConfig& cfg, - const IntentLlmFn& llm = nullptr); + const IntentLlmFn& llm = nullptr, + const LabelScorer& scorer = nullptr, + double decision_margin = 0); // One-line (plus optional GOAL) preamble injected ahead of the user text. // Empty when there is nothing useful to show the executor. diff --git a/include/label_score.h b/include/label_score.h new file mode 100644 index 00000000..660f44b7 --- /dev/null +++ b/include/label_score.h @@ -0,0 +1,123 @@ +#pragma once +// arbiter/include/label_score.h — Closed-set next-token label distribution. +// +// A local model scores a short state against single-letter labels. Callers +// act only when the renormalized distribution is peaked; otherwise they +// keep the generative path they already had. The scorer never writes prose. +// +// Hosted providers are not consulted. An empty model, a non-ollama id, a +// missing label in top_logprobs, or a transport error yields ok=false, and +// callers fall through. + +#include +#include +#include +#include +#include + +namespace arbiter { + +class ApiClient; + +// Peak must clear this floor before a margin check can skip a generative +// call. The margin itself comes from the environment and defaults to off. +inline constexpr double kDecisionPeakFloor = 0.9; + +// Intent utterances passed to the scorer are clipped here. The heuristic +// and the advisor completion still see the full text. +inline constexpr std::size_t kIntentDecisionStateMaxBytes = 2048; + +struct LabelSpec { + char letter = 0; // one ASCII letter + std::string name; // stable id callers switch on ("silent", "research") + std::string description; // the only instruction the model is given +}; + +struct TokenLogprob { + std::string token; + double logprob = 0; +}; + +struct LabelDistribution { + bool ok = false; + std::string argmax; // LabelSpec::name; empty when !ok + double peak = 0; // probability of argmax after renormalizing + double margin = 0; // peak − second; 0 when the top two tie + // name → probability, in spec order. Empty when !ok. + std::vector> probs; +}; + +// Softmax over the supplied labels only. ok is false when any letter is +// absent, a letter is duplicated, or a spec is not a single ASCII letter. +// Tokens match a letter only when, after trimming whitespace, they are +// exactly that letter. "Silent" does not count as S. +LabelDistribution distribution_from_logprobs( + const std::vector& specs, + const std::vector& tokens); + +// Parse choices[0].logprobs.content[0].top_logprobs from an OpenAI-compatible +// chat completion body. Any other shape returns ok=false. +LabelDistribution distribution_from_openai_body( + const std::vector& specs, + const std::string& body); + +// True when the distribution is peaked enough to skip a generative call. +// margin_floor <= 0 never passes: the filter ships dark. +bool label_distribution_is_extreme(const LabelDistribution& d, + double margin_floor); + +const char* label_score_system_prompt(); + +std::string label_score_user_prompt(const std::string& state, + const std::vector& specs); + +// Clip to max_bytes without splitting a trailing UTF-8 sequence. +std::string clip_utf8(std::string text, std::size_t max_bytes); + +struct DecisionFilterConfig { + std::string model; // empty, or an ollama/… id + double presence_silence_margin = 0; + double intent_route_margin = 0; + + bool enabled_for_presence() const; + bool enabled_for_intent() const; +}; + +// Reads ARBITER_DECISION_MODEL, ARBITER_PRESENCE_SILENCE_MARGIN, and +// ARBITER_INTENT_ROUTE_MARGIN. Unset or unparsable margins are 0 (off). +// Values are clamped to [0, 1]. +DecisionFilterConfig decision_filter_from_env(); + +// Same parse, for tests that must not touch the process environment. +DecisionFilterConfig decision_filter_from_values(const char* model, + const char* presence_margin, + const char* intent_margin); + +// Closed kind set for the intent filter, including unknown. multi is +// omitted: a multi-cue heuristic result never reaches the scorer. +std::vector intent_decision_labels(); + +using LabelScorer = std::function& specs)>; + +// Network call. Defined in api_client.cpp. Non-ollama models and failed +// completions return a distribution with ok=false and do not throw. +struct LabelScoreCall { + LabelDistribution distribution; + // called is true only when complete() ran, so callers can attribute + // cost. A refused non-ollama id leaves called false. + bool called = false; + bool response_ok = false; + int input_tokens = 0; + int output_tokens = 0; + int cache_read_tokens = 0; + int cache_creation_tokens = 0; + std::string model; +}; + +LabelScoreCall score_labels(ApiClient& client, + const std::string& model, + const std::string& state, + const std::vector& specs); + +} // namespace arbiter diff --git a/include/orchestrator.h b/include/orchestrator.h index c9c79c6a..02993d83 100644 --- a/include/orchestrator.h +++ b/include/orchestrator.h @@ -218,6 +218,14 @@ class Orchestrator { std::string kind; std::string detail; bool malformed = false; + // Set when the local silence filter ran before the review call. + // filter_action is "skip" or "fallthrough". peak/margin/label are + // the renormalized distribution so a later pass can refit the floor. + bool filter_consulted = false; + std::string filter_action; + double filter_peak = 0; + double filter_margin = 0; + std::string filter_label; }; using PresenceEventCallback = std::function; void set_presence_event_callback(PresenceEventCallback cb) { @@ -547,6 +555,7 @@ class Orchestrator { PresenceEventCallback presence_event_cb_; std::string intent_source_hint_; std::string ingress_channel_; + DecisionFilterConfig decision_filter_; HistoryCheckpointCallback history_checkpoint_cb_; std::atomic stream_counter_{-1}; // next_stream_id returns 0 first int next_stream_id(); diff --git a/include/presence.h b/include/presence.h index ad9bb855..8a3d26fc 100644 --- a/include/presence.h +++ b/include/presence.h @@ -25,6 +25,8 @@ // envelope) is owned by the orchestrator. Fail-open: transport or parse // errors become SILENT so a watcher never blocks the working agent. +#include "label_score.h" + #include #include #include @@ -109,6 +111,18 @@ PresenceOutput parse_presence_signal(const std::string& reply); std::string format_presence_injection(const std::string& watcher_name, const std::string& note); +// Reduced snapshot for the silence filter. First line of the task, capped +// at 240 bytes, plus tool names only. Arguments and result bodies stay out. +std::string presence_filter_state(const std::string& original_task, + const std::string& tool_summary); + +std::vector presence_filter_labels(); + +// True only for an extreme "silent" label. A peaked "context" does not +// skip the review — the filter cannot invent a note. +bool presence_filter_skips_review(const LabelDistribution& d, + double margin_floor); + // History-less review call. Empty model or transport error → Silent // (fail-open). on_response fires with the raw ApiResponse for cost // attribution, matching run_advisor_gate. diff --git a/src/api_client.cpp b/src/api_client.cpp index 8fd08a9b..c2db07cf 100644 --- a/src/api_client.cpp +++ b/src/api_client.cpp @@ -2,6 +2,7 @@ // See api_client.h for the routing model. #include "api_client.h" #include "circuit_breaker.h" +#include "label_score.h" #include "metrics.h" #include @@ -741,6 +742,12 @@ std::string ApiClient::build_body_openai(const Provider& prov, if (is_openai) m["max_completion_tokens"] = jnum(static_cast(req.max_tokens)); else m["max_tokens"] = jnum(static_cast(req.max_tokens)); if (req.include_temperature && !reasoning) m["temperature"] = jnum(req.temperature); + // Opt-in. The decision filter is the only caller, and only for ollama/. + if (req.logprobs) { + m["logprobs"] = jbool(true); + if (req.top_logprobs > 0) + m["top_logprobs"] = jnum(static_cast(req.top_logprobs)); + } if (streaming) { m["stream"] = jbool(true); // OpenAI only reports usage on the terminal stream chunk when you opt @@ -1917,4 +1924,40 @@ ApiResponse ApiClient::stream(const ApiRequest& req, StreamCallback cb) { return r; } +LabelScoreCall score_labels(ApiClient& client, + const std::string& model, + const std::string& state, + const std::vector& specs) { + LabelScoreCall call; + call.model = model; + if (model.rfind("ollama/", 0) != 0 || specs.empty()) return call; + + ApiRequest req; + req.model = model; + req.max_tokens = 1; + req.temperature = 0; + req.include_temperature = true; + req.logprobs = true; + int top = static_cast(specs.size()) + 5; + if (top < 8) top = 8; + if (top > 20) top = 20; + req.top_logprobs = top; + req.system_prompt = label_score_system_prompt(); + Message msg; + msg.role = "user"; + msg.content = label_score_user_prompt(state, specs); + req.messages.push_back(std::move(msg)); + + ApiResponse resp = client.complete(req); + call.called = true; + call.response_ok = resp.ok; + call.input_tokens = resp.input_tokens; + call.output_tokens = resp.output_tokens; + call.cache_read_tokens = resp.cache_read_tokens; + call.cache_creation_tokens = resp.cache_creation_tokens; + if (!resp.ok) return call; + call.distribution = distribution_from_openai_body(specs, resp.raw_body); + return call; +} + } // namespace arbiter diff --git a/src/api_server.cpp b/src/api_server.cpp index bbbea064..0d12e0b4 100644 --- a/src/api_server.cpp +++ b/src/api_server.cpp @@ -4874,7 +4874,18 @@ void handle_intent_classify(int fd, const HttpRequest& req, } } - Intent intent = resolve_intent(in, cfg, llm); + LabelScorer scorer; + DecisionFilterConfig decision = decision_filter_from_env(); + if (decision.enabled_for_intent() && + (cfg.mode == "hybrid" || cfg.mode == "llm")) { + const std::string model = decision.model; + scorer = [&](const std::string& state, + const std::vector& specs) { + return score_labels(orch->client(), model, state, specs).distribution; + }; + } + + Intent intent = resolve_intent(in, cfg, llm, scorer, decision.intent_route_margin); if (llm_rate_limited) { write_429_response(fd, llm_retry_after, "intent_llm_rate_limit", metrics, tenant.id); @@ -4897,6 +4908,11 @@ void handle_intent_classify(int fd, const HttpRequest& req, m["llm_used"] = jbool(intent.llm_used); m["malformed"] = jbool(intent.malformed); m["applied"] = jbool(applied); + if (intent.decision_consulted) { + m["decision_peak"] = jnum(intent.decision_peak); + m["decision_margin"] = jnum(intent.decision_margin); + m["decision_label"] = jstr(intent.decision_label); + } m["requested_agent"] = jstr(in.requested_agent); if (!intent.todo_seeds.empty()) { auto arr = jarr(); @@ -10333,6 +10349,12 @@ void handle_orchestrate(int fd, const HttpRequest& req, m["kind"] = jstr(ev.kind); if (!ev.detail.empty()) m["detail"] = jstr(ev.detail); if (ev.malformed) m["malformed"] = jbool(true); + if (ev.filter_consulted) { + m["filter_action"] = jstr(ev.filter_action); + m["filter_peak"] = jnum(ev.filter_peak); + m["filter_margin"] = jnum(ev.filter_margin); + m["filter_label"] = jstr(ev.filter_label); + } emit("presence", p); }); @@ -10359,6 +10381,11 @@ void handle_orchestrate(int fd, const HttpRequest& req, m["plan_seed_count"] = jnum(static_cast(ev.intent.plan_seeds.size())); if (ev.intent.llm_used) m["llm_used"] = jbool(true); if (ev.intent.malformed) m["malformed"] = jbool(true); + if (ev.intent.decision_consulted) { + m["decision_peak"] = jnum(ev.intent.decision_peak); + m["decision_margin"] = jnum(ev.intent.decision_margin); + m["decision_label"] = jstr(ev.intent.decision_label); + } emit("intent", p); }); diff --git a/src/intent.cpp b/src/intent.cpp index 56767f7d..09c0a722 100644 --- a/src/intent.cpp +++ b/src/intent.cpp @@ -466,9 +466,23 @@ bool intent_should_apply(const IntentConfig& cfg, return true; } +namespace { + +void stamp_decision_trace(Intent& out, const Intent& trace) { + if (!trace.decision_consulted) return; + out.decision_consulted = true; + out.decision_peak = trace.decision_peak; + out.decision_margin = trace.decision_margin; + out.decision_label = trace.decision_label; +} + +} // namespace + Intent resolve_intent(const IntentInput& in, const IntentConfig& cfg, - const IntentLlmFn& llm) { + const IntentLlmFn& llm, + const LabelScorer& scorer, + double decision_margin) { Intent out; out.kind = "unknown"; out.source = "none"; @@ -507,6 +521,40 @@ Intent resolve_intent(const IntentInput& in, return drop_unknown_agent(std::move(out), in.roster); } + // No cue hit. A peaked local distribution may route with the canned + // brief. Flat, unknown, or a kind with no roster agent falls through + // to the tagged completion, which is the only writer of seeds. + Intent trace; + if ((mode == "hybrid" || mode == "llm") && scorer && + decision_margin > 0.0 && hint.kind == "unknown") { + LabelDistribution dist; + try { + std::string state = clip_utf8(in.text, kIntentDecisionStateMaxBytes); + dist = scorer(state, intent_decision_labels()); + } catch (...) { + dist = {}; + } + trace.decision_consulted = true; + trace.decision_peak = dist.peak; + trace.decision_margin = dist.margin; + trace.decision_label = dist.argmax; + if (label_distribution_is_extreme(dist, decision_margin) && + dist.argmax != "unknown" && dist.argmax != "multi" && + intent_kind_is_valid(dist.argmax)) { + std::string agent = match_roster_agent(in.roster, dist.argmax); + if (!agent.empty()) { + out.kind = dist.argmax; + out.target_agent = agent; + out.confidence = dist.peak; + out.source = "decision"; + out.brief = default_brief(out.kind, out.target_agent); + out.llm_used = false; + stamp_decision_trace(out, trace); + return drop_unknown_agent(std::move(out), in.roster); + } + } + } + if ((mode == "hybrid" || mode == "llm") && llm) { std::string reply = llm(build_llm_user_prompt(in, hint)); if (reply.empty()) { @@ -515,6 +563,7 @@ Intent resolve_intent(const IntentInput& in, out = hint; out.llm_used = true; if (in.source_hint == "event") out.source = "event"; + stamp_decision_trace(out, trace); return drop_unknown_agent(std::move(out), in.roster); } out = parse_intent_signal(reply); @@ -528,12 +577,14 @@ Intent resolve_intent(const IntentInput& in, if (in.source_hint == "event") out.source = "event"; if (out.brief.empty()) out.brief = default_brief(out.kind, out.target_agent); + stamp_decision_trace(out, trace); return out; } // hybrid/llm with no llm fn, or heuristic-only miss. out = hint; if (in.source_hint == "event") out.source = "event"; + stamp_decision_trace(out, trace); return drop_unknown_agent(std::move(out), in.roster); } diff --git a/src/label_score.cpp b/src/label_score.cpp new file mode 100644 index 00000000..59ec1dbb --- /dev/null +++ b/src/label_score.cpp @@ -0,0 +1,241 @@ +// src/label_score.cpp — Closed-set label distribution. No provider I/O. + +#include "label_score.h" + +#include "json.h" + +#include +#include +#include +#include +#include + +namespace arbiter { + +namespace { + +bool is_ascii_letter(char c) { + return (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'); +} + +char upper_ascii(char c) { + if (c >= 'a' && c <= 'z') return static_cast(c - 'a' + 'A'); + return c; +} + +// Trimmed token that is exactly one letter, uppercased. Empty otherwise. +std::string letter_token(const std::string& token) { + size_t b = 0; + size_t e = token.size(); + while (b < e && std::isspace(static_cast(token[b]))) ++b; + while (e > b && std::isspace(static_cast(token[e - 1]))) --e; + if (e != b + 1) return {}; + if (!is_ascii_letter(token[b])) return {}; + return std::string(1, upper_ascii(token[b])); +} + +bool ollama_model(const std::string& model) { + return model.rfind("ollama/", 0) == 0 && model.size() > std::string("ollama/").size(); +} + +double parse_margin(const char* s) { + if (!s || !*s) return 0; + char* end = nullptr; + double v = std::strtod(s, &end); + if (end == s) return 0; + if (!std::isfinite(v) || v <= 0) return 0; + if (v > 1) return 1; + return v; +} + +DecisionFilterConfig config_from(const char* model, + const char* presence_margin, + const char* intent_margin) { + DecisionFilterConfig c; + if (model && *model) c.model = model; + c.presence_silence_margin = parse_margin(presence_margin); + c.intent_route_margin = parse_margin(intent_margin); + return c; +} + +} // namespace + +LabelDistribution distribution_from_logprobs( + const std::vector& specs, + const std::vector& tokens) { + LabelDistribution out; + if (specs.size() < 2) return out; + + std::vector letters; + letters.reserve(specs.size()); + for (const auto& spec : specs) { + if (!is_ascii_letter(spec.letter) || spec.name.empty()) return out; + char letter = upper_ascii(spec.letter); + if (std::find(letters.begin(), letters.end(), letter) != letters.end()) + return out; + letters.push_back(letter); + } + + std::vector best(specs.size(), 0); + std::vector seen(specs.size(), false); + for (const auto& tok : tokens) { + std::string letter = letter_token(tok.token); + if (letter.size() != 1) continue; + if (!std::isfinite(tok.logprob)) continue; + auto it = std::find(letters.begin(), letters.end(), letter[0]); + if (it == letters.end()) continue; + size_t i = static_cast(it - letters.begin()); + if (!seen[i] || tok.logprob > best[i]) { + best[i] = tok.logprob; + seen[i] = true; + } + } + for (bool hit : seen) { + if (!hit) return out; + } + + double m = best[0]; + for (double lp : best) m = std::max(m, lp); + double sum = 0; + for (double lp : best) sum += std::exp(lp - m); + if (!(sum > 0) || !std::isfinite(sum)) return out; + + out.probs.reserve(specs.size()); + size_t winner = 0; + double second = -1; + for (size_t i = 0; i < specs.size(); ++i) { + double p = std::exp(best[i] - m) / sum; + out.probs.emplace_back(specs[i].name, p); + if (i == 0 || p > out.probs[winner].second) winner = i; + } + out.peak = out.probs[winner].second; + for (size_t i = 0; i < out.probs.size(); ++i) { + if (i == winner) continue; + second = std::max(second, out.probs[i].second); + } + out.margin = (second < 0) ? out.peak : (out.peak - second); + if (out.margin < 0) out.margin = 0; + out.argmax = specs[winner].name; + out.ok = true; + return out; +} + +LabelDistribution distribution_from_openai_body( + const std::vector& specs, + const std::string& body) { + try { + auto root = json_parse(body); + auto choices = root->get("choices"); + if (!choices || !choices->is_array() || choices->as_array().empty()) + return {}; + auto ch0 = choices->as_array().front(); + if (!ch0) return {}; + auto logprobs = ch0->get("logprobs"); + if (!logprobs || !logprobs->is_object()) return {}; + auto content = logprobs->get("content"); + if (!content || !content->is_array() || content->as_array().empty()) + return {}; + auto first = content->as_array().front(); + if (!first) return {}; + auto top = first->get("top_logprobs"); + if (!top || !top->is_array()) return {}; + + std::vector tokens; + tokens.reserve(top->as_array().size()); + for (const auto& entry : top->as_array()) { + if (!entry || !entry->is_object()) continue; + TokenLogprob tp; + tp.token = entry->get_string("token"); + auto lp = entry->get("logprob"); + if (!lp || !lp->is_number()) continue; + tp.logprob = lp->as_number(); + tokens.push_back(std::move(tp)); + } + return distribution_from_logprobs(specs, tokens); + } catch (...) { + return {}; + } +} + +bool label_distribution_is_extreme(const LabelDistribution& d, + double margin_floor) { + if (!d.ok) return false; + if (!(margin_floor > 0.0)) return false; + if (d.peak + 1e-12 < kDecisionPeakFloor) return false; + if (d.margin + 1e-12 < margin_floor) return false; + if (d.argmax.empty()) return false; + return true; +} + +const char* label_score_system_prompt() { + return "Choose one option. Reply with that option's letter and nothing else."; +} + +std::string label_score_user_prompt(const std::string& state, + const std::vector& specs) { + std::ostringstream ss; + ss << state << "\n\n"; + for (const auto& spec : specs) { + ss << upper_ascii(spec.letter) << ". " << spec.name << " — " + << spec.description << "\n"; + } + return ss.str(); +} + +std::string clip_utf8(std::string text, std::size_t max_bytes) { + if (text.size() <= max_bytes) return text; + std::size_t keep = max_bytes; + while (keep > 0 && + (static_cast(text[keep]) & 0xC0) == 0x80) { + --keep; + } + text.resize(keep); + return text; +} + +bool DecisionFilterConfig::enabled_for_presence() const { + return ollama_model(model) && presence_silence_margin > 0.0; +} + +bool DecisionFilterConfig::enabled_for_intent() const { + return ollama_model(model) && intent_route_margin > 0.0; +} + +DecisionFilterConfig decision_filter_from_env() { + return config_from(std::getenv("ARBITER_DECISION_MODEL"), + std::getenv("ARBITER_PRESENCE_SILENCE_MARGIN"), + std::getenv("ARBITER_INTENT_ROUTE_MARGIN")); +} + +DecisionFilterConfig decision_filter_from_values(const char* model, + const char* presence_margin, + const char* intent_margin) { + return config_from(model, presence_margin, intent_margin); +} + +std::vector intent_decision_labels() { + return { + {'R', "research", + "The user wants sources, facts, or a survey."}, + {'V', "review", + "The user wants a critique of code or a diff."}, + {'W', "write", + "The user wants prose, docs, or a draft."}, + {'O', "ops", + "The user wants infrastructure, deploy, or runtime work."}, + {'F', "frontend", + "The user wants UI or client-side code."}, + {'B', "backend", + "The user wants server, API, or data-store work."}, + {'P', "plan", + "The user wants the work decomposed into steps."}, + {'M', "market", + "The user wants positioning or a campaign."}, + {'S', "social", + "The user wants a social post or audience copy."}, + {'U', "unknown", + "None of the other labels fit, or the request is too unclear to route."}, + }; +} + +} // namespace arbiter diff --git a/src/orchestrator.cpp b/src/orchestrator.cpp index 1d26edf8..68dbd99f 100644 --- a/src/orchestrator.cpp +++ b/src/orchestrator.cpp @@ -101,6 +101,7 @@ Orchestrator::Orchestrator(std::map api_keys) master.model = fallback; } index_master_ = std::make_unique("index", master, client_); + decision_filter_ = decision_filter_from_env(); } Agent& Orchestrator::create_agent(const std::string& id, Constitution config) { @@ -738,15 +739,47 @@ std::string Orchestrator::collect_presence_notes( in.tool_summary = tool_summary; PresenceOutput out; - try { - out = run_presence_review( - client_, w.model, w.cfg.prompt, in, - [this, wid = w.id, model = w.model](const ApiResponse& resp) { - if (cost_cb_) cost_cb_(wid, model, resp); - }); - } catch (...) { - out.kind = PresenceOutput::Kind::Silent; - out.malformed = true; + bool filter_consulted = false; + std::string filter_action; + LabelDistribution filter_dist; + if (decision_filter_.enabled_for_presence()) { + filter_consulted = true; + filter_action = "fallthrough"; + try { + LabelScoreCall scored = score_labels( + client_, decision_filter_.model, + presence_filter_state(original_task, tool_summary), + presence_filter_labels()); + filter_dist = scored.distribution; + if (scored.called && cost_cb_) { + ApiResponse cost; + cost.ok = scored.response_ok; + cost.input_tokens = scored.input_tokens; + cost.output_tokens = scored.output_tokens; + cost.cache_read_tokens = scored.cache_read_tokens; + cost.cache_creation_tokens = scored.cache_creation_tokens; + cost_cb_(w.id, decision_filter_.model, cost); + } + } catch (...) { + filter_dist = {}; + } + if (presence_filter_skips_review( + filter_dist, decision_filter_.presence_silence_margin)) { + filter_action = "skip"; + out.kind = PresenceOutput::Kind::Silent; + } + } + if (filter_action != "skip") { + try { + out = run_presence_review( + client_, w.model, w.cfg.prompt, in, + [this, wid = w.id, model = w.model](const ApiResponse& resp) { + if (cost_cb_) cost_cb_(wid, model, resp); + }); + } catch (...) { + out.kind = PresenceOutput::Kind::Silent; + out.malformed = true; + } } if (presence_event_cb_) { @@ -758,7 +791,14 @@ std::string Orchestrator::collect_presence_notes( ? "context" : "silent"; if (out.kind == PresenceOutput::Kind::Context && !out.text.empty()) ev.detail = out.text; + else if (filter_action == "skip") + ev.detail = "filter"; ev.malformed = out.malformed; + ev.filter_consulted = filter_consulted; + ev.filter_action = filter_action; + ev.filter_peak = filter_dist.peak; + ev.filter_margin = filter_dist.margin; + ev.filter_label = filter_dist.argmax; presence_event_cb_(ev); } @@ -903,8 +943,30 @@ Orchestrator::apply_intent_ingress(const std::string& agent_id, } } + LabelScorer scorer; + if (decision_filter_.enabled_for_intent() && + (icfg.mode == "hybrid" || icfg.mode == "llm")) { + const std::string model = decision_filter_.model; + const std::string cost_id = result.agent_id; + scorer = [this, model, cost_id](const std::string& state, + const std::vector& specs) { + LabelScoreCall scored = score_labels(client_, model, state, specs); + if (scored.called && cost_cb_) { + ApiResponse cost; + cost.ok = scored.response_ok; + cost.input_tokens = scored.input_tokens; + cost.output_tokens = scored.output_tokens; + cost.cache_read_tokens = scored.cache_read_tokens; + cost.cache_creation_tokens = scored.cache_creation_tokens; + cost_cb_(cost_id, model, cost); + } + return scored.distribution; + }; + } + try { - result.intent = resolve_intent(in, icfg, llm); + result.intent = resolve_intent( + in, icfg, llm, scorer, decision_filter_.intent_route_margin); } catch (...) { return result; // never break dispatch } diff --git a/src/presence.cpp b/src/presence.cpp index cb3fdf92..8a56b239 100644 --- a/src/presence.cpp +++ b/src/presence.cpp @@ -232,4 +232,64 @@ PresenceOutput run_presence_review( return parse_presence_signal(resp.content); } +std::string presence_filter_state(const std::string& original_task, + const std::string& tool_summary) { + std::string task; + for (char c : original_task) { + if (c == '\n' || c == '\r') break; + task.push_back(c); + } + task = clip_utf8(std::move(task), 240); + + std::vector names; + std::string line; + auto flush = [&](std::string current) { + if (names.size() >= 12) return; + size_t i = 0; + while (i < current.size() && + std::isspace(static_cast(current[i]))) ++i; + if (i >= current.size() || current[i] != '-') return; + ++i; + while (i < current.size() && + std::isspace(static_cast(current[i]))) ++i; + size_t b = i; + while (i < current.size() && + !std::isspace(static_cast(current[i]))) ++i; + if (i > b) names.emplace_back(current.substr(b, i - b)); + }; + for (char c : tool_summary) { + if (c == '\n') { + flush(std::move(line)); + line.clear(); + } else { + line.push_back(c); + } + } + if (!line.empty()) flush(std::move(line)); + + std::ostringstream ss; + ss << "Task: " << task << "\nTools:"; + if (names.empty()) { + ss << " (none)"; + } else { + for (const auto& name : names) ss << " " << name; + } + return ss.str(); +} + +std::vector presence_filter_labels() { + return { + {'S', "silent", + "Nothing here would change the working agent's next action."}, + {'C', "context", + "A fact in this snapshot would save the working agent's next action."}, + }; +} + +bool presence_filter_skips_review(const LabelDistribution& d, + double margin_floor) { + return label_distribution_is_extreme(d, margin_floor) && + d.argmax == "silent"; +} + } // namespace arbiter diff --git a/tests/test_api_client.cpp b/tests/test_api_client.cpp index 8fc96c9d..e410bf4b 100644 --- a/tests/test_api_client.cpp +++ b/tests/test_api_client.cpp @@ -273,6 +273,32 @@ TEST_CASE("openai body emits inline base64 as a data: URL") { CHECK(body.find("\"type\":\"text\"") != std::string::npos); } +TEST_CASE("ollama body emits logprobs only when the decision filter asks") { + ApiRequest req; + req.model = "ollama/qwen3:0.6b"; + req.max_tokens = 1; + req.logprobs = true; + req.top_logprobs = 8; + Message m; + m.role = "user"; + m.content = "S or C"; + req.messages.push_back(m); + + Provider prov; + prov.name = "ollama"; + prov.format = Provider::FORMAT_OPENAI_CHAT; + + auto body = ApiClient::build_body_openai(prov, req, /*streaming=*/false); + CHECK(body.find("\"logprobs\":true") != std::string::npos); + CHECK(body.find("\"top_logprobs\":8") != std::string::npos); + CHECK(body.find("\"max_tokens\":1") != std::string::npos); + + req.logprobs = false; + req.top_logprobs = 0; + auto plain = ApiClient::build_body_openai(prov, req, /*streaming=*/false); + CHECK(plain.find("logprobs") == std::string::npos); +} + TEST_CASE("openrouter body keeps canonical hosted model slug") { ApiRequest req; req.model = "openrouter/openai/gpt-5.2"; diff --git a/tests/test_intent.cpp b/tests/test_intent.cpp index fbf73d67..b4e21057 100644 --- a/tests/test_intent.cpp +++ b/tests/test_intent.cpp @@ -430,3 +430,233 @@ TEST_CASE("standalone heuristic does not invoke llm on unconfident text") { CHECK_FALSE(out.llm_used); CHECK(out.source == "heuristic"); } + +static LabelDistribution peaked(const std::string& name, double peak, double margin) { + LabelDistribution d; + d.ok = true; + d.argmax = name; + d.peak = peak; + d.margin = margin; + return d; +} + +TEST_CASE("no-cue peaked research skips the llm") { + IntentInput in; + in.text = "Hello, how is the weather in the garden?"; + in.requested_agent = "index"; + in.roster = starter_roster(); + bool llm_called = false; + bool scored = false; + auto scorer = [&](const std::string& state, const std::vector& specs) { + scored = true; + CHECK(state.find("garden") != std::string::npos); + bool unknown = false, multi = false, research = false; + for (const auto& s : specs) { + if (s.name == "unknown") unknown = true; + if (s.name == "multi") multi = true; + if (s.name == "research") research = true; + } + CHECK(unknown); + CHECK(research); + CHECK_FALSE(multi); + return peaked("research", 0.96, 0.9); + }; + auto llm = [&](const std::string&) { + llm_called = true; + return std::string("write0.9" + "quillno"); + }; + auto out = resolve_intent(in, hybrid_cfg(), llm, scorer, 0.5); + CHECK(scored); + CHECK_FALSE(llm_called); + CHECK(out.source == "decision"); + CHECK(out.kind == "research"); + CHECK(out.target_agent == "scout"); + CHECK(out.confidence == doctest::Approx(0.96)); + CHECK(out.llm_used == false); + CHECK(out.decision_consulted); + CHECK(out.decision_label == "research"); + CHECK(out.brief.find("scout") != std::string::npos); + CHECK(intent_should_apply(hybrid_cfg(), out, "index")); +} + +TEST_CASE("flat distribution still calls the llm") { + IntentInput in; + in.text = "Hello, how is the weather in the garden?"; + in.requested_agent = "index"; + in.roster = starter_roster(); + bool llm_called = false; + auto scorer = [&](const std::string&, const std::vector&) { + return peaked("research", 0.4, 0.05); + }; + auto llm = [&](const std::string&) { + llm_called = true; + return std::string("write0.91" + "quilldraft it"); + }; + auto out = resolve_intent(in, hybrid_cfg(), llm, scorer, 0.5); + CHECK(llm_called); + CHECK(out.source == "llm"); + CHECK(out.kind == "write"); + CHECK(out.target_agent == "quill"); + CHECK(out.decision_consulted); + CHECK(out.decision_label == "research"); + CHECK(out.decision_peak == doctest::Approx(0.4)); +} + +TEST_CASE("unknown argmax does not route") { + IntentInput in; + in.text = "Hello, how is the weather in the garden?"; + in.requested_agent = "index"; + in.roster = starter_roster(); + bool llm_called = false; + auto scorer = [&](const std::string&, const std::vector&) { + return peaked("unknown", 0.99, 0.95); + }; + auto llm = [&](const std::string&) { + llm_called = true; + return std::string("research0.9" + "scoutx"); + }; + auto out = resolve_intent(in, hybrid_cfg(), llm, scorer, 0.5); + CHECK(llm_called); + CHECK(out.source == "llm"); + CHECK(out.kind == "research"); +} + +TEST_CASE("a below-floor peak falls through") { + IntentInput in; + in.text = "Hello, how is the weather in the garden?"; + in.requested_agent = "index"; + in.roster = starter_roster(); + bool llm_called = false; + auto scorer = [&](const std::string&, const std::vector&) { + return peaked("research", 0.85, 0.8); + }; + auto llm = [&](const std::string&) { + llm_called = true; + return std::string("ops0.9" + "forgex"); + }; + auto out = resolve_intent(in, hybrid_cfg(), llm, scorer, 0.5); + CHECK(llm_called); + CHECK(out.source == "llm"); + CHECK(out.kind == "ops"); +} + +TEST_CASE("multi-cue text does not consult the scorer") { + IntentInput in; + in.text = "Look up primary sources and write a Dockerfile"; + in.requested_agent = "index"; + in.roster = starter_roster(); + bool scored = false; + bool llm_called = false; + auto scorer = [&](const std::string&, const std::vector&) { + scored = true; + return peaked("research", 0.99, 0.99); + }; + auto llm = [&](const std::string&) { + llm_called = true; + return std::string("ops0.9" + "forgex"); + }; + auto hint = heuristic_classify(in); + CHECK(hint.kind == "multi"); + auto out = resolve_intent(in, hybrid_cfg(), llm, scorer, 0.5); + CHECK_FALSE(scored); + CHECK(llm_called); + CHECK(out.source == "llm"); + CHECK_FALSE(out.decision_consulted); +} + +TEST_CASE("a confident cue never consults the scorer") { + IntentInput in; + in.text = "Look up primary sources on SPARC v9"; + in.requested_agent = "index"; + in.roster = starter_roster(); + bool scored = false; + bool llm_called = false; + auto scorer = [&](const std::string&, const std::vector&) { + scored = true; + return peaked("write", 0.99, 0.99); + }; + auto llm = [&](const std::string&) { + llm_called = true; + return std::string{}; + }; + auto out = resolve_intent(in, hybrid_cfg(), llm, scorer, 0.5); + CHECK_FALSE(scored); + CHECK_FALSE(llm_called); + CHECK(out.source == "heuristic"); + CHECK(out.target_agent == "scout"); +} + +TEST_CASE("a zero margin leaves the scorer unconsulted") { + IntentInput in; + in.text = "Hello, how is the weather in the garden?"; + in.requested_agent = "index"; + in.roster = starter_roster(); + bool scored = false; + auto scorer = [&](const std::string&, const std::vector&) { + scored = true; + return peaked("research", 0.99, 0.99); + }; + auto out = resolve_intent(in, hybrid_cfg(), nullptr, scorer, 0.0); + CHECK_FALSE(scored); + CHECK(out.source == "heuristic"); + CHECK(out.kind == "unknown"); +} + +TEST_CASE("heuristic mode does not consult the scorer") { + IntentInput in; + in.text = "Hello, how is the weather in the garden?"; + in.requested_agent = "index"; + in.roster = starter_roster(); + bool scored = false; + auto scorer = [&](const std::string&, const std::vector&) { + scored = true; + return peaked("research", 0.99, 0.99); + }; + IntentConfig cfg = hybrid_cfg(); + cfg.mode = "heuristic"; + auto out = resolve_intent(in, cfg, nullptr, scorer, 0.5); + CHECK_FALSE(scored); + CHECK(out.source == "heuristic"); +} + +TEST_CASE("a peaked kind with no roster agent falls through") { + IntentInput in; + in.text = "Hello, how is the weather in the garden?"; + in.requested_agent = "index"; + in.roster = {{"scout", "research-analyst", "Find facts", {"/search"}}}; + bool llm_called = false; + auto scorer = [&](const std::string&, const std::vector&) { + return peaked("frontend", 0.97, 0.9); + }; + auto llm = [&](const std::string&) { + llm_called = true; + return std::string("research0.9" + "scoutx"); + }; + auto out = resolve_intent(in, hybrid_cfg(), llm, scorer, 0.5); + CHECK(llm_called); + CHECK(out.source == "llm"); + CHECK(out.target_agent == "scout"); + CHECK(out.decision_consulted); +} + +TEST_CASE("the scorer sees a clipped utterance") { + IntentInput in; + in.text = std::string(kIntentDecisionStateMaxBytes, 'a') + "TAIL"; + in.requested_agent = "index"; + in.roster = starter_roster(); + auto scorer = [&](const std::string& state, const std::vector&) { + CHECK(state.size() <= kIntentDecisionStateMaxBytes); + CHECK(state.find("TAIL") == std::string::npos); + return peaked("unknown", 0.5, 0.1); + }; + auto out = resolve_intent(in, hybrid_cfg(), nullptr, scorer, 0.5); + CHECK(out.decision_consulted); + CHECK(out.source == "heuristic"); + CHECK(out.kind == "unknown"); +} diff --git a/tests/test_label_score.cpp b/tests/test_label_score.cpp new file mode 100644 index 00000000..b390a4e0 --- /dev/null +++ b/tests/test_label_score.cpp @@ -0,0 +1,159 @@ +// tests/test_label_score.cpp — Renormalized label distributions and the +// env gates. No provider client. + +#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN +#include "doctest.h" +#include "label_score.h" + +using namespace arbiter; + +static std::vector two() { + return { + {'S', "silent", "stay quiet"}, + {'C', "context", "a note would help"}, + }; +} + +static std::vector three() { + return { + {'R', "research", "sources"}, + {'W', "write", "prose"}, + {'U', "unknown", "none of these"}, + }; +} + +TEST_CASE("softmax over the supplied letters only") { + auto d = distribution_from_logprobs(two(), { + {"S", 0.0}, + {"C", -10.0}, + {"Z", 5.0}, // not a label; ignored + }); + REQUIRE(d.ok); + CHECK(d.argmax == "silent"); + CHECK(d.peak > 0.999); + CHECK(d.margin > 0.99); + CHECK(d.probs.size() == 2); + CHECK(d.probs[0].first == "silent"); +} + +TEST_CASE("a missing letter is not a zero") { + auto d = distribution_from_logprobs(three(), { + {"R", 0.0}, + {"W", -0.5}, + }); + CHECK_FALSE(d.ok); + CHECK(d.argmax.empty()); + CHECK(d.probs.empty()); +} + +TEST_CASE("whitespace around a letter matches; a word does not") { + auto spaced = distribution_from_logprobs(two(), { + {" S", 0.0}, + {"\nC", -1.0}, + }); + CHECK(spaced.ok); + CHECK(spaced.argmax == "silent"); + + auto word = distribution_from_logprobs(two(), { + {"Silent", 0.0}, + {"C", -0.2}, + }); + CHECK_FALSE(word.ok); +} + +TEST_CASE("the higher logprob wins when a letter is repeated") { + auto d = distribution_from_logprobs(two(), { + {"s", -3.0}, + {"S", 0.0}, + {"C", -1.0}, + }); + REQUIRE(d.ok); + CHECK(d.argmax == "silent"); + CHECK(d.peak > d.probs[1].second); +} + +TEST_CASE("openai body reads top_logprobs and ignores other shapes") { + const char* body = R"({ + "choices": [{ + "message": {"content": "R"}, + "logprobs": { + "content": [{ + "token": "R", + "logprob": -0.1, + "top_logprobs": [ + {"token": "R", "logprob": -0.1}, + {"token": "W", "logprob": -2.0}, + {"token": "U", "logprob": -3.0} + ] + }] + } + }] + })"; + auto d = distribution_from_openai_body(three(), body); + REQUIRE(d.ok); + CHECK(d.argmax == "research"); + CHECK(d.peak > 0.5); + + CHECK_FALSE(distribution_from_openai_body(three(), "{}").ok); + CHECK_FALSE(distribution_from_openai_body(three(), "not json").ok); +} + +TEST_CASE("extreme requires the peak floor and a positive margin") { + LabelDistribution d; + d.ok = true; + d.argmax = "silent"; + d.peak = 0.96; + d.margin = 0.9; + CHECK(label_distribution_is_extreme(d, 0.5)); + CHECK_FALSE(label_distribution_is_extreme(d, 0.0)); + d.peak = 0.85; + CHECK_FALSE(label_distribution_is_extreme(d, 0.5)); + d.peak = 0.96; + d.margin = 0.2; + CHECK_FALSE(label_distribution_is_extreme(d, 0.5)); + d.ok = false; + d.margin = 0.9; + CHECK_FALSE(label_distribution_is_extreme(d, 0.5)); +} + +TEST_CASE("margins parse off by default and only ollama ids arm the filter") { + auto off = decision_filter_from_values(nullptr, nullptr, nullptr); + CHECK_FALSE(off.enabled_for_presence()); + CHECK_FALSE(off.enabled_for_intent()); + + auto local = decision_filter_from_values("ollama/qwen3:0.6b", "0.5", "0.4"); + CHECK(local.enabled_for_presence()); + CHECK(local.enabled_for_intent()); + CHECK(local.presence_silence_margin == doctest::Approx(0.5)); + + auto hosted = decision_filter_from_values("anthropic/claude-haiku-4-5", "0.5", "0.5"); + CHECK_FALSE(hosted.enabled_for_presence()); + CHECK_FALSE(hosted.enabled_for_intent()); + + auto clamped = decision_filter_from_values("ollama/qwen", "2", "nope"); + CHECK(clamped.presence_silence_margin == doctest::Approx(1.0)); + CHECK(clamped.intent_route_margin == doctest::Approx(0.0)); + CHECK(clamped.enabled_for_presence()); + CHECK_FALSE(clamped.enabled_for_intent()); +} + +TEST_CASE("prompt lists the letter, the name, and the description") { + auto prompt = label_score_user_prompt("Task: hello", two()); + CHECK(prompt.find("Task: hello") != std::string::npos); + CHECK(prompt.find("S. silent — stay quiet") != std::string::npos); + CHECK(prompt.find("C. context — a note would help") != std::string::npos); +} + +TEST_CASE("intent labels include unknown and omit multi") { + auto specs = intent_decision_labels(); + bool unknown = false; + bool multi = false; + for (const auto& s : specs) { + if (s.name == "unknown") unknown = true; + if (s.name == "multi") multi = true; + CHECK(s.letter != 0); + CHECK_FALSE(s.description.empty()); + } + CHECK(unknown); + CHECK_FALSE(multi); +} diff --git a/tests/test_presence.cpp b/tests/test_presence.cpp index ffdc9934..83147cf9 100644 --- a/tests/test_presence.cpp +++ b/tests/test_presence.cpp @@ -173,3 +173,38 @@ TEST_CASE("parse: note cap does not split trailing UTF-8") { const unsigned char last = static_cast(out.text.back()); CHECK((last & 0xC0) != 0x80); // not a continuation byte } + +TEST_CASE("filter state keeps the first task line and tool names only") { + std::string task = "Ship the login form\npassword=hunter2 result=TOKENBODY"; + std::string tools = + "- fetch args=https://secret.example result=TOKENBODY\n" + "- exec args=rm -rf result=boom\n"; + auto state = presence_filter_state(task, tools); + CHECK(state.find("Ship the login form") != std::string::npos); + CHECK(state.find("fetch") != std::string::npos); + CHECK(state.find("exec") != std::string::npos); + CHECK(state.find("hunter2") == std::string::npos); + CHECK(state.find("TOKENBODY") == std::string::npos); + CHECK(state.find("secret.example") == std::string::npos); + CHECK(state.find("boom") == std::string::npos); +} + +TEST_CASE("silence skip is only an extreme silent") { + LabelDistribution silent; + silent.ok = true; + silent.argmax = "silent"; + silent.peak = 0.97; + silent.margin = 0.9; + CHECK(presence_filter_skips_review(silent, 0.5)); + + LabelDistribution context = silent; + context.argmax = "context"; + CHECK_FALSE(presence_filter_skips_review(context, 0.5)); + + LabelDistribution flat = silent; + flat.peak = 0.55; + flat.margin = 0.1; + CHECK_FALSE(presence_filter_skips_review(flat, 0.5)); + + CHECK_FALSE(presence_filter_skips_review(silent, 0.0)); +}