From 0f11800d645aa6f125682e1e1d14299db6faa6e3 Mon Sep 17 00:00:00 2001 From: Yogthos Date: Thu, 24 Sep 2026 22:16:18 -0400 Subject: [PATCH] Native tools for DeepSeek and GLM, DSML calls, GLM business codes deepseek-v4-flash wrote its DSML call markup into the content, and every such turn was a no-call shown raw in the agent log. The parser now rewrites DSML tags (fullwidth or ASCII bars, doubled, spaced) to the XML call form it already reads, and honours string="true"/"false" on parameters. The likely cause was that tools only travelled natively when a gate forced one; DeepSeek's and Zhipu's own harnesses send them on every request. The tool surface is now derived from the signatures in the branch's system prompt and sent as `tools` wherever the endpoint has :native-tools (DeepSeek, GLM), except beside a prefill, which DeepSeek rejects. Native calls fold back into fences with the prose around them kept. Measured live on both providers: replies come back as tool_calls. GLM business codes are classified by code, not only by status: the retryable ones (from ZCode's table) retry under a 400 or inside a 200, 3008-3010 and 1304 are now caps, errors in 200 bodies go through the same path, and an error frame mid-stream is returned instead of reading as an empty reply. The Anthropic-compatible endpoints were evaluated live and bring nothing over chat-completions with native tools while history is text. --- docs/RFCS/RFC-005-provider-layer.md | 1 + resources/gates.edn | 9 ++ resources/manual.edn | 2 +- resources/wordlists.edn | 28 ++++-- src/samizdat/agent/infer.clj | 13 ++- src/samizdat/config.clj | 12 ++- src/samizdat/llm/adapter/openai.clj | 63 +++++++++--- src/samizdat/llm/client.clj | 102 +++++++++++++------ src/samizdat/llm/fence.clj | 59 ++++++++--- src/samizdat/llm/stream.clj | 7 +- src/samizdat/llm/toolspec.clj | 113 +++++++++++++++++++++ test/samizdat/infer_test.clj | 15 +++ test/samizdat/llm_stream_test.clj | 11 +++ test/samizdat/llm_test.clj | 147 +++++++++++++++++++++++++++- test/samizdat/test_runner.clj | 2 + test/samizdat/toolspec_test.clj | 76 ++++++++++++++ 16 files changed, 585 insertions(+), 75 deletions(-) create mode 100644 src/samizdat/llm/toolspec.clj create mode 100644 test/samizdat/toolspec_test.clj diff --git a/docs/RFCS/RFC-005-provider-layer.md b/docs/RFCS/RFC-005-provider-layer.md index 0491cfb0..2ddb7885 100644 --- a/docs/RFCS/RFC-005-provider-layer.md +++ b/docs/RFCS/RFC-005-provider-layer.md @@ -156,6 +156,7 @@ is the question a cell, a gate or an adapter asks. |---|---| | `:prefill` | a trailing assistant message is continued (DeepSeek /beta, llama.cpp --jinja) | | `:native-tool-choice` | tools + `tool_choice {type function}` | +| `:native-tools` | the whole tool surface as `tools` on every turn not forced by a prefill or grammar | | `:grammar` | a GBNF `grammar` applied at sampling (llama.cpp) | | `:cache-prompt` | `cache_prompt` / `id_slot` prefix reuse (llama.cpp) | | `:reasoning-budget` | `reasoning_budget_tokens` per call (llama.cpp) | diff --git a/resources/gates.edn b/resources/gates.edn index 88b345b5..0dc9c7c2 100644 --- a/resources/gates.edn +++ b/resources/gates.edn @@ -4054,6 +4054,15 @@ :prediction "the branch commits a change, runs a test, or ships" :window 3}] + :native-tool-description-chars + {:value 240 :kind :threshold :capability-tunable? true + :provenance ["karamazov-jl1f"] + :doc "How many characters of each tool's description go in the native + `tools` array (samizdat.llm.toolspec), cut at a sentence end. The + array rides in every request and the full prose is already in the + system prompt, so a sentence or two is what it is for; eval's + paragraph is ~1500 characters. Absent, descriptions go whole."} + :forceable-tools ;; Schemas for the tools a gate may FORCE via native tool_choice (tier 3b: ;; data). Only the terminal tools — forcing a mid-task tool would be the diff --git a/resources/manual.edn b/resources/manual.edn index 8777e064..ac9d273e 100644 --- a/resources/manual.edn +++ b/resources/manual.edn @@ -213,7 +213,7 @@ {:name samizdat.llm.grammar/fence-grammar :summary "The GBNF under which a llama.cpp reply's tool calls are well-formed fences naming only the listed tools — required (the force) or optional (the restriction). Cache-safe: applied at sampling, never in the prompt. gates.edn :local-grammar says when it is sent."} {:name samizdat.config/supports? - :summary "Whether this run's endpoint has a feature (:prefill :grammar :cache-prompt :reasoning-budget :thinking-toggle :native-tool-choice :reasoning-effort) — declared per provider preset, added to by the startup probe, editable in a config file's :llm :features. The one question to ask before choosing a mechanism; the adapter emits only what the features allow."} + :summary "Whether this run's endpoint has a feature (:prefill :grammar :cache-prompt :reasoning-budget :thinking-toggle :native-tool-choice :native-tools :reasoning-effort) — declared per provider preset, added to by the startup probe, editable in a config file's :llm :features. The one question to ask before choosing a mechanism; the adapter emits only what the features allow."} {:name samizdat.config/features-of :summary "The features a provider preset declares, before discovery."} {:name samizdat.agent.infer/force-mechanism diff --git a/resources/wordlists.edn b/resources/wordlists.edn index 07f3da18..ce847823 100644 --- a/resources/wordlists.edn +++ b/resources/wordlists.edn @@ -165,19 +165,31 @@ ;; body with a message that names nothing in the English list. The set is ;; the vendor's own — ZCode (Zhipu's client, karamazov-luqc.3) keeps it as ;; the codes a workflow stops on to wait for quota rather than retry: 1005 - ;; 1308 1310 1313 1316-1321 2056 20097 — and its OpenAI/Anthropic-family - ;; strings for the same wall (credit_balance_exhausted, the spend_limit and - ;; usage_limit families). What that table marks RETRYABLE is deliberately - ;; NOT here: 1302/1303/1305 are per-minute limits and 3008-3010 are - ;; concurrency caps, both of which clear on their own and are the retry - ;; ladder's and the rate-limit latch's case. Unmeasured on this side; the - ;; codes are bounded on both ends so a token count cannot match them. + ;; 1304 1308 1310 1313 1316-1321 2056 20097 3008-3010 — and its + ;; OpenAI/Anthropic-family strings for the same wall (credit_balance_exhausted, + ;; the spend_limit and usage_limit families). 3008-3010 and 1304 were once + ;; read as retryable concurrency caps; the table, read directly on + ;; 2026-09-24, marks all four retryable false (karamazov-jvdu). What it + ;; marks RETRYABLE is :retryable-business-codes below. The codes are bounded + ;; on both ends so a token count cannot match them. ;; ;; Here rather than in the adapter because it is a fact about PROVIDERS, and ;; providers change their error text without asking. A user hitting a new ;; gateway's wording should be able to add it and keep working, not wait for ;; a release. Kept as a string because EDN has no regex literal. - "(?i)insufficient|quota|billing|exceeded your current|payment|per-day|daily limit|credit_balance_exhausted|spend_limit_exceeded|usage_limit_exceeded|使用上限|每日|\\b(1005|1308|1310|1313|131[6-9]|132[01]|2056|20097)\\b" + "(?i)insufficient|quota|billing|exceeded your current|payment|per-day|daily limit|credit_balance_exhausted|spend_limit_exceeded|usage_limit_exceeded|使用上限|每日|\\b(1005|1304|1308|1310|1313|131[6-9]|132[01]|2056|20097|300[89]|3010)\\b" + + :retryable-business-codes + ;; BigModel business codes that mean TRY AGAIN — a per-minute limit (1302 + ;; 1303 1305), an overload (1312), a server or network hiccup (1120 1230 + ;; 1234 2007 3002) — from ZCode's own table (Zhipu's client, + ;; failure-provider-business-codes.ts, read 2026-09-24). Retried whatever + ;; HTTP status carries them: GLM answers some under a 400, or inside a 200, + ;; and the status ladder alone called those fatal (karamazov-jvdu). Matched + ;; against the CODE — the body's code / error_code field, or a `[1302]` + ;; prefix on its message — never against free text, where a figure is not a + ;; code. A set of strings. + #{"1120" "1230" "1234" "1302" "1303" "1305" "1312" "2007" "3002"} :tool-version ;; A tool name followed by its version. Stripped BEFORE tokenizing, diff --git a/src/samizdat/agent/infer.clj b/src/samizdat/agent/infer.clj index 1637c5fe..2605a723 100644 --- a/src/samizdat/agent/infer.clj +++ b/src/samizdat/agent/infer.clj @@ -63,6 +63,7 @@ [samizdat.llm.fence :as fence] [samizdat.llm.grammar :as grammar] [samizdat.llm.message :as message] + [samizdat.llm.toolspec :as toolspec] [samizdat.store.journal :as journal] [samizdat.tape :as tape])) @@ -378,7 +379,16 @@ ;; endpoint's features and policy (force-mechanism). {:keys [grammar] forced-prefill :prefill forced-tool :force-tool} (force-mechanism ctx tape) - reasoning-budget (reasoning-budget-for ctx tape)] + reasoning-budget (reasoning-budget-for ctx tape) + ;; The tool surface as native specs, read off the documentation + ;; this branch's system prompt gives it, gates.edn's exact schemas + ;; winning (llm.toolspec). Only for an endpoint measured to take + ;; them; the adapter still leaves them off a prefilled or grammar + ;; turn (karamazov-jl1f). + tools (when (config/supports? (:llm-config ctx) :native-tools) + (not-empty (toolspec/for-messages (:messages tape) + (gates/threshold :forceable-tools) + (gates/threshold :native-tool-description-chars))))] (loop [attempt 1] (let [;; Somebody may be watching the run: its calls stream, and what ;; the model writes goes onto the bus as it is written. Not a @@ -408,6 +418,7 @@ ;; A per-call thinking cap on llama.cpp ;; (reasoning-budget-for). reasoning-budget (assoc :reasoning-budget reasoning-budget) + tools (assoc :tools tools) ;; The stable conversation key an endpoint ;; pins its prefix cache to. Only the local ;; adapter emits it; see LR-5. diff --git a/src/samizdat/config.clj b/src/samizdat/config.clj index 575572bc..205c4dbe 100644 --- a/src/samizdat/config.clj +++ b/src/samizdat/config.clj @@ -257,7 +257,11 @@ ;; URL continues a flagged assistant prefix; `thinking {type ;; disabled}` reliably yields no reasoning; a native tool_choice ;; is honoured, but only with thinking off (the adapter knows). - :features #{:prefill :native-tool-choice :thinking-toggle :reasoning-effort :stream} + ;; :native-tools — the tool surface sent on every request — + ;; measured 2026-09-24: native tool_calls back with thinking on, + ;; and no DSML in the content (samizdat.llm.toolspec). + :features #{:prefill :native-tool-choice :native-tools :thinking-toggle + :reasoning-effort :stream} :key-env "DEEPSEEK_API_KEY" ;; deepseek-v4-flash is the development and test model: cheap ;; enough to run the beam repeatedly. deepseek-v4-pro is the @@ -275,7 +279,9 @@ ;; Measured 2026-09-06: ignores a trailing assistant prefix ;; entirely, cannot be told not to think (effort low is the ;; least), honours tool_choice {type function} despite its docs. - :features #{:native-tool-choice :reasoning-effort :stream} + ;; :native-tools measured 2026-09-24 as for DeepSeek, and it is + ;; how Zhipu's own harness (ZCode) calls every GLM model. + :features #{:native-tool-choice :native-tools :reasoning-effort :stream} :key-env "ZHIPU_API_KEY" :model "glm-5.3" ;; GLM benefits from a low temperature on coding tasks (dirge @@ -320,6 +326,8 @@ ;; :prefill continues a trailing assistant message (the fence ;; force; the reply content may or may not repeat it) ;; :native-tool-choice tools + tool_choice {type function} +;; :native-tools the whole tool surface sent as `tools` on every turn +;; a prefill or grammar does not force (llm.toolspec) ;; :grammar a GBNF `grammar` field applied at sampling (llama.cpp) ;; :cache-prompt `cache_prompt` / `id_slot` prefix-cache reuse (llama.cpp) ;; :reasoning-budget `reasoning_budget_tokens` per call (llama.cpp) diff --git a/src/samizdat/llm/adapter/openai.clj b/src/samizdat/llm/adapter/openai.clj index ebeea180..01d5989f 100644 --- a/src/samizdat/llm/adapter/openai.clj +++ b/src/samizdat/llm/adapter/openai.clj @@ -46,17 +46,34 @@ (defn- tool-call->fence "A native OpenAI tool_call turned into the harness's text-fence convention, so - a forced (tool_choice) response parses through exactly the same path a normal - fenced response does. `arguments` is a JSON string; it is parsed and re-embedded - as `args` so the downstream fence parser reads one object." + a native call parses through exactly the same path a fenced one does. + `arguments` is a JSON string, re-embedded as `args` when it parses — and + embedded AS WRITTEN when it does not, so the fence parser's repair ladder + gets its chance and a failure is reported to the model as the parse error + it is, rather than becoming an empty `{}` the tool then calls missing." [tc] (let [name (get-in tc [:function :name]) - args (try (json/read-str (str (get-in tc [:function :arguments]))) - (catch Throwable _ {}))] + raw (str (get-in tc [:function :arguments])) + args (try (json/read-str raw) (catch Throwable _ ::unparsed))] (str "```tool-call\n" - (json/write-str {:name name :args args}) + (if (= ::unparsed args) + (str "{\"name\": " (json/write-str name) ", \"args\": " raw "}") + (json/write-str {:name name :args args})) "\n```"))) +(defn- native-content + "What a reply said, with every native call it made appended as a fence: + the words a model wrote beside its call are kept (glm-5.3 writes a + sentence and a call), and several calls stay several fences, for the fence + rule to pick from and count rather than for this to drop quietly." + [msg] + (let [calls (:tool_calls msg) + said (str/trim (str (:content msg)))] + (if (seq calls) + (str/join "\n\n" (cond->> (map tool-call->fence calls) + (seq said) (cons said))) + (:content msg)))) + (defn- supports-prefill? "Whether this endpoint continues a flagged trailing assistant message: config :llm :features says (samizdat.config), not the provider id and not @@ -158,7 +175,7 @@ {})) (chat-body [this config {:keys [messages max-tokens temperature prefill force-tool - cache-key reasoning-effort grammar reasoning-budget]}] + cache-key reasoning-effort grammar reasoning-budget tools]}] ;; The gate is the protocol method on THIS adapter (provenance R3-14), so the ;; answer a caller can query and the answer chat-body acts on are one ;; path and cannot drift apart. @@ -182,7 +199,17 @@ ;; because tool_choice is incompatible with DeepSeek's thinking mode; ;; and where a grammar forces it, the tools array is not sent either, ;; because the template would render it into the prefix. - forcing? (and force-tool (not use-prefill?) (not grammar*))] + forcing? (and force-tool (not use-prefill?) (not grammar*)) + ;; The whole tool surface, natively, on every turn the endpoint takes + ;; it (config :native-tools, measured) — the way DeepSeek's and + ;; Zhipu's own harnesses call their models (samizdat.llm.toolspec). + ;; Never beside a prefill: DeepSeek answers `Function call should + ;; not be used with prefix` (400, measured 2026-09-24). Never beside + ;; a grammar, which forces the call at sampling on an endpoint that + ;; renders tools into the prefix. + native (when (and (seq tools) (config/supports? config :native-tools) + (not use-prefill?) (not grammar*)) + (mapv (fn [t] {:type "function" :function t}) tools))] (cond-> {:model (:model config) :messages (if use-prefill? ;; `:prefix true` is what makes the provider CONTINUE @@ -205,9 +232,16 @@ grammar* (assoc :grammar grammar*) - ;; Only the forced tool is exposed. + native (assoc :tools native) + + ;; A force names its tool. With the surface already native the array + ;; stays whole, so the forced turn's prefix is the same bytes as every + ;; other turn's; otherwise only the forced tool is exposed. forcing? - (assoc :tools [{:type "function" :function force-tool}] + (assoc :tools (cond + (nil? native) [{:type "function" :function force-tool}] + (some #(= (:name force-tool) (get-in % [:function :name])) native) native + :else (conj native {:type "function" :function force-tool})) :tool_choice {:type "function" :function {:name (:name force-tool)}}) @@ -230,12 +264,9 @@ (parse-chat [_ body] (when-let [choice (first (:choices body))] (let [msg (:message choice)] - {;; A forced tool_choice response carries the call in tool_calls, not - ;; content — fold it into the fence convention so downstream is blind to - ;; how the call was produced. - :content (if-let [tc (first (:tool_calls msg))] - (tool-call->fence tc) - (:content msg)) + {;; A native call arrives in tool_calls, not content — folded into the + ;; fence convention so downstream is blind to how the call was made. + :content (native-content msg) :reasoning (get msg reasoning-key) :finish-reason (or (:finish_reason choice) "stop") ;; The model that ANSWERED, as the provider names it, or nil when diff --git a/src/samizdat/llm/client.clj b/src/samizdat/llm/client.clj index c4eecda2..e550b6d4 100644 --- a/src/samizdat/llm/client.clj +++ b/src/samizdat/llm/client.clj @@ -169,15 +169,39 @@ overhead)] (min ceiling (max floor derived)))) +(defn business-codes + "The provider business codes an error body carries, as strings: its + `error.code`, `code` or `error_code`, and a `[1302]`-style prefix on its + message — how BigModel reports them (ZCode reads the same places). Never a + number found in free text: a token count is not a code." + [body] + (let [fields [(get-in body [:error :code]) (:code body) (:error_code body)] + texts [(get-in body [:error :message]) (:message body) (:msg body) (:error_msg body)]] + (into #{} + (concat (keep #(when (and (some? %) (re-matches #"\d{3,6}" (str %))) (str %)) fields) + (keep #(second (re-find #"^\s*\[(\d{3,6})\]" (str %))) texts))))) + +(defn- retryable-code? + "Whether the body names a business code wordlists.edn + :retryable-business-codes lists — a per-minute limit, an overload, a + server hiccup — which is retried whatever status carries it + (karamazov-jvdu). An older project without the list retries none this way." + [body] + (boolean (some (or (lexicon/wordlist :retryable-business-codes) #{}) + (business-codes body)))) + (defn classify - "Decide what to do about a non-2xx response. + "Decide what to do about a response that failed. :retry — transient, try again. :fatal — do not retry, the answer will not change. Anything unrecognized is fatal, because retrying an error we do not - understand spends budget to learn nothing." + understand spends budget to learn nothing. A retryable business code wins + over the status: GLM carries per-minute limits under a 400, and inside a + 200." [adapter status body] (cond (and (= 429 status) (adapter/usage-cap? adapter status body)) :fatal + (retryable-code? body) :retry (= 429 status) :retry (>= status 500) :retry (= 408 status) :retry @@ -238,6 +262,38 @@ (cancel/after-blocking!) r)))) +(defn- failure + "The outcome of a response that failed — a non-2xx, or a 2xx whose body is + an error: the retry decision, the error text, and the reason when there is + one (a context overflow, a usage cap)." + [adapter status decoded resp] + ;; A context overflow outranks the status-code ladder: it is the one + ;; 5xx that is deterministic, and the reason travels from where it is + ;; detected so the loop can respond by compacting rather than retrying + ;; (karamazov-d41). + (let [overflow? (context-overflow? (:body resp)) + ;; A wall, not a window: DeepSeek answers 402 "Insufficient + ;; Balance", and any status can carry the usage-cap wording the + ;; adapter knows. Retrying spends the run's budget against + ;; something that will not move (dirge PR 689), so it is fatal and + ;; the reason travels so a supervisor tells a wall from a bug. + cap? (and (not overflow?) + (or (= 402 status) + (adapter/usage-cap? adapter status decoded)))] + (cond-> {:outcome (if (or overflow? cap?) + :fatal + (classify adapter status decoded)) + :headers (:headers resp) + :error (str (adapter/display-name adapter) " error " status + (when-let [m (or (adapter/error-message adapter decoded) + (some-> (:msg decoded) str not-empty))] + (str " — " m)) + (when-not decoded + (str " — " (subs (str (:body resp)) + 0 (min 300 (count (str (:body resp))))))))} + overflow? (assoc :reason :context-overflow) + cap? (assoc :reason :usage-cap)))) + (defn- post-once* [adapter config request url] (let [;; Whether the prefill in the request was actually sent — the adapter ;; drops it where the endpoint cannot continue a trailing assistant @@ -277,9 +333,13 @@ (log/warn (adapter/display-name adapter) "rate-limited for" (long (/ ms 1000)) "s — latched for every branch")) (if (<= 200 status 299) - (if-let [err (adapter/error-message adapter decoded)] - ;; Some providers return 200 with an error object in the body. - {:outcome :fatal :error (str (adapter/display-name adapter) " API error: " err)} + (if (or (adapter/error-message adapter decoded) + (and (false? (:success decoded)) (seq (business-codes decoded)))) + ;; Some providers return 200 with an error object in the body — GLM + ;; with its business codes. Classified like any other failure, so a + ;; per-minute limit there is retried and a 1261 is an overflow + ;; rather than a flat "API error" (karamazov-jvdu). + (failure adapter status decoded resp) (if-let [parsed (adapter/parse-chat adapter decoded)] (let [merged (message/merge-reasoning (:content parsed) (:reasoning parsed))] (if (str/blank? merged) @@ -341,30 +401,7 @@ :error (str (adapter/display-name adapter) " reply had no completion in it: " (subs (str (:body resp)) 0 (min 300 (count (str (:body resp))))))})) - ;; A context overflow outranks the status-code ladder: it is the one - ;; 5xx that is deterministic, and the reason travels from where it is - ;; detected so the loop can respond by compacting rather than retrying - ;; (karamazov-d41). - (let [overflow? (context-overflow? (:body resp)) - ;; A wall, not a window: DeepSeek answers 402 "Insufficient - ;; Balance", and any status can carry the usage-cap wording the - ;; adapter knows. Retrying spends the run's budget against - ;; something that will not move (dirge PR 689), so it is fatal and - ;; the reason travels so a supervisor tells a wall from a bug. - cap? (and (not overflow?) - (or (= 402 status) - (adapter/usage-cap? adapter status decoded)))] - (cond-> {:outcome (if (or overflow? cap?) - :fatal - (classify adapter status decoded)) - :headers (:headers resp) - :error (str (adapter/display-name adapter) " error " status - (when-let [m (adapter/error-message adapter decoded)] (str " — " m)) - (when-not decoded - (str " — " (subs (str (:body resp)) - 0 (min 300 (count (str (:body resp))))))))} - overflow? (assoc :reason :context-overflow) - cap? (assoc :reason :usage-cap)))))) + (failure adapter status decoded resp)))) ;; --- the public surface ----------------------------------------------------- @@ -379,7 +416,7 @@ ([adapter config messages] (chat adapter config messages nil)) ([adapter config messages {:keys [max-tokens temperature max-retries prefill force-tool cache-key reasoning-effort grammar reasoning-budget - on-delta]}] + on-delta tools]}] (let [request {:messages (message/prepare messages) :max-tokens (or max-tokens (:max-tokens config)) :temperature (or temperature (:temperature config)) @@ -414,7 +451,10 @@ ;; Somebody watching the reply as it is written: with an ;; endpoint that can stream (:stream), each delta goes to ;; it as it arrives (samizdat.llm.stream). - :on-delta on-delta} + :on-delta on-delta + ;; The tool surface as native specs (samizdat.llm.toolspec); + ;; the adapter sends it where the endpoint's features say. + :tools tools} ;; The read timeout is sized to the budget being asked for: a big ;; max-tokens legitimately takes longer than a small one, and a fixed ;; bound cut off long generations and re-billed them (see diff --git a/src/samizdat/llm/fence.clj b/src/samizdat/llm/fence.clj index f790eeaa..4b9e3658 100644 --- a/src/samizdat/llm/fence.clj +++ b/src/samizdat/llm/fence.clj @@ -470,11 +470,17 @@ ;; contains a literal `]*>(.*?)]*>") + ;; + ;; The attributes are captured: `string="false"` is the model saying the + ;; body is a JSON value (see xml-value). + #"(?s)]*)>(.*?)]*>") (defn- xml-value - "A parameter's value. Verbatim, except that something which is entirely a - number becomes one. + "A parameter's value, given the tag's attributes. DeepSeek's form says + which it is: `string=\"true\"` is text, kept verbatim, and `string=\"false\"` + is a JSON value — a boolean, a number, a list — read as one (verbatim if it + does not parse). Without the attribute: verbatim, except that something + which is entirely a number becomes one. Values here are NOT JSON-escaped — that is the whole reason a model reaches for this form when handing over a Lean proof — so the text is kept exactly @@ -482,9 +488,15 @@ not cosmetic: `top_k` reaches `(take k)` and a string throws there. Anchored and strict, so `s#1392` and a claim that merely mentions a figure stay strings." - [s] - (let [t (str/trim s)] + [attrs s] + (let [t (str/trim s) + declared (second (re-find #"string=\"(true|false)\"" (str attrs)))] (cond + (= "true" declared) s + + (= "false" declared) + (let [{:keys [ok value]} (read-json t)] (if ok value s)) + (re-matches #"-?\d+" t) (parse-long t) ;; A JSON array or object. The XML parameter form has no way to express @@ -504,6 +516,25 @@ :else s))) +;; DeepSeek's DSML: the same XML call form with each tag name prefixed +;; `｜DSML｜`, in FULLWIDTH bars (U+FF5C) — documented as +;; `<｜DSML｜function_calls>`, and written by deepseek-v4-flash in run ab935047 +;; as `<｜｜DSML｜｜ calls>`, bars doubled and a space after. Neither matched +;; , so every such call was a no-call (11 turns in endless-flight). +;; Rewritten to the plain form rather than given patterns of its own, so it +;; gets every tolerance the XML form has already earned. ASCII bars too. +(def ^:private dsml-calls-re #"(") +(def ^:private dsml-tag-re #"( (str s) + (str/replace dsml-calls-re "$1function_calls>") + (str/replace dsml-tag-re "$1"))) + (defn- xml-call "The response's last complete , as {:name :args}, or nil. @@ -511,12 +542,15 @@ model that drafts one call while reasoning and then issues the real one puts the real one last." [response] - (when-let [m (last (re-seq invoke-re (or response "")))] - (let [[_ nm body] m] - (when-not (str/blank? nm) - {:name nm - :args (reduce (fn [acc [_ k v]] (assoc acc (keyword k) (xml-value v))) - {} (re-seq parameter-re (or body "")))})))) + (let [dsml (dsml? response)] + (when-let [m (last (re-seq invoke-re (undsml response)))] + (let [[_ nm body] m] + (when-not (str/blank? nm) + (cond-> {:name nm + :args (reduce (fn [acc [_ k attrs v]] (assoc acc (keyword k) (xml-value attrs v))) + {} (re-seq parameter-re (or body "")))} + dsml (assoc :dsml? true))))))) + ;; Qwen's chat template wraps its native calls in tags around ;; plain JSON. Under a prefilled fence the model emits prose then its native @@ -660,7 +694,6 @@ #"(?s).*?(?:|\z)" #"(?s).*?(?:|\z)" #"(?s)]*>.*?(?:|\z)" - #"(?s)<\|DSML\|[^>]*>.*?(?:]*>|\z)" #"]*>"]) (defn prose @@ -669,7 +702,7 @@ the other — and the blank runs the cuts leave collapsed. Display only; the transcript keeps the reply as it came." [s] - (-> (reduce #(str/replace %1 %2 "") (strip-think s) call-markup-res) + (-> (reduce #(str/replace %1 %2 "") (undsml (strip-think s)) call-markup-res) (str/replace #"\n[ \t]*(?:\n[ \t]*){2,}" "\n\n") str/trim)) diff --git a/src/samizdat/llm/stream.clj b/src/samizdat/llm/stream.clj index de811206..f6f18f55 100644 --- a/src/samizdat/llm/stream.clj +++ b/src/samizdat/llm/stream.clj @@ -56,6 +56,10 @@ (let [{:keys [delta finish_reason]} (first (:choices chunk))] (cond-> (merge acc (select-keys chunk [:id :model :created])) (:usage chunk) (assoc :usage (:usage chunk)) + ;; An error frame mid-stream — GLM sends its business codes this way. + ;; It has no choices, so it folded into nothing and the call read as an + ;; empty reply (karamazov-jvdu); kept, it comes back as the reply. + (:error chunk) (assoc :error (:error chunk)) finish_reason (assoc :finish-reason finish_reason) delta (update :message @@ -79,7 +83,8 @@ (update :role #(or % "assistant"))) (seq calls) (assoc :tool_calls (mapv val (sort-by key calls)))) :finish_reason finish-reason}]} - usage (assoc :usage usage)))) + usage (assoc :usage usage) + (:error acc) (assoc :error (:error acc))))) (defn delta "What a chunk adds that a reader sees: {:text} and/or {:reasoning}, or nil." diff --git a/src/samizdat/llm/toolspec.clj b/src/samizdat/llm/toolspec.clj new file mode 100644 index 00000000..5e257074 --- /dev/null +++ b/src/samizdat/llm/toolspec.clj @@ -0,0 +1,113 @@ +;; samizdat - a claim-first verification harness +;; Copyright (C) 2026 Dmitri Sotnikov +;; +;; This program is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. +;; +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; +;; You should have received a copy of the GNU General Public License +;; along with this program. If not, see . +;; +;; SPDX-License-Identifier: GPL-3.0-or-later + +(ns samizdat.llm.toolspec + "Native tool specs — the `tools` array of an OpenAI-compatible request — + read off the tool documentation a branch was actually given. + + WHY NATIVE AT ALL. The harness's protocol is a fenced ```tool-call block in + the text, and the tools used to travel natively only when a gate forced + one. DeepSeek's and Zhipu's own harnesses send `tools` on every request and + parse nothing out of text (2026-09-24 review); with no `tools` in a + request, deepseek-v4-flash wrote its trained call format, DSML, into the + content instead. Measured the same day: with `tools` present, both + deepseek-v4-flash and glm-5.3 answered with native tool_calls — even with + fenced calls in the history and a system prompt asking for fences. + + WHY FROM THE PROMPT. Every tool is documented to the model as a signature + line — `edit_file({path, old_text, new_text, replace_all?})` — followed by + an indented description, in resources/prompts. That prose is the single + source: userspace, editable at runtime, and exactly the set this branch's + role was shown. A second hand-written schema per tool would drift from it. + So a derived spec names the arguments and REQUIRES NONE: the prose says + which are required, the tool checks, and a schema that wrongly required one + would have the model inventing a value to satisfy it. Where gates.edn + :forceable-tools holds an exact schema, that one is sent instead. + + Pure: text in, specs out." + (:require [clojure.string :as str])) + +(def ^:private signature-re + ;; A signature is a whole line: a tool name, then `({args})` or `()`. + ;; Anchored so a mention inside a sentence — `call done({answer}) when…` — + ;; is not a second definition. + #"(?m)^([a-z_][a-z0-9_]*)\((?:\{([^}]*)\})?\)\s*$") + +(defn- clip + "`s` cut at the last sentence end within `limit` characters, or at the + limit; whole when `limit` is nil." + [s limit] + (if (or (nil? limit) (<= (count s) limit)) + s + (let [head (subs s 0 limit) + end (str/last-index-of head ". ")] + (if (and end (pos? end)) (subs head 0 (inc end)) (str (str/trimr head) "…"))))) + +(defn- description + "The indented lines right after a signature, joined — up to the first line + that is not indented — and clipped to `limit`." + [lines limit] + (clip (->> lines + (take-while #(re-find #"^\s+\S" %)) + (map str/trim) + (str/join " ")) + limit)) + +(defn- arg-names [args] + (->> (str/split (str args) #",") + (map str/trim) + (remove #(or (str/blank? %) (= "..." %))) + (map #(str/replace % #"\?$" "")) + (filter #(re-matches #"[A-Za-z_][A-Za-z0-9_]*" %)))) + +(defn signatures + "Every tool signature documented in `text`, first definition of each name + kept, as {:name :description :parameters}. `limit` caps a description's + characters (nil: whole) — it rides in every request's tools array, and the + full prose is in the system prompt already." + ([text] (signatures text nil)) + ([text limit] + (let [lines (str/split-lines (str text))] + (->> (map-indexed vector lines) + (keep (fn [[i line]] + (when-let [[_ nm args] (re-matches signature-re line)] + {:name nm + :description (description (drop (inc i) lines) limit) + :parameters {:type "object" + :properties (into {} + (map (fn [a] [(keyword a) {}])) + (arg-names args))}}))) + (reduce (fn [acc s] (if (some #(= (:name s) (:name %)) acc) acc (conj acc s))) []))))) + +(defn for-messages + "The tool specs for a request whose history is `messages`: the signatures + in its SYSTEM messages — the documentation the branch was given — with + `exact` (name -> spec, gates.edn :forceable-tools) taking precedence. + Sorted by name, so the array, which lands in the cached prefix, is the same + bytes every turn. `limit` as for `signatures`." + ([messages exact] (for-messages messages exact nil)) + ([messages exact limit] + (let [doc (->> messages + (filter #(= "system" (str (:role %)))) + (map #(str (:content %))) + (str/join "\n")) + derived (signatures doc limit)] + (->> derived + (map (fn [s] (or (get exact (:name s)) s))) + (sort-by :name) + vec)))) diff --git a/test/samizdat/infer_test.clj b/test/samizdat/infer_test.clj index 053e555f..b97d618c 100644 --- a/test/samizdat/infer_test.clj +++ b/test/samizdat/infer_test.clj @@ -503,3 +503,18 @@ (is (nil? @seen) "no watcher asked for, so the call is not streamed") (is (empty? (filter #(= :delta (:kind %)) (events/collect sub)))) (finally (events/unsubscribe! sub))))) + +(deftest a-call-carries-the-branchs-tools-where-the-endpoint-takes-them + ;; The specs come from the branch's own system prompt (llm.toolspec), and + ;; only an endpoint with :native-tools is handed them (karamazov-jl1f). + (let [seen (atom nil) + tape {:id "B1" :messages [{:role "system" :content "Tools:\n```\nread_file({path})\n Read a file.\n```"} + {:role "user" :content "go"}] + :turns []}] + (with-redefs [samizdat.llm.client/chat + (fn [_ _ _ opts] (reset! seen opts) + {:content (fenced "read_file" "{\"path\": \"a\"}") :finish-reason "stop"})] + ((infer/complete-fn {:llm-adapter ::a :llm-config {:features #{:native-tools}}} {:journal? false}) tape) + (is (= ["read_file"] (mapv :name (:tools @seen)))) + ((infer/complete-fn {:llm-adapter ::a :llm-config {:features #{}}} {:journal? false}) tape) + (is (nil? (:tools @seen)) "an endpoint not measured to take them is not sent them")))) diff --git a/test/samizdat/llm_stream_test.clj b/test/samizdat/llm_stream_test.clj index 836b7f3e..05d491eb 100644 --- a/test/samizdat/llm_stream_test.clj +++ b/test/samizdat/llm_stream_test.clj @@ -99,3 +99,14 @@ (fn [_] (throw (ex-info "no deltas on an error" {}))))] (is (= 400 (:status r))) (is (= "{\"error\":{\"message\":\"bad request\"}}" (:body r))))))) + +(deftest an-error-frame-mid-stream-is-the-reply + ;; A stream can end in `data: {"error": {...}}` — GLM sends its business + ;; codes that way. The frame has no choices, so it folded into nothing and + ;; the call read as an empty reply, misreported and never classified + ;; (karamazov-jvdu). The error is what comes back. + (let [acc (reduce stream/accumulate nil + [(chunk {:content "Hel"}) + {:error {:code "1302" :message "请求过快"}}])] + (is (= {:error {:code "1302" :message "请求过快"}} + (select-keys (stream/completion acc) [:error]))))) diff --git a/test/samizdat/llm_test.clj b/test/samizdat/llm_test.clj index 4b60164a..0483b7dc 100644 --- a/test/samizdat/llm_test.clj +++ b/test/samizdat/llm_test.clj @@ -1040,9 +1040,49 @@ "credit_balance_exhausted" "organization_spend_limit_exceeded" "project_spend_limit_exceeded" "organization_usage_limit_exceeded") ;; and the codes that same table marks RETRYABLE stay a window: a - ;; concurrency cap or a per-minute limit clears on its own. + ;; per-minute limit, an overload or a server hiccup clears on its own. (are [code] (= :retry (client/classify a 429 {:error {:code code :message "请求过快"}})) - "1302" "1303" "1305" "3008" "3010")))) + "1302" "1303" "1305" "1312")) + + (testing "3008-3010 and 1304 are walls: ZCode's table marks them terminal" + ;; An earlier reading recorded them as retryable concurrency caps; the + ;; table, read directly on 2026-09-24, has retryable false for all four + ;; (karamazov-jvdu). + (are [code] (= :fatal (client/classify a 429 {:error {:code code :message "限额"}})) + "1304" "3008" "3009" "3010")) + + (testing "a retryable business code is retried whatever status carries it" + ;; GLM answers some of these under a 400 (or a 200); the status ladder + ;; alone called them fatal and the branch lost the turn. + (are [body] (= :retry (client/classify a 400 body)) + {:error {:code "1302" :message "并发数过高"}} + {:error {:code 1312 :message "overloaded"}} + {:error {:message "[1305][rate limit] slow down"}} + {:code 1234 :msg "network error" :success false} + {:error_code "2007" :error_msg "server busy"}) + (is (= :fatal (client/classify a 400 {:error {:code "1214" :message "bad param"}})) + "a code the table does not list keeps the status's answer") + (is (= :fatal (client/classify a 400 {:error {:message "token count 13022 too long"}})) + "a figure in the text is not a code")))) + +(deftest an-error-in-a-200-body-is-classified-like-any-other + ;; GLM can answer 200 with {success false, code ...}; that was always a + ;; fatal "API error", so a per-minute limit ended the call. + (let [a (registry/adapter-for :glm) + calls (atom 0)] + (with-redefs [http/post (fn [& _] + (swap! calls inc) + (if (= 1 @calls) + {:status 200 :body (json/write-str {:error {:code "1302" :message "请求过快"}})} + {:status 200 :body (json/write-str {:choices [{:message {:content "ok"} + :finish_reason "stop"}]})})) + samizdat.cancel/sleep! (fn [_] nil)] + (is (= "ok" (:content (client/chat a {:model "m" :max-retries 1} [{:role "user" :content "x"}])))) + (is (= 2 @calls))) + (testing "and a context overflow in one is an overflow" + (with-redefs [http/post (fn [& _] {:status 200 :body (json/write-str {:error {:code "1261" :message "Prompt exceeds max length"}})})] + (let [e (try (client/chat a {:model "m"} [{:role "user" :content "x"}]) (catch Exception ex ex))] + (is (= :context-overflow (:reason (ex-data e))))))))) (deftest a-402-is-a-usage-cap-and-is-not-retried ;; DeepSeek answers 402 "Insufficient Balance" when the account is out of @@ -1942,3 +1982,106 @@ (with-redefs [client/probe-llama-cpp (fn [_] (throw (ex-info "probed" {})))] (is (= {:provider :local} (client/with-discovery {:provider :local}))))) (client/forget-discoveries!))) + +(deftest deepseeks-dsml-call-is-the-call-it-unmistakably-is + ;; deepseek-v4-flash writes its native call syntax into the text: the XML + ;; form with every tag prefixed `｜DSML｜` in FULLWIDTH bars (U+FF5C), in + ;; run ab935047 doubled and followed by a space. Nothing matched it — the + ;; call parser looked for , the display for an ASCII `|DSML|` — so + ;; all 11 such turns in endless-flight were answered "No ```tool-call + ;; block", and the agent log showed the raw markup. + (let [doubled (str "Let me read the file.\n" + "<｜｜DSML｜｜ calls>\n" + "<｜｜DSML｜｜ invoke name=\"read_file\">\n" + "<｜｜DSML｜｜ parameter name=\"path\" string=\"true\">.samizdat/manual.edn\n" + "<｜｜DSML｜｜ parameter name=\"anchors\" string=\"false\">true\n" + "\n" + "\n") + single (str "Reading it.\n" + "<｜DSML｜function_calls>\n" + "<｜DSML｜invoke name=\"patch\">\n" + "<｜DSML｜parameter name=\"path\" string=\"true\">src/flight/draw.clj\n" + "<｜DSML｜parameter name=\"edits\" string=\"false\">[{\"from\": \"24:bf3\", \"to\": \"24:bf3\"}]\n" + "<｜DSML｜parameter name=\"count\" string=\"false\">3\n" + "\n" + "")] + (testing "the doubled form run ab935047 wrote" + (let [p (fence/parse-tool-call doubled {})] + (is (= "read_file" (:name p))) + (is (= ".samizdat/manual.edn" (get-in p [:args :path]))) + (is (true? (get-in p [:args :anchors])) "string=\"false\" is a JSON value: a boolean") + (is (:dsml? p) "recorded, not quietly normalised"))) + (testing "the documented single-bar form" + (let [p (fence/parse-tool-call single {})] + (is (= "patch" (:name p))) + (is (= [{:from "24:bf3" :to "24:bf3"}] (get-in p [:args :edits]))) + (is (= 3 (get-in p [:args :count]))))) + (testing "a reader sees the words, not the markup" + (is (= "Reading it." (fence/prose single))) + (is (= "" (fence/prose doubled)))))) + +(deftest string-true-keeps-a-json-looking-value-a-string + ;; `string="true"` is the model saying the body is TEXT; a body that happens + ;; to parse as JSON (an EDN vector, a numeric id) stays the string it is. + (let [p (fence/parse-tool-call + (str "\n" + "[\"a\" \"b\"]\n" + "42\n" + "") {})] + (is (= "[\"a\" \"b\"]" (get-in p [:args :content]))) + (is (= "42" (get-in p [:args :id]))))) + +(deftest the-tools-travel-natively-where-the-endpoint-takes-them + ;; DeepSeek's and Zhipu's own harnesses send `tools` on every request; with + ;; none, deepseek-v4-flash wrote DSML into its content (karamazov-jl1f). + (let [specs [{:name "done" :description "Finish." :parameters {:type "object" :properties {:answer {}}}} + {:name "read_file" :description "Read." :parameters {:type "object" :properties {:path {}}}}] + ds (registry/adapter-for :deepseek) + glm (registry/adapter-for :glm) + ds-cfg {:base-url "https://api.deepseek.com/beta" :model "deepseek-v4-flash" + :features (config/features-of :deepseek)} + glm-cfg {:base-url "https://open.bigmodel.cn/api/coding/paas/v4" :model "glm-5.3" + :features (config/features-of :glm)} + msgs [{:role "user" :content "x"}]] + (testing "every turn, not only a forced one" + (doseq [[a cfg] [[ds ds-cfg] [glm glm-cfg]]] + (let [body (adapter/chat-body a cfg {:messages msgs :tools specs})] + (is (= (mapv (fn [s] {:type "function" :function s}) specs) (:tools body))) + (is (nil? (:tool_choice body)) "the model chooses")))) + (testing "not where the endpoint was not measured to take them" + (is (nil? (:tools (adapter/chat-body ds (assoc ds-cfg :features #{}) {:messages msgs :tools specs}))))) + (testing "not beside a prefill: DeepSeek 400s `Function call should not be used with prefix`" + (let [body (adapter/chat-body ds ds-cfg {:messages msgs :tools specs + :prefill "```tool-call\n{\"name\": \"done\""})] + (is (nil? (:tools body))) + (is (some :prefix (:messages body))))) + (testing "a native force keeps the whole array — the prefix it lands in unchanged — and names the tool" + (let [body (adapter/chat-body glm glm-cfg {:messages msgs :tools specs + :force-tool {:name "done" :description "Finish." + :parameters {:type "object"}}})] + (is (= ["done" "read_file"] (mapv #(get-in % [:function :name]) (:tools body)))) + (is (= {:type "function" :function {:name "done"}} (:tool_choice body))))))) + +(deftest a-native-call-keeps-the-words-around-it + ;; glm-5.3 answered a tools request with a sentence AND a tool_call. The + ;; sentence is what the agent log shows; replacing the content with the + ;; fence dropped it. + (let [glm (registry/adapter-for :glm) + reply (fn [msg] {:choices [{:message msg :finish_reason "tool_calls"}]})] + (let [p (adapter/parse-chat glm (reply {:content "I'll add inc2." + :tool_calls [{:function {:name "write_file" + :arguments "{\"path\":\"a.clj\"}"}}]}))] + (is (str/starts-with? (:content p) "I'll add inc2.")) + (is (= {:name "write_file" :args {:path "a.clj"}} + (select-keys (fence/parse-tool-call (:content p) {}) [:name :args])))) + (testing "arguments that are not JSON stay as written, for the repair ladder and the model to see" + (let [p (adapter/parse-chat glm (reply {:tool_calls [{:function {:name "write_file" + :arguments "{\"path\": \"a.clj\",}"}}]})) + parsed (fence/parse-tool-call (:content p) {})] + (is (str/includes? (:content p) "\"path\": \"a.clj\",}")) + (is (= "a.clj" (get-in parsed [:args :path])) "a trailing comma is the ladder's to fix"))) + (testing "every call the reply made is kept; the fence rule picks and counts" + (let [p (adapter/parse-chat glm (reply {:tool_calls [{:function {:name "read_file" :arguments "{\"path\":\"a\"}"}} + {:function {:name "read_file" :arguments "{\"path\":\"b\"}"}}]})) + parsed (fence/parse-tool-call (:content p) {})] + (is (= 2 (:fences parsed))))))) diff --git a/test/samizdat/test_runner.clj b/test/samizdat/test_runner.clj index bd01521a..a3dd05f1 100644 --- a/test/samizdat/test_runner.clj +++ b/test/samizdat/test_runner.clj @@ -175,6 +175,7 @@ [samizdat.kanban-test] [samizdat.security.policy-test] [samizdat.llm-test] + [samizdat.toolspec-test] [samizdat.llm-stream-test] [samizdat.prompt-test] [samizdat.server-test] @@ -274,6 +275,7 @@ samizdat.tui-widgets-test samizdat.store-test samizdat.llm-test + samizdat.toolspec-test samizdat.llm-stream-test samizdat.agent-test samizdat.approval-test diff --git a/test/samizdat/toolspec_test.clj b/test/samizdat/toolspec_test.clj new file mode 100644 index 00000000..a1d0cad7 --- /dev/null +++ b/test/samizdat/toolspec_test.clj @@ -0,0 +1,76 @@ +;; samizdat - a claim-first verification harness +;; Copyright (C) 2026 Dmitri Sotnikov +;; +;; This program is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. +;; +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; +;; You should have received a copy of the GNU General Public License +;; along with this program. If not, see . +;; +;; SPDX-License-Identifier: GPL-3.0-or-later + +(ns samizdat.toolspec-test + "Native tool specs, read off the tool documentation the branch was given." + (:require [clojure.test :refer [deftest testing is]] + [samizdat.llm.toolspec :as toolspec])) + +(def ^:private doc + (str "## Tools\n\n```\n" + "done({answer})\n" + " Ship. `answer` is REQUIRED and is the run's actual output.\n" + " A `done` with no answer is refused.\n" + "read_file({path, offset, limit, outline})\n" + " Read a file, a window of it at a time.\n" + "edit_file({path, old_text, new_text, replace_all?})\n" + " Replace one exact piece of text.\n" + "task({action, ...})\n" + " The task board.\n" + "eval({code, timeout_ms?})\n" + " Evaluate Clojure in the {% if x %}live{% endif %} image.\n" + "```\n\nSome prose that mentions done({answer}) in passing.\n")) + +(deftest a-signature-is-a-tool-its-arguments-and-its-description + (let [specs (into {} (map (juxt :name identity)) (toolspec/signatures doc))] + (is (= #{"done" "read_file" "edit_file" "task" "eval"} (set (keys specs)))) + (is (= "Ship. `answer` is REQUIRED and is the run's actual output. A `done` with no answer is refused." + (:description (specs "done")))) + (is (= #{:path :old_text :new_text :replace_all} + (set (keys (get-in specs ["edit_file" :parameters :properties]))))) + (testing "nothing is required: the prose says what is, and the tool checks" + (is (not (contains? (get-in specs ["read_file" :parameters]) :required)))) + (testing "`...` is the open end of an action tool: its properties are the named ones" + (is (= #{:action} (set (keys (get-in specs ["task" :parameters :properties])))))) + (is (= "object" (get-in specs ["task" :parameters :type]))))) + +(deftest a-mention-in-prose-is-not-a-second-definition + (is (= 1 (count (filter #(= "done" (:name %)) (toolspec/signatures doc)))))) + +(deftest the-branchs-own-system-messages-are-the-source-and-exact-schemas-win + (let [messages [{:role "system" :content doc} + {:role "user" :content "read_file({path}) — a user message is not documentation"}] + exact {"done" {:name "done" :description "Finish." + :parameters {:type "object" :properties {:answer {:type "string"}} + :required ["answer"]}}} + specs (toolspec/for-messages messages exact)] + (is (= ["done" "edit_file" "eval" "read_file" "task"] (mapv :name specs)) + "sorted by name, so the array — and the prefix it lands in — is stable") + (is (= ["answer"] (get-in (first specs) [:parameters :required])) + "a tool gates.edn gives an exact schema to is sent with it") + (is (empty? (toolspec/for-messages [{:role "user" :content doc}] exact)) + "no system message, no documentation, no tools"))) + +(deftest a-long-description-is-cut-at-a-sentence-end + (let [text (str "```\neval({code})\n" + " Evaluate Clojure. This is how to work: try a form and iterate.\n" + " More prose follows here.\n```") + [spec] (toolspec/signatures text 40)] + (is (= "Evaluate Clojure." (:description spec))) + (is (= 1 (count (toolspec/signatures text nil))) "no limit, the whole paragraph") + (is (clojure.string/includes? (:description (first (toolspec/signatures text nil))) "More prose"))))