diff --git a/docs/RFCS/RFC-002-manifests-and-cells.md b/docs/RFCS/RFC-002-manifests-and-cells.md index 0f9fce94..e755c4a9 100644 --- a/docs/RFCS/RFC-002-manifests-and-cells.md +++ b/docs/RFCS/RFC-002-manifests-and-cells.md @@ -56,6 +56,8 @@ glob-scoped interceptors match on. ; [branch-kw pattern guard], or a (fn [data] pred) form :constraints [{:type :must-follow :if node :then node}] :subworkflows {cell-id manifest-name} ; optional: a nested manifest as one node + ; (routing only: the child's :prompt is not + ; read, and compile warns if it has one) :prompt "name" ; optional: prompt appended to the base :turn-sliceable? false ; optional, default true — see below :extends "manifest-name" ; optional: carry only what differs from it @@ -280,6 +282,21 @@ workflow/compile-loop ## Known gaps +- **A file edit is checked, not soaked or replayed.** The `cell`, `manifest` + and `policy` tools run the whole protocol: compile, soak (cells), and the + held-out battery (RFC-014) before an edit goes live. A project's workflow + is also FILES under `.samizdat/`, and an edit made to one directly — by a + person, or by the agent's own `write_file` — is recorded as a version and + goes live on its next read once it passes that kind's validator: it must + read and compile (and a cell must load), but it is not soaked and not + replayed against the battery. That is deliberate: a person editing their + project's workflow is not gated by the agent's own safety net, and the + file tools' refusal to write outside the project applies as ever. What it + means for the agent is that the tools are the gated route and a direct + write is not; RFC-001 says a direct edit is honoured. The shipped + `resources/cells` are templates only once a project has files: editing + one changes nothing that runs, and is offered to the supervisor for + adoption (karamazov-95u5 observed the opposite before files mode). - **Every declared invariant is enforced.** Every ordering rule a manifest claims is declared in its `:invariants`, each saying what it `:protects` and whether it is `:enforced`; the enforced ones are DERIVED into the diff --git a/resources/cells/loop.clj b/resources/cells/loop.clj index 21c2f61c..4d545268 100644 --- a/resources/cells/loop.clj +++ b/resources/cells/loop.clj @@ -13,7 +13,8 @@ ;; Naming is load-bearing: :llm/*, :tool/*, :journal/*, :gate/* are what ;; glob-scoped interceptors match on. (ns cells.loop - (:require [mycelium.cell :as cell] + (:require [clojure.string :as str] + [mycelium.cell :as cell] [samizdat.agent.compaction :as cmp] [samizdat.agent.gates :as gates] [samizdat.agent.instructions :as instr] @@ -223,7 +224,9 @@ {:doc "The single boundary: at most one steer, chosen in priority, plus the context block of shared artifacts and similar failures." :effects [:db] - :requires [] + ;; The run's billed spend, read here because this cell may reach the db and + ;; route may not (karamazov-lq57). + :requires [:conn :run-id :token-budget] ;; :settled is REQUIRED and is the invariant: only :gate/settle writes it, ;; so a manifest that reaches the arbiter without closing this turn's ;; predictions first — crediting a gate with an outcome that preceded it — @@ -232,10 +235,19 @@ :input [:map [:settled :map] [:branch :map] [:turn :int] [:parsed {:optional true} :any] [:result {:optional true} :any]] - :output [:map [:branch :map]]} - (fn [ctx {:keys [branch turn parsed result] :as data}] - (assoc data :branch (turn/steer-step ctx branch turn - {:parsed parsed :result result})))) + :output [:map [:branch :map] [:over-budget? {:optional true} :boolean]]} + (fn [{:keys [conn run-id token-budget] :as ctx} {:keys [branch turn parsed result] :as data}] + (let [;; THE RUN'S TOKEN BUDGET, in every loop that runs a turn + ;; (karamazov-lq57). It was read only at the beam's round boundary, + ;; so a feature, team or decompose run — one "turn" to the beam — + ;; spent without bound inside its nested loops. The sum is the + ;; run's whole bill off the journal, the same number the beam + ;; holds; a run with no budget pays for no query. + spent (when (and token-budget conn run-id) + (:total-tokens (journal/run-usage conn run-id)))] + (cond-> (assoc data :branch (turn/steer-step ctx branch turn + {:parsed parsed :result result})) + (and spent (>= spent token-budget)) (assoc :over-budget? true))))) (cell/defcell :loop/route {:doc "Decide the turn's verdict: :continue (next turn), :done, :abandoned, @@ -244,7 +256,7 @@ branch and the configured cap — no side effects." :pure true :requires [:max-turns] - :input [:map [:branch :map] [:turn :int]] + :input [:map [:branch :map] [:turn :int] [:over-budget? {:optional true} :any]] ;; :turn as well as :verdict, because the :continue branch increments it. ;; The dissoc of the per-turn products is invisible to mycelium — it models ;; what a cell ADDS, never what it drops — and that is safe here only @@ -268,12 +280,27 @@ (gates/threshold :provider-error-limit)) :abandoned + ;; The run has spent its token budget (the arbiter read + ;; it): every loop's branch ends, nested ones included. + (:over-budget? data) :exhausted + (>= turn max-turns) :exhausted :else :continue)] (cond-> (assoc data :verdict verdict) + ;; An ending with NO reason is how an oversight pass came to record + ;; `abandoned` with `ended` and `notes` both null (karamazov-n6ql): the + ;; provider-error arm and an inactive branch with no answer set none. + ;; Named here, where the verdict is decided, when nothing else did. + (and (= verdict :abandoned) (nil? (:inactive-reason branch))) + (assoc-in [:branch :inactive-reason] + (str/trim (prompt/render "abandoned-reason" + (if (state/active? branch) + {:provider-errors (:consecutive-provider-errors branch)} + {:status (some-> (:status branch) name)})))) + (= verdict :continue) (-> (update :turn inc) - (dissoc :before :call :parsed :signals :said :result :tool :settled) + (dissoc :before :call :parsed :signals :said :result :tool :settled :over-budget?) ;; Each mycelium trace entry snapshots the whole data map — branch ;; message history included — so an uncapped trace grows ;; quadratically over a run. The journal is the durable record; the diff --git a/resources/gates.edn b/resources/gates.edn index a3b30127..ff2f5d02 100644 --- a/resources/gates.edn +++ b/resources/gates.edn @@ -358,6 +358,16 @@ Raise it on a slow filesystem or a very large repo; 0 disables the cache and reads on every request."} + :image-start + {:value {:attempts 2 :image-log-lines 20} + :provenance ["karamazov-69p0" "karamazov-tetz"] + :kind :policy :capability-tunable? false + :doc "How a project image is started. :attempts — tries, each on a fresh + port, before the eval fails: the free-port gap is a real race and a + load spike is not a reason to fail an eval. :image-log-lines — how + much of the child's own output a failed start logs, so a failure + says why rather than only where its profile is."} + :image-connect-ms {:value 20000 :kind :threshold :capability-tunable? true :provenance ["karamazov-zrq"] diff --git a/resources/prompts/abandoned-reason.md b/resources/prompts/abandoned-reason.md new file mode 100644 index 00000000..186bbb7b --- /dev/null +++ b/resources/prompts/abandoned-reason.md @@ -0,0 +1 @@ +{% if provider-errors %}the provider failed {{provider-errors}} calls in a row{% else %}the branch stopped ({{status}}) without an answer{% endif %} diff --git a/resources/prompts/shell-refused.md b/resources/prompts/shell-refused.md index fb127442..99d0dddc 100644 --- a/resources/prompts/shell-refused.md +++ b/resources/prompts/shell-refused.md @@ -8,7 +8,7 @@ this one is fine: {{blocked}} -`{{blockedhead}}` is not on the allow list. Reissue the command without that +{% if blockedform %}This form of `{{blockedhead}}` is not on the allow list — other forms of it are.{% else %}`{{blockedhead}}` is not on the allow list.{% endif %} Reissue the command without that part, or use a tool that does the same job: `read_file` and `grep` to look around, `eval` to run Clojure. {% else %}{% if promoted %} @@ -25,7 +25,7 @@ This is a COMPOUND command — it contains {{markers}} — so it is judged as on whole claim rather than by its first word, and `{{head}}` is not allowed on its own either. Split it up and check the parts. {% else %} -`{{head}}` is not on the allow list, so it needs a human to grant it — and if +{% if headform %}This form of `{{head}}` is not on the allow list — other forms of it are — so it needs a human{% else %}`{{head}}` is not on the allow list, so it needs a human{% endif %} to grant it — and if this run has no human watching, it will not be granted. Prefer a tool that does the same job without the shell: `read_file` and `grep` to look around, `eval` to run Clojure, including this project's own tests once you have required the diff --git a/resources/prompts/task-claim-refused.md b/resources/prompts/task-claim-refused.md new file mode 100644 index 00000000..55064ed5 --- /dev/null +++ b/resources/prompts/task-claim-refused.md @@ -0,0 +1 @@ +Cannot {{verb}} `{{id}}`: {% if missing %}there is no task with that id. `task list` shows the board.{% endif %}{% if closed %}it is already {{status}}{% if yours %} — you closed it yourself{% endif %}. A closed task stays closed; if work is left, create a task for it rather than reopening this one.{% endif %}{% if held %}{% if other-run %}another run holds it{% else %}branch {{holder}} holds it{% endif %}. Pick another task, or leave this one to whoever is working it.{% endif %} diff --git a/resources/userspace.edn b/resources/userspace.edn index 61002165..948672a8 100644 --- a/resources/userspace.edn +++ b/resources/userspace.edn @@ -57,6 +57,7 @@ :prompts {:acceptance-failed "prompts/acceptance-failed.md" + :abandoned-reason "prompts/abandoned-reason.md" :acceptance-judge "prompts/acceptance-judge.md" :adopt-tool "prompts/adopt-tool.md" :battery-tool "prompts/battery-tool.md" @@ -64,6 +65,7 @@ :heldout-pending "prompts/heldout-pending.md" :heldout-refused "prompts/heldout-refused.md" :refused-again "prompts/refused-again.md" + :task-claim-refused "prompts/task-claim-refused.md" :tests-not-run "prompts/tests-not-run.md" :adoption-offer "prompts/adoption-offer.md" :architect "prompts/architect.md" diff --git a/resources/wordlists.edn b/resources/wordlists.edn index 658f446a..5c4cf9cb 100644 --- a/resources/wordlists.edn +++ b/resources/wordlists.edn @@ -25,6 +25,20 @@ "clpfd" "prolog" "smt" "lean" "works" "available" "loaded" "basic" "supports" "simple" "test" "check" "verify" "verified" "example"} + :truncated-finish-reasons + ;; finish_reason values that mean the reply was cut short, not finished: the + ;; fix is more tokens or a retry, never steering (fence/signals :truncated). + ;; DeepSeek sends insufficient_system_resource when it stops generating + ;; under load (karamazov-6uyv). + #{"length" "insufficient_system_resource"} + + :surviving-findings + ;; The heading a verify pass puts over the findings it kept (karamazov-kdoj). + ;; When a reply has one, only what is under it is read as findings: the + ;; reasoning above it quotes candidates' severity tags and used to be stored + ;; as the findings and handed to the retry. + #{"surviving findings" "findings that survive" "remaining findings"} + :request-framing ;; Words that frame a REQUEST rather than name its subject — what to do and ;; where to say it. The problem-relevance rung drops them from the PROBLEM's diff --git a/src/samizdat/agent/beam.clj b/src/samizdat/agent/beam.clj index 0df5b267..1b0eca6d 100644 --- a/src/samizdat/agent/beam.clj +++ b/src/samizdat/agent/beam.clj @@ -693,10 +693,12 @@ (reset! started pending) (reduce (fn [acc p] (conj acc (settle p))) [] pending))) (catch Throwable e - ;; The round itself was cancelled (an abort) with turns in flight: - ;; every turn goes down with it before the signal travels on. - (when (cancel/control-signal? e) - (doseq [[_ t] @started :when (map? t)] ((:cancel t)))) + ;; The round failed with turns in flight — an abort, or anything else + ;; that broke the wait: every turn it started goes down with it + ;; before the throw travels on. Only a cancel signal used to, so any + ;; other failure left spawned turns running past the driver that + ;; owned them (karamazov-odyx). + (doseq [[_ t] @started :when (map? t)] ((:cancel t))) (throw e))))) (defn dispose-branch-engines! diff --git a/src/samizdat/agent/judge.clj b/src/samizdat/agent/judge.clj index abbbafb1..795cec4d 100644 --- a/src/samizdat/agent/judge.clj +++ b/src/samizdat/agent/judge.clj @@ -39,6 +39,7 @@ judge is told to say has to touch both." (:require [clojure.string :as str] [samizdat.agent.gates :as gates] + [samizdat.lexicon :as lexicon] [samizdat.llm.message :as message] [samizdat.prompt :as prompt] [samizdat.util :as util])) @@ -444,14 +445,31 @@ f)))) (defn- severity-line? - "Whether `line` opens a finding: it carries one of gates.edn - :review-severities as a bracketed tag." + "Whether `line` opens a finding: it STARTS with one of gates.edn + :review-severities as a bracketed tag, after an optional bullet, number or + emphasis. Anywhere in the line was too loose: a reasoning paragraph quoting + a candidate's `[low]` became a finding and carried the prose after it + (karamazov-kdoj)." [line] (let [sevs (gates/threshold :review-severities)] (boolean (and (seq sevs) - (re-find (re-pattern (str "(?i)\\[(" (str/join "|" sevs) ")\\]")) + (re-find (re-pattern (str "(?i)^\\s*(?:[-*+]|\\d+[.)])?\\s*\\**\\[(" + (str/join "|" sevs) ")\\]")) (str line)))))) +(defn- surviving-section + "The part of a verify reply under its surviving-findings heading (wordlists + :surviving-findings), or the reply whole when it has none." + [reply] + (let [heads (lexicon/wordlist :surviving-findings) + lines (str/split-lines (str reply)) + head? (fn [l] (let [t (str/lower-case (str/trim (str/replace (str l) #"[#*:]" "")))] + (contains? (set heads) t))) + after (rest (drop-while (complement head?) lines))] + (if (and (seq heads) (some head? lines)) + (str/trim (str/join "\n" after)) + reply))) + (defn finding-segments "A findings text split into one segment per finding. @@ -569,8 +587,10 @@ (clean-pass? reply) nil (str/blank? (str reply)) (dedupe-findings candidates) :else - (let [segs (finding-segments reply) - dropped (count (filterv false-positive? segs)) + (let [;; Only what the judge listed as surviving, when it listed it: + ;; the deliberation above that heading is not findings. + segs (finding-segments (surviving-section reply)) + dropped (count (filterv false-positive? (finding-segments reply))) ;; A SURVIVOR HAS TO BE A FINDING. Filtering only on ;; false-positive? let any prose the judge emitted through as the ;; findings text — "Hmm, hard to say." would have REPLACED two real @@ -579,7 +599,9 @@ ;; narration, and narration never fabricates a finding. kept (->> segs (remove false-positive?) - (filter #(severity-line? (first (str/split-lines %))))) + ;; Its first NON-BLANK line: a segment that opens on a + ;; blank line is still the finding under it. + (filter #(severity-line? (first (remove str/blank? (str/split-lines %)))))) survivors (not-empty (str/trim (str/join kept)))] (cond survivors (dedupe-findings survivors) diff --git a/src/samizdat/agent/tools/tasks.clj b/src/samizdat/agent/tools/tasks.clj index 107b167c..fd2ef9b0 100644 --- a/src/samizdat/agent/tools/tasks.clj +++ b/src/samizdat/agent/tools/tasks.clj @@ -103,6 +103,23 @@ " list, show {id}, update {id, ...fields}, claim {id}," " switch {id, reason}, close {id, status?}.")) +(defn- claim-refused + "Why `claim!` returned nil, from the task's own row: there is none, it is + closed (and by this branch), or someone else holds it. One sentence used to + cover all three, so a branch that had closed its own task read that another + run held it, and made a duplicate (karamazov-fjrq)." + [conn id run-id branch verb] + (let [row (tasks/get-task conn id) + closed? (and row (or (:closed_at row) (#{"done" "cancelled"} (str (:status row)))))] + (prompt/render "task-claim-refused" + (cond + (nil? row) {:verb verb :id id :missing true} + closed? {:verb verb :id id :closed true :status (:status row) + :yours (= (str (:branch_id row)) (str (:id branch)))} + :else {:verb verb :id id :held true + :other-run (not= (str (:run_id row)) (str run-id)) + :holder (:branch_id row)})))) + (defmethod base/run-tool "task" [{:keys [branch conn run-id] :as ctx}] ;; Every action is `ok` (:neutral) on purpose: working the board is ;; bookkeeping, and bookkeeping is not progress — the same reasoning as @@ -176,8 +193,7 @@ (base/ok (take-task branch t) (str "Claimed " (task-line t)) :progress? true) - (base/malformed branch (str "Cannot claim " (base/arg ctx :id) - ": no such task, or another run holds it."))))) + (base/malformed branch (claim-refused conn (base/arg ctx :id) run-id branch "claim"))))) "switch" (or (want :id) (want :reason) @@ -215,8 +231,7 @@ (task-line t) "\nRecorded why: " reason) :progress? true)) - (base/malformed branch (str "Cannot switch to " (base/arg ctx :id) - ": no such task, or another run holds it.")))))) + (base/malformed branch (claim-refused conn (base/arg ctx :id) run-id branch "switch to")))))) "close" (or (want :id) diff --git a/src/samizdat/llm/fence.clj b/src/samizdat/llm/fence.clj index 53b81d6b..086c8b25 100644 --- a/src/samizdat/llm/fence.clj +++ b/src/samizdat/llm/fence.clj @@ -878,7 +878,12 @@ fix is more tokens, not more steering. It was the first thing a live deepseek-v4-flash call did here, so it is not a hypothetical." [{:keys [finish-reason content]} parsed] - (let [truncated (= "length" finish-reason) + (let [;; Every finish reason that means the reply was CUT SHORT rather than + ;; finished (wordlists :truncated-finish-reasons): `length` everywhere, + ;; and DeepSeek's `insufficient_system_resource`, which ended a reply + ;; mid-generation and read as a no-call (karamazov-6uyv). + truncated (contains? (or (lexicon/wordlist :truncated-finish-reasons) #{"length"}) + (str finish-reason)) ;; A reply repeating itself, checked only where it matters — a ;; truncated reply, or one that made no call — so a long healthy ;; reply that reached its fence is not scanned (karamazov-o4wm.5). diff --git a/src/samizdat/manifests.clj b/src/samizdat/manifests.clj index 28f787e5..eb089f51 100644 --- a/src/samizdat/manifests.clj +++ b/src/samizdat/manifests.clj @@ -692,10 +692,20 @@ ;; see them where they already look for :undeclared-effects. (let [unguarded (for [c (unguarded-cycles definition)] (assoc c :type :unguarded-cycle)) + ;; COMPOSITION IS ROUTING-ONLY (karamazov-r6x1): a composed child + ;; runs as a node of its parent, under the parent's prompt, and its + ;; own :prompt is never read. Said, rather than silently dropped. + composed-prompts (for [[cell-id mname] (:subworkflows definition) + :let [child (try (read-definition (manifest-body! mname)) + (catch Throwable _ nil))] + :when (:prompt child)] + {:type :composed-prompt-ignored :cell-id cell-id + :manifest mname :prompt (:prompt child)}) + warnings (concat unguarded composed-prompts) compiled (cond-> compiled - (and (seq unguarded) (:compiled-fsm compiled)) + (and (seq warnings) (:compiled-fsm compiled)) (update-in [:compiled-fsm :mycelium/compile-warnings] - (fnil into []) unguarded))] + (fnil into []) warnings))] (when-let [warnings (:mycelium/compile-warnings (:compiled-fsm compiled))] (log/warn "loop definition compiled with warnings:" (pr-str warnings))) compiled)))) diff --git a/src/samizdat/repl.clj b/src/samizdat/repl.clj index d3ff6895..c977ccd5 100644 --- a/src/samizdat/repl.clj +++ b/src/samizdat/repl.clj @@ -134,7 +134,10 @@ missing (remove (set current) added)] (when (seq missing) (host/set-source-roots! (into current missing)) - (log/info "eval can now reach" (str/join ", " missing))) + ;; The HARNESS image's search path: the supervisor evaluates here. + ;; Every other role evaluates in the project image, which route adds + ;; the same roots to when it starts one (karamazov-1b37). + (log/info "the harness image's eval can now reach" (str/join ", " missing))) (vec missing)))) (def ^:private session-counter (atom 0)) diff --git a/src/samizdat/repl/image.clj b/src/samizdat/repl/image.clj index ba7f8280..ac266b23 100644 --- a/src/samizdat/repl/image.clj +++ b/src/samizdat/repl/image.clj @@ -98,17 +98,35 @@ (sandbox/wrap backend confinement ["jolt" "-Sdeps" nrepl-sdeps "nrepl-server" (str port)])) (defn- await-port! - "Block until `port` accepts a connection, or `deadline-ms` passes. True when - it came up." - [port deadline-ms] + "Block until `port` accepts a connection, `deadline-ms` passes, or `proc` + has exited — a child that died will not come up, and waiting out the whole + deadline for it only hides why. True when it came up." + [port deadline-ms proc] (let [end (+ (System/currentTimeMillis) deadline-ms)] (loop [] - (if (try (with-open [_ (java.net.Socket. "127.0.0.1" (int port))] true) - (catch Exception _ false)) + (cond + (try (with-open [_ (java.net.Socket. "127.0.0.1" (int port))] true) + (catch Exception _ false)) true - (when (< (System/currentTimeMillis) end) - (Thread/sleep 100) - (recur)))))) + + (and proc (not (process/alive? proc))) + false + + (< (System/currentTimeMillis) end) + (do (Thread/sleep 100) (recur)))))) + +(defn- logged-argv + "`argv` with the child's stdout and stderr sent to `log`. They were pipes + nobody read (jolt.process's default), so a child that printed enough while + starting — a cold cache's warnings — could fill one and stall before it + bound its port (karamazov-69p0). The redirect is made by a shell OUTSIDE the + sandbox wrapper, so the confinement profile is not touched." + [argv log] + (into ["sh" "-c" "exec \"$@\" >>\"$0\" 2>&1" (str log)] argv)) + +(defn- tail-of [f n] + (try (->> (slurp f) str/split-lines (take-last n) (str/join "\n")) + (catch Exception _ ""))) (defn start! "Start a project image at `root` and return it, or nil when it could not be @@ -131,7 +149,9 @@ {:project-root root :scratch-paths [scratch]} (sandbox/deny-read-kinds (:deny-read sandbox-spec)))] (sandbox/write-profile! backend profile spec) - (let [argv (spawn-argv backend {:profile profile :spec spec} port) + (loop [attempt 1 port port] + (let [argv (logged-argv (spawn-argv backend {:profile profile :spec spec} port) + (io/file scratch "image.log")) ;; THE CHILD SEES ONLY A SCRUBBED ENVIRONMENT, the same one the shell ;; tool's subprocess gets. boundary-test's security map records the ;; rule this is obeying: a tool whose reach is :spawns-process "must @@ -144,7 +164,7 @@ proc (process/process argv {:dir (str root) :env (secrets/scrub-env (into {} (System/getenv)))})] - (if (await-port! port (connect-timeout-ms)) + (if (await-port! port (connect-timeout-ms) proc) (do (log/info "project image up on" port "rooted at" root (if (= :none backend) "(unsandboxed)" (str "under " (name backend)))) {:proc proc :port port :root (str root) :backend backend @@ -154,10 +174,22 @@ ;; shared one crossed concurrent branches' replies and let a ;; timed-out eval poison every later one. :sessions (atom #{})}) - (do (log/error "project image did not come up on" port - "— profile at" profile) - (try (process/destroy-tree proc) (catch Exception _ nil)) - nil))))) + (let [exit (when-not (process/alive? proc) + (try (.exitValue ^java.lang.Process (:proc proc)) (catch Exception _ nil)))] + (try (process/destroy-tree proc) (catch Exception _ nil)) + ;; WHY, where it used to say only where the profile was: the + ;; deadline in force, whether the child had exited and how, and + ;; what it printed. A start that failed under full-suite load left + ;; nothing to go on (karamazov-69p0, karamazov-tetz). + (log/error "project image did not come up on" port + (str "(attempt " attempt ", waited up to " (connect-timeout-ms) " ms" + (if exit (str ", child exited " exit) ", child still running") ")") + "— profile at" profile "\n" (tail-of (io/file scratch "image.log") + (:image-log-lines (gates/threshold :image-start)))) + ;; Once more on a fresh port: the free-port gap is a real race, and + ;; a load spike is not a reason to fail the eval. + (when (< attempt (:attempts (gates/threshold :image-start))) + (recur (inc attempt) (free-port))))))))) (defn- reap-timeout-ms "How long to wait for a killed image to actually be gone. gates.edn diff --git a/src/samizdat/repl/route.clj b/src/samizdat/repl/route.clj index 21e44bf4..91750a3b 100644 --- a/src/samizdat/repl/route.clj +++ b/src/samizdat/repl/route.clj @@ -120,6 +120,15 @@ {:root root :backend backend :sandbox-spec (sandbox-spec (System/getenv "HOME") (str (fs/cwd)))})] + ;; THE PROJECT'S OWN ROOTS, in the image that evaluates for it. + ;; The image starts as a bare `jolt nrepl-server` — no alias — + ;; so a test tree declared only under an alias's :extra-paths was + ;; not on its search path, and a namespace a branch had just + ;; written there could not be required (karamazov-1b37). Added, + ;; never replacing, like the harness image's. + (let [roots (repl/declared-roots root)] + (image/eval-in im (str "(jolt.host/set-source-roots! (into (vec (jolt.host/source-roots)) " + "(remove (set (jolt.host/source-roots)) " (pr-str roots) ")))"))) (swap! images assoc root im) im))))) diff --git a/src/samizdat/security/policy.clj b/src/samizdat/security/policy.clj index e504277b..488bd363 100644 --- a/src/samizdat/security/policy.clj +++ b/src/samizdat/security/policy.clj @@ -375,6 +375,12 @@ ["jolt -e **" :allow] ["jolt -A **" :allow] ["jolt -M **" :allow] ["jolt -A:test **" :allow] ["jolt -M:test **" :allow] ["jolt -A:dev **" :allow] ["jolt -A:test -e **" :allow] ["jolt -M:test -e **" :allow] + ;; ANY alias, not one row per alias (karamazov-9nsf): a project's own + ;; aliases run its own code, the same trust as its test alias, and a row per + ;; name meant every alias a project added was refused until somebody wrote + ;; one. A mid-pattern `*` matches the alias name; compound splitting, the + ;; wrapper promotion and the hard denies still apply per segment. + ["jolt -A:* **" :allow] ["jolt -M:* **" :allow] ;; The project's RUN alias, beside its test alias. `cargo run` and `go run` ;; were already here and this was not, purely because the colon-alias quirk ;; above needs one pattern per alias and nobody had needed this one. Live @@ -470,6 +476,16 @@ [] {:structural structural-rules :table base-rules}) +(defn head-has-allow? + "Whether some allow rule starts with command head `h` — so a refusal can say + that THIS form of it is not allowed, rather than that the command is closed. + `jolt` refused on `-A:foo` while `jolt -M:test` runs read as \"jolt is not + on the allow list\" (karamazov-9nsf)." + [h] + (boolean (and h (some (fn [[pat eff]] + (and (= :allow eff) (= h (first (str/split (str pat) #"\s+"))))) + base-rules)))) + (defn decide "The decision for a shell command: {:effect :allow|:ask|:deny :head :raw :rule}, where :rule names which rule made it — see `rules`. @@ -684,6 +700,8 @@ ;; command whose other parts were never refused. :blocked blocked-segment :blockedhead (some-> blocked-segment command-head) + :blockedform (head-has-allow? (some-> blocked-segment command-head)) + :headform (head-has-allow? head) :markers (when complex? (str/join " or " (map #(str "`" % "`") diff --git a/src/samizdat/server.clj b/src/samizdat/server.clj index 7263ff98..cc98ac23 100644 --- a/src/samizdat/server.clj +++ b/src/samizdat/server.clj @@ -37,6 +37,7 @@ [samizdat.api.stream :as stream] [samizdat.approval :as approval] [samizdat.config :as config] + [samizdat.engine.proc :as proc] [samizdat.llm.client :as llm-client] [samizdat.store.db :as db] [samizdat.system :as system] @@ -96,12 +97,35 @@ ;; --- handlers --------------------------------------------------------------- +(defonce ^:private started-at (str (java.time.Instant/now))) + +(def harness-identity + "Which harness THIS process is: the checkout's commit and whether its tree + had uncommitted changes, read once, on the first /health — so a checkout + that moves on afterwards shows as a revision the process is not running — + and when the process started. A stale + `serve` answered /health like a current one, so a client could not tell it + was talking to old code until a run failed (karamazov-uk77). Nil fields + where there is no checkout to read, as from a built binary." + (let [identity* (delay + (let [dir (System/getProperty "user.dir") + git (fn [& args] + (let [r (apply proc/run {:timeout-ms 5000} "git" "-C" dir args)] + (when (and (not (:timeout r)) (zero? (long (or (:exit r) 1)))) + (str/trim (str (:out r)))))) + rev (not-empty (git "rev-parse" "HEAD"))] + {:revision rev + :dirty (when rev (boolean (seq (git "status" "--porcelain" "--untracked-files=no"))))}))] + (fn [] (assoc @identity* :started_at started-at)))) + (defn- health [_req] (let [cfg (system/config)] (json-response {:status "ok" :schema_version (db/schema-version (system/conn)) :active_runs (count @control/active) + ;; Which code is answering (karamazov-uk77). + :harness (harness-identity) ;; DEFAULTS, said plainly. :run/share-artifacts? in particular is not ;; what a given run is doing — beam/run! forces it on for any seeded run ;; — and reporting it flat once said sharing was off during a run that diff --git a/src/samizdat/store/interventions.clj b/src/samizdat/store/interventions.clj index 379cb9d1..56671cc4 100644 --- a/src/samizdat/store/interventions.clj +++ b/src/samizdat/store/interventions.clj @@ -117,6 +117,17 @@ (when-not (contains? kinds kind) (throw (ex-info (str "Unknown intervention kind: " kind) {:kind kind :known (sort (keys kinds))}))) + ;; A directive to a branch that has ENDED would sit pending forever: nothing + ;; drains a closed branch's boundary (karamazov-amem). Refused, saying how it + ;; ended, so the issuer can target a live one. `exhausted` is the one ending + ;; that is not final — `extend` reopens it — so it still takes directives. + (when-let [row (when branch-id + (db/fetch-one conn ["SELECT status, inactive_reason FROM branches + WHERE run_id = ? AND id = ?" run-id branch-id]))] + (when-not (#{"active" "exhausted"} (str (:status row))) + (throw (ex-info (str "branch " branch-id " has already ended (" (:status row) + (when-let [r (:inactive_reason row)] (str ": " r)) ")") + {:branch-id branch-id :status (:status row)})))) (let [id (db/with-writer (db/execute! conn ["INSERT INTO interventions (run_id, branch_id, kind, payload, diff --git a/test/samizdat/base_test.clj b/test/samizdat/base_test.clj index f8742201..9444ee16 100644 --- a/test/samizdat/base_test.clj +++ b/test/samizdat/base_test.clj @@ -756,10 +756,6 @@ #{ "Skills — load one with `skill load {name}` when it is" } - "src/samizdat/agent/tools/tasks.clj" - #{ - ": no such task, or another run holds it." - } "src/samizdat/agent/verify.clj" #{ "(java.lang.System/exit (if (clojure.core/pos? (+ (:fail s) (:error s))) 1 0)))" diff --git a/test/samizdat/beam_cancel_test.clj b/test/samizdat/beam_cancel_test.clj index 06faeb98..0710911f 100644 --- a/test/samizdat/beam_cancel_test.clj +++ b/test/samizdat/beam_cancel_test.clj @@ -270,3 +270,19 @@ "each turn ends before the next begins, in list order") (is (= ["B1" "B1.2" "B1.3"] (mapv :id out)) "results in the branches' order") (is (every? :ran out)))))) + +(deftest a-round-that-throws-cancels-the-turns-it-started + ;; karamazov-odyx: advance-all cancelled its in-flight turns only when the + ;; throw was a cancel signal, so any other failure while waiting left the + ;; turns it had spawned running past the driver that owned them. + (let [finished (atom #{}) + bs (mapv #(state/new-branch {:id % :problem "p"}) ["B1" "B2"])] + (with-redefs [beam/advance-branch (fn [_ b _] + (ebb/? (ebb/sleep 1500)) + (swap! finished conj (:id b)) + b) + cancel/await-or-cancel (fn [& _] (throw (ex-info "the barrier broke" {})))] + (is (thrown-with-msg? Exception #"the barrier broke" + (beam/advance-all (ctx 5000) bs 1))) + (Thread/sleep 2000) + (is (empty? @finished) "the turns it had started were cancelled, not left running")))) diff --git a/test/samizdat/cells_test.clj b/test/samizdat/cells_test.clj index 40314ac6..08f309d4 100644 --- a/test/samizdat/cells_test.clj +++ b/test/samizdat/cells_test.clj @@ -25,6 +25,10 @@ [jolt.fs :as fs] [mycelium.cell :as cell] [samizdat.cells :as cells] + [samizdat.agent.state] + [samizdat.store.db] + [samizdat.store.journal] + [samizdat.store.runs] [samizdat.store.db :as db] [samizdat.userspace :as userspace])) @@ -439,3 +443,43 @@ "(ns cells.gen.ok (:require [mycelium.cell :as cell])) (cell/defcell :gen/ok {:doc \"d\" :requires [:run-id]} (fn [{:keys [run-id]} data] (assoc data :r run-id)))"))))) + +(deftest a-run-over-its-token-budget-ends-inside-a-nested-loop + ;; karamazov-lq57: the budget was read only at the beam's round boundary, so + ;; a feature, team or decompose run — whose whole job is one "turn" — was + ;; unbounded. The turn cells every loop runs now read it: the arbiter sums + ;; what the run has billed, and route ends the branch :exhausted once the + ;; budget is reached. + (cells/load-cells!) + (let [c (samizdat.store.db/open! ":memory:") + rid (samizdat.store.runs/start-run! c {:problem "p"}) + arbiter (:handler (cell/get-cell! :gate/arbiter)) + route (:handler (cell/get-cell! :loop/route)) + b (samizdat.agent.state/new-branch {:id "W1" :problem "p"}) + ctx {:conn c :run-id rid :token-budget 1000 :max-turns 50} + spend! (fn [n] (samizdat.store.journal/record-turn! + c rid {:branch-id "W1" :turn 1 :tool-name "shell" :category :neutral + :result "" :usage {:prompt-tokens n :completion-tokens 0 + :total-tokens n}}))] + (spend! 400) + (let [d (arbiter ctx {:settled {} :branch b :turn 1})] + (is (not (:over-budget? d)) "under the budget") + (is (= :continue (:verdict (route ctx d))))) + (spend! 700) + (let [d (arbiter ctx {:settled {} :branch b :turn 2})] + (is (:over-budget? d) "1100 billed against 1000") + (is (= :exhausted (:verdict (route ctx d))) "the branch ends, whichever loop it is in")) + (testing "a run with no budget pays for no query and runs on" + (let [d (arbiter (dissoc ctx :token-budget) {:settled {} :branch b :turn 3})] + (is (not (:over-budget? d))))))) + +(deftest an-abandoned-branch-always-says-why + ;; karamazov-n6ql: the provider-error arm ended a branch with no + ;; :inactive-reason, and an oversight pass recorded abandoned with ended null. + (cells/load-cells!) + (let [route (:handler (cell/get-cell! :loop/route)) + b (assoc (samizdat.agent.state/new-branch {:id "S" :problem "p"}) + :consecutive-provider-errors 99) + d (route {:max-turns 50} {:branch b :turn 3})] + (is (= :abandoned (:verdict d))) + (is (re-find #"provider failed 99" (str (get-in d [:branch :inactive-reason])))))) diff --git a/test/samizdat/control_test.clj b/test/samizdat/control_test.clj index 1fc4594b..5126a85b 100644 --- a/test/samizdat/control_test.clj +++ b/test/samizdat/control_test.clj @@ -1046,3 +1046,22 @@ (let [r (api-control/start-run! {:conn c :config cfg} {:problem "p"})] (is (= 503 (:status r))) (is (str/includes? (get-in r [:body :error :message]) "connection refused: 127.0.0.1:8080"))))))))) + +(deftest a-directive-to-a-branch-that-has-ended-is-refused + ;; karamazov-amem: the supervisor queued directives to branches that had + ;; already ended; nothing would ever drain them. + (let [c (samizdat.store.db/open! ":memory:") + rid (samizdat.store.runs/start-run! c {:problem "p"}) + submit! (fn [m] (samizdat.store.interventions/submit! c rid m))] + (doseq [b ["B1" "B2" "B3"]] (samizdat.store.runs/open-branch! c rid {:branch-id b})) + (samizdat.store.runs/close-branch! c rid "B1" :abandoned "gave up") + (samizdat.store.runs/close-branch! c rid "B2" :exhausted "turn cap") + (let [e (try (submit! {:branch-id "B1" :kind "message" :payload "hello"}) nil + (catch Throwable e e))] + (is (some? e) "an abandoned branch will never drain it") + (is (re-find #"B1" (str (ex-message e)))) + (is (re-find #"gave up" (str (ex-message e))) "and the refusal says how it ended")) + (is (number? (submit! {:branch-id "B2" :kind "extend" :payload {:turns 5}})) + "an exhausted branch can still be extended — the one ending that is not final") + (is (number? (submit! {:branch-id "B3" :kind "message" :payload "hi"})) "a live one") + (is (number? (submit! {:branch-id nil :kind "pause" :payload {}})) "run-wide is unaffected"))) diff --git a/test/samizdat/image_test.clj b/test/samizdat/image_test.clj index 61da9d35..ee5e6854 100644 --- a/test/samizdat/image_test.clj +++ b/test/samizdat/image_test.clj @@ -196,3 +196,18 @@ (is (= "42" (:value (route/eval-for ctx "(+ 40 2)" nil 25000))) "the image never recovered from a timed-out eval") (finally (route/release! root))))) + +(deftest a-child-that-dies-fails-fast-and-is-tried-again + ;; karamazov-69p0 / tetz: a start that failed waited out the whole connect + ;; deadline and logged only the profile path. A child that has exited will + ;; not come up, so the wait ends there, and the start is tried once more on + ;; a fresh port before the eval fails. + (let [spawned (atom 0)] + (with-redefs [image/spawn-argv (fn [_ _ _] (swap! spawned inc) ["sh" "-c" "echo boom; exit 7"])] + (let [t0 (System/currentTimeMillis) + img (image/start! {:root (System/getProperty "java.io.tmpdir") :backend :none + :sandbox-spec {}})] + (is (nil? img) "no image") + (is (= 2 @spawned) "tried again on a fresh port") + (is (< (- (System/currentTimeMillis) t0) 10000) + "and did not wait out the connect deadline for a child that had died"))))) diff --git a/test/samizdat/judge_test.clj b/test/samizdat/judge_test.clj index 4cac7d6e..abdf5e0a 100644 --- a/test/samizdat/judge_test.clj +++ b/test/samizdat/judge_test.clj @@ -667,3 +667,21 @@ Some trailing prose that is not a bullet. (is (<= (count cut) (+ 80 60))) (is (str/includes? cut "sources truncated at 80 chars")))) (is (nil? (judge/focus-sources {} "anything" 100)) "no sources, no section"))) + +(deftest the-second-pass-keeps-only-what-survived-not-its-deliberation + ;; karamazov-kdoj, run 40c57a2a: the verify reply reasoned in long + ;; one-line paragraphs that quoted the candidates' tags, then listed what + ;; survived under its own heading; the whole deliberation was stored as the + ;; findings and handed to the retry. + (let [candidates "- [low] the constants are not in the diff\n- [low] two runner omissions reported\n" + reply (str "Looking at the first: the claim that [low] the constants are not in the diff is" + " accurate, the diff shows calls but not definitions. But wait, is it blocking? No." + " — VERIFIED\n\n" + "The second, [low] two runner omissions reported, is out of scope. — VERIFIED\n\n" + "---\n\nSurviving findings:\n\n" + "- [low] the constants are not in the diff — VERIFIED\n" + "- [low] two runner omissions reported — VERIFIED\n") + out (judge/verified-findings {:reply reply :candidates candidates})] + (is (str/includes? (str out) "the constants are not in the diff")) + (is (not (str/includes? (str out) "But wait")) "the deliberation is not a finding") + (is (not (str/includes? (str out) "Looking at the first"))))) diff --git a/test/samizdat/llm_test.clj b/test/samizdat/llm_test.clj index dcd3f476..9bca0691 100644 --- a/test/samizdat/llm_test.clj +++ b/test/samizdat/llm_test.clj @@ -2129,3 +2129,10 @@ "```tool-call\n{\"name\":\"shell\",\"args\":{\"command\":\"echo \\\"a\\\",\\\"b\\\"\"}}\n```")] (is (= {:command "echo \"a\",\"b\""} (:args p))) (is (not (:auto-repaired? p)))))) + +(deftest a-reply-cut-short-by-the-provider-reads-as-truncated + ;; karamazov-6uyv: DeepSeek's insufficient_system_resource ends a reply + ;; mid-generation, like length; it read as a model that made no call. + (is (:truncated (fence/signals {:finish-reason "insufficient_system_resource"} nil))) + (is (:truncated (fence/signals {:finish-reason "length"} nil))) + (is (not (:truncated (fence/signals {:finish-reason "stop"} nil))))) diff --git a/test/samizdat/manifest_test.clj b/test/samizdat/manifest_test.clj index 77ea28cc..dfc67352 100644 --- a/test/samizdat/manifest_test.clj +++ b/test/samizdat/manifest_test.clj @@ -927,3 +927,13 @@ (is (some? e)) (is (re-find #":answerr" (str (ex-message e)))) (is (re-find #":answerer" (str (ex-message e))) "and lists the roles there are")))) + +(deftest a-composed-child-with-a-prompt-is-warned-about + ;; karamazov-r6x1: composing a manifest that declares :prompt ran it under + ;; the parent's prompt with nothing saying the child's was dropped. + (let [parent (assoc (wf/read-definition (slurp (io/resource "manifests/orchestrator.edn"))) + :subworkflows {:loop/worker "worker" :test/reviewing "review"}) + compiled (try (manifests/compile-loop parent) (catch Throwable _ nil)) + warnings (get-in compiled [:compiled-fsm :mycelium/compile-warnings])] + (is (some #(and (= :composed-prompt-ignored (:type %)) (= "review" (:manifest %))) warnings) + (pr-str warnings)))) diff --git a/test/samizdat/repl_confinement_test.clj b/test/samizdat/repl_confinement_test.clj index 95266157..1aadf9fd 100644 --- a/test/samizdat/repl_confinement_test.clj +++ b/test/samizdat/repl_confinement_test.clj @@ -195,3 +195,18 @@ "the refusal did not name the tool to reach for instead")) (is (:ok r) "WITHOUT a sandbox the shell is reachable from the REPL — zrq.8")))) + +(deftest the-project-image-can-require-what-the-project-declares + ;; karamazov-1b37: the harness put the project's declared roots (:paths and + ;; every alias's :extra-paths) on the HARNESS image and logged "eval can now + ;; reach …/test", but every role but the supervisor evaluates in the project + ;; image, which starts with no alias — so a namespace the branch just wrote + ;; under test/ could not be required. + (let [root (str (fs/create-temp-dir))] + (spit (str root "/deps.edn") (pr-str {:paths ["src"] :aliases {:test {:extra-paths ["test"]}}})) + (.mkdirs (java.io.File. (str root "/test/probe"))) + (spit (str root "/test/probe/just_written_test.clj") + "(ns probe.just-written-test)\n(defn answer [] 42)\n") + (let [r (ev root "(require 'probe.just-written-test) (probe.just-written-test/answer)")] + (is (str/includes? (str (:result r) (:value r) (:output r)) "42") (pr-str r))) + (route/release! root))) diff --git a/test/samizdat/security/policy_test.clj b/test/samizdat/security/policy_test.clj index 84ef78ae..36ea5387 100644 --- a/test/samizdat/security/policy_test.clj +++ b/test/samizdat/security/policy_test.clj @@ -136,6 +136,17 @@ allowed all along" (is (= :allow (:effect (policy/decide {} "jolt -M:run")))) (is (= :allow (:effect (policy/decide {} "jolt -M:run 2>&1"))))) + (testing "and ANY project alias, not one pattern per alias (karamazov-9nsf): + a project's own aliases are the same trust as its test alias" + (is (= :allow (:effect (policy/decide {} "jolt -M:camera2d")))) + (is (= :allow (:effect (policy/decide {} "jolt -A:bench -e '(run)'")))) + (is (= :allow (:effect (policy/decide {} "jolt -M:dev:test")))) + (is (not= :allow (:effect (policy/decide {} "sudo jolt -M:camera2d"))) + "a wrapper still does not ride the allow")) + (testing "a refused form of an allowed head says so, not that the tool is closed" + (let [r (policy/run-shell {:args {:command "jolt nrepl-server 7000"}})] + (is (:needs-approval r)) + (is (str/includes? (:result r) "This form of `jolt`") (:result r)))) (testing "a leading VAR=value assignment does not defeat an allow — it sets a variable for the very command the rule reads, unlike an exec wrapper which stands in front of a different one. Run a3566c73 was diff --git a/test/samizdat/server_test.clj b/test/samizdat/server_test.clj index 3defd2a6..a31ea66c 100644 --- a/test/samizdat/server_test.clj +++ b/test/samizdat/server_test.clj @@ -342,3 +342,12 @@ (is (= "qwen" (:model r))))) (testing "a colon that names no provider is part of the model's name" (is (= "qwen3:32b" (:model (control/run-llm-config config base {:model "qwen3:32b"}))))))) + +(deftest health-says-which-harness-is-answering + ;; karamazov-uk77: a stale serve process answered /health exactly like a + ;; current one. + (let [id (server/harness-identity)] + (is (re-matches #"[0-9a-f]{40}" (str (:revision id))) "the checkout's commit, run from one") + (is (boolean? (:dirty id))) + (is (string? (:started_at id))) + (is (= id (server/harness-identity)) "read once: the code the process loaded"))) diff --git a/test/samizdat/tasks_test.clj b/test/samizdat/tasks_test.clj index e8a39a95..ed2a3ef7 100644 --- a/test/samizdat/tasks_test.clj +++ b/test/samizdat/tasks_test.clj @@ -363,3 +363,24 @@ (is (= 1 (tasks/attempted! c id))) (is (= 2 (tasks/attempted! c id))) (is (= 2 (:attempts (tasks/get-task c id))) "and it is on the row, not in a process")))) + +(deftest a-refused-claim-says-which-case-it-is + ;; karamazov-fjrq: one sentence — "no such task, or another run holds it" — + ;; covered a missing task, a closed one and a held one, so a branch that + ;; had closed its own task read that another run held it and made a + ;; duplicate. + (with-db [c] + (let [rid (runs/start-run! c {:problem "p"}) + claim (fn [bid id] (tools/run-tool {:tool-name "task" :args {:action "claim" :id id} + :conn c :run-id rid + :branch (state/new-branch {:id bid :problem "p"})})) + mine (tasks/create! c {:title "mine"}) + theirs (tasks/create! c {:title "theirs"})] + (is (str/includes? (:result (claim "B1" "sz-nope")) "no task with that id")) + (tasks/claim! c mine rid "B1") + (tasks/close! c mine "done") + (let [r (:result (claim "B1" mine))] + (is (str/includes? r "already done") r) + (is (str/includes? r "you closed it yourself") r)) + (tasks/claim! c theirs rid "B2") + (is (str/includes? (:result (claim "B1" theirs)) "branch B2 holds it")))))