diff --git a/dev/samizdat/dev/arena.clj b/dev/samizdat/dev/arena.clj index 1b8855b2..9389eba3 100644 --- a/dev/samizdat/dev/arena.clj +++ b/dev/samizdat/dev/arena.clj @@ -54,6 +54,12 @@ [samizdat.agent.beam :as beam] [samizdat.agent.gates :as gates] [samizdat.agent.verify :as verify] + [samizdat.config :as config] + [samizdat.leakage :as leakage] + [samizdat.llm.client :as client] + [samizdat.llm.fence :as fence] + [samizdat.llm.registry :as registry] + [samizdat.prompt :as prompt] [samizdat.server :as server] [samizdat.session :as session] [samizdat.stats :as stats] @@ -458,6 +464,17 @@ (do (kill! p) false))) (catch Throwable _ nil))) +(defn subject-terms + "The subject's own words, for the leakage screen: its top-level namespace + prefixes (src/flight -> \"flight.\") and the project directory's name." + [root] + (let [src (io/file root "src")] + (vec (distinct + (concat (for [f (or (some-> src .listFiles seq) []) + :when (.isDirectory ^java.io.File f)] + (str (str/replace (.getName ^java.io.File f) "_" "-") ".")) + [(.getName (io/file root))]))))) + (defn edits "What the SUPERVISOR changed about the loop during this run, and what became of each change. @@ -482,9 +499,13 @@ [conn run-id] (let [rows (db/fetch conn ["SELECT kind, name, version, rationale, success_count, failure_count, error_count, - length(body) AS body_len + length(body) AS body_len, body FROM userspace WHERE source = 'project' ORDER BY kind, name, version"]) + seed-body (into {} (for [r (db/fetch conn ["SELECT kind, name, body FROM userspace + WHERE source = 'factory'"])] + [[(:kind r) (:name r)] (:body r)])) + terms (subject-terms (System/getProperty "user.dir")) ;; The factory seed for each name, to size the edit against the ;; TEMPLATE rather than against the previous edit: three small saves ;; that walk a cell a long way from the shipped one is the shape drift @@ -501,7 +522,16 @@ ;; not the version's doing (karamazov-a6mj.1). :standing [(:success_count r) (:failure_count r) (:error_count r)] :chars-vs-seed (when-let [b (get seed [(:kind r) (:name r)])] - (- (:body_len r) b))}) + (- (:body_len r) b)) + ;; Whether it could be promoted as it stands: does it + ;; name this subject, or drop a guard the template had + ;; (karamazov-na2k.13). Against the factory seed, which + ;; is what a promotion would replace. + :leakage (when-let [t (get seed-body [(:kind r) (:name r)])] + (let [f (leakage/screen {:kind (:kind r) :template t + :edited (:body r) + :subject-terms terms})] + (when-not (leakage/clean? f) f)))}) rows) ;; journal/notes returns the note's DATA map already parsed; wrapping it ;; in (mapv :data ...) yields a vector of nils, which reads as 'edits @@ -850,7 +880,11 @@ :wall-ms (- (System/currentTimeMillis) started) :budget budget :load [load-at-start (load-average)] - :at (str (java.time.Instant/now))} + :at (str (java.time.Instant/now)) + ;; where this run's database is kept, so a later reader + ;; (contrast, karamazov-na2k.11) can open the run itself + :recording (when recordings + (str recordings "/" (.getName (io/file dest)) ".sqlite3"))} (harness-revision) extra))] (try ;; WHAT THIS ARM ALREADY KNOWS, before the child opens the database. @@ -1131,7 +1165,14 @@ :turns-to-first-artifact (med (map :turns-to-first-artifact rs))} :spread {:turns [(med (map :turns rs)) (apply min (or (seq (keep :turns rs)) [nil])) - (apply max (or (seq (keep :turns rs)) [nil]))]}}])) + (apply max (or (seq (keep :turns rs)) [nil]))] + ;; Tokens are the one column steady enough to read + ;; at the n this rig affords, so an arm's cost is + ;; shown with its range, not only its median + ;; (karamazov-na2k.3). + :tokens [(med (map :tokens rs)) + (apply min (or (seq (keep :tokens rs)) [nil])) + (apply max (or (seq (keep :tokens rs)) [nil]))]}}])) ;; Pairwise, on the same rows the arms' own :reliability reads — and ;; only where both arms decided something. Read it with the :by of ;; each arm: two arms decided by different columns are not comparable @@ -1170,9 +1211,17 @@ (mapv (fn [[k rids]] {:surface k :runs (count rids) :of n :rate (double (/ (count rids) (max 1 n))) - :run-ids (vec rids)}))))] + :run-ids (vec rids)})))) + ;; Any run's version of a surface that the screen flagged: a + ;; recurring fix is a promotion candidate only if it does not name + ;; the subject or drop a guard (karamazov-na2k.13). + leaks (->> rows + (mapcat (comp :saves :edits)) + (keep (fn [e] (when (:leakage e) [[(:kind e) (:name e)] (:leakage e)]))) + (into {}))] {:runs n - :saved (tally :saves) + :saved (mapv #(cond-> % (leaks (:surface %)) (assoc :leakage (leaks (:surface %)))) + (tally :saves)) :refused (tally (fn [e] (map (fn [d] {:kind "attempt" :name (str (:reason d))}) (:refused e)))) :rationales (->> rows @@ -1182,6 +1231,52 @@ (sort-by (comp - val)) vec)})) +(defn contrast-pairs + "A passed and a failed run of the SAME task, one pair per task that has + both — the corpus SIMBA distils rules from (a good and a bad trajectory of + one input), which a sweep produces for free: n runs of a task from one + pinned sha (karamazov-na2k.11). Only rows the rig or the loop decided, and + only rows with a recording to read." + [rows] + (->> rows + (filter :recording) + (group-by :task) + (keep (fn [[task rs]] + (let [good (first (filter #(true? (first (passed? %))) rs)) + bad (first (filter #(false? (first (passed? %))) rs))] + (when (and good bad) {:task task :good good :bad bad})))) + vec)) + +(defn run-summary + "One recorded run as text a reader can compare: its outcome, and each + branch's turns as `turn tool category`, with the answer it shipped." + [db-path] + (let [c (db/open! db-path)] + (try + (let [run (first (db/fetch c ["SELECT id, status, final_answer FROM runs ORDER BY started_at DESC LIMIT 1"])) + turns (db/fetch c ["SELECT branch_id, turn, tool_name, category FROM turns + WHERE run_id = ? ORDER BY id" (:id run)])] + (str "status: " (:status run) "\n" + (str/join "\n" (map #(str (:branch_id %) " t" (:turn %) " " (:tool_name %) " " (:category %)) + turns)) + "\nanswer: " (some-> (:final_answer run) (subs 0 (min 1500 (count (:final_answer run))))))) + (finally (db/close c))))) + +(defn contrast! + "For each passed/failed pair of one task, ask `chat` (fn [prompt] -> text) + what the passing run did that the failing one did not, and return + [{:task :good :bad :difference}]. A report for the operator, like + `recurring-edits`: a difference seen once is an anecdote, and promoting one + into a memory or the shipped templates is a person's decision." + [rows chat] + (vec (for [{:keys [task good bad]} (contrast-pairs rows)] + {:task task :good (:run-id good) :bad (:run-id bad) + :difference (chat (prompt/render-str + (slurp (io/resource "samizdat/dev/contrast.md")) + {:task (name task) + :good (run-summary (:recording good)) + :bad (run-summary (:recording bad))}))}))) + (defn tasks "The sweep task set, with `:standing-requirements` already appended to each problem. One place builds the prompt so two runs of the same task id can @@ -1345,5 +1440,13 @@ (clojure.pprint/pprint (summarize rows)) (println "\n=== RECURRING EDITS ===") (clojure.pprint/pprint (recurring-edits rows)) + ;; Opt-in, because it spends a model call per task that has both a + ;; passed and a failed run (karamazov-na2k.11). + (when (System/getenv "ARENA_CONTRAST") + (let [cfg (:llm (config/load-config)) + ad (registry/adapter-for (:provider cfg)) + chat (fn [p] (fence/prose (:content (client/chat ad cfg [{:role "user" :content p}]))))] + (println "\n=== WHAT THE PASSING RUN DID ===") + (clojure.pprint/pprint (contrast! rows chat)))) (flush) (System/exit 0))) diff --git a/dev/samizdat/dev/contrast.md b/dev/samizdat/dev/contrast.md new file mode 100644 index 00000000..785d88c4 --- /dev/null +++ b/dev/samizdat/dev/contrast.md @@ -0,0 +1,9 @@ +Two runs of the same task, "{{task}}", from the same starting tree and the same harness. The first passed and the second did not. + +Name what the passing run DID that the failing one did not — a step it took, a check it ran, an order it did things in — as one or two rules a future run could follow. Only differences you can point to in the turns below; not luck, not the model's mood. If the runs differ only by chance, say that. + +PASSED: +{{good}} + +FAILED: +{{bad}} diff --git a/docs/RFCS/RFC-002-manifests-and-cells.md b/docs/RFCS/RFC-002-manifests-and-cells.md index a99f084a..ea349828 100644 --- a/docs/RFCS/RFC-002-manifests-and-cells.md +++ b/docs/RFCS/RFC-002-manifests-and-cells.md @@ -63,11 +63,11 @@ glob-scoped interceptors match on. `:turn-sliceable?` declares that a manifest may **not** be a run's loop. The slice cuts every edge returning to `:start` into `:end`, which is the definition of a turn for an iterating loop and silent data loss for anything -else; two shipped manifests have that back edge *structurally*, and both say -so here. `beam.edn` is the scheduler — it advances the branches that make -model calls rather than making one — and `repl.edn` is a shape whose four -pure cells classify a branch, with the binding enforcement in `phases.edn`. -Such a manifest stays in `catalog` (the inventory) but is off `render-catalog` +else; one shipped manifest has that back edge *structurally*, and says so +here: `beam.edn` is the scheduler — it advances the branches that make model +calls rather than making one. (`repl.edn`, a shape whose enforcement lived in +`phases.edn` and which no driver could run, was the second until it was +removed, karamazov-na2k.9.) Such a manifest stays in `catalog` (the inventory) but is off `render-catalog` (the supervisor's switch menu) and off selection's candidates, and both drivers refuse it. Beam's exemption used to be an artifact of the fact that nothing called `turn-manifest` on it, which is an invariant held by absence — diff --git a/resources/cells/board.clj b/resources/cells/board.clj index c0d0929c..4afdd9a9 100644 --- a/resources/cells/board.clj +++ b/resources/cells/board.clj @@ -1219,6 +1219,9 @@ (journal/note! conn run-id :board-review {:data {:task task :attempt attempts :verdict verdict :decision decision :landed (boolean landed?) + ;; the verdict is the fail-open default: the + ;; judge's reply had no verdict line (na2k.6) + :unread (boolean (:unread reviewed)) ;; WHY, beside the verdict (karamazov-3htz). :reason (judge/for-the-record :reply-chars det) ;; The VERIFIED findings, which is what the retry diff --git a/resources/cells/feature.clj b/resources/cells/feature.clj index 7463845c..a75a5aa9 100644 --- a/resources/cells/feature.clj +++ b/resources/cells/feature.clj @@ -341,7 +341,7 @@ :usage (:usage r)}) (catch Throwable _ nil)) (:content r))) - {:keys [verdict findings]} + {:keys [verdict findings unread]} (judge/review {:chat chat :requirement (:problem branch) :evidence (judge/evidence rows) @@ -361,12 +361,16 @@ (judge/blocking-findings (str "FINDINGS:\n" all)))] {:decision (if (and (= :complete verdict) (not blocking)) :ship :revise) :verdict verdict + :unread unread :findings (judge/for-the-record :reply-chars all)})) decision (:decision judged)] (journal/note! conn run-id :critique {:data {:decision decision :deterministic (boolean det) :reason (judge/for-the-record :reply-chars det) :verdict (:verdict judged) + ;; true when no verdict line was read and + ;; :verdict is the fail-open default + :unread (boolean (:unread judged)) :findings (:findings judged)}}) (assoc data :critic/decision decision :critique/findings (or det "")))) ;; fail-open: a broken critic ships rather than wedging the loop diff --git a/resources/cells/oversight.clj b/resources/cells/oversight.clj index d152d6f6..2b7e1f04 100644 --- a/resources/cells/oversight.clj +++ b/resources/cells/oversight.clj @@ -113,6 +113,44 @@ sent-back-green? (seq offers)))) +(defn since-last-look + "The signals worth-a-look? judges, reduced to what is NEW since the last + pass that spent a model call (karamazov-na2k.16). + + `now` is this gather's reading as counts and flags; `prev` is the :signals + the last spent pass recorded, nil when there was none, in which case the + levels are the news. worth-a-look? read levels on every pass, so evidence a + pass had already read woke the next one too: run 390dcd17 spent all 12 + passes in 24 minutes on unmet 5 over a floor of 2, then went two hours with + no supervision left. A quiet pass records nothing, so `prev` is always the + last look that cost something. + + - unmet: the firings since, not the total. + - idle: a stall the last pass already saw wakes the next only after another + floor's worth of idle turns; one it did not see, or a new one after a + write reset the count, counts whole. + - errors and refusals: the entries past the ones already read. Both lists + come from the journal in order, so the count is a position. + - the round's flags (at the cap, nothing shipped, green work sent back): + once per round, the round being how many :route notes there are. + - offers are already once per run." + [{:keys [unmet idle errors refused round at-cap? nothing-shipped? + sent-back-green? offers]} + prev {:keys [idle-floor]}] + (let [idle (or idle 0) + p-idle (or (:idle prev) 0) + new-round? (or (nil? prev) (not= round (:round prev)))] + {:unmet-gates (- (or unmet 0) (if prev (or (:unmet prev) 0) 0)) + :idle-turns (if (and prev (>= p-idle idle-floor) (>= idle p-idle)) + (- idle p-idle) + idle) + :errors (seq (drop (if prev (or (:errors prev) 0) 0) errors)) + :refused (seq (drop (if prev (or (:refused prev) 0) 0) refused)) + :at-cap? (and at-cap? new-round?) + :nothing-shipped? (and nothing-shipped? new-round?) + :sent-back-green? (and sent-back-green? new-round?) + :offers offers})) + (defn sent-back-green? "Whether the last round passed its own tests and was sent back regardless. @@ -197,6 +235,7 @@ [:oversight/crashes :any] [:oversight/results :any] [:oversight/self-graded :any] [:oversight/refused :any] + [:oversight/signals :any] [:oversight/worth-a-look? :boolean]] :quiet [:map [:oversight/worth-a-look? :boolean]]}]} (fn [{:keys [conn run-id]} data] @@ -259,7 +298,15 @@ ;; WHAT THE PROJECT HAS NOT TAKEN, until a pass of this run has ;; shown it (the :adoption-offered note the reason cell leaves). offers (when-not (journal/last-note conn run-id :adoption-offered) - (safely :offers userspace/offers []))] + (safely :offers userspace/offers [])) + errors (vec (concat (filter :error findings) crashes)) + refusals (vec (concat refused rejected)) + idle-floor (gates/threshold :oversight-idle-floor) + ;; What a pass that spends a call records, so the next gather + ;; can tell new evidence from evidence already read. + signals {:unmet unmet :idle since + :errors (count errors) :refused (count refusals) + :round (count (journal/notes conn run-id :route))}] (assoc data :oversight/turns turns :oversight/firings firings @@ -273,16 +320,19 @@ :oversight/rejected rejected :oversight/crashes crashes :oversight/offers (vec offers) + :oversight/signals signals :oversight/worth-a-look? - (worth-a-look? {:unmet-gates unmet :idle-turns since - :errors (seq (concat (filter :error findings) crashes)) - :at-cap? (at-cap? round) - :nothing-shipped? (nothing-shipped? results) - :refused (seq (concat refused rejected)) - :sent-back-green? (sent-back-green? round) - :offers offers} + (worth-a-look? (since-last-look + (assoc signals + :errors errors :refused refusals + :at-cap? (at-cap? round) + :nothing-shipped? (nothing-shipped? results) + :sent-back-green? (sent-back-green? round) + :offers offers) + (:signals (journal/last-note conn run-id :oversight)) + {:idle-floor idle-floor}) {:unmet-floor (gates/threshold :oversight-unmet-floor) - :idle-floor (gates/threshold :oversight-idle-floor)})))) + :idle-floor idle-floor})))) (assoc data :oversight/worth-a-look? false)))) ;; --- reason ----------------------------------------------------------------- @@ -457,6 +507,37 @@ [])] (when (seq rs) (prompt/render "retirement" {:gates rs}))) + ;; What the reviews said while each open + ;; change was in force, beside it — the + ;; verdict is a number, these are the why + ;; (GEPA's feedback, karamazov-na2k.12). + :reviews (let [open (safely :experiments + #(vec (remove :settled (session/experiments))) + []) + rows (for [e open + :let [rs (safely :reviews + #(journal/notes-since + conn run-id [:critique :board-review] (:at e)) + [])] + :when (seq rs)] + {:name (:name e) + :reviews (mapv (fn [r] + {:kind (:kind r) + :verdict (:verdict r) + :findings (some-> (:findings r) + (clip (gates/threshold :oversight-note-chars)))}) + (take-last 3 rs))})] + (when (seq rows) + (prompt/render "experiment-reviews" {:changes rows}))) + ;; The same question for a project edit: + ;; current, failed runs, never shipped one + ;; (karamazov-na2k.10). + :prune (let [ps (safely :pruning + #(store-us/pruning-candidates + conn (gates/threshold :pruning)) + [])] + (when (seq ps) + (prompt/render "pruning" {:edits ps}))) :catalog (safely :catalog #(wf/render-catalog conn) "")}) ;; ONE branch for the run, carried by the stream. Opened once; ;; re-opening an existing id is a no-op that returns the row. @@ -527,6 +608,7 @@ :input [:map [:oversight/verdict :keyword] [:oversight/answer :any] [:oversight/idle {:optional true} :any] [:oversight/unmet {:optional true} :any] + [:oversight/signals {:optional true} :any] [:oversight/branch {:optional true} :any]] ;; Returns `data`: the record is the effect, and the row is the product. :output [:map]} @@ -564,6 +646,7 @@ (journal/note! conn run-id :oversight {:data {:idle (:oversight/idle data) :unmet (:oversight/unmet data) + :signals (:oversight/signals data) ;; A blank note means one of several things and ;; the verdict is which: `:done` said nothing, ;; `:exhausted` ran out of turns, `:error` threw. diff --git a/resources/cells/repl.clj b/resources/cells/repl.clj deleted file mode 100644 index e2201280..00000000 --- a/resources/cells/repl.clj +++ /dev/null @@ -1,99 +0,0 @@ -;; samizdat - a self-hosting agentic harness -;; SPDX-License-Identifier: GPL-3.0-or-later -;; -;; THE REPL SESSION as cells. A session is orient -> declare -> explore -> -;; land, and the `repl` manifest is where that order is written down. -;; -;; Why it exists, from run bd56a286: a strong model spent 238 turns in the -;; REPL hunting a defect that was in its own tests, and a sibling branch read -;; for 316 turns and wrote nothing at all. Neither was reachable by steering — -;; nudges are 0-for-6 across two model tiers. What was missing was not -;; encouragement but a CONTRACT: name the files you are about to change before -;; you start exploring, and land them before you stop. -;; -;; The two conditions are enforced as phases.edn refusals (eval needs a plan; -;; done needs the plan landed) because a withhold holds and advice does not. -;; These cells compute the session's SHAPE — which step a branch is in and -;; what it still owes — so the digest and any workflow can read it, and so the -;; order is data an agent can rewrite rather than control flow in src/. -(ns cells.repl - (:require [mycelium.cell :as cell] - [samizdat.agent.state :as state])) - -(cell/defcell :repl/orient - {:doc "Entry. The branch has no plan yet: reading is open (read_file, grep, - lsp, shell) and the REPL is closed. This is where the hypothesis comes - from — read the failing assertion and the code it calls, and decide - which of the two is lying." - :pure true - :requires [] - ;; The entry node, so it requires only what the driver seeds. Every cell in - ;; this file writes :repl/step — the session's position, which the phases - ;; refusals read — and each adds the one fact its own step establishes. - :input [:map [:branch :map]] - :output [:map [:repl/step :keyword] [:repl/may-eval? :boolean]]} - (fn [_ data] - (assoc data :repl/step :orient - :repl/may-eval? false))) - -(cell/defcell :repl/declare - {:doc "The commitment, made with the `plan` tool: - - plan({\"files\": [\"src/…\"], \"tests\": [\"test/…\"], \"goal\": \"one line\"}) - - `files` is what will be created or edited, `tests` what will be - written; both land in the same debt, so a declared test nobody writes - blocks the exit exactly as a declared source file does. At least one - path is required — an empty declaration is not one, and naming no file - is exactly the state this step exists to rule out. - - Enforced by the phases.edn refusal :repl-needs-a-plan, which withholds - `eval` until this has happened. Calling plan again REPLACES the - previous declaration rather than adding to it: the contract is commit - to a hypothesis, not never change your mind." - :pure true - :requires [] - :input [:map [:branch :map]] - ;; :repl/planned? is what the manifest dispatches on — :planned to the - ;; REPL, :empty back to :start. The cell writes the key and the edge reads - ;; it, which is the rule the manifests are held to. - :output [:map [:repl/step :keyword] [:repl/planned? :boolean] - [:repl/plan :any]]} - (fn [_ {:keys [branch] :as data}] - (assoc data :repl/step :declare - :repl/planned? (state/planned? branch) - :repl/plan (state/plan branch)))) - -(cell/defcell :repl/explore - {:doc "The REPL, open. Bounded by having said what it is for rather than by - a turn budget: a branch exploring against a named file is doing the - work, and a branch exploring against nothing is the failure this - workflow was built after." - :pure true - :requires [] - :input [:map [:branch :map]] - :output [:map [:repl/step :keyword] [:repl/may-eval? :boolean]]} - (fn [_ {:keys [branch] :as data}] - (assoc data :repl/step :explore - :repl/may-eval? (state/planned? branch)))) - -(cell/defcell :repl/land - {:doc "Exit. What the branch said it would change and has not yet written. - Empty means the session may close; anything else is the debt the - phases.edn refusal :plan-not-landed withholds `done` on, naming the - outstanding paths back. Discharged by a SUCCESSFUL write from any - :file-write tool (write_file, edit_file, patch) — reading a file you - promised to change is not changing it. The REPL dies with the branch; - only a file survives it." - :pure true - :requires [] - :input [:map [:branch :map]] - ;; :repl/complete? is the exit condition the manifest loops :land on until - ;; it goes true — the debt is empty and the session may end. - :output [:map [:repl/step :keyword] [:repl/unwritten :any] - [:repl/complete? :boolean]]} - (fn [_ {:keys [branch] :as data}] - (let [owed (state/unwritten branch)] - (assoc data :repl/step :land - :repl/unwritten owed - :repl/complete? (empty? owed))))) diff --git a/resources/gates.edn b/resources/gates.edn index d0d25b08..56a0e04b 100644 --- a/resources/gates.edn +++ b/resources/gates.edn @@ -1201,6 +1201,17 @@ :provider-retry -0.5} :min-turns-for-verdict 6 :meaningful-delta 0.15 + ;; RRSI's cost rule (karamazov-na2k.3): a gain may add tokens per turn only + ;; up to base + per-fitness * the fitness it gained, relative; past that the + ;; verdict is :costly. Inside the deadband, tokens cut by more than base + ;; make it :cheaper. So the deadband's own 0.15 of gain buys 25% more tokens. + ;; Provisional: set by argument, not yet measured here. + :cost-rule {:base 0.10 :per-fitness 1.0} + ;; The deadband measured rather than fixed (karamazov-na2k.4): z standard + ;; errors of the before/after difference, each side's spread estimated + ;; from its own counts, never under :min-band. :meaningful-delta above is + ;; what stands in while either side is shorter than :min-turns-for-verdict. + :noise {:z 2.0 :min-band 0.02} :max-open-experiments 1} :kind :policy :capability-tunable? false :provenance ["RFC-012 F3"] @@ -1955,6 +1966,16 @@ judgement — it hands over the episodes that have earned the question."} + :pruning + {:value {:min-runs 3 :limit 5} + :provenance ["karamazov-na2k.10" "2609.24972v2"] + :kind :policy :capability-tunable? false + :doc "WHEN A PROJECT EDIT HAS EARNED THE QUESTION OF BEING REMOVED: the + version in force for at least :min-runs failed runs and no shipped + one. RRSI's pruning rule (a component with no positive gain over a + window is offered for deletion), the edit-level twin of :retirement + for gates. At most :limit are shown per pass. Crashes are excluded."} + :retirement {:value {:min-runs 3 :limit 5} :provenance ["karamazov-ylte.3" "2609.09153v1"] @@ -2307,9 +2328,10 @@ forced native tool_choice come off the turn row (migration v30)."} :repetition - {:value {:window 80 :step 1 :min-repeats 3 :min-period 20 :max-gap-variance 0.5} + {:value {:window 80 :step 1 :min-repeats 3 :min-period 20 :max-gap-variance 0.5 + :short-min-period 8 :short-min-repeats 20} :kind :threshold :capability-tunable? true - :provenance ["karamazov-o4wm.5"] + :provenance ["karamazov-o4wm.5" "karamazov-na2k.19"] :doc "WHEN A REPLY COUNTS AS REPEATING ITSELF (samizdat.llm.repetition). A :window-character slice that recurs :min-repeats times at gaps @@ -2319,7 +2341,11 @@ taken every :step characters — 1, because a coarser step sees only the periods it divides — and a period under :min-period is a run (a line of dashes, a column of zeros), not a model repeating - itself. The shape is floatboat's LoopDetector + itself — unless the period is at least :short-min-period and it + recurs :short-min-repeats times: GLM-5.3 once wrote + "action":"close", (a 17-character period) about 900 times in one + reply and nothing called it a loop (run 390dcd17, na2k.19). The + scan also runs on a reply that ended in a parse error. The shape is floatboat's LoopDetector (350-char windows, three near-matches, period variance under half) over exact text rather than SimHash, since samizdat reads the finished reply and a degenerate repeat is verbatim. @@ -3060,6 +3086,24 @@ it matches :decompose-max-parts — each piece is an agent, and a parent that lists eight has not thought about the boundary."} + :reply-formats + {:value {"judge" ["VERDICT" "FINDINGS" "COMPLETE" "INCOMPLETE" "ABSTAIN"] + "judge-plan" ["VERDICT" "FINDINGS" "COMPLETE" "INCOMPLETE" "ABSTAIN"] + "judge-verify" ["VERIFIED" "FALSE_POSITIVE" "No issues found"] + "acceptance-judge" ["YES" "NO"] + "critic-system" ["SCORE"] + "workflow-select-system" ["kind:" "size:" "workflow:"]} + :kind :policy :capability-tunable? false + :provenance ["karamazov-na2k.6"] + :doc "WHAT EACH SIDE CALL'S PARSER READS, per prompt: the words the prompt + has to keep asking for. A side call's prompt and its parser are two + halves of one format (judge.clj says so), both are userspace, and the + judges fail OPEN — so a reworded judge prompt that stopped asking for + a VERDICT line switched the judge off with no error anywhere. A prompt + save that drops one of these tokens is refused, naming it (matched + without regard to case). Change a format by changing the parser rule + (:judge-rules), this list and the prompt together."} + :judge-rules ;; drg-4026 #55: the judge's deterministic finalization rules — what counts ;; as a test run, a code edit, an outside claim — as data. A fixed @@ -3525,6 +3569,8 @@ Above :storm because a branch making no calls at all is further gone than one repeating a call, and the storm window cannot even see it: nothing reaches dispatch." + ;; the quantity the :when compares, recorded at every check (na2k.7) + :measure (or (:consecutive-mechanics-failures branch) 0) :when (and (state/active? branch) (>= (or (:consecutive-mechanics-failures branch) 0) (threshold :mechanics-streak-threshold))) @@ -3828,6 +3874,8 @@ until the branch puts a different one on record. A gate is data and cannot mutate the branch, so that effect lives in the loop beside the turn-budget bookkeeping." + ;; the quantity the :when compares, recorded at every check (na2k.7) + :measure (:consecutive-failures branch) :when (>= (:consecutive-failures branch) (threshold :stuck-threshold)) ;; Rendered rather than read raw: the prose names `split` only where the ;; branch may call it, which is a {% if %} over message-context's @@ -3865,6 +3913,8 @@ Between prologue-cap and progress-stalled: sharper than the generic stall (it names the files), broader than the build-phase prologue. Settles by rule, not by tool: met when the streak actually broke." + ;; the quantity the :when compares, recorded at every check (na2k.7) + :measure (get-in branch [:file-touch :streak] 0) :when (>= (get-in branch [:file-touch :streak] 0) (threshold :file-thrash-threshold)) :message-form (str (prompt "file-thrash") @@ -3883,6 +3933,8 @@ :doc "Turns passing with no progress event, after the branch has shown it can make progress. Arms only after the first, so exploration is never nudged." + ;; the quantity the :when compares, recorded at every check (na2k.7) + :measure (:turns-since-progress branch) :when (and (:any-progress? branch) ;; Only a role that can write is told to (karamazov-1wv9): ;; an answerer reading is doing its job. diff --git a/resources/manifests/repl.edn b/resources/manifests/repl.edn deleted file mode 100644 index 4d229d35..00000000 --- a/resources/manifests/repl.edn +++ /dev/null @@ -1,63 +0,0 @@ -;; THE REPL SESSION — a mini workflow with an entry and an exit condition. -;; -;; orient -> declare -> explore -> land, and a branch cannot skip a step: -;; `eval` is refused until it has declared (phases.edn :repl-needs-a-plan) and -;; `done` is refused until it has landed (:plan-not-landed). Those two -;; refusals are the enforcement; THIS file is the shape, and the shape is what -;; an agent may rewrite. -;; -;; The failure it was built after (run bd56a286, GLM-5.3): 238 turns in the -;; REPL chasing a defect that was in the branch's own tests, because nothing -;; ever made it say where it thought the defect was; and a sibling branch that -;; read for 316 turns and wrote nothing. Neither responded to steering — -;; suggestion-only gates are 0-for-6 across two model tiers (karamazov-qic). -;; A declared file is a hypothesis, and having to state one is the point. -;; -;; :declare loops back to :orient when nothing was named, so an empty plan -;; cannot open the REPL. :land loops back to itself while files are owed, -;; which is the exit condition holding. -{:description "The REPL session: orient, declare, explore, land. The declaration is made with the `plan` tool - plan({files: [...], tests: [...], goal: \"one line\"}) - and at least one file is required; an empty plan is refused. Entry and exit are enforced as phases.edn refusals, :repl-needs-a-plan (eval is withheld until a file is named) and :plan-not-landed (done is withheld while a declared file is unwritten), so exploration cannot start without a hypothesis or end without a file. Calling plan again REPLACES the declaration, which is how a branch corrects a wrong guess. Reorder or drop a step here and the next session takes the new shape; the two refusals live in phases.edn and are what actually bind." - ;; NOT TURN-SLICEABLE, and this is the file that has to say so. :declare - ;; routes its :empty edge back to :start and the manifest does not iterate, - ;; which is the shape turn-manifest refuses — so a run pointed at "repl" - ;; threw at run start under the beam and, worse, HUNG under the single-branch - ;; driver: all four cells are pure functions of an unchanging branch, so the - ;; cycle has no model call to break it. That is not a graph to repair, it is - ;; a category error. This manifest is a SHAPE, written down so an agent can - ;; rewrite the shape; what binds is the two phases.edn refusals, and nothing - ;; drives it as a run's loop (karamazov-4sx). - :turn-sliceable? false - - :input-schema - ;; What a driver hands this manifest to start a run. mycelium checks it in - ;; run-compiled BEFORE any cell runs, so a caller that builds the map wrongly - ;; is told which key, instead of a nil surfacing several cells later. - [:map [:branch :map] [:turn :int]] - - :cells {:start :repl/orient - :declare :repl/declare - :explore :repl/explore - :land :repl/land} - - :edges {:start :declare - :declare {:planned :explore - :empty :start} - :explore :land - :land {:complete :end - :owing :land}} - - :dispatches {:declare [[:planned {:repl/planned? true}] - [:empty _]] - :land [[:complete {:repl/complete? true}] - [:owing _]]} - - :invariants - [{:type :must-precede :cell :declare :before :explore :enforced true - :protects "The REPL never opens before a file has been named. This is the - whole contract: exploring is cheap and a REPL session dies with - the branch, so the only thing that makes it worth doing is - knowing which file it is going to end in."} - - {:type :must-precede :cell :explore :before :land :enforced true - :protects "Landing follows exploring rather than replacing it — the point - is not to skip the REPL, it is to bound it at both ends."}]} diff --git a/resources/manual.edn b/resources/manual.edn index 322a6d81..f275eba2 100644 --- a/resources/manual.edn +++ b/resources/manual.edn @@ -94,7 +94,7 @@ {:name samizdat.workflow/turn-manifest :summary "A whole-run manifest reduced to one turn — the slice the beam schedules. Refuses a manifest that declares :turn-sliceable? false, and one that routes an edge back to :start without iterating (that edge is cut into :end, so each cycle would run as a fresh turn with a fresh data map)."} {:name samizdat.workflow/turn-sliceable? - :summary "Whether a manifest may be a run's LOOP at all. Default true; declare :turn-sliceable? false at the top of a manifest for one whose back edge to :start is structural rather than a turn — beam.edn (the scheduler, which drives branches rather than being driven) and repl.edn (a shape whose four pure cells classify a branch, enforced by phases.edn refusals). Such a manifest stays in the catalogue but is off the supervisor's switch menu and off selection's candidate list, and both drivers refuse to run it."} + :summary "Whether a manifest may be a run's LOOP at all. Default true; declare :turn-sliceable? false at the top of a manifest for one whose back edge to :start is structural rather than a turn — beam.edn, the scheduler, which drives branches rather than being driven. Such a manifest stays in the catalogue but is off the supervisor's switch menu and off selection's candidate list, and both drivers refuse to run it."} {:name samizdat.workflow/workflow-prompt :summary "A manifest's own instructions: `:prompt \"name\"` at the top names resources/prompts/.md, appended to the base system prompt for every branch the run opens. This is how a manifest declares what its graph is FOR — review.edn is the same loop framed as a code review — so a new task shape is a manifest plus a prompt rather than a code path. A manifest may also name `:role` (roles.edn): every branch it opens runs as that role, on its tools and its prompt — answer.edn runs as the read-only `answerer`. Read from the LOADED definition, so an edit to the manifest reframes the next run."} {:name samizdat.agent.select/triage! @@ -324,6 +324,10 @@ :summary "Every part the per-turn context block can carry, in reading order. A branch holds what each cost last turn as :context-sizes; `introspect` renders both, so a cap in gates.edn :context-budget can be retuned against a measurement."} {:name samizdat.store.journal/gate-tally :summary "Fired / met / met-late / unmet per gate for a run — whether the steering is being obeyed, per steer."} + {:name samizdat.leakage/screen + :summary "Whether an edit is fit to promote into the shipped templates: the subject-specific terms it introduces and the guards (gates, refusals, invariants) the template had and it dropped. Flags only; promoting is a person's call."} + {:name samizdat.store.journal/gate-checks + :summary "Every check of a gate that declares :measure — the value it compared, whether its :when held, whether it fired. Both sides of a threshold's cut, which gate firings alone never had; the data a threshold is fitted from."} {:name samizdat.store.journal/retirement-candidates :summary "Gates that fired across several runs and were never once met — deletion candidates. It surfaces them; deciding is yours."} {:name samizdat.agent.telemetry/failure-exemplars @@ -333,7 +337,9 @@ {:name samizdat.session/branch-fitness :summary "One branch's fitness per turn, by the same weights the session and every experiment are scored with — the number the cull reads and the digest shows (RFC-012 F3)."} {:name samizdat.session/verdict - :summary "What an experiment measured, and every reason the number might not be a measurement of it: `confounded` when the reading with the provider's own failures discounted disagrees with the full one, `regraded` when the run moved the weights it is being scored by, and a count of the branches that went backwards under an aggregate that did not."} + :summary "What an experiment measured, and every reason the number might not be a measurement of it: `confounded` when the reading with the provider's own failures discounted disagrees with the full one, `regraded` when the run moved the weights it is being scored by, and a count of the branches that went backwards under an aggregate that did not. The deadband is measured from each side's own spread (`:band`); a gain that added more tokens per turn than it earned is `costly`, a flat change that cut them is `cheaper`, and a result under the best a kept change reached is `below-best`. An experiment started with kind/target/predicts carries the version it tested and whether the predicted signal moved."} + {:name samizdat.session/signal-names + :summary "The fitness signals an experiment may predict a move in — `predicts` takes one of them and up or down."} {:name samizdat.events/subscribe :summary "A channel of every event published from now on, for watching a run without polling."}]} diff --git a/resources/prompts/drift.md b/resources/prompts/drift.md index 93524bd4..1265ec8d 100644 --- a/resources/prompts/drift.md +++ b/resources/prompts/drift.md @@ -1,4 +1,4 @@ {% if no-db %}No run database in this context, so no edit history to read.{% else %}{% if surfaces %}## What has been tuned lately Per surface, over the last {{window}} runs: names edited, saves, reverts, and the standing those versions earned (runs that ended shipped or not while one was current). A surface reverted about as often as it is saved is oscillating — read the rationales before touching it again. When a threshold or a prompt would do, prefer it to a cell or a manifest: the smaller the surface, the cheaper being wrong. -{% for d in surfaces %}- {{d.kind}}: {{d.names}} edited, {{d.saves}} saves, {{d.reverts}} reverts, standing {{d.shipped}} green / {{d.failed}} failed{% if d.churn %} — most edited: {% for n in d.churn %}{{n.name}} ({{n.saves}} saves, {{n.reverts}} reverts){% if not forloop.last %}, {% endif %}{% endfor %}{% endif %} +{% for d in surfaces %}- {{d.kind}}: {{d.names}} edited, {{d.saves}} saves, {{d.reverts}} reverts, standing {{d.shipped}} green / {{d.failed}} failed{% if d.churn %} — most edited: {% for n in d.churn %}{{n.name}} ({{n.saves}} saves, {{n.reverts}} reverts{% if n.why %}; latest: "{{n.why}}"{% endif %}){% if not forloop.last %}, {% endif %}{% endfor %}{% endif %} {% endfor %}{% else %}Nothing in userspace has been edited in the last {{window}} runs.{% endif %}{% endif %} \ No newline at end of file diff --git a/resources/prompts/experiment-reviews.md b/resources/prompts/experiment-reviews.md new file mode 100644 index 00000000..98129102 --- /dev/null +++ b/resources/prompts/experiment-reviews.md @@ -0,0 +1,9 @@ +## What the reviews said while your open changes were in force + +The verdict on a change is one number. These are the reviews written since it +went in, which say what the work actually got wrong or right — read them +before deciding the number moved because of you. + +{% for c in changes %}`{{c.name}}`: +{% for r in c.reviews %}- {{r.kind}}: {{r.verdict}}{% if r.findings %} — {{r.findings}}{% endif %} +{% endfor %}{% endfor %} \ No newline at end of file diff --git a/resources/prompts/experiment-tool.md b/resources/prompts/experiment-tool.md index a99c5bcd..2ee26d9f 100644 --- a/resources/prompts/experiment-tool.md +++ b/resources/prompts/experiment-tool.md @@ -1,9 +1,11 @@ -{% if usage %}`experiment` binds a change you are making to what you expect it to do, so the next round can tell you whether it worked: {name, change, hypothesis}. Start one whenever you edit a cell, manifest, prompt or threshold. -`verdict {name}` reads it back: better / worse / unchanged / too early, with the fitness before and after. +{% if usage %}`experiment` binds a change you are making to what you expect it to do, so the next round can tell you whether it worked: {name, change, hypothesis, kind?, target?, predicts?}. Start one whenever you edit a cell, manifest, prompt or threshold. `kind` and `target` name what you edited (cell / manifest / prompt / policy and its name), and the ledger records the version. `predicts` is the one signal you expect to move and which way, e.g. "parse-error down" or "gate-unmet down"; the verdict says whether it did. +`verdict {name}` reads it back: better / worse / unchanged / costly / cheaper / below best / too early, with the fitness and the tokens per turn before and after. `verdict {name, action, why}` settles it once you have acted — action is `reverted` or `kept`. A losing change you have not settled keeps being raised, because a modification the evidence says is not helping, left in place because nobody got back to it, is the thing this measurement exists to prevent.{% endif %}{% if started %}Experiment `{{name}}` started. Measuring from here. changed: {{change}} - expected: {{hypothesis}} + expected: {{hypothesis}}{% if target %} + edit: {{target}}{% endif %}{% if predicts %} + predicts: {{predicts}}{% endif %} You will see the verdict on your next turn. Change nothing else until then — a second change measured alongside this one tells you nothing about either.{% endif %}{% if settled %}`{{name}}` settled as {{action}}.{% if why %} Reason: {{why}}{% endif %} @@ -11,13 +13,17 @@ It will stop being raised. `remember` what it taught you if you have not already Two changes measured over the same interval tell you nothing about either, so this is enforced rather than trusted to discipline. Settle what is open first — `verdict {name}` to read it, then `verdict {name, action: reverted|kept, why}` — and the slot frees.{% if unsettled %} -Waiting on: {{unsettled}}{% endif %}{% endif %}{% if no-experiment %}No experiment named `{{name}}`. `experiment` starts one.{% endif %}{% if reported %}`{{name}}` — {{verdict}}{% if numbers %} ({{numbers}}){% endif %} +Waiting on: {{unsettled}}{% endif %}{% endif %}{% if bad-prediction %}`predicts` is one signal and a direction, e.g. "parse-error down". The signals: {{signals}}.{% endif %}{% if no-experiment %}No experiment named `{{name}}`. `experiment` starts one.{% endif %}{% if reported %}`{{name}}` — {{verdict}}{% if numbers %} ({{numbers}}){% endif %} changed: {{change}} - expected: {{hypothesis}} + expected: {{hypothesis}}{% if prediction %} + predicted {{prediction.said}}: {{prediction.moved}} — {% if prediction.hit %}it went that way{% else %}it did not{% endif %}{% endif %} {% if better %} It earned its place. Leave it, and `remember` what it fixed so the next run starts from it.{% endif %}{% if worse %} -Revert it. That is the experiment working, not a mistake — and `remember` that this lever made things worse, so nobody spends a round trying it again.{% endif %}{% if unchanged %} +Revert it. That is the experiment working, not a mistake — and `remember` that this lever made things worse, so nobody spends a round trying it again.{% endif %}{% if costly %} +It gained, but it added more tokens per turn than that gain pays for. Revert it, or say why the gain is worth the price — a harness that grows more expensive for small wins ends up slow for nothing.{% endif %}{% if below-best %} +It moved up from the stretch just before it, but it is still below the {{below-best}} a change you kept already reached. Beating the previous step while sitting under the best one is how a run walks downhill a small step at a time; revert it.{% endif %}{% if cheaper %} +Fitness did not move measurably, and the turns got clearly cheaper. That is worth keeping; say so and leave it.{% endif %}{% if unchanged %} The change was not the fix. Revert it rather than leaving a change nobody can justify, and look somewhere else.{% endif %}{% if too-early %} Not enough has happened to tell yet. Wait — do not stack another change on top of an unmeasured one.{% endif %}{% if confounded %} Not attributable. Discounting the provider's own failures, the same two stretches read {{confounded.verdict}} ({{confounded.numbers}}) — so the direction above belongs to the endpoint as much as to your change. From inside one run, a change that stopped empty replies and an endpoint that simply came back look identical, and no number here separates them. Decide on the reasoning, say which you believe, and settle it — `verdict {name, action: reverted|kept, why}`. It holds the slot until you do.{% endif %}{% if regraded %} diff --git a/resources/prompts/manifest-tool.md b/resources/prompts/manifest-tool.md index ef47a2cc..7e60f041 100644 --- a/resources/prompts/manifest-tool.md +++ b/resources/prompts/manifest-tool.md @@ -12,4 +12,4 @@ set-dispatches {{args.set-dispatches}} Replace a node's dispatch table: "[[label pattern] ...]", first match wins. set-edge {{args.set-edge}} - Set a node's outgoing edge, or one labelled transition of it. A label on a plain edge makes it a map — add the dispatch in the same batch.{% endif %} + Set a node's outgoing edge, or one labelled transition of it. A label on a plain edge makes it a map — add the dispatch in the same batch.{% endif %}{% if no-versions %}No stored versions of manifest {{name}} in this project — it is still the shipped template.{% endif %} diff --git a/resources/prompts/oversight-pass.md b/resources/prompts/oversight-pass.md index 82493e2e..e9f5b496 100644 --- a/resources/prompts/oversight-pass.md +++ b/resources/prompts/oversight-pass.md @@ -63,7 +63,9 @@ as one; if it is noise, `retire` it with the reason. Left alone it stays an episode and ranks as one. {% for m in candidates %}- {{m.id}} seen in {{m.corroborations}} runs: {{m.content}} {% endfor %}{% endif %}{% if retire %} -{{retire}}{% endif %}{% if offers %} +{{retire}}{% endif %}{% if reviews %} +{{reviews}}{% endif %}{% if prune %} +{{prune}}{% endif %}{% if offers %} {{offers}}{% endif %}{% if rejected %} {{rejected}}{% endif %}{% if refused %} {{refused}}{% endif %}{% if drift %} diff --git a/resources/prompts/parse-error-causes.md b/resources/prompts/parse-error-causes.md index fc537a92..bf679f94 100644 --- a/resources/prompts/parse-error-causes.md +++ b/resources/prompts/parse-error-causes.md @@ -1 +1 @@ -{{error}}{% if line %} — the block stops parsing at line {{line}}, column {{column}}: right after `{{before}}`, at `{{after}}`{% endif %}. Common causes: (a) a raw newline inside a string value — use \n, (b) an unescaped quote inside a string — use \", (c) an unescaped backslash — use \\, (d) a missing closing brace — count the `{` and `}`, the outer object needs one of its own after `args` closes. For multi-line content, skip JSON escaping entirely: use the XML call form (raw text). +{% if bad-escape %}Invalid escape `{{bad-escape}}` in a string: after a backslash JSON allows only \" \\ \/ \b \f \n \r \t or \uXXXX, so write `\\` for a literal backslash, or drop the backslash if you did not mean one{% else %}{{error}}{% endif %}{% if line %} — the block stops parsing at line {{line}}, column {{column}}: right after `{{before}}`, at `{{after}}`{% endif %}. Common causes: (a) a raw newline inside a string value — use \n, (b) an unescaped quote inside a string — use \", (c) an unescaped backslash — use \\, (d) a missing closing brace — count the `{` and `}`, the outer object needs one of its own after `args` closes. For multi-line content, skip JSON escaping entirely: use the XML call form (raw text). diff --git a/resources/prompts/parse-error-repaired.md b/resources/prompts/parse-error-repaired.md index c011b468..cbdb1958 100644 --- a/resources/prompts/parse-error-repaired.md +++ b/resources/prompts/parse-error-repaired.md @@ -1 +1 @@ -{{error}}{% if line %} — the block stops parsing at line {{line}}, column {{column}}: right after `{{before}}`, at `{{after}}`{% endif %}. The harness auto-repaired what it could (control characters inside strings, trailing commas, a dangling key, missing closers) and the result still did not parse — escape \n, \r, \t, \\ and \" inside string values, and check the braces. For multi-line content, skip JSON escaping entirely: use the XML call form (raw text). +{% if bad-escape %}Invalid escape `{{bad-escape}}` in a string: after a backslash JSON allows only \" \\ \/ \b \f \n \r \t or \uXXXX, so write `\\` for a literal backslash, or drop the backslash if you did not mean one{% else %}{{error}}{% endif %}{% if line %} — the block stops parsing at line {{line}}, column {{column}}: right after `{{before}}`, at `{{after}}`{% endif %}. The harness auto-repaired what it could (control characters inside strings, trailing commas, a dangling key, missing closers) and the result still did not parse — escape \n, \r, \t, \\ and \" inside string values, and check the braces. For multi-line content, skip JSON escaping entirely: use the XML call form (raw text). diff --git a/resources/prompts/pruning.md b/resources/prompts/pruning.md new file mode 100644 index 00000000..f5031012 --- /dev/null +++ b/resources/prompts/pruning.md @@ -0,0 +1,13 @@ +## Edits of this project's workflow that have not earned their place + +Each is the version currently in force, and every run that ended while it was +current ended without shipping. A change that stops producing a gain is not +neutral: it costs context, turns and attention, and nothing makes it leave on +its own. + +{% for e in edits %}- {{e.kind}} `{{e.name}}` v{{e.version}} — {{e.failed}} failed runs, none shipped{% if e.rationale %}; made because: "{{e.rationale}}"{% endif %} +{% endfor %} +If the reason it was made no longer holds, revert it and say what the record +showed. If it is guarding something the failures are not about, keep it and +say so — the record cannot tell those apart for you. Crashed runs are not +counted here; they are the harness's, not the edit's. diff --git a/resources/prompts/reply-format-dropped.md b/resources/prompts/reply-format-dropped.md new file mode 100644 index 00000000..9a4f8de8 --- /dev/null +++ b/resources/prompts/reply-format-dropped.md @@ -0,0 +1 @@ +The prompt {{name}} no longer asks for {{missing}}, which is what its parser reads (gates.edn :reply-formats). A reply without it is read as a default, silently — for a judge, as a pass. Keep asking for it, or change the parser rule in gates.edn :judge-rules and the list in :reply-formats together. diff --git a/resources/prompts/session-block.md b/resources/prompts/session-block.md index 84ad9aef..73cd28a9 100644 --- a/resources/prompts/session-block.md +++ b/resources/prompts/session-block.md @@ -17,9 +17,9 @@ Nothing has happened since you looked: no turn has been taken. Whatever you chan ### Changes you have made, and what happened {{experiments}} -{% if unsettled %}**{{unsettled}} change(s) above measured worse or unchanged and you have not acted on them.** Do that first, before anything else: revert them, or say why you are keeping one. A modification the evidence says is not helping, left in place because nobody got back to it, is exactly what this measurement exists to prevent — and the next supervisor inherits it with no sign it was ever questioned. +{% if unsettled %}**{{unsettled}} change(s) above measured worse, unchanged, costly or below the best already kept, and you have not acted on them.** Do that first, before anything else: revert them, or say why you are keeping one. A modification the evidence says is not helping, left in place because nobody got back to it, is exactly what this measurement exists to prevent — and the next supervisor inherits it with no sign it was ever questioned. -{% endif %}A change with verdict `better` earned its place — say so and leave it. `worse` means REVERT it; that is not a failure, it is the experiment working. `unchanged` means the change was not the fix, so revert it too rather than leaving a change nobody can justify. `too early` means wait — do not stack another change on an unmeasured one. `confounded` means the number moved but not because of you: the world moved under the measurement, so decide on the reasoning and say that is what you did. It still holds the slot until you settle it, the same as any other change in flight. +{% endif %}A change with verdict `better` earned its place — say so and leave it. `worse` means REVERT it; that is not a failure, it is the experiment working. `unchanged` means the change was not the fix, so revert it too rather than leaving a change nobody can justify. `costly` means it gained less than the tokens it added are worth: revert it, or say why the price is right. `cheaper` means nothing measurable moved and the turns cost less — keep it. `below best` means it beat the stretch before it but not what a change you already kept reached — revert it. `too early` means wait — do not stack another change on an unmeasured one. `confounded` means the number moved but not because of you: the world moved under the measurement, so decide on the reasoning and say that is what you did. It still holds the slot until you settle it, the same as any other change in flight. {% endif %}{% if findings %} ### Worth your attention {{findings}} diff --git a/resources/prompts/system-tools.md b/resources/prompts/system-tools.md index 5a33c713..67fa982d 100644 --- a/resources/prompts/system-tools.md +++ b/resources/prompts/system-tools.md @@ -282,10 +282,11 @@ intervene({kind, branch?, text?}) Say the specific thing to do next, not that it seems stuck — a branch that could tell it was stuck would have stopped already. Watch what it does with one directive before sending another. -experiment({name, change, hypothesis}) +experiment({name, change, hypothesis, kind?, target?, predicts?}) Bind a change you are making to what you expect it to do, so the next round can tell you whether it worked. Start one whenever you edit a cell, - manifest, prompt or threshold. A change with no stated expectation cannot + manifest, prompt or threshold; name the edit with kind and target, and + the one signal you expect to move with predicts ("parse-error down"). A change with no stated expectation cannot be wrong, and a change that cannot be wrong teaches nothing. verdict({name}) Read an experiment back: better / worse / unchanged / too early, with the diff --git a/resources/prompts/task-board-owned.md b/resources/prompts/task-board-owned.md new file mode 100644 index 00000000..41268925 --- /dev/null +++ b/resources/prompts/task-board-owned.md @@ -0,0 +1,4 @@ +{{id}} — {{title}} — is the board's work, dispatched to you, and it is not +closed here: the board closes it when the review of your diff passes. Finish +it by calling `done` with your answer. If it genuinely cannot be done, say so +with `give_up`, or close it with `"status": "cancelled"` and the reason. diff --git a/resources/prompts/task-busy.md b/resources/prompts/task-busy.md index e6c1380d..c18ff016 100644 --- a/resources/prompts/task-busy.md +++ b/resources/prompts/task-busy.md @@ -1,4 +1,5 @@ -You are already working on {{current-id}} — {{current-title}}. Finish it with -`task close {{current-id}}`, or if you genuinely need to change what you are -working on, use `task switch` with a `reason` — switching is recorded, so say -why the current task is being set down. +You are already working on {{current-id}} — {{current-title}}. {% if board %}It is +the board's work, so finish it by calling `done`; the board closes it when your +diff passes review.{% else %}Finish it with `task close {{current-id}}`.{% endif %} If +you genuinely need to change what you are working on, use `task switch` with a +`reason` — switching is recorded, so say why the current task is being set down. diff --git a/resources/userspace.edn b/resources/userspace.edn index 0c844260..e94225fc 100644 --- a/resources/userspace.edn +++ b/resources/userspace.edn @@ -28,8 +28,7 @@ :board-bt "manifests/board-bt.edn" :feature "manifests/feature.edn" :decompose "manifests/decompose.edn" - :repair "manifests/repair.edn" - :repl "manifests/repl.edn"} + :repair "manifests/repair.edn"} :policies {:gates "gates.edn" @@ -51,7 +50,6 @@ "cells/probe.clj" "cells/repair.clj" "cells/oversight.clj" - "cells/repl.clj" "cells/team.clj"] :prompts @@ -193,7 +191,11 @@ :system-structure "prompts/system-structure.md" :system-tools "prompts/system-tools.md" :system-turn "prompts/system-turn.md" + :experiment-reviews "prompts/experiment-reviews.md" + :pruning "prompts/pruning.md" + :reply-format-dropped "prompts/reply-format-dropped.md" :task-busy "prompts/task-busy.md" + :task-board-owned "prompts/task-board-owned.md" :task-claimed "prompts/task-claimed.md" :task-current "prompts/task-current.md" :task-folded "prompts/task-folded.md" diff --git a/src/samizdat/agent/arbiter.clj b/src/samizdat/agent/arbiter.clj index eaecfbd9..aabc0f44 100644 --- a/src/samizdat/agent/arbiter.clj +++ b/src/samizdat/agent/arbiter.clj @@ -54,6 +54,22 @@ (not (gates/budget-exceeded? g (:branch ctx)))))) (sort-by :priority))) +(defn measurements + "Every gate that declares a :measure, with the value it measured this check + and whether its precondition held — fired or not (karamazov-na2k.7). + gate_firings keeps only the checks that fired, which is one side of the + data a threshold is fitted from. A measure that throws is left out: a + counter must not cost a turn." + [ctx] + (into [] + (keep (fn [g] + (when-let [m (:measure g)] + (try {:gate (:gate g) + :value (m ctx) + :held (boolean ((:when g) ctx))} + (catch Throwable _ nil))))) + (gates/gates))) + (defn decide "Pick at most one steer for this boundary. diff --git a/src/samizdat/agent/gates.clj b/src/samizdat/agent/gates.clj index 0c7ea433..26275a5a 100644 --- a/src/samizdat/agent/gates.clj +++ b/src/samizdat/agent/gates.clj @@ -350,7 +350,11 @@ :message (if (:message-form entry) (compile-form (:message-form entry)) (compile-message entry)) - :prediction (let [p (:prediction entry)] (fn [_] p)))) + :prediction (let [p (:prediction entry)] (fn [_] p)) + ;; The quantity a threshold gate compares, when it declares one — + ;; recorded at every check so the threshold can be fitted from data + ;; rather than set by hand (karamazov-na2k.7). + :measure (some-> (:measure entry) compile-form))) (def gates "The steer table, compiled from gates.edn :gates — all data since tier 3b. diff --git a/src/samizdat/agent/judge.clj b/src/samizdat/agent/judge.clj index ba69bec7..abbbafb1 100644 --- a/src/samizdat/agent/judge.clj +++ b/src/samizdat/agent/judge.clj @@ -558,7 +558,12 @@ explicit FALSE_POSITIVE verdicts account for every candidate. Anything else — nothing parseable, a partial answer, silence — keeps pass 1." [{:keys [reply candidates]}] - (let [cands (finding-segments candidates)] + ;; Through `usable`, like pass 1: a verify reply with its reasoning inline + ;; kept a thought that mentioned "[low]" as the findings (run 390dcd17, + ;; karamazov-na2k.21). An unterminated think block reads as blank, which + ;; keeps the candidates — the fail-safe case. + (let [reply (usable reply) + cands (finding-segments candidates)] (cond (empty? cands) nil (clean-pass? reply) nil @@ -631,6 +636,11 @@ :diff diff :answer answer :transcript transcript})) verdict (if reply (parse-verdict reply) :complete) + ;; Nothing on a verdict line: the verdict above is the default, not + ;; the judge's (karamazov-na2k.6). Still fail-open; now visible. + unread (boolean (and reply + (not-any? #(re-find (re-pattern (:verdict-line-regex (rules))) %) + (str/split-lines (usable reply))))) candidates (when reply (findings reply)) verified (if (and candidates (gates/threshold :judge-verify?)) ;; Skipped when pass 1 found nothing, so a clean review @@ -640,7 +650,7 @@ (verify-prompt {:candidates candidates :diff diff})) :candidates candidates}) candidates)] - {:verdict verdict :candidates candidates :findings verified})) + {:verdict verdict :candidates candidates :findings verified :unread unread})) (defn review-plan "Judge a PLAN against a requirement, both passes, against an injected `chat`. diff --git a/src/samizdat/agent/loop.clj b/src/samizdat/agent/loop.clj index 82d0a3c1..4e17daed 100644 --- a/src/samizdat/agent/loop.clj +++ b/src/samizdat/agent/loop.clj @@ -1026,11 +1026,14 @@ parsed — for the process and for this branch, whose own tally is the number the cull and the supervisor share (RFC-012 F3). Never allowed to throw: a counter must not be able to cost a turn." - [run-id branch tool result signals] + [run-id branch tool result signals tokens] (try (session/observe-turn! {:tool tool :category (:category result) :signals signals + ;; what the turn cost, for a verdict's cost rule + ;; (karamazov-na2k.3) + :tokens tokens :branch (when (and run-id (:id branch)) [run-id (:id branch)])}) (catch Throwable _ nil))) @@ -1047,7 +1050,8 @@ ;; and those are the ones a supervisor is least ;; able to infer from outcomes. {:parse-error (= "__parse_error__" (:name parsed)) - :auto-repaired (:auto-repaired? parsed)})) + :auto-repaired (:auto-repaired? parsed)}) + (some-> response :usage :total-tokens)) (journal/record-turn! conn run-id {:branch-id (:id branch) :turn turn ;; The task the turn served: the caller's, else the @@ -1241,16 +1245,21 @@ (assoc branch :lines-written (gitdiff/changed-lines (:root ctx) (:git-baseline ctx))) branch) - decision (arbiter/decide - {:branch branch - :max-turns max-turns - ;; How wide the beam already is, so the reproduction - ;; rung knows whether the run can afford offspring. - :branch-count (or (:branch-count ctx) 1) - :done-block (:done-block result) - :directive (or (:pending-directive branch) - (:directive ctx)) - :safe-state-coverage coverage}) + gate-ctx {:branch branch + :max-turns max-turns + ;; How wide the beam already is, so the reproduction + ;; rung knows whether the run can afford offspring. + :branch-count (or (:branch-count ctx) 1) + :done-block (:done-block result) + :directive (or (:pending-directive branch) + (:directive ctx)) + :safe-state-coverage coverage} + decision (arbiter/decide gate-ctx) + ;; Every measured gate's value this check, fired or not, so its + ;; threshold can be fitted later (karamazov-na2k.7). + _ (journal/record-gate-checks! conn run-id (:id branch) turn + (arbiter/measurements gate-ctx) + (:gate decision)) {ctx-block :block branch :branch} (context-block conn run-id branch (get-in parsed [:args :claim]) diff --git a/src/samizdat/agent/tools/base.clj b/src/samizdat/agent/tools/base.clj index 493c700c..d78a70ab 100644 --- a/src/samizdat/agent/tools/base.clj +++ b/src/samizdat/agent/tools/base.clj @@ -22,7 +22,10 @@ missing-argument complaints, the :default method, tool-names, and the per-phase refusal the branch loop consults before dispatch. Tool groups require this namespace; nothing here requires a group back." - (:require [clojure.string :as str] + (:require ;; the java.time host shim, before data.json — see samizdat.store.journal + [jolt.time] + [clojure.data.json :as json] + [clojure.string :as str] [jolt.fs :as fs] [samizdat.agent.files :as files] ;; gates and storm are not used by name here: the phases.edn @@ -123,6 +126,20 @@ [branch capability e] (ok branch (str capability " is unavailable: " (ex-message e)))) +(defn listed + "`v` as the list it stands for when it is a STRING holding a JSON array, + else `v` unchanged. GLM-5.3 sends array arguments as the string + \"[\\\"src/a.clj\\\"]\" (run 390dcd17, karamazov-na2k.15): plan then took one + entry as the path [\"src/a.clj\"], and split read its parts as none." + [v] + (if (string? v) + (let [t (str/trim v)] + (or (when (str/starts-with? t "[") + (let [d (try (json/read-str t) (catch Throwable _ nil))] + (when (sequential? d) d))) + v)) + v)) + (defn arg "A tool call's argument `k`, however the model spelled it. diff --git a/src/samizdat/agent/tools/experiments.clj b/src/samizdat/agent/tools/experiments.clj index b3339600..83dfa75f 100644 --- a/src/samizdat/agent/tools/experiments.clj +++ b/src/samizdat/agent/tools/experiments.clj @@ -20,23 +20,64 @@ (:require [clojure.string :as str] [samizdat.agent.tools.base :as base] [samizdat.prompt :as prompt] - [samizdat.session :as session])) + [samizdat.session :as session] + [samizdat.userspace :as userspace])) (defn- msg [ctx] (prompt/render "experiment-tool" ctx)) (def ^:private usage (delay (msg {:usage true}))) +(defn- read-prediction + "`predicts` as {:signal :direction}: \" up\" or \" down\", + the signal one of the counts the fitness weighs. nil for nothing given, + :bad for something that is not that shape." + [s] + (when-let [t (some-> s str str/trim not-empty)] + (let [[sig dir] (str/split (str/lower-case t) #"\s+") + sig (keyword sig)] + (if (and (contains? (set (session/signal-names)) sig) (#{"up" "down"} dir)) + {:signal sig :direction (keyword dir)} + :bad)))) + +(defn- target-of + "The userspace version an experiment tests: the latest stored version of + `kind`/`name` in this project, so the ledger names the edit and not only + the idea behind it (karamazov-na2k.5)." + [kind name] + (when (and kind name) + {:kind kind :name name + :version (try (some-> (userspace/versions (keyword kind) name) last :version) + (catch Throwable _ nil))})) + (defmethod base/run-tool "experiment" [{:keys [branch] :as ctx}] (let [name (some-> (base/arg ctx :name) str str/trim not-empty) change (some-> (base/arg ctx :change) str str/trim not-empty) - hypothesis (some-> (base/arg ctx :hypothesis) str str/trim not-empty)] - (if-not (and name change hypothesis) + hypothesis (some-> (base/arg ctx :hypothesis) str str/trim not-empty) + predicts (read-prediction (base/arg ctx :predicts)) + target (target-of (some-> (base/arg ctx :kind) str str/trim not-empty) + (some-> (base/arg ctx :target) str str/trim not-empty))] + (cond + (not (and name change hypothesis)) (base/malformed branch (str (base/missing ctx :name :change :hypothesis) "\n\n" @usage)) + + (= :bad predicts) + (base/malformed branch (msg {:bad-prediction true + :signals (str/join ", " (map clojure.core/name + (session/signal-names)))})) + + :else (try - (session/experiment! name {:change change :hypothesis hypothesis}) + (session/experiment! name {:change change :hypothesis hypothesis + :target target :predicts predicts}) (base/ok branch (msg {:started true :name name - :change change :hypothesis hypothesis}) + :change change :hypothesis hypothesis + :target (when target + (str (:kind target) " " (:name target) + (when (:version target) (str " v" (:version target))))) + :predicts (when predicts + (str (clojure.core/name (:signal predicts)) " " + (clojure.core/name (:direction predicts))))}) :progress? true) (catch clojure.lang.ExceptionInfo e (if (= :samizdat.session/too-many-open (:type (ex-data e))) @@ -76,9 +117,23 @@ :verdict (clojure.core/name (:verdict v)) :change (:change v) :hypothesis (:hypothesis v) :numbers (when (and (:before v) (:after v)) - (format "fitness %.2f -> %.2f over %d turns" - (:before v) (:after v) (:turns-since v))) + (str (format "fitness %.2f -> %.2f over %d turns" + (:before v) (:after v) (:turns-since v)) + (when-let [c (:cost v)] + (format ", tokens/turn %.0f -> %.0f" + (:before c) (:after c))))) :better (= :better (:verdict v)) + :costly (= :costly (:verdict v)) + :cheaper (= :cheaper (:verdict v)) + :prediction (when-let [p (:prediction v)] + {:said (str (clojure.core/name (:signal p)) " " + (clojure.core/name (:direction p))) + :moved (if (and (:before p) (:after p)) + (format "%.2f -> %.2f per turn" (:before p) (:after p)) + "not measurable yet") + :hit (:hit? p)}) + :below-best (when (= :below-best (:verdict v)) + (format "%.2f" (double (:best v)))) :worse (= :worse (:verdict v)) :unchanged (= :unchanged (:verdict v)) :too-early (= :too-early (:verdict v)) diff --git a/src/samizdat/agent/tools/manifest.clj b/src/samizdat/agent/tools/manifest.clj index 4577eda5..18cc3843 100644 --- a/src/samizdat/agent/tools/manifest.clj +++ b/src/samizdat/agent/tools/manifest.clj @@ -148,7 +148,7 @@ ", its cells promise " (pr-str (vec (sort produces))))))})))) (def ^:private usage - "Actions: list, show {name, version?}, save {name, edn | file, rationale}, patch {name, ops, rationale, expect-version?}, refs {name, cell}, diff {name, from?, to?}. A manifest is the loop as data — a :cells map, :edges, and dispatch patterns. Save and patch validate by compiling before they store; the run that uses it is chosen by config :run :loop. Show, save and patch report where only branch order decides and name every dispatch entry written as a form, which the analysis cannot read. Prefer patch to save for an edit: it names the change and keeps the rest of the file, comments included. rationale: one sentence on why — the history shows it to the next supervisor deciding whether your change stays.") + "Actions: list, show {name, version?}, versions {name}, save {name, edn | file, rationale}, patch {name, ops, rationale, expect-version?}, refs {name, cell}, diff {name, from?, to?}. A manifest is the loop as data — a :cells map, :edges, and dispatch patterns. Save and patch validate by compiling before they store; the run that uses it is chosen by config :run :loop. Show, save and patch report where only branch order decides and name every dispatch entry written as a form, which the analysis cannot read. Prefer patch to save for an edit: it names the change and keeps the rest of the file, comments included. rationale: one sentence on why — the history shows it to the next supervisor deciding whether your change stays.") (defn- deflag "The vendored op refusals speak the CLI's `--rewire`; here an argument is a @@ -309,6 +309,20 @@ "list" (base/ok branch (render-list conn)) + ;; The history with its rationales and how each version has fared — + ;; the one kind that had no way to read it (karamazov-na2k.5). + "versions" + (let [name (base/arg ctx :name)] + (if (str/blank? (str name)) + (base/malformed branch (base/missing ctx :name)) + (let [rows (if (userspace/bound?) + (userspace/versions :manifest name) + (us/versions conn :manifest name))] + (base/ok branch + (if (seq rows) + (str/join "\n" (map base/version-line rows)) + (say :no-versions :name name)))))) + "show" (let [name (base/arg ctx :name) v (some-> (base/arg ctx :version) str str/trim not-empty parse-long)] diff --git a/src/samizdat/agent/tools/plan.clj b/src/samizdat/agent/tools/plan.clj index 10d93c5d..c5918d60 100644 --- a/src/samizdat/agent/tools/plan.clj +++ b/src/samizdat/agent/tools/plan.clj @@ -26,7 +26,8 @@ (defmethod base/run-tool "plan" [{:keys [branch] :as ctx}] (let [coerce (fn [k] - (let [v (or (base/arg ctx k) (get (:args ctx) (name k)))] + (let [v (or (base/arg ctx k) (get (:args ctx) (name k))) + v (base/listed v)] (cond (nil? v) [] (coll? v) (vec (remove empty? (map str v))) :else [(str v)]))) @@ -45,7 +46,9 @@ ;; `done` from turn 117 was refused for it after the real files had ;; landed (karamazov-j9ow). Refused on the turn it is declared, ;; naming the entry, rather than withheld a hundred turns later. - not-paths (filterv #(re-find #"\s" %) (concat files tests))] + ;; List syntax left in an entry ([, ], a quote) is the same failure + ;; by another road: a list the call half-encoded. + not-paths (filterv #(re-find #"[\s\[\]\"]" %) (concat files tests))] (cond (seq not-paths) (base/malformed branch (msg {:not-a-path (first not-paths)})) diff --git a/src/samizdat/agent/tools/prompts.clj b/src/samizdat/agent/tools/prompts.clj index 05ba4098..d8e098d4 100644 --- a/src/samizdat/agent/tools/prompts.clj +++ b/src/samizdat/agent/tools/prompts.clj @@ -57,10 +57,15 @@ with an unbalanced `{% if %}` throws where it is USED, which for a gate message is in the middle of a run and for the system prompt is at the top of every branch. Better to refuse the save." - [body] - (try (prompt/render-str body {}) nil - (catch Throwable e - (str "this does not render: " (ex-message e))))) + [name body] + ;; The registered prompt validator, not a second copy of it: it also + ;; refuses a side call's prompt that stopped asking for what its parser + ;; reads (gates.edn :reply-formats, karamazov-na2k.6). + (when-let [p (try ((userspace/validator :prompt) name body) + (catch Throwable e {:stage :render :message (ex-message e)}))] + (if (= :render (:stage p)) + (str "this does not render: " (:message p)) + (:message p)))) (defn- shipped? [name] (some? (userspace/template :prompt name))) @@ -183,7 +188,7 @@ (nil? body) (base/malformed branch (base/missing ctx :body)) (nil? why) (base/malformed branch (base/missing ctx :rationale)) :else - (if-let [complaint (render-check (str body))] + (if-let [complaint (render-check name (str body))] ;; The render check is this surface's whole validation, so a ;; body that fails it is a rejected edit — nothing was stored. (base/rejected branch (msg {:bad-render true :name name diff --git a/src/samizdat/agent/tools/split.clj b/src/samizdat/agent/tools/split.clj index ae0a46c5..6b24ca40 100644 --- a/src/samizdat/agent/tools/split.clj +++ b/src/samizdat/agent/tools/split.clj @@ -86,7 +86,8 @@ (not (and (string? v) (str/blank? v)))) v)) ks)) - coll (fn [v] (cond (nil? v) [] (coll? v) (vec (map str v)) :else [(str v)]))] + coll (fn [v] (let [v (base/listed v)] + (cond (nil? v) [] (coll? v) (vec (map str v)) :else [(str v)])))] {:name (some-> (g :name) str str/trim) :description (some-> (g :description :desc) str str/trim) :file (some-> (g :file :path) str str/trim) @@ -153,7 +154,7 @@ (defmethod base/run-tool "split" [{:keys [branch conn run-id root] :as ctx}] - (let [raw (or (base/arg ctx :parts) (base/arg ctx :subtasks)) + (let [raw (base/listed (or (base/arg ctx :parts) (base/arg ctx :subtasks))) parts (mapv part-of (if (coll? raw) raw [])) reason (some-> (base/arg ctx :reason) str str/trim) {:keys [min-parts max-parts]} (gates/threshold :split-parts) diff --git a/src/samizdat/agent/tools/tasks.clj b/src/samizdat/agent/tools/tasks.clj index 7c660285..107b167c 100644 --- a/src/samizdat/agent/tools/tasks.clj +++ b/src/samizdat/agent/tools/tasks.clj @@ -14,6 +14,14 @@ ;; --- the task board ---------------------------------------------------------- +(defn- board-task? + "Whether the board dispatched task `id` to an owner in this run — its work + finishes through `done` and the board's review, not through `task close`." + [conn run-id id] + (boolean (and conn run-id id + (some #(= (str id) (str (:task %))) + (journal/notes conn run-id :board-task))))) + (defn- task-line "One line for a task. Shows the HOLDER when a branch has claimed it, which is what makes the board usable by a team: several implementors fanned out @@ -162,7 +170,8 @@ (base/ok branch (str "Already working on " (task-line held))) (base/malformed branch (prompt/render "task-busy" {:current-id (:id held) - :current-title (:title held)}))) + :current-title (:title held) + :board (board-task? conn run-id (:id held))}))) (if-let [t (tasks/claim! conn (base/arg ctx :id) run-id (:id branch))] (base/ok (take-task branch t) (str "Claimed " (task-line t)) @@ -228,6 +237,17 @@ (:branch_id row) "; only the holder" " closes it.")) + ;; The board's own work closes when its per-diff review + ;; passes (cells/board.clj), and `done` is what that review + ;; reads. An owner that closed it here marked it done with no + ;; review, no critic and no acceptance check, and every later + ;; round read it as shipped (karamazov-na2k.18). Cancelling it + ;; is still the owner's call. + (and (= "done" (str (or (base/arg ctx :status) "done"))) + (board-task? conn run-id (:id row))) + (base/malformed branch (prompt/render "task-board-owned" + {:id (:id row) :title (:title row)})) + :else (let [t (tasks/close! conn (base/arg ctx :id) (or (base/arg ctx :status) "done")) ;; Closing the CURRENT task clears the slot, so the next diff --git a/src/samizdat/capabilities.clj b/src/samizdat/capabilities.clj index 0493f4cf..319fd02a 100644 --- a/src/samizdat/capabilities.clj +++ b/src/samizdat/capabilities.clj @@ -60,6 +60,7 @@ [samizdat.events] [samizdat.export] [samizdat.hashline] + [samizdat.leakage] [samizdat.layers] [samizdat.lisp] [samizdat.llm.client] diff --git a/src/samizdat/claims.clj b/src/samizdat/claims.clj index 4d4fb911..999727dc 100644 --- a/src/samizdat/claims.clj +++ b/src/samizdat/claims.clj @@ -105,7 +105,16 @@ drive it with three names and what keeps a store-level check from dragging in the agent's whole capability surface." [claim turns tools] - (if-not (denial? claim) + ;; PER SENTENCE. The denial and the tool it denies have to share one: a + ;; steer that said "run the suite via shell" and, two clauses on, "no + ;; test_runner change needed" was refused as saying shell does not work + ;; (run 390dcd17, karamazov-na2k.17). A period only ends a sentence when + ;; a space follows it, so flight.render stays one word. + (let [denials (->> (str/split (str claim) #"(?<=[.!?])\s+|[;\n]+") + (filter denial?) + (mapcat #(map vector (subject-tools % (map str tools)) (repeat %))) + (reduce (fn [m [t sentence]] (if (contains? m t) m (assoc m t sentence))) {}))] + (if (empty? denials) [] (let [tuples (record-facts turns) db (sym/facts tuples) @@ -116,8 +125,7 @@ ;; how much, and an escalation saying \"it worked 14 times in this ;; run\" is a different argument from one saying it worked. n-worked (frequencies (keep (fn [[rel t]] (when (= :worked rel) t)) tuples)) - tools (subject-tools claim (map str tools)) - hits (for [t tools + hits (for [[t sentence] denials :when (seq (sym/query db [[:worked t]]))] - {:tool t :worked (get n-worked t 0) :claim (str claim)})] - (vec hits)))) + {:tool t :worked (get n-worked t 0) :claim (str/trim sentence)})] + (vec hits))))) diff --git a/src/samizdat/leakage.clj b/src/samizdat/leakage.clj new file mode 100644 index 00000000..7b8c37f4 --- /dev/null +++ b/src/samizdat/leakage.clj @@ -0,0 +1,48 @@ +;; samizdat - a self-hosting agentic harness +;; SPDX-License-Identifier: GPL-3.0-or-later + +(ns samizdat.leakage + "WHETHER A PROJECT'S EDIT IS FIT TO PROMOTE into the shipped templates + (karamazov-na2k.13). + + Inside a project an edit that names the project is the point: the workflow + is the project's. Promoting it into resources/ changes every project, and + there RRSI's critic applies (2609.24972, rrsi/critic.py): an edit that + encodes the task it was made on is a fit to that task, not a mechanism, + and an edit that removes a safety mechanism without a replacement is a + regression that happened to score. This is the deterministic half of that + screen — pure over the edit's text and its template, flags only. Deciding + is the person promoting it." + (:require [clojure.edn :as edn] + [clojure.string :as str])) + +(defn- read-edn [s] (try (edn/read-string (str s)) (catch Throwable _ nil))) + +(defn- guards + "The guards a body carries, by kind: a policy table's gates, a manifest's + declared invariants, a phases table's refusals." + [kind text] + (let [v (read-edn text)] + (when (map? v) + (case (str kind) + "policy" (set (concat (keep #(some-> (:gate %) str) (:gates v)) + (keep #(some-> (:refusal %) str) (:refusals v)))) + "manifest" (set (map pr-str (:invariants v))) + nil)))) + +(defn screen + "Flags for promoting `edited` over `template`: {:names-subject [terms the + edit introduces] :drops-guard [guards the template had and the edit lost]}. + `subject-terms` are the project's own words — namespace prefixes, task ids, + file names — which the caller knows and this namespace does not." + [{:keys [kind template edited subject-terms]}] + (let [t (str template) e (str edited) + introduced (vec (filter #(and (str/includes? e %) (not (str/includes? t %))) + (map str subject-terms))) + before (guards kind t) + after (guards kind e)] + {:names-subject introduced + :drops-guard (if (and before after) (vec (sort (remove after before))) [])})) + +(defn clean? [flags] + (and (empty? (:names-subject flags)) (empty? (:drops-guard flags)))) diff --git a/src/samizdat/llm/fence.clj b/src/samizdat/llm/fence.clj index 4b9e3658..bee09c6c 100644 --- a/src/samizdat/llm/fence.clj +++ b/src/samizdat/llm/fence.clj @@ -320,6 +320,34 @@ :before (subs s (max 0 (- index excerpt-chars)) index) :after (subs s index (min (count s) (+ index excerpt-chars)))})))) +(defn invalid-escape + "The first backslash inside a JSON string that starts no valid escape, as + {:escape \"\\\\&\" :index :line :column :before :after}, or nil. data.json + reports one as \"No matching clause: 38\" (the character code) and the + grammar above places it at the start of the string, which named nothing a + model could fix (run 390dcd17, karamazov-na2k.20)." + [^String s] + (let [n (count s)] + (loop [i 0 in? false] + (when (< i n) + (let [c (.charAt s i)] + (cond + (not in?) (recur (inc i) (= c \")) + (= c \") (recur (inc i) false) + (= c \\) + (let [nx (when (< (inc i) n) (.charAt s (inc i)))] + (if (and nx (contains? #{\" \\ \/ \b \f \n \r \t \u} nx)) + (recur (+ i 2) true) + (let [pre (subs s 0 i) + nl (str/last-index-of pre "\n")] + {:escape (str "\\" (or nx "")) + :index i + :line (inc (count (filter #(= \newline %) pre))) + :column (inc (- i (if nl (inc nl) 0))) + :before (subs s (max 0 (- i excerpt-chars)) i) + :after (subs s i (min n (+ i excerpt-chars)))}))) + :else (recur (inc i) true))))))) + (defn- read-json [s] (try {:ok true :value (json/read-str s :key-fn keyword)} @@ -786,8 +814,12 @@ ;; one, whose positions have drifted by every character the repair ;; inserted. The same location serves both complaints below, which ;; is why it is computed once here. - (let [where (locate-json-failure body) - complaint (merge {:error (:error first-try)} where) + (let [bad (invalid-escape body) + where (if bad + (dissoc bad :escape) + (locate-json-failure body)) + complaint (merge {:error (:error first-try) :bad-escape (:escape bad)} + where) fail (fn [msg] (if-let [x (or (some-> (tagged-call body) (assoc :tagged-call? true)) @@ -826,7 +858,11 @@ ;; A reply repeating itself, checked only where it matters — a ;; truncated reply, or one that made no call — so a long healthy ;; reply that reached its fence is not scanned (karamazov-o4wm.5). - periodic (when (or truncated (nil? parsed)) + periodic (when (or truncated (nil? parsed) + ;; a loop that ended in broken JSON is still a + ;; loop, and a JSON complaint is the wrong steer + ;; for it (karamazov-na2k.19) + (= "__parse_error__" (:name parsed))) (repetition/periodic content (lexicon/policy :repetition)))] {:no-fence (and (nil? parsed) (not truncated)) :truncated truncated diff --git a/src/samizdat/llm/repetition.clj b/src/samizdat/llm/repetition.clj index 5caea838..acea9806 100644 --- a/src/samizdat/llm/repetition.clj +++ b/src/samizdat/llm/repetition.clj @@ -41,7 +41,8 @@ within `max-gap-variance` of their mean and a period of at least `min-period`, else nil. Slices are taken every `step` characters; a step of 1 sees every period, a coarser one only periods it divides." - [text {:keys [window step min-repeats min-period max-gap-variance]}] + [text {:keys [window step min-repeats min-period max-gap-variance + short-min-period short-min-repeats]}] (let [text (str text) n (count text)] (when (and (pos? (or window 0)) (pos? (or step 0)) @@ -68,7 +69,13 @@ ;; A period under :min-period is a RUN — a line of ;; dashes, a column of zeros — which repeats ;; perfectly and is not a model repeating itself. - (when (>= period (or min-period 1)) + ;; Unless it is a short phrase repeated far past + ;; the floor: GLM wrote "action":"close", (17 + ;; characters) some 900 times (karamazov-na2k.19). + (when (or (>= period (or min-period 1)) + (and short-min-period short-min-repeats + (>= period short-min-period) + (>= (count ps) short-min-repeats))) {:repeats (count ps) :period period}))))) (sort-by (juxt (comp - :repeats) :period)) first))))) diff --git a/src/samizdat/prompt.clj b/src/samizdat/prompt.clj index 75d3c48f..18c22c63 100644 --- a/src/samizdat/prompt.clj +++ b/src/samizdat/prompt.clj @@ -147,9 +147,23 @@ ;; A prompt edit is checked before it is what renders (karamazov-1a51.8): the ;; template has to parse. Whether every variable it names is supplied is the -;; caller's business and not knowable from the text alone. +;; caller's business and not knowable from the text alone. A side call's +;; prompt also has to keep asking for what its parser reads, gates.edn +;; :reply-formats (karamazov-na2k.6). +(defn- dropped-tokens + "The reply tokens prompt `nm` no longer asks for." + [nm text] + (let [formats (try ((requiring-resolve 'samizdat.agent.gates/threshold) :reply-formats) + (catch Throwable _ nil)) + l (str/lower-case (str text))] + (vec (remove #(str/includes? l (str/lower-case (str %))) (get formats (str nm)))))) + (userspace/register-validator! :prompt - (fn [_ text] - (try (render-str text {}) nil - (catch Throwable e (userspace/problem :render e))))) + (fn [nm text] + (or (try (render-str text {}) nil + (catch Throwable e (userspace/problem :render e))) + (when-let [missing (seq (dropped-tokens nm text))] + {:stage :format + :message (render-str (userspace/body :prompt "reply-format-dropped") + {:name nm :missing (str/join ", " missing)})})))) diff --git a/src/samizdat/session.clj b/src/samizdat/session.clj index ae512c22..72cf02b3 100644 --- a/src/samizdat/session.clj +++ b/src/samizdat/session.clj @@ -132,8 +132,13 @@ nil)) (defn- count-turn - [t {:keys [tool category signals]}] + [t {:keys [tool category signals tokens]}] (cond-> (update t :turns (fnil inc 0)) + ;; What the turn cost, so a verdict can ask whether a gain paid for it + ;; (karamazov-na2k.3). Absent on a turn with no usage, never counted as 0. + (and (number? tokens) (pos? tokens)) + (update :tokens (fnil + 0) tokens) + (and tool category) (update-in [:tools (str tool) (keyword (name category))] (fnil inc 0)) @@ -220,6 +225,52 @@ ;; --- distillation ----------------------------------------------------------- +(declare named-counts) + +(defn- weighted-counts + "Every count the fitness weighs, with its weight: [[count weight] ...]." + [tally p] + (let [w (:weights p)] + (map (fn [[k n]] [n (or (get w k) 0)]) (named-counts tally)))) + +(defn- named-counts + "Every count the fitness weighs, keyed by the name of its weight." + [tally] + (let [sig (:signals tally) + tool-counts (reduce (fn [acc [_ cats]] + (merge-with + acc (select-keys cats + [:success :failure :mechanics]))) + {} (:tools tally)) + gate-counts (reduce (fn [acc [_ o]] + (merge-with + acc (select-keys o [:met :met-late :unmet]))) + {} (:gates tally)) + v (:verify tally) + prov (:provider tally)] + (into (array-map) + (map (fn [[k n]] [k (or n 0)])) + [[:provider-empty (:empty-reply prov)] + [:provider-failed (+ (or (:call-failed prov) 0) (or (:usage-cap prov) 0))] + [:provider-retry (:retried prov)] + [:tool-success (:success tool-counts)] + [:tool-failure (:failure tool-counts)] + [:tool-mechanics (:mechanics tool-counts)] + [:parse-error (:parse-error sig)] + [:no-fence (:no-fence sig)] + [:truncated (:truncated sig)] + [:auto-repaired (:auto-repaired sig)] + [:verify-green (:green v)] + [:verify-red (:red v)] + [:verify-skipped (:skipped v)] + [:gate-met (:met gate-counts)] + [:gate-met-late (:met-late gate-counts)] + [:gate-unmet (:unmet gate-counts)]]))) + +(defn signal-names + "The signals an experiment may predict a move in: the counts the fitness + weighs, by the names gates.edn :fitness :weights gives them." + [] + (keys (named-counts {}))) + (defn fitness-of "How well the loop ran, per turn, over a tally-shaped map. @@ -236,35 +287,27 @@ ([tally p] (let [turns (or (:turns tally) 0)] (when (pos? turns) - (let [w (:weights p) - sig (:signals tally) - tool-counts (reduce (fn [acc [_ cats]] - (merge-with + acc (select-keys cats - [:success :failure :mechanics]))) - {} (:tools tally)) - gate-counts (reduce (fn [acc [_ o]] - (merge-with + acc (select-keys o [:met :met-late :unmet]))) - {} (:gates tally)) - v (:verify tally) - prov (:provider tally) - score (+ (* (or (:empty-reply prov) 0) (:provider-empty w)) - (* (+ (or (:call-failed prov) 0) (or (:usage-cap prov) 0)) - (:provider-failed w)) - (* (or (:retried prov) 0) (:provider-retry w)) - (* (or (:success tool-counts) 0) (:tool-success w)) - (* (or (:failure tool-counts) 0) (:tool-failure w)) - (* (or (:mechanics tool-counts) 0) (:tool-mechanics w)) - (* (or (:parse-error sig) 0) (:parse-error w)) - (* (or (:no-fence sig) 0) (:no-fence w)) - (* (or (:truncated sig) 0) (:truncated w)) - (* (or (:auto-repaired sig) 0) (:auto-repaired w)) - (* (or (:green v) 0) (:verify-green w)) - (* (or (:red v) 0) (:verify-red w)) - (* (or (:skipped v) 0) (:verify-skipped w)) - (* (or (:met gate-counts) 0) (:gate-met w)) - (* (or (:met-late gate-counts) 0) (:gate-met-late w)) - (* (or (:unmet gate-counts) 0) (:gate-unmet w)))] - (/ score (double turns))))))) + (/ (reduce + (map (fn [[n w]] (* n w)) (weighted-counts tally p))) + (double turns)))))) + +(defn fitness-variance + "How much one turn's fitness varies, estimated from the same counts + `fitness-of` sums — nil for an empty tally. + + Each weighted count is read as an event that happens on a turn with rate + count / turns, so a turn's fitness varies by sum w^2 p(1-p). It is an + estimate (a signal can fire twice in a turn, and signals are not + independent), but it needs nothing the tally does not already hold, and it + is the right order: a stretch where half the calls fail is noisy and one + where every call succeeds is not (karamazov-na2k.4)." + ([tally] (fitness-variance tally (lexicon/policy :fitness))) + ([tally p] + (let [turns (or (:turns tally) 0)] + (when (pos? turns) + (reduce + (map (fn [[n w]] + (let [q (min 1.0 (/ (double n) turns))] + (* w w q (- 1.0 q)))) + (weighted-counts tally p))))))) (defn fitness "The session's fitness as it stands." @@ -307,7 +350,7 @@ and what the supervisor expected. The hypothesis matters as much as the measurement: a change with no stated expectation cannot be wrong, and a change that cannot be wrong teaches nothing whichever way the numbers go." - [name {:keys [change hypothesis]}] + [name {:keys [change hypothesis target predicts]}] (let [in-flight (filterv #(and (nil? (:settled %)) (not= :too-early (:verdict %))) (experiments)) @@ -343,7 +386,12 @@ ;; `snapshot` deliberately drops them, and a regression count ;; needs a before per branch, not one for the session. :before-branches (:branches @tally) - :before-fitness (fitness-of before)}) + :before-fitness (fitness-of before) + ;; WHAT is being tested and WHAT should move (karamazov-na2k.5): + ;; the userspace version the change made, and one fitness signal + ;; with the direction it is expected to go. + :target target + :predicts predicts}) (mark! name) nil)) @@ -370,14 +418,44 @@ (update p :weights #(reduce (fn [w k] (assoc w k 0.0)) % health-weights))) (defn- direction - "Which way a per-turn move counts as having gone, inside `p`'s deadband." - [move p] + "Which way a per-turn move counts as having gone, inside `band`." + [move band] (cond (nil? move) :too-early - (>= move (:meaningful-delta p)) :better - (<= move (- (:meaningful-delta p))) :worse + (>= move band) :better + (<= move (- band)) :worse :else :unchanged)) +(defn band + "The deadband for a move between two tallies: z standard errors of the + difference of their per-turn means, from each side's own spread + (karamazov-na2k.4, RRSI's calibrated delta), never under :min-band. With + either side shorter than :min-turns-for-verdict the spread is not worth + estimating, and the fixed :meaningful-delta stands in." + [before-tally delta p] + (let [{:keys [z min-band]} (:noise p) + nb (or (:turns before-tally) 0) + na (or (:turns delta) 0) + vb (fitness-variance before-tally p) + va (fitness-variance delta p)] + (if (and z vb va (>= nb (:min-turns-for-verdict p)) (>= na (:min-turns-for-verdict p))) + (max (or min-band 0.0) + (* z (Math/sqrt (+ (/ vb nb) (/ va na))))) + (:meaningful-delta p)))) + +(defn- floor-band + "How far under the best kept fitness a stretch may sit and still be read as + noise: z standard errors of the stretch's OWN per-turn mean. Not the + difference band — the floor compares one estimate against a number, and a + noisy stretch before the change says nothing about this one." + [delta p] + (let [{:keys [z min-band]} (:noise p) + n (or (:turns delta) 0) + v (fitness-variance delta p)] + (if (and z v (>= n (:min-turns-for-verdict p))) + (max (or min-band 0.0) (* z (Math/sqrt (/ v n)))) + (:meaningful-delta p)))) + (defn- reading "One scoring of an experiment's two tallies: before, after, and the direction between them. Both sides go through `fitness-of` with the SAME policy, which @@ -388,8 +466,9 @@ [before-tally delta p] (let [b (fitness-of before-tally p) a (fitness-of delta p) - move (when (and a b) (- a b))] - {:before b :after a :delta move :verdict (direction move p)})) + move (when (and a b) (- a b)) + width (band before-tally delta p)] + {:before b :after a :delta move :band width :verdict (direction move width)})) (defn- branch-regressions "How many branches measured on both sides of the change went backwards, and @@ -410,6 +489,47 @@ {:measured (count moves) :regressed (count (filter #(<= % (- (:meaningful-delta p))) moves))}))) +(defn- cost + "Tokens per turn before the change and since it, and the relative change — + nil when either side has no token count to divide." + [before delta] + (let [per (fn [t] (let [n (:turns t) k (:tokens t)] + (when (and (number? k) (pos? k) (number? n) (pos? n)) + (/ (double k) n)))) + b (per before) + a (per delta)] + (when (and b a) + {:before b :after a :change (/ (- a b) b)}))) + +(defn- prediction + "An experiment's predicted move, settled: the signal's per-turn rate before + and since, and whether it went the stated way. nil with no prediction." + [{:keys [signal direction]} before delta] + (when signal + (let [rate (fn [t] (let [n (or (:turns t) 0)] + (when (pos? n) (/ (double (get (named-counts t) signal 0)) n)))) + b (rate before) + a (rate delta)] + {:signal signal :direction direction :before b :after a + :hit? (boolean (and a b (case direction + :up (> a b) + :down (< a b) + false)))}))) + +(defn- priced + "A verdict with RRSI's cost rule applied (2609.24972 eq. 7, karamazov-na2k.3): + a gain has to pay for the tokens it added, relative change dC <= base + + per-fitness * dS, or it is `:costly`; a change that moved fitness inside + the deadband but cut tokens by more than `base` is `:cheaper`. Anything + else, and any verdict with no token count, is left as it was." + [v {:keys [cost delta]} {:keys [base per-fitness]}] + (let [dc (:change cost)] + (cond + (or (nil? dc) (nil? base)) v + (and (= :better v) (> dc (+ base (* (or per-fitness 0) (or delta 0))))) :costly + (and (= :unchanged v) (< dc (- base))) :cheaper + :else v))) + (defn verdict "What happened after an experiment, or nil when there is no such experiment. @@ -442,6 +562,9 @@ (when-let [e (get-in @tally [:experiments (str name)])] (let [delta (deep-diff (:before e) (snapshot)) turns (or (:turns delta) 0) + ;; The best fitness any OTHER experiment reached and was kept at. + best (some->> (vals (dissoc (:experiments @tally) (str name))) + (keep :kept-after) seq (apply max)) full (reading (:before e) delta p) blind (reading (:before e) delta (health-blind p)) stamped (:before-fitness e) @@ -452,7 +575,7 @@ regraded? (boolean (and stamped (:before full) (not= (:before full) stamped)))] (merge (select-keys e [:change :hypothesis :at]) - (select-keys full [:before :after :delta]) + (select-keys full [:before :after :delta :band]) {:turns-since turns :full full :health-blind blind @@ -462,12 +585,23 @@ ;; both numbers rather than only asserting the scale moved. :before-as-stamped stamped :branches (branch-regressions e p) + :cost (cost (:before e) delta) + :target (:target e) + :prediction (prediction (:predicts e) (:before e) delta) :verdict (cond (< turns (:min-turns-for-verdict p)) :too-early (nil? (:delta full)) :too-early (not= (:verdict full) (:verdict blind)) :confounded - :else (:verdict full))}))))) + ;; RRSI's floor (eq. 5): under the best a kept change + ;; reached, less the band, is a loss whichever way it moved + ;; from the stretch just before it. + (and best (:after full) (< (:after full) (- best (floor-band delta p)))) + :below-best + :else (priced (:verdict full) + {:cost (cost (:before e) delta) :delta (:delta full)} + (:cost-rule p))) + :best best}))))) (defn reverted! "Record that the supervisor acted on a verdict — reverted the change, or @@ -478,8 +612,12 @@ answer was clear, and the change stayed anyway. Marking it settles it and stops the block nagging; NOT marking it is what makes the nagging escalate." [name kept?] - (swap! tally update-in [:experiments (str name)] - (fn [e] (when e (assoc e :settled (if kept? :kept :reverted))))) + (let [after (when kept? (:after (verdict name)))] + (swap! tally update-in [:experiments (str name)] + (fn [e] (when e (cond-> (assoc e :settled (if kept? :kept :reverted)) + ;; what a kept change reached: the floor later + ;; changes are held to (karamazov-na2k.4) + after (assoc :kept-after after)))))) nil) (defn experiments @@ -491,7 +629,8 @@ (:experiments @tally)))) (defn unsettled-losses - "Experiments that measured `worse` or `unchanged` and have not been acted on. + "Experiments that measured `worse`, `unchanged` or `costly` and have not + been acted on. The one thing a supervisor under selection pressure must not be allowed to quietly skip. A change that was measured and found wanting, and then left in @@ -500,7 +639,7 @@ supervisor inherits it with no way to tell it was ever questioned." [] (vec (filter #(and (nil? (:settled %)) - (contains? #{:worse :unchanged} (:verdict %))) + (contains? #{:worse :unchanged :costly :below-best} (:verdict %))) (experiments)))) (defn- detail diff --git a/src/samizdat/store/journal.clj b/src/samizdat/store/journal.clj index a97a827f..ada7682e 100644 --- a/src/samizdat/store/journal.clj +++ b/src/samizdat/store/journal.clj @@ -837,6 +837,29 @@ WHERE run_id = ? AND branch_id = ? AND outcome IS NULL ORDER BY id" run-id branch-id])) +(defn record-gate-checks! + "One row per measured gate check this turn: value, whether it held, and + whether it was the gate chosen (karamazov-na2k.7). Best effort — a lost + measurement must not cost the turn." + [conn run-id branch-id turn checks fired-gate] + (when (and conn run-id (seq checks)) + (try + (db/with-writer + (doseq [{:keys [gate value held]} checks + :when (number? value)] + (db/execute! conn + ["INSERT INTO gate_checks (run_id, branch_id, turn, gate, value, held, fired, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)" + run-id branch-id turn (name gate) (double value) + (if held 1 0) (if (= gate fired-gate) 1 0) (db/now)]))) + (catch Throwable e (log/warn "gate checks not recorded:" (ex-message e)))))) + +(defn gate-checks + "Every recorded check of a measured gate on this run, oldest first." + [conn run-id] + (db/fetch conn ["SELECT gate, branch_id, turn, value, held, fired FROM gate_checks + WHERE run_id = ? ORDER BY id" run-id])) + (defn gate-firings [conn run-id] (db/fetch conn ["SELECT * FROM gate_firings WHERE run_id = ? ORDER BY id" run-id])) @@ -983,6 +1006,23 @@ ORDER BY id" run-id (name kind)]))) +(defn notes-since + "Notes of any of `kinds` on this run created at or after `since` (an ISO + instant), oldest first, each its parsed data with :kind beside it." + [conn run-id kinds since] + (when (seq kinds) + (into [] + (keep (fn [row] + (try (assoc (json/read-str (str (:data row)) :key-fn keyword) + :kind (:kind row)) + (catch Throwable _ nil)))) + (db/fetch conn (into [(str "SELECT kind, data FROM events + WHERE run_id = ? AND created_at >= ? AND kind IN (" + (str/join ", " (repeat (count kinds) "?")) + ") ORDER BY id") + run-id (str since)] + (map name kinds)))))) + (def ^:private record-tables "The tables holding a run's account of itself, and how each one names its run. Ordered so an FTS mirror goes before the rows it indexes — deleted the diff --git a/src/samizdat/store/knowledge.clj b/src/samizdat/store/knowledge.clj index 10ffd23b..1cc27027 100644 --- a/src/samizdat/store/knowledge.clj +++ b/src/samizdat/store/knowledge.clj @@ -608,7 +608,7 @@ discovered it. The verdict becomes the memory's RECORD, not just its text: `better` is a - success and `worse` or `unchanged` a failure, so a lever tried repeatedly + success (and so is `cheaper`) and `worse`, `unchanged` or `costly` a failure, so a lever tried repeatedly without result accumulates a negative record and sinks in the ranking on its own. `too-early` is not written at all — an unfinished experiment has concluded nothing, and recording it would teach the next session that the @@ -626,16 +626,25 @@ ;; call under the lazy seq's realization lock is a forbidden park when the ;; connection is contended (ADR-001 rule 1). Same as distill-findings!. (reduce - (fn [acc {:keys [name change hypothesis verdict before after regraded]}] + (fn [acc {:keys [name change hypothesis verdict before after regraded target prediction]}] (if-not (and change (not regraded) (not (#{:too-early :confounded} verdict))) acc (let [pattern (lever-key change) content (str "[lever] " change " — " (clojure.core/name verdict) (when (and before after) (format " (fitness %.2f -> %.2f)" before after)) - ". Expected: " hypothesis) + ". Expected: " hypothesis + ;; The ledger's two extra columns (karamazov-na2k.5): + ;; which edit, and whether what it said would move did. + (when target + (str " Edit: " (:kind target) " " (:name target) + (when (:version target) (str " v" (:version target))) ".")) + (when-let [{:keys [signal direction hit?]} prediction] + (str " Predicted " (clojure.core/name signal) " " + (clojure.core/name direction) ": " + (if hit? "it did." "it did not.")))) existing (by-pattern conn pattern) - worked? (= :better verdict)] + worked? (contains? #{:better :cheaper} verdict)] (conj acc (if existing ;; Here the outcome IS earned: the lever was pulled and measured. diff --git a/src/samizdat/store/migrations.clj b/src/samizdat/store/migrations.clj index a4b3e3e2..4bba1895 100644 --- a/src/samizdat/store/migrations.clj +++ b/src/samizdat/store/migrations.clj @@ -861,7 +861,30 @@ column." ["ALTER TABLE turns ADD COLUMN task_id TEXT"]) +(def v36 + "EVERY CHECK of a gate that declares a :measure, fired or not + (karamazov-na2k.7). + + gate_firings holds the checks that fired, so a threshold's data was + censored: the value it compared was never kept, and the checks under it + not at all. A threshold cannot be fitted from one side of its own cut + (DSPy's ReAnchor fits cut points at midpoints of observed values and keeps + one only if it wins a held-out check). value is what the :measure form + returned, held whether the :when held, fired whether this gate was the + one chosen." + ["CREATE TABLE IF NOT EXISTS gate_checks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + run_id TEXT NOT NULL, + branch_id TEXT, + turn INTEGER, + gate TEXT NOT NULL, + value REAL, + held INTEGER NOT NULL, + fired INTEGER NOT NULL, + created_at TEXT NOT NULL)" + "CREATE INDEX IF NOT EXISTS idx_gate_checks_run ON gate_checks(run_id, gate)"]) + (def migrations "Ordered. Index 0 is migration 1; PRAGMA user_version holds the count applied." [v1 v2 v3 v4 v5 v6 v7 v8 v9 v10 v11 v12 v13 v14 v15 v16 v17 v18 v19 v20 v21 v22 v23 v24 - v25 v26 v27 v28 v29 v30 v31 v32 v33 v34 v35]) + v25 v26 v27 v28 v29 v30 v31 v32 v33 v34 v35 v36]) diff --git a/src/samizdat/store/userspace.clj b/src/samizdat/store/userspace.clj index e561fce1..6b0d5fda 100644 --- a/src/samizdat/store/userspace.clj +++ b/src/samizdat/store/userspace.clj @@ -246,7 +246,11 @@ (let [churn (->> (group-by :name rs) (mapv (fn [[nm vs]] {:name nm :saves (count vs) - :reverts (count (filter revert? vs))})) + :reverts (count (filter revert? vs)) + ;; why the latest version was made, + ;; which the block says to read + ;; (karamazov-na2k.5) + :why (some-> (last vs) :rationale str not-empty)})) (sort-by (juxt (comp - :reverts) (comp - :saves) :name)) vec)] {:kind kind @@ -259,6 +263,30 @@ (sort-by (juxt (comp - :saves) :kind)) vec))) +(defn pruning-candidates + "Project edits that have not earned their place: the CURRENT project + version of a name that has been current for at least `min-runs` failed runs + and never once for a shipped one, most-failed first, at most `limit`. + + RRSI's structural pruning (2609.24972 eq. 14): a component with no + positive measured gain over a window is offered for deletion, because + score-only evolution has no incentive to remove anything. retirement + candidates do it for gates; this is the same for an edit (karamazov- + na2k.10). Crashes are not counted — a run the harness could not finish is + evidence about the harness, not the edit. It surfaces them; deciding is the + supervisor's." + [conn {:keys [min-runs limit]}] + (db/fetch conn ["SELECT u.kind, u.name, u.version, u.failure_count AS failed, u.rationale + FROM userspace u + WHERE u.source = 'project' + AND u.version = (SELECT MAX(v.version) FROM userspace v + WHERE v.kind = u.kind AND v.name = u.name) + AND u.success_count = 0 + AND u.failure_count >= ? + ORDER BY u.failure_count DESC, u.kind, u.name + LIMIT ?" + min-runs limit])) + (defn names "Every name at `kind`, with its latest version and how many versions it has — the catalogue a tool lists." diff --git a/test/samizdat/agent_test.clj b/test/samizdat/agent_test.clj index 913de632..9288caa2 100644 --- a/test/samizdat/agent_test.clj +++ b/test/samizdat/agent_test.clj @@ -92,6 +92,17 @@ (is (string? (:message d))) (is (string? (:prediction d)) "every gate declares what it expects next"))) + (testing "a gate that declares :measure reports the quantity it compares" + ;; karamazov-na2k.7: gate_firings recorded only that a gate fired, not the + ;; value it compared or the checks where it did not, so its threshold + ;; could never be fitted from data (DSPy's ReAnchor needs both sides). + (let [b (branch-with :any-progress? true :turns-since-progress 9 + :turns (vec (repeat 5 {}))) + ms (arbiter/measurements {:branch b :max-turns 40}) + ps (first (filter #(= :progress-stalled (:gate %)) ms))] + (is (= 9 (:value ps))) + (is (boolean? (:held ps))))) + (testing "a human directive outranks every machine gate" ;; dirge PR 717 as a design property rather than a bug fix. (let [b (branch-with :consecutive-failures 5 diff --git a/test/samizdat/claims_test.clj b/test/samizdat/claims_test.clj index 0c48957d..ce93f489 100644 --- a/test/samizdat/claims_test.clj +++ b/test/samizdat/claims_test.clj @@ -50,6 +50,25 @@ "do not ship until the ring course test is re-pinned" record ["eval" "shell"])))))) +(deftest a-denial-has-to-be-about-the-tool-it-names + ;; karamazov-na2k.17, run 390dcd17 supervisor turn 6: a steer that told the + ;; branch to run the suite VIA shell was refused as saying shell does not + ;; work, because it said "no test_runner change needed" two clauses later. + ;; The denial and the tool have to share a sentence. + (let [record (turns ["shell" "success"] ["shell" "success"]) + directive (str "Good recovery after the REPL went down \u2014 the fade math in " + "flight.render is on disk now. Next steps, concretely: (1) run " + "`jolt -M:test` via shell and get the suite green before anything " + "else; (2) the new pure functions are exactly the kind of no-raylib " + "pure code this project unit-tests \u2014 add cases to the existing " + "test/flight/render_test.clj (no test_runner change needed since the " + "namespace exists), especially edge cases.")] + (is (empty? (claims/contradicted-by-record directive record ["shell" "eval"]))) + (testing "the same denial in one sentence with the tool is still caught" + (is (seq (claims/contradicted-by-record + "Fine work. shell does not work in this sandbox, so stop using it." + record ["shell" "eval"])))))) + (deftest a-directive-that-contradicts-the-run-is-refused-with-the-count ;; The wiring, in the tool where karamazov-ko5b actually landed — an ;; intervene directive, not a knowledge row. diff --git a/test/samizdat/judge_test.clj b/test/samizdat/judge_test.clj index 38dffc8e..4cac7d6e 100644 --- a/test/samizdat/judge_test.clj +++ b/test/samizdat/judge_test.clj @@ -127,6 +127,31 @@ (is (empty? (judge/finding-segments ""))) (is (empty? (judge/finding-segments nil)))) +(deftest a-reply-with-no-verdict-line-is-marked-unread-not-just-passed + ;; karamazov-na2k.6. Fail-open stays — a judge that cannot answer must not + ;; wedge the loop — but a pass because nothing was read must not look like + ;; a pass because the judge said so. + (let [said (judge/review {:chat (fn [_ _] "VERDICT: COMPLETE") :requirement "r" :diff "d"}) + mute (judge/review {:chat (fn [_ _] "Looks fine to me overall.") :requirement "r" :diff "d"})] + (is (= :complete (:verdict said))) + (is (not (:unread said))) + (is (= :complete (:verdict mute)) "still fails open") + (is (true? (:unread mute)) "and says it read no verdict"))) + +(deftest the-verify-pass-reads-what-the-judge-said-not-what-it-thought + ;; karamazov-na2k.21, run 390dcd17: GLM-5.3's verify reply carried its + ;; reasoning inline, the reasoning mentioned "[low]" while weighing a + ;; candidate, and that paragraph survived as the findings the next round + ;; was handed. Pass 1 already reads through `usable`; pass 2 did not. + (let [candidates "- [low] The fade never quite completes at the cull radius.\n" + reply (str "The candidates are mostly positive observations plus one [low]\n" + "finding. Let me verify each against the diff.\n" + "- [low] The fade never quite completes at the cull radius. VERIFIED.\n") + out (judge/verified-findings {:reply reply :candidates candidates})] + (is (str/includes? (str out) "cull radius")) + (is (not (str/includes? (str out) "think")) out) + (is (not (str/includes? (str out) "Let me verify")) out))) + (deftest the-verify-pass-drops-only-what-it-explicitly-calls-a-false-positive ;; Pass 2, ported from dirge (roborev's VerifyDedupePreamble). The judge ;; re-reads each candidate against the diff and marks it VERIFIED or diff --git a/test/samizdat/kanban_test.clj b/test/samizdat/kanban_test.clj index cbc1d477..7f4306da 100644 --- a/test/samizdat/kanban_test.clj +++ b/test/samizdat/kanban_test.clj @@ -17,6 +17,7 @@ [samizdat.agent.tools :as tools] [samizdat.llm.message :as message] [samizdat.store.db :as db] + [samizdat.store.journal :as journal] [samizdat.store.runs :as runs] [samizdat.store.tasks :as tasks] [samizdat.tape :as tape])) @@ -94,6 +95,32 @@ "so the next turn asks for the next task rather than pointing at finished work") (is (= "done" (:status (tasks/get-task c id))))))) +(deftest a-board-owner-finishes-its-task-with-done-not-task-close + ;; karamazov-na2k.18, run 390dcd17 turn 74: after a compaction the owner + ;; closed the task the board had dispatched to it with `task close`. The + ;; board closes a task only when its per-diff review passes, so that close + ;; skipped review, critic and acceptance, and every later round read the + ;; task as done. + (with-run [c rid] + (let [id (tasks/create! c {:title "the board's task"}) + _ (journal/note! c rid :board-task {:branch-id "B1" :data {:task id :title "t"}}) + held (:branch (run-task c rid (branch) {:action "claim" :id id})) + r (run-task c rid held {:action "close" :id id})] + (is (= :mechanics (:category r))) + (is (str/includes? (:result r) "done") "it names the way to finish") + (is (not= "done" (:status (tasks/get-task c id))) "and the row is not closed") + (testing "cancelling it is still the owner's to say" + (let [r2 (run-task c rid held {:action "close" :id id :status "cancelled"})] + (is (= "cancelled" (:status (tasks/get-task c id))) (:result r2)))))) + (testing "a claimed task that is busy points a board owner at done too" + (with-run [c rid] + (let [a (tasks/create! c {:title "board work"}) + b (tasks/create! c {:title "another"}) + _ (journal/note! c rid :board-task {:branch-id "B1" :data {:task a :title "t"}}) + held (:branch (run-task c rid (branch) {:action "claim" :id a})) + r (run-task c rid held {:action "claim" :id b})] + (is (not (str/includes? (:result r) "task close")) (:result r)))))) + (deftest closing-somebody-elses-task-does-not-free-your-slot (with-run [c rid] (let [mine (tasks/create! c {:title "mine"}) diff --git a/test/samizdat/leakage_test.clj b/test/samizdat/leakage_test.clj new file mode 100644 index 00000000..0daf03c4 --- /dev/null +++ b/test/samizdat/leakage_test.clj @@ -0,0 +1,37 @@ +;; samizdat - a self-hosting agentic harness +;; SPDX-License-Identifier: GPL-3.0-or-later + +(ns samizdat.leakage-test + "karamazov-na2k.13: what makes a project's edit unfit to promote into the + shipped templates, from RRSI's critic: it names the project, or it removes + a safety mechanism without a replacement." + (:require [clojure.test :refer [deftest is testing]] + [samizdat.leakage :as leakage])) + +(deftest an-edit-that-names-its-project-is-flagged + (let [f (leakage/screen {:kind "prompt" + :template "Run the tests before you ship." + :edited "Run the tests before you ship. flight.draw is the only namespace that calls rl/." + :subject-terms ["flight." "horizon-fade"]})] + (is (= ["flight."] (:names-subject f)))) + (testing "a term the template already had is not the edit's" + (is (empty? (:names-subject (leakage/screen {:kind "prompt" + :template "flight. is mentioned already" + :edited "flight. is mentioned already, still" + :subject-terms ["flight."]})))))) + +(deftest an-edit-that-drops-a-guard-is-flagged + (testing "a gate removed from gates.edn" + (let [f (leakage/screen {:kind "policy" :name "gates" + :template (pr-str {:gates [{:gate :stuck} {:gate :storm}]}) + :edited (pr-str {:gates [{:gate :stuck}]})})] + (is (= [":storm"] (:drops-guard f))))) + (testing "an invariant removed from a manifest" + (let [f (leakage/screen {:kind "manifest" + :template (pr-str {:invariants [{:type :must-follow :if :dispatch :then :journal}]}) + :edited (pr-str {:invariants []})})] + (is (seq (:drops-guard f))))) + (testing "an edit that keeps every guard is clean" + (is (leakage/clean? (leakage/screen {:kind "policy" :name "gates" + :template (pr-str {:gates [{:gate :stuck}]}) + :edited (pr-str {:gates [{:gate :stuck} {:gate :new}]})}))))) diff --git a/test/samizdat/llm_test.clj b/test/samizdat/llm_test.clj index 0483b7dc..4279ff91 100644 --- a/test/samizdat/llm_test.clj +++ b/test/samizdat/llm_test.clj @@ -1683,6 +1683,31 @@ (is (nil? (:position (fence/parse-tool-call (fenced "[1, 2, 3]"))))))) +(deftest an-invalid-escape-is-named-where-it-is + ;; karamazov-na2k.20, run 390dcd17: the model wrote 2>\\&1 in a shell command + ;; and was told "No matching clause: 38" — data.json's reader hitting `&` + ;; after a backslash — pointed at the start of the string. Not repaired: + ;; whether the backslash was meant (a regex's \\d) or not (\\&) changes + ;; what the command does, so the model is told exactly what to fix. + (let [p (fence/parse-tool-call + (fenced "{\"name\":\"shell\",\"args\":{\"command\":\"jolt -M:test 2>\\&1 | tail\"}}"))] + (is (= "__parse_error__" (:name p))) + (is (not (str/includes? (:parse-error p) "No matching clause")) (:parse-error p)) + (is (str/includes? (:parse-error p) "\\&") "it names the escape") + (is (= 51 (get-in p [:position :column])) "and points at the backslash itself"))) + +(deftest a-parse-error-that-is-a-loop-is-flagged-as-one + ;; karamazov-na2k.19: the degenerate reply parsed as __parse_error__, and + ;; the repetition scan ran only on replies with no call or a truncation, so + ;; the branch was told about JSON syntax instead of that it was looping. + (let [content (fenced (str "{\"name\": \"task\", \"args\": {" + (apply str (repeat 300 "\"action\":\"close\",")) + "\"actionclose}")) + parsed (fence/parse-tool-call content) + sig (fence/signals {:content content :finish-reason "stop"} parsed)] + (is (= "__parse_error__" (:name parsed))) + (is (true? (:periodic sig))))) + ;; --- a drifted closing tag must not swallow the next parameter -------------- ;; Run c377260b turn 300: the model wrote a complete, correct boundary_test.clj ;; and closed its parameters with `` — mirroring the `name=` diff --git a/test/samizdat/manifest_test.clj b/test/samizdat/manifest_test.clj index f186ce5d..5db7efac 100644 --- a/test/samizdat/manifest_test.clj +++ b/test/samizdat/manifest_test.clj @@ -128,14 +128,14 @@ (str nm " declares itself unsliceable, so slicing it must be refused rather than quietly producing a turn"))))))) -(deftest exactly-two-shipped-manifests-declare-themselves-unsliceable +(deftest exactly-one-shipped-manifest-declares-itself-unsliceable ;; Declaring it is a real decision and not a way out of fixing a graph, so ;; the set is pinned: `beam` is the scheduler, which drives branches rather - ;; than being driven, and `repl` is a SHAPE — four pure cells classifying a - ;; branch, with the enforcement in phases.edn — that no driver should ever - ;; be pointed at. A third name here is a decision somebody has to make on - ;; purpose. - (is (= #{"beam" "repl"} + ;; than being driven. `repl` was the second — a SHAPE whose enforcement + ;; lived in phases.edn and which no driver could run — and was removed as + ;; machinery that could not earn its place (karamazov-na2k.9). A second + ;; name here is a decision somebody has to make on purpose. + (is (= #{"beam"} (set (remove #(manifests/turn-sliceable? (shipped-definition %)) (manifests/shipped-manifests)))))) @@ -172,7 +172,7 @@ ;; compiles the whole-run form. (let [menu (wf/render-catalog nil)] (is (str/includes? menu "loop")) - (doseq [nm ["beam" "repl"]] + (doseq [nm ["beam"]] (is (not (str/includes? menu (str "- " nm " "))) (str nm " is offered as a workflow to switch to, and cannot run as one"))))) @@ -184,7 +184,7 @@ ;; anywhere. Under workflow/run! `:run :loop "repl"` did not fail, it HUNG, ;; which is worse than the beam's throw. (let [conn (db/open! ":memory:")] - (doseq [nm ["repl" "beam"]] + (doseq [nm ["beam"]] (testing nm (let [e (try (wf/compile-turn-loop conn nm) nil (catch Throwable t t))] (is (some? e) (str nm " sliced")) @@ -251,6 +251,21 @@ (is (= :mechanics (:category r))) (is (nil? (us/load-latest conn :manifest "bad")) "nothing broken was stored"))))))) +(deftest the-manifest-history-can-be-read-with-its-rationales + ;; karamazov-na2k.5: cell, prompt and policy had `versions`; manifest did + ;; not, so the supervisor could not read why a manifest was changed. + (with-db + (fn [conn] + (let [good (slurp (io/resource "manifests/loop.edn"))] + (base/run-tool {:branch {:id "B1"} :conn conn :tool-name "manifest" + :args {:action "save" :name "loop2" :edn good + :rationale "a second loop to compare"}}) + (let [r (base/run-tool {:branch {:id "B1"} :conn conn :tool-name "manifest" + :args {:action "versions" :name "loop2"}})] + (is (= :neutral (:category r)) (:result r)) + (is (str/includes? (:result r) "v1")) + (is (str/includes? (:result r) "a second loop to compare"))))))) + (deftest a-composed-manifest-registers-and-compiles-its-sub-loops (with-db (fn [conn] diff --git a/test/samizdat/mechanics_test.clj b/test/samizdat/mechanics_test.clj index f6500125..06dc09ad 100644 --- a/test/samizdat/mechanics_test.clj +++ b/test/samizdat/mechanics_test.clj @@ -922,6 +922,33 @@ (is (= ["src/flight/draw.clj" "test/flight/ghost_test.clj"] (:files (state/plan (:branch r))))))) +(deftest a-plan-list-sent-as-a-json-string-is-read-as-the-list + ;; karamazov-na2k.15, run 390dcd17 on GLM-5.3: files arrived as the STRING + ;; "[\"src/flight/render.clj\"]". Several entries were refused as not a + ;; path (6 of 8 plan calls wasted); ONE entry had no whitespace and was + ;; declared as the path ["src/flight/render.clj"], which no write could ever + ;; discharge — done was withheld for it at turn 99 of 100 and the branch + ;; ended with no answer. + (let [b (state/new-branch {:id "b1" :problem "p"}) + plan-of (fn [args] (tools/run-tool {:tool-name "plan" :branch b :args args}))] + (testing "several entries" + (let [r (plan-of {"files" "[\"src/flight/render.clj\", \"src/flight/draw.clj\"]" + "tests" "[\"test/flight/render_test.clj\"]" + "goal" "fade"})] + (is (not= :mechanics (:category r)) (:result r)) + (is (= ["src/flight/render.clj" "src/flight/draw.clj" "test/flight/render_test.clj"] + (:files (state/plan (:branch r))))))) + (testing "one entry is the path inside, not the brackets" + (let [r (plan-of {"files" "[\"src/flight/render.clj\"]" "goal" "fade"})] + (is (= ["src/flight/render.clj"] (:files (state/plan (:branch r))))))) + (testing "an empty list string is an empty plan, refused as one" + (let [r (plan-of {"files" "[]" "tests" "[]" "goal" "report"})] + (is (= :mechanics (:category r))) + (is (not (state/planned? (or (:branch r) b)))))) + (testing "an entry that still carries list syntax is not a path" + (let [r (plan-of {"files" ["[\"src/a.clj\"" "b.clj"] "goal" "g"})] + (is (= :mechanics (:category r))))))) + ;; --- a refusal a branch cannot discharge is a deadlock ---------------------- ;; ;; karamazov-iwm4, seen live in run b8ae2b1f. The supervisor stream's first diff --git a/test/samizdat/oversight_test.clj b/test/samizdat/oversight_test.clj index 09846e7b..3e152c88 100644 --- a/test/samizdat/oversight_test.clj +++ b/test/samizdat/oversight_test.clj @@ -326,6 +326,86 @@ (is (str/includes? (str prob) "REVISION CAP REACHED")) (is (str/includes? (str prob) "revision 3"))))) +(defn- gather-now [conn rid] + (cells/load-cells!) + ((:handler (cell/get-cell! :oversight/gather)) {:conn conn :run-id rid :config {}} {})) + +(defn- spent-pass! [conn rid g] + ;; What the reason cell leaves behind a pass that spent a model call: the + ;; signals it was woken on, so the next gather can tell new from seen. + (journal/note! conn rid :oversight + {:data {:verdict "done" :idle (:oversight/idle g) :unmet (:oversight/unmet g) + :signals (:oversight/signals g)}})) + +(deftest a-pass-is-not-spent-again-on-evidence-it-already-read + ;; karamazov-na2k.16. Run 390dcd17 spent all 12 passes in 24 minutes: after + ;; one real pass, every gather still saw unmet 5 over the floor of 2 and + ;; woke the supervisor to say 'already done', and the run then went two + ;; hours with no supervision at all. The trigger has to be what CHANGED + ;; since the last pass that spent a call, not the level. + (testing "a crash wakes it once" + (let [conn (db/open! ":memory:") + rid (runs/start-run! conn {:problem "p"})] + (journal/note! conn rid :run-error {:data {:error "boom" :node ":spawn"}}) + (let [g1 (gather-now conn rid)] + (is (true? (:oversight/worth-a-look? g1))) + (spent-pass! conn rid g1) + (is (false? (:oversight/worth-a-look? (gather-now conn rid))) + "the same crash, already read, is not worth another call") + (journal/note! conn rid :stage-error {:data {:error "boom again" :node ":board"}}) + (is (true? (:oversight/worth-a-look? (gather-now conn rid))) + "a new crash is")))) + (testing "the soft cap wakes it once per round" + (let [conn (db/open! ":memory:") + rid (runs/start-run! conn {:problem "p"})] + (journal/note! conn rid :route {:data {:decision "revise" :revision 3 :soft-cap 3}}) + (let [g1 (gather-now conn rid)] + (is (true? (:oversight/worth-a-look? g1))) + (spent-pass! conn rid g1) + (is (false? (:oversight/worth-a-look? (gather-now conn rid)))) + (journal/note! conn rid :route {:data {:decision "revise" :revision 4 :soft-cap 3}}) + (is (true? (:oversight/worth-a-look? (gather-now conn rid))) + "the next round at the cap is a new decision")))) + (testing "a refusal wakes it once" + (let [conn (db/open! ":memory:") + rid (runs/start-run! conn {:problem "p"})] + (journal/note! conn rid :mutation-rolled-back {:data {:reason "no" :target "manifest x"}}) + (let [g1 (gather-now conn rid)] + (is (true? (:oversight/worth-a-look? g1))) + (spent-pass! conn rid g1) + (is (false? (:oversight/worth-a-look? (gather-now conn rid)))))))) + +(deftest the-apply-cell-records-what-the-pass-was-woken-on + ;; The same claim through the real cells: gather, a stubbed reason, apply's + ;; note, and the next gather sees the evidence as read. + (cells/load-cells!) + (let [conn (db/open! ":memory:") + rid (runs/start-run! conn {:problem "p"}) + ctx {:conn conn :run-id rid :config {}}] + (journal/note! conn rid :run-error {:data {:error "boom" :node ":spawn"}}) + (let [g (gather-now conn rid) + r (with-redefs [myc/run-compiled (fn [_ _ data] + {:branch (assoc (:branch data) :final-answer "looked")})] + ((:handler (cell/get-cell! :oversight/reason)) ctx g))] + ((:handler (cell/get-cell! :oversight/apply)) ctx (assoc r :oversight/verdict :done)) + (is (= 1 (:errors (:signals (journal/last-note conn rid :oversight))))) + (is (false? (:oversight/worth-a-look? (gather-now conn rid))))))) + +(deftest unmet-and-idle-count-from-the-last-look + (let [since @(ns-resolve 'cells.oversight 'since-last-look) + floors {:idle-floor 25}] + (testing "no earlier pass: the levels, as before" + (is (= 5 (:unmet-gates (since {:unmet 5 :idle 3} nil floors))))) + (testing "unmet counts only the firings since" + (is (= 0 (:unmet-gates (since {:unmet 5} {:unmet 5} floors)))) + (is (= 3 (:unmet-gates (since {:unmet 8} {:unmet 5} floors))))) + (testing "a stall already seen needs another floor's worth to wake it" + (is (= 5 (:idle-turns (since {:idle 30} {:idle 25} floors)))) + (is (= 25 (:idle-turns (since {:idle 50} {:idle 25} floors))))) + (testing "a stall not yet seen counts whole, and a write resets it" + (is (= 30 (:idle-turns (since {:idle 30} {:idle 10} floors)))) + (is (= 26 (:idle-turns (since {:idle 26} {:idle 40} floors))))))) + (deftest a-round-under-the-cap-with-nothing-else-wrong-is-still-quiet (let [conn (db/open! ":memory:") rid (runs/start-run! conn {:problem "p"})] @@ -828,6 +908,36 @@ (is (str/includes? (str prob) "manifest broken") "and the brief names what was refused")))) +(deftest an-edit-that-has-not-earned-its-place-reaches-the-supervisor + ;; karamazov-na2k.10: the brief offers it for removal, with why it was made. + (let [conn (db/open! ":memory:") + rid (runs/start-run! conn {:problem "p"})] + ((requiring-resolve 'samizdat.store.userspace/save!) conn :prompt "system" "v2" "project" "tightened the split rule") + (dotimes [_ 3] ((requiring-resolve 'samizdat.store.userspace/record-run-outcome!) conn :failed)) + (journal/note! conn rid :run-error {:data {:error "boom" :node ":x"}}) + (let [{:keys [prob]} (reasoning-over conn rid)] + (is (str/includes? (str prob) "not earned their place")) + (is (str/includes? (str prob) "tightened the split rule"))))) + +(deftest an-open-experiment-is-shown-with-what-the-reviews-said-since + ;; karamazov-na2k.12, from GEPA: reflection reads {inputs, outputs, + ;; feedback}, not a bare score. The verdict is a fitness number; the + ;; reviews written while the change was in force say why it moved. + (session/reset!) + (try + (let [conn (db/open! ":memory:") + rid (runs/start-run! conn {:problem "p"})] + (dotimes [_ 6] (session/observe-turn! {:tool "eval" :category :success :signals {}})) + (session/experiment! "tighter-fence" {:change "one example call" :hypothesis "fewer parse errors"}) + (journal/note! conn rid :critique {:data {:decision "revise" :verdict "incomplete" + :findings "- [high] the screenshot was never described"}}) + (journal/note! conn rid :run-error {:data {:error "boom" :node ":x"}}) + (let [{:keys [prob]} (reasoning-over conn rid)] + (is (str/includes? (str prob) "tighter-fence")) + (is (str/includes? (str prob) "the screenshot was never described") + "the finding reaches the brief beside the change it was measured under"))) + (finally (session/reset!)))) + (deftest a-refused-mutation-is-worth-a-model-call ;; A refusal means the supervisor tried to change the harness and the ;; protocol said no. That is precisely a moment where the next pass will diff --git a/test/samizdat/repetition_test.clj b/test/samizdat/repetition_test.clj index 8dc1813c..c553fe08 100644 --- a/test/samizdat/repetition_test.clj +++ b/test/samizdat/repetition_test.clj @@ -43,3 +43,18 @@ "a run of whitespace repeats perfectly and means nothing") (is (nil? (rep/periodic (apply str (repeat 200 "-=")) policy)) "so does a rule of dashes: a period under the floor is a run, not a loop")) + +(deftest a-short-period-repeated-hundreds-of-times-is-a-loop + ;; karamazov-na2k.19, run 390dcd17 T0 turn 73: GLM-5.3 emitted + ;; "action":"close", about 900 times — a 17-character period, under the + ;; :min-period floor meant for rules of dashes — and nothing called it a + ;; loop. A short period counts when it repeats far past the long floor. + (let [policy (assoc policy :short-min-period 8 :short-min-repeats 20) + loop-text (str "{\"name\": \"task\", \"args\": {" + (apply str (repeat 300 "\"action\":\"close\",")))] + (is (some? (rep/periodic loop-text policy))) + (testing "a few short repeats are still not a loop" + (is (nil? (rep/periodic (str "{" (apply str (repeat 5 "\"action\":\"close\",")) "}") + (assoc policy :window 20))))) + (testing "and a rule of dashes is still a run" + (is (nil? (rep/periodic (apply str (repeat 200 "-=")) policy)))))) diff --git a/test/samizdat/session_test.clj b/test/samizdat/session_test.clj index 4696d210..8651d5f8 100644 --- a/test/samizdat/session_test.clj +++ b/test/samizdat/session_test.clj @@ -183,6 +183,111 @@ (is (= "budget 50k -> 80k" (:change v))) (is (seq (:hypothesis v)) "a change with no stated expectation cannot be wrong"))) +(deftest tokens-are-counted-per-turn + (session/observe-turn! {:tool "eval" :category :success :signals {} :tokens 1200}) + (session/observe-turn! {:tool "eval" :category :success :signals {} :tokens 800}) + (session/observe-turn! {:tool "eval" :category :success :signals {}}) + (is (= 2000 (:tokens (session/snapshot))))) + +(defn- turns! [n tool cat tokens & [signals]] + (dotimes [_ n] (session/observe-turn! {:tool tool :category cat + :signals (or signals {}) :tokens tokens}))) + +(deftest a-verdict-weighs-what-the-change-cost + ;; karamazov-na2k.3, RRSI eq. 7: added tokens have to be paid for by + ;; measured gain, dC <= base + per-fitness * dS with dC relative. Their + ;; acceptance rules cut tokens per trial from 3.59M to 2.42M in ablation; + ;; samizdat's fitness had no cost term and its verdicts no token count. + (testing "the verdict carries tokens per turn on both sides" + (turns! 6 "eval" :success 1000) + (session/experiment! "same" {:change "c" :hypothesis "h"}) + (turns! 8 "eval" :success 1000) + (let [v (session/verdict "same")] + (is (= 1000.0 (get-in v [:cost :before]))) + (is (= 1000.0 (get-in v [:cost :after]))) + (is (= :unchanged (:verdict v))))) + (session/reset!) + (testing "a gain bought with far more tokens than it earned is costly" + (turns! 6 "eval" :mechanics 1000 {:parse-error true}) + (session/experiment! "pricey" {:change "c" :hypothesis "h"}) + (turns! 8 "eval" :success 9000) + (is (= :costly (:verdict (session/verdict "pricey"))))) + (session/reset!) + (testing "a gain that paid for its tokens is better" + (turns! 6 "eval" :mechanics 1000 {:parse-error true}) + (session/experiment! "fair" {:change "c" :hypothesis "h"}) + (turns! 8 "eval" :success 1300) + (is (= :better (:verdict (session/verdict "fair"))))) + (session/reset!) + (testing "no measurable change in fitness at a clearly lower cost is cheaper" + (turns! 6 "eval" :success 1000) + (session/experiment! "lean" {:change "c" :hypothesis "h"}) + (turns! 8 "eval" :success 600) + (is (= :cheaper (:verdict (session/verdict "lean"))))) + (session/reset!) + (testing "with no token counts the rule stays out of it" + (turns! 6 "eval" :mechanics nil {:parse-error true}) + (session/experiment! "blind" {:change "c" :hypothesis "h"}) + (turns! 8 "eval" :success nil) + (is (= :better (:verdict (session/verdict "blind")))))) + +(deftest the-deadband-is-the-noise-the-tally-shows + ;; karamazov-na2k.4. :meaningful-delta 0.15 was one number for every run. + ;; RRSI calibrates its band from how much the baseline itself varies, and + ;; a stretch where half the calls fail moves by more than 0.15 on noise. + (turns! 3 "eval" :success nil) + (turns! 3 "eval" :failure nil) + (session/experiment! "noisy" {:change "c" :hypothesis "h"}) + (turns! 5 "eval" :success nil) + (turns! 3 "eval" :failure nil) + (let [v (session/verdict "noisy")] + (is (> (:delta v) 0.15) "a move the fixed deadband would have called better") + (is (> (:band v) (:delta v)) "is inside the band this tally's own spread gives") + (is (= :unchanged (:verdict v))))) + +(deftest a-kept-gain-sets-the-floor-the-next-change-is-held-to + ;; karamazov-na2k.4, RRSI eq. 5: a candidate has to clear the best score + ;; seen so far minus the band, or a run walks downhill through changes each + ;; of which beat only the one before it. + (turns! 6 "eval" :mechanics nil {:parse-error true}) + (session/experiment! "first" {:change "c1" :hypothesis "h"}) + (turns! 8 "eval" :success nil) + (is (= :better (:verdict (session/verdict "first")))) + (session/reverted! "first" true) + (session/experiment! "second" {:change "c2" :hypothesis "h"}) + (turns! 5 "eval" :success nil) + (turns! 3 "eval" :failure nil) + (let [v (session/verdict "second")] + (is (pos? (:delta v)) "better than the stretch before it") + (is (= :below-best (:verdict v)) "and still below what a kept change reached") + (is (some #(= "second" (:name %)) (session/unsettled-losses)) + "so it is a loss to act on"))) + +(deftest an-experiment-states-what-should-move-and-hears-whether-it-did + ;; karamazov-na2k.5. RRSI records per edit what it predicted would change + ;; and scores the hit; samizdat's gates carry predictions, its edits did + ;; not. A prediction is one fitness signal and a direction. + (turns! 6 "eval" :mechanics nil {:parse-error true}) + (session/experiment! "fence" {:change "c" :hypothesis "h" + :target {:kind "prompt" :name "system-turn" :version 3} + :predicts {:signal :parse-error :direction :down}}) + (turns! 8 "eval" :success nil) + (let [v (session/verdict "fence") + pr (:prediction v)] + (is (= {:kind "prompt" :name "system-turn" :version 3} (:target v))) + (is (= :parse-error (:signal pr))) + (is (> (:before pr) (:after pr))) + (is (true? (:hit? pr)))) + (session/reverted! "fence" true) + (session/experiment! "wrong-way" {:change "c2" :hypothesis "h2" + :predicts {:signal :tool-success :direction :down}}) + (turns! 8 "eval" :success nil) + (is (false? (get-in (session/verdict "wrong-way") [:prediction :hit?])))) + +(deftest a-prediction-names-a-signal-the-fitness-weighs + (is (contains? (set (session/signal-names)) :parse-error)) + (is (contains? (set (session/signal-names)) :gate-unmet))) + (deftest a-change-that-moved-nothing-is-unchanged-not-better (dotimes [_ 6] (session/observe-turn! {:tool "eval" :category :success :signals {}})) (session/experiment! "noop" {:change "reworded a prompt" :hypothesis "nothing"}) @@ -266,6 +371,23 @@ "the finding has to name the fix it rules OUT, or it will be read as an argument for a wider beam"))) +(deftest the-lever-memory-names-the-edit-and-the-prediction + ;; karamazov-na2k.5: the next session reads which version was tried and + ;; whether what it was meant to move did. + (let [c (db/open! ":memory:")] + (try + (turns! 6 "eval" :mechanics nil {:parse-error true}) + (session/experiment! "fence" {:change "one example call in system-turn" + :hypothesis "fewer parse errors" + :target {:kind "prompt" :name "system-turn" :version 4} + :predicts {:signal :parse-error :direction :down}}) + (turns! 8 "eval" :success nil) + (let [w (knowledge/distill-verdicts! c (session/experiments) {:run-id "r1"}) + row (knowledge/get-by-id c (:id (first w)))] + (is (str/includes? (:content row) "prompt system-turn v4")) + (is (str/includes? (:content row) "Predicted parse-error down: it did."))) + (finally (db/close c))))) + (deftest a-verdict-becomes-a-lever-fact-that-outlives-the-session ;; The heredity of selection. Without it a lever that was tried and made ;; things worse is forgotten by the next session, which is free to try it diff --git a/test/samizdat/split_test.clj b/test/samizdat/split_test.clj index a4b645f9..45a2f129 100644 --- a/test/samizdat/split_test.clj +++ b/test/samizdat/split_test.clj @@ -171,6 +171,16 @@ (is (= 2 (count kids)) "the pieces are children of the held task") (is (= #{"parse-line" "render-report"} (set (map :title kids))))))))) +(deftest parts-sent-as-a-json-string-are-the-parts + ;; karamazov-na2k.15: GLM-5.3 sends array arguments as JSON strings. Read + ;; as-is, `parts` was not a collection and the split saw no parts at all. + (with-project + (fn [{:keys [conn run-id] :as p}] + (let [r (call p {:reason "two responsibilities" + :parts ((requiring-resolve 'clojure.data.json/write-str) good-parts)})] + (is (= :neutral (:category r)) (str "accepted: " (:result r))) + (is (= 2 (count (filter :parent_id (tasks/board conn {:run-id run-id}))))))))) + (deftest a-split-parks-the-branch-and-blocks-the-row-it-was-working ;; karamazov-ioo.15.4. The split IS the branch's last act. It had nothing ;; left to do until its pieces came back, and the worst thing it could do diff --git a/test/samizdat/store_test.clj b/test/samizdat/store_test.clj index 51cf4353..c7805ac6 100644 --- a/test/samizdat/store_test.clj +++ b/test/samizdat/store_test.clj @@ -1594,3 +1594,17 @@ (journal/last-note c rid :forfeit)))) (testing "another branch's dispatch is not this branch's" (is (nil? (journal/in-flight-dispatch c rid "B2"))))))) + +(deftest gate-checks-are-kept-with-their-value-and-outcome + ;; karamazov-na2k.7: every check of a measured gate, fired or not. + (let [c (db/open! ":memory:") + rid (runs/start-run! c {:problem "p"})] + (journal/record-gate-checks! c rid "B1" 7 + [{:gate :progress-stalled :value 9 :held true} + {:gate :stuck :value 1 :held false}] + :progress-stalled) + (let [rows (journal/gate-checks c rid)] + (is (= 2 (count rows))) + (is (= #{["progress-stalled" 9.0 1 1] ["stuck" 1.0 0 0]} + (set (map (juxt :gate #(double (:value %)) :held :fired) rows))))))) + diff --git a/test/samizdat/test_runner.clj b/test/samizdat/test_runner.clj index a3dd05f1..e68378ef 100644 --- a/test/samizdat/test_runner.clj +++ b/test/samizdat/test_runner.clj @@ -162,6 +162,7 @@ [samizdat.user-sim-test] [samizdat.userspace-test] [samizdat.userspace-files-test] + [samizdat.leakage-test] [samizdat.userspace-validation-test] [samizdat.userspace-adoption-test] [samizdat.event-stream-test] @@ -373,6 +374,7 @@ samizdat.user-sim-test samizdat.userspace-test samizdat.userspace-files-test + samizdat.leakage-test samizdat.userspace-validation-test samizdat.userspace-adoption-test samizdat.event-stream-test diff --git a/test/samizdat/userspace_test.clj b/test/samizdat/userspace_test.clj index 70fb9ada..ecab5c74 100644 --- a/test/samizdat/userspace_test.clj +++ b/test/samizdat/userspace_test.clj @@ -310,6 +310,19 @@ (is (= :mechanics (:category r)) (str tool " " (:action args))) (is (re-find #"rationale" (str (:result r))) (str tool " " (:action args)))))) +(deftest the-prompt-tool-refuses-a-judge-prompt-that-drops-its-format + ;; karamazov-na2k.6, through the tool the supervisor actually uses. + (us/bind! *conn*) + (let [r (tools/run-tool {:tool-name "prompt" + :branch (state/new-branch {:id "B1" :problem "p"}) + :conn *conn* + :args {:action "save" :name "judge" + :body "Say whether the work is done, in your own words." + :rationale "friendlier"}})] + (is (:edit-rejected? r) (str (:result r))) + (is (re-find #"VERDICT" (str (:result r)))) + (is (empty? (store/versions *conn* :prompt "judge")) "nothing was stored"))) + (deftest a-rationale-rides-the-version-and-shows-in-the-history (us/bind! *conn*) (let [run-prompt (fn [args] @@ -965,6 +978,26 @@ project's — the shipped copy does not answer for it")) (finally (rm-rf (java.io.File. root)))))) +;; --- pruning: project edits that have not earned their place (na2k.10) ------ + +(deftest a-project-edit-that-never-shipped-a-run-is-a-pruning-candidate + ;; RRSI's Lasso step: machinery that stops producing a gain is offered for + ;; deletion. retirement-candidates does it for gates; this is the same for + ;; an edit: the current project version, on runs that ended failed and + ;; never once shipped. Crashes are the harness's, not the edit's. + (store/seed! *conn* :prompt "system" "the template") + (store/save! *conn* :prompt "system" "v2" "project" "tightened the split rule") + (store/save! *conn* :cell "loop" "c2" "project" "a cell edit") + (dotimes [_ 3] (store/record-run-outcome! *conn* :failed)) + (store/record-run-outcome! *conn* :error) + (store/save! *conn* :cell "loop" "c3" "project" "fixed the cell edit") + (let [cs (store/pruning-candidates *conn* {:min-runs 3 :limit 5})] + (is (= [["prompt" "system"]] (mapv (juxt :kind :name) cs)) + "the prompt's current version failed three runs and shipped none; the + cell's current version is new and has no record yet") + (is (= 3 (:failed (first cs))) "the crash is not counted against it") + (is (= "tightened the split rule" (:rationale (first cs)))))) + ;; --- drift: how much each surface has moved (karamazov-00qw) ----------------- (deftest drift-counts-saves-reverts-and-standing-per-surface @@ -985,7 +1018,11 @@ (select-keys (by-kind "prompt") [:names :saves :reverts :shipped :failed])) "three project versions of one name, one of them a revert; the current one has survived a green run") - (is (= [{:name "system" :saves 3 :reverts 1}] (:churn (by-kind "prompt")))) + (is (= [{:name "system" :saves 3 :reverts 1}] + (mapv #(dissoc % :why) (:churn (by-kind "prompt"))))) + (is (str/includes? (str (:why (first (:churn (by-kind "prompt"))))) + "v3 made the model hedge") + "and why the latest version was made, which the block says to read") (is (= "prompt" (:kind (first d))) "the surface that moved most comes first")) (testing "the window bounds it, and top-names bounds the churn list" (is (empty? (store/drift *conn* {:since "9999-01-01T00:00:00.000Z"}))) diff --git a/test/samizdat/userspace_validation_test.clj b/test/samizdat/userspace_validation_test.clj index 94c17a84..d03f04a8 100644 --- a/test/samizdat/userspace_validation_test.clj +++ b/test/samizdat/userspace_validation_test.clj @@ -158,6 +158,25 @@ (is (= :compile (:stage p))) (is (str/includes? (:message p) "no-such/cell"))))) +(deftest a-judge-prompt-that-stops-asking-for-what-its-parser-reads-is-rejected + ;; karamazov-na2k.6. The judge's parser reads a VERDICT line, and a judge + ;; reply with none falls through to :complete — fail-open by design. So a + ;; reworded prompts/judge.md that stopped asking for VERDICT turned the + ;; judge off with no error anywhere. Both halves are userspace; the save + ;; that separates them is refused, naming what the parser reads. + (with-project [root] + (let [text (str/replace (us/body :prompt "judge") "VERDICT" "OUTCOME") + p (problem-for root :prompt "judge" "prompts/judge.md" text)] + (is (= :format (:stage p))) + (is (str/includes? (:message p) "VERDICT"))))) + +(deftest every-shipped-side-call-prompt-asks-for-what-its-parser-reads + (with-project [_root] + (doseq [[nm tokens] ((requiring-resolve 'samizdat.agent.gates/threshold) :reply-formats) + t tokens] + (is (str/includes? (str/lower-case (us/body :prompt nm)) (str/lower-case t)) + (str "prompts/" nm ".md does not ask for " t))))) + (deftest a-prompt-that-does-not-render-is-rejected (with-project [root] (let [p (problem-for root :prompt "critic" "prompts/critic.md" "{% if x %} never closed")]