diff --git a/dev/samizdat/dev/arena.clj b/dev/samizdat/dev/arena.clj index 9389eba3..75c971aa 100644 --- a/dev/samizdat/dev/arena.clj +++ b/dev/samizdat/dev/arena.clj @@ -53,6 +53,7 @@ [samizdat.agent.acceptance :as acceptance] [samizdat.agent.beam :as beam] [samizdat.agent.gates :as gates] + [samizdat.heldout :as heldout] [samizdat.agent.verify :as verify] [samizdat.config :as config] [samizdat.leakage :as leakage] @@ -1438,6 +1439,19 @@ :verify-timeout-ms 600000})] (println "\n=== SUMMARY ===") (clojure.pprint/pprint (summarize rows)) + ;; THE HELD-OUT GATE, STAGE 2 (karamazov-7mo.4): the candidate arm read + ;; against the baseline arm, per task, floor -> cost -> guards. + ;; ARENA_ACCEPT=,; ARENA_STRUCTURAL=1 when the + ;; candidate adds a component, which is admissible inside the band. + (when-let [spec (System/getenv "ARENA_ACCEPT")] + (let [[b c] (map keyword (str/split spec #",")) + fitness (gates/threshold :fitness)] + (println "\n=== HELD-OUT STAGE 2 ===") + (clojure.pprint/pprint + (heldout/live-verdict rows {:baseline b :candidate c + :structural? (= "1" (System/getenv "ARENA_STRUCTURAL")) + :cost-rule (:cost-rule fitness) + :noise (:noise fitness)})))) (println "\n=== RECURRING EDITS ===") (clojure.pprint/pprint (recurring-edits rows)) ;; Opt-in, because it spends a model call per task that has both a diff --git a/docs/RFCS/RFC-002-manifests-and-cells.md b/docs/RFCS/RFC-002-manifests-and-cells.md index ea349828..0f9fce94 100644 --- a/docs/RFCS/RFC-002-manifests-and-cells.md +++ b/docs/RFCS/RFC-002-manifests-and-cells.md @@ -57,9 +57,34 @@ glob-scoped interceptors match on. :constraints [{:type :must-follow :if node :then node}] :subworkflows {cell-id manifest-name} ; optional: a nested manifest as one node :prompt "name" ; optional: prompt appended to the base - :turn-sliceable? false} ; optional, default true — see below + :turn-sliceable? false ; optional, default true — see below + :extends "manifest-name" ; optional: carry only what differs from it + :replaces [:invariants] ; optional: keys that are this file's alone + :fragment? true} ; optional: a base, not a workflow to run ``` +`:extends` names another manifest this one is built on, and the file carries +only what differs (karamazov-xtd3). `manifests/turn.edn` holds the turn chain +— assemble, the context-budget ladder, infer, parse, dispatch, journal, +settle, arbiter, route — and `loop`, `worker`, `reviewer` and `supervisor` +extend it and add their tail: the supervisor nothing, worker and reviewer a +`:distil` on every ending, loop a `:distil` then `:finish`. They were four +copies of one graph until then, and copies drift. `read-definition` resolves +the link, so every reader sees the whole graph: `:cells`, `:edges` and +`:dispatches` merge key by key with the child winning and a `nil` removing the +base's entry; the base's `:invariants` hold and the child's are added; any +other key is the child's where it has one; a key listed in `:replaces` is the +child's alone, which is how a child that renames a node drops the base's +invariants naming the old one. A base that resolves to nothing, and a chain +that returns to itself, are refused by name. The base is a manifest role like +any other — editable, versioned, validated — and saving it compiles every +manifest that extends it against the candidate first, refusing, by name, a +change that breaks one (`manifests/validate-edit!`). `manifest patch` applies +its ops to the resolved graph and writes back only the delta +(`manifests/extension-delta`), so the file keeps its `:extends`. `:fragment?` +marks a base that is not itself a workflow: it stays in `catalog`, is off the +switch menu and selection, and is not inherited. + `:turn-sliceable?` declares that a manifest may **not** be a run's loop. The slice cuts every edge returning to `:start` into `:end`, which is the definition of a turn for an iterating loop and silent data loss for anything @@ -95,7 +120,8 @@ checked (karamazov-viht.2; the same disclosure rides `policy show gates` and manifests/beam.edn the ROUND advance · score · cull · settle · repopulate · spawn · tick · back edge └─ manifests/loop.edn the TURN assemble · infer · parse · dispatch · - (per-turn slice) journal · settle · arbiter · route + (per-turn slice, journal · settle · arbiter · route + extends turn.edn) ``` `turn-manifest` **derives** the per-turn slice from a whole-run manifest by diff --git a/docs/RFCS/RFC-003-security-model.md b/docs/RFCS/RFC-003-security-model.md index 3df211c1..cc1f6e92 100644 --- a/docs/RFCS/RFC-003-security-model.md +++ b/docs/RFCS/RFC-003-security-model.md @@ -108,6 +108,12 @@ flowchart LR webfetch --> egress egress --> redact + heldout[held-out gate: battery tool, and cell/manifest/policy saves — + git archive a recorded fixture, replay it in a jolt child] + toolcall --> heldout + scrub --> heldout + heldout --> redact + plan[plan tool: records declared paths on the branch] toolcall --> plan plan --> redact diff --git a/docs/RFCS/RFC-014-validation-and-procedure.md b/docs/RFCS/RFC-014-validation-and-procedure.md index b6fe6ac1..3e34b18b 100644 --- a/docs/RFCS/RFC-014-validation-and-procedure.md +++ b/docs/RFCS/RFC-014-validation-and-procedure.md @@ -1,10 +1,10 @@ # RFC-014 — Validation and procedure: measuring a change to the loop **Status:** partially implemented. The replay substrate, the battery's -expectations, the gate inside the mutation protocol, and the procedural graph's -mechanism are built and tested (karamazov-ylte). The battery's CASES and the -graph itself are not; both are data, and where they come from is specified -below. +expectations, the held-out gate in production (stage 1 on every cell, manifest +and policy save; stage 2 over the arena's arms — karamazov-7mo.4 / ylte.4) and +the procedural graph's mechanism are built and tested. A project's CASES are +its own data, added with the `battery` tool; the graph is not built. ## Purpose @@ -182,6 +182,54 @@ at all* on ALFWorld — 54.48 against a 72.58 baseline, at 5.3× the tokens — silently serving the whole graph is the measured-worse option and the caller decides. +### The gate in production (karamazov-7mo.4 / ylte.4) + +`samizdat.heldout` is what runs the battery. Every `cell save`, `manifest +save`/`patch` and `policy save` calls `heldout/check-edit` before it stores: +the battery is replayed with the candidate in place and without it, and an +edit under which a target that passed fails is refused with the targets named +(`prompts/heldout-refused.md`). Nothing is saved; what runs is unchanged. + +**One child process per case.** A replay must read the candidate as the +project's userspace from every branch fiber, and a dynamic binding does not +cross a fiber while a global override would leak the candidate into the live +run proposing it. So the candidate is materialized: the case's fixture — the +recorded run's `:git-baseline`, unpacked with `git archive` — gets a copy of +the project's `.samizdat/` with the candidate file written in, and +`samizdat.heldout.child` starts the harness there on an in-memory database, +replays the recording through `beam/run!`, and runs `battery/check`. Side +calls a cell makes on its own (critic, judges) were never recorded and are +refused identically on both sides. + +**The rule is non-compensatory, per target.** No target that passed at +baseline may fail on the candidate, and both sides must cover the same +targets. Ties are accepted. A case that does not run at baseline is set aside +by name; one that ran and no longer runs is a regression. + +**One replay per edit, not two.** Measurements are cached by the project and +the CONTENT of its userspace, so the candidate's measurement is the baseline +of the state a commit of it produces. + +**The battery may grow and may not be weakened — now enforced.** +`battery_cases` (v37) is the authority: a case is written once under its id. +A case file edited under `.samizdat/battery/` runs as stored and the edit is +named; a deleted one still runs. The `battery` tool adds a case from a +finished run (`heldout/draft!`, which pins its baseline under +`refs/samizdat/battery/`) and has no remove. + +**Recorded beside every number.** `heldout_checks` holds one row per target +per edit: before, after, the verdict, and the fixture sha, harness revision, +model and scorer it was measured on. + +**What stage 1 does not judge.** Prompt edits: the replies are fixed. That is +stage 2 — `heldout/live-verdict` over the arena's baseline and candidate +arms, per task, in RRSI's order: floor (median fitness no lower than the best +kept score less the baseline's own noise band), cost (a gain pays for its +tokens by `:fitness :cost-rule`; inside the band only a cost cut or a declared +structural change is admissible), guards (no acceptance criterion the baseline +always met is lost, no suite goes red). The arena prints it with +`ARENA_ACCEPT=,`. + ## API | fn | contract | @@ -243,15 +291,19 @@ is a worse failure than the one the gate prevents. ## What this does not do -**The battery has no cases yet**, and until it does the gate is inert wherever -nobody wires `battery-fn`. Cases come from two subjects, and both are required: -recorded `endless-flight` runs and recorded runs of samizdat working on its own -repo. A battery of one subject would pass an edit that breaks the other, and -self-modification is the project's reason for existing. +**A project with no cases is not gated**: the tools consult the battery on +every save, and with no cases it passes nothing and refuses nothing. + +**A battery is per project.** Userspace is per project, so the cases that +gate an edit are that project's own runs: an edit tested on endless-flight's +cases is tested on endless-flight, and samizdat working on its own repo keeps +its own battery in its own `.samizdat/`. Promoting a project's edit into the +shipped templates is where both subjects have to be replayed, and that is not +automated. -**"May add, may not weaken or delete" is stated, not enforced.** The rule that -a running agent may add a case from an observed failure and may never weaken -one is policy with nothing behind it yet. +**"May add, may not weaken or delete" is enforced by the table, not the +files** (`battery_cases`); a person with the database can still delete a row, +and that is deliberate. **No graph ships.** Building one by hand is the 58.93 row. It should be grown by a refiner from `Start → End` against the gate — scratch-with-evolution beat diff --git a/resources/cells/oversight.clj b/resources/cells/oversight.clj index 540b6287..83c80ea9 100644 --- a/resources/cells/oversight.clj +++ b/resources/cells/oversight.clj @@ -347,9 +347,24 @@ instantly, so it spoke once and went quiet for the rest of the run. Concluding is not the same as having nothing left to say. A pass ends; the - stream does not." - [b] - (-> b (dissoc :final-answer :verdict :done? :status) (assoc :advisory? true))) + stream does not. + + And THIS PASS'S BRIEF is appended (karamazov-3keg). Without it the resumed + branch woke to a conversation ending in its own accepted `done` and nothing + new: run bcd61b39's second pass, woken by three unmet gates, answered \"the + pass is complete\", and no later pass could see a round sent back, a + failing criterion or anything else the brief exists to carry." + [b brief] + ;; ACTIVE, not merely unfinished: state/active? is (= :active status), and + ;; dissoc'ing :status left a branch route read as inactive, so every + ;; resumed pass ran one turn and ended :abandoned (run bcd61b39, passes 2 + ;; and 3). The last pass's ending reason and failure streaks go with it — + ;; a pass is not charged for how the one before it ended. + (cond-> (-> b + (dissoc :final-answer :verdict :done? :inactive-reason + :consecutive-mechanics-failures :consecutive-provider-errors) + (assoc :status :active :advisory? true)) + brief (update :messages (fnil conj []) {:role "user" :content brief}))) (cell/defcell :oversight/reason {:doc "One turn of the supervisor ROLE, in the stream's OWN branch. @@ -554,7 +569,7 @@ ;; run-scoped resources every driver provides, and the carry is ;; this pass's value. Putting it in ctx would have meant claiming ;; the beam driver provides it, which it does not. - b (or (some-> (:oversight/carry data) resume-branch) + b (or (some-> (:oversight/carry data) (resume-branch prob)) (assoc (state/new-branch {:id bid :problem prob :messages (turn/initial-messages prob suffix :supervisor)}) diff --git a/resources/gates.edn b/resources/gates.edn index f2c25c3c..becc31c2 100644 --- a/resources/gates.edn +++ b/resources/gates.edn @@ -1966,6 +1966,46 @@ judgement — it hands over the episodes that have earned the question."} + :heldout + {:value {:enabled? true + :dir "battery" + :max-cases 5 + :case-timeout-ms 600000 + :stage-timeout-ms 120000 + :turns-slack 5 + :harness-dir nil + :command "jolt" + :trail-turns 12 + :trail-chars 300 + :skip "\\.sqlite3(-wal|-shm|-journal)?$"} + :provenance ["karamazov-7mo.4" "karamazov-ylte.4" "2609.24972v2" "2609.09153v1"] + :kind :policy :capability-tunable? false + :doc "THE HELD-OUT GATE, STAGE 1 (samizdat.heldout). Every cell, manifest + and policy edit a tool saves is first replayed against the project's + frozen battery — .samizdat/<:dir>//.edn, stored in + battery_cases the first time it is read — with the edit in place and + without it, each case in a child process on a copy of the tree the + recorded run started from. An edit under which a target that passed + fails is refused and nothing is saved. Ties are accepted. + + A project with no cases is not gated: it tunes itself on the compile + and the soak, as before. :enabled? false turns the gate off where + cases exist. At most :max-cases run per edit; each has + :case-timeout-ms, and unpacking its fixture :stage-timeout-ms. A + replay runs the recording's longest branch plus :turns-slack turns, + so a candidate that takes more turns exhausts the recording (a named + result) rather than being cut off. :harness-dir is the samizdat + checkout a child is started from; nil means the server's working + directory, and :command the jolt it is started with. A .samizdat file + whose path matches :skip (the databases) is not copied into a replay. + A replay hands back the last :trail-turns turns of its run, each + result cut at :trail-chars, since its database dies with it. + + Prompt edits are not replayed: the replies are fixed, so a prompt + cannot change what the model says under replay. That is stage 2's — + the arena's live arms (heldout/live-verdict), read with :fitness + :noise and :cost-rule."} + :pruning {:value {:min-runs 3 :limit 5} :provenance ["karamazov-na2k.10" "2609.24972v2"] diff --git a/resources/manifests/loop.edn b/resources/manifests/loop.edn index 11162354..899ebdd4 100644 --- a/resources/manifests/loop.edn +++ b/resources/manifests/loop.edn @@ -4,125 +4,29 @@ ;; definition carries the routing — which is exactly the part an agent can ;; safely rewrite, behind compile-time validation. ;; -;; Dispatch entries are PATTERNS over the data map — {:verdict :done} matches -;; any data map carrying that key with that value, `_` matches anything, and -;; the first entry that matches wins. The cell computes the decision INTO the -;; data map and the pattern only reads it, so the routing stays visible here -;; where it can be edited, and a table of patterns can be checked for a -;; branch nothing can reach. A (fn [d] ...) form is still accepted where a -;; pattern cannot say it. +;; The turn itself — assemble through route — is manifests/turn.edn, which this +;; extends. What this file adds is how a run ENDS: every terminal verdict goes +;; through :distil and then :finish, the whole-run teardown. {:description "The factory loop: a single branch iterating a turn at a time until it ships or gives up. The simplest driver — no fan-out, no critic, no supervisor. The baseline every other workflow specializes." - :input-schema - ;; What a driver hands this manifest to start a run. mycelium checks it in - ;; run-compiled BEFORE any cell runs, so a caller that builds the map wrongly - ;; is told which key, instead of a nil surfacing several cells later. - [:map [:branch :map] [:turn :int]] + :extends "turn" - :cells {:start :loop/assemble - ;; THE CONTEXT-BUDGET LADDER, as nodes. Compaction used to be a - ;; side effect of building the request — invisible to every - ;; manifest, unreachable by introspect, tunable only by moving a - ;; character count. It runs between assembling the turn and calling - ;; the model, which is the only place the cheap rungs can act, and - ;; each rung is its own node so a project can rewire or drop one. - :measure :compaction/measure - :cap :compaction/cap - :prune :compaction/prune - :fold :compaction/fold - :infer :llm/infer - :parse :llm/parse - :provider-error :loop/provider-error - :no-call :loop/no-call - :dispatch :tool/dispatch - :journal :journal/record - :settle :gate/settle - :arbiter :gate/arbiter - :route :loop/route - :distil :memory/distil - :finish :loop/finish} + :cells {:distil :memory/distil + :finish :loop/finish} - :edges {:start :measure - ;; measure WRITES :compaction/route and the dispatch READS it — - ;; never the other way round. A predicate that computed the tier - ;; here would put the routing back inside code and leave the - ;; manifest showing an opaque box. - :measure {:cap :cap - :none :infer} - :cap {:prune :prune - :none :infer} - :prune :fold - :fold :infer - :infer :parse - :parse {:provider-error :provider-error - :no-call :no-call - :tool :dispatch} - :provider-error :route - :no-call :route - :dispatch :journal - :journal :settle - :settle :arbiter - :arbiter :route - ;; The back edge: one turn ends where the next begins. - ;; Every ending goes through :distil first. A run that gave up has - ;; the most valuable gotchas in it, so reflection is not a reward for - ;; success — it is what a finished task owes the next one. - :route {:continue :start - :done :distil - :abandoned :distil - :exhausted :distil} - :distil :finish - :finish :end} + ;; Every ending goes through :distil first. A run that gave up has the most + ;; valuable gotchas in it, so reflection is not a reward for success — it is + ;; what a finished task owes the next one. + :edges {:route {:continue :start + :done :distil + :abandoned :distil + :exhausted :distil} + :distil :finish + :finish :end} - :dispatches {:measure [[:cap {:compaction/route :cap}] - [:none _]] - :cap [[:prune {:compaction/route :prune}] - [:none _]] - ;; ORDER MATTERS in this one table. A provider failure passes - ;; the data through untouched, so :parsed is then ABSENT, and - ;; a map pattern needs its key present: {:parsed nil} would not - ;; match that data map, and the data would fall through to :tool. - ;; The provider-error branch has to be tried first. `manifest show` - ;; reports this table as order-dependent, which is correct. - :parse [[:provider-error {:call {:ok false}}] - [:no-call {:parsed nil}] - [:no-call {:parsed {:name "__parse_error__"}}] - [:tool _]] - :route [[:continue {:verdict :continue}] - [:done {:verdict :done}] - [:abandoned {:verdict :abandoned}] - [:exhausted {:verdict :exhausted}]]} - - ;; EVERY ordering rule this turn claims, and which of them the compiler - ;; catches. `:constraints` is DERIVED from the `:enforced true` entries, so - ;; an editor can read this list and know what is actually defended rather - ;; than having to go and read the cells. :invariants - [{:type :must-follow :if :dispatch :then :journal :enforced true - :protects "A dispatched tool call is always recorded."} - - {:type :must-follow :if :journal :then :arbiter :enforced true - :protects "A recorded turn always faces a gate."} - - {:type :must-precede :cell :dispatch :before :arbiter :enforced true - :protects "The arbiter reads what the turn DID. A gate deciding before the - tool ran would steer on the previous turn's state. The - provider-error and no-call routes reach neither node, which is - why this is expressible as a precedence rather than as a - must-follow: it binds only the paths where a gate actually - fires."} - - {:type :must-precede :cell :parse :before :dispatch :enforced true - :protects "Nothing is dispatched that was not parsed out of a reply."} - - {:type :must-precede :cell :distil :before :finish :enforced true + [{:type :must-precede :cell :distil :before :finish :enforced true :protects "A task that ended wrote down what it learned about the project before the branch closed. Asking the model to remember things produced nothing across 46 turns of live runs; a step in the graph runs whether or not it feels like it, and putting it before - :finish is what stops a future edit routing an ending around it."} - - {:type :must-precede :cell :settle :before :arbiter :enforced true - :protects "A gate cannot be credited with an outcome that preceded it: - :gate/settle closes this turn's open predictions BEFORE the - arbiter chooses, so a resolution closes against the gate that - asked for it and not the one about to."}]} + :finish is what stops a future edit routing an ending around it."}]} diff --git a/resources/manifests/reviewer.edn b/resources/manifests/reviewer.edn index 9af04217..90ea898d 100644 --- a/resources/manifests/reviewer.edn +++ b/resources/manifests/reviewer.edn @@ -1,108 +1,23 @@ -;; The REVIEWER role loop. Structurally the per-turn worker graph — assemble, -;; infer, parse, dispatch, journal, arbiter, route — because a reviewer works -;; the same way an implementor does: a turn at a time until a terminal verdict, -;; then control returns to whatever composed it. What makes it the REVIEWER is -;; its role prompt (prompts/roles/reviewer.md, injected by :feature/review) and -;; its position in feature.edn (it runs on the implementors' finished work). +;; The REVIEWER role loop. Structurally the per-turn worker graph — the turn +;; chain (manifests/turn.edn, which it extends) with task reflection on every +;; ending — because a reviewer works the same way an implementor does: a turn +;; at a time until a terminal verdict, then control returns to whatever +;; composed it. What makes it the REVIEWER is its role prompt +;; (prompts/roles/reviewer.md, injected by :feature/review) and its position in +;; feature.edn (it runs on the implementors' finished work). ;; ;; It is a SEPARATE, independently-versioned manifest from `worker` on purpose: ;; the reviewer role owns this loop and can tune it (manifest save "reviewer") ;; without touching how implementors work. Today it mirrors `worker`; it is free ;; to diverge. {:description "The reviewer role loop: reads the implementors' finished work and returns PASS or REVISE. A component of the feature loop, not usually run standalone." - :input-schema - ;; What a driver hands this manifest to start a run. mycelium checks it in - ;; run-compiled BEFORE any cell runs, so a caller that builds the map wrongly - ;; is told which key, instead of a nil surfacing several cells later. - [:map [:branch :map] [:turn :int]] + :extends "turn" - :cells {:start :loop/assemble - ;; THE CONTEXT-BUDGET LADDER, as nodes. Compaction used to be a - ;; side effect of building the request — invisible to every - ;; manifest, unreachable by introspect, tunable only by moving a - ;; character count. It runs between assembling the turn and calling - ;; the model, which is the only place the cheap rungs can act, and - ;; each rung is its own node so a project can rewire or drop one. - :measure :compaction/measure - :cap :compaction/cap - :prune :compaction/prune - :fold :compaction/fold - :infer :llm/infer - :parse :llm/parse - :provider-error :loop/provider-error - :no-call :loop/no-call - :dispatch :tool/dispatch - :journal :journal/record - :settle :gate/settle - :arbiter :gate/arbiter - :route :loop/route - ;; Task reflection on every ending (blt.26). - :distil :memory/distil} + ;; Task reflection on every ending (blt.26). + :cells {:distil :memory/distil} - :edges {:start :measure - ;; measure WRITES :compaction/route and the dispatch READS it — - ;; never the other way round. A predicate that computed the tier - ;; here would put the routing back inside code and leave the - ;; manifest showing an opaque box. - :measure {:cap :cap - :none :infer} - :cap {:prune :prune - :none :infer} - :prune :fold - :fold :infer - :infer :parse - :parse {:provider-error :provider-error - :no-call :no-call - :tool :dispatch} - :provider-error :route - :no-call :route - :dispatch :journal - :journal :settle - :settle :arbiter - :arbiter :route - :route {:continue :start - :done :distil - :abandoned :distil - :exhausted :distil} - :distil :end} - - :dispatches {:measure [[:cap {:compaction/route :cap}] - [:none _]] - :cap [[:prune {:compaction/route :prune}] - [:none _]] - ;; ORDER MATTERS here, as in loop.edn: a provider failure passes - ;; the data through with :parsed ABSENT, and a map pattern needs - ;; its key present, so :provider-error is tried first. - :parse [[:provider-error {:call {:ok false}}] - [:no-call {:parsed nil}] - [:no-call {:parsed {:name "__parse_error__"}}] - [:tool _]] - :route [[:continue {:verdict :continue}] - [:done {:verdict :done}] - [:abandoned {:verdict :abandoned}] - [:exhausted {:verdict :exhausted}]]} - - ;; EVERY ordering rule this turn claims, and which of them the compiler - ;; catches. `:constraints` is DERIVED from the `:enforced true` entries - ;; (workflow/enforced-constraints), so the documented set and the checked set - ;; cannot drift apart. The turn-shaped manifests all carry the same four — - ;; they are the shape of a turn, not of this particular workflow. - :invariants - [{:type :must-follow :if :dispatch :then :journal :enforced true - :protects "A dispatched tool call is always recorded."} - - {:type :must-follow :if :journal :then :arbiter :enforced true - :protects "A recorded turn always faces a gate."} - - {:type :must-precede :cell :dispatch :before :arbiter :enforced true - :protects "The arbiter reads what the turn DID; a gate deciding before the - tool ran would steer on the previous turn's state."} - - {:type :must-precede :cell :settle :before :arbiter :enforced true - :protects "A gate cannot be credited with an outcome that preceded it: - :gate/settle closes this turn's open predictions BEFORE the - arbiter chooses, so a resolution closes against the gate that - asked for it and not the one about to."} - - {:type :must-precede :cell :parse :before :dispatch :enforced true - :protects "Nothing is dispatched that was not parsed out of a reply."}]} + :edges {:route {:continue :start + :done :distil + :abandoned :distil + :exhausted :distil} + :distil :end}} diff --git a/resources/manifests/supervisor.edn b/resources/manifests/supervisor.edn index fc13ad9c..dac6e70d 100644 --- a/resources/manifests/supervisor.edn +++ b/resources/manifests/supervisor.edn @@ -1,106 +1,16 @@ ;; The SUPERVISOR role loop — the harness's introspection agent. Structurally -;; the per-turn worker graph (assemble -> infer -> parse -> dispatch -> journal -;; -> arbiter -> route), because the supervisor works the same way every role -;; does: a turn at a time until it decides. What makes it the SUPERVISOR is its -;; role prompt (prompts/roles/supervisor.md, injected by :oversight/reason) and -;; that it is handed a run-health digest to introspect on rather than a part to -;; build. It has the FULL tool surface on purpose — it must be able to read the -;; journal and transcripts to diagnose, and edit manifests / cells / prompts -;; (the mutation protocol, which validates every change) to tune the harness. +;; the turn chain itself (manifests/turn.edn, which it extends with nothing), +;; because the supervisor works the same way every role does: a turn at a time +;; until it decides, then every terminal verdict returns to the oversight cell. +;; What makes it the SUPERVISOR is its role prompt (prompts/roles/supervisor.md, +;; injected by :oversight/reason) and that it is handed a run-health digest to +;; introspect on rather than a part to build. It has the FULL tool surface on +;; purpose — it must be able to read the journal and transcripts to diagnose, +;; and edit manifests / cells / prompts (the mutation protocol, which validates +;; every change) to tune the harness. ;; ;; A separate, independently-versioned manifest so the supervisor role owns and -;; can tune its own loop (manifest save "supervisor"). +;; can tune its own loop (manifest save "supervisor") without changing the turn +;; every other role runs. {:description "The supervisor role loop: the harness's introspection agent, handed a run-health digest to diagnose and steer from. A component of the feature loop, not usually run standalone." - :input-schema - ;; What a driver hands this manifest to start a run. mycelium checks it in - ;; run-compiled BEFORE any cell runs, so a caller that builds the map wrongly - ;; is told which key, instead of a nil surfacing several cells later. - [:map [:branch :map] [:turn :int]] - - :cells {:start :loop/assemble - ;; THE CONTEXT-BUDGET LADDER, as nodes. Compaction used to be a - ;; side effect of building the request — invisible to every - ;; manifest, unreachable by introspect, tunable only by moving a - ;; character count. It runs between assembling the turn and calling - ;; the model, which is the only place the cheap rungs can act, and - ;; each rung is its own node so a project can rewire or drop one. - :measure :compaction/measure - :cap :compaction/cap - :prune :compaction/prune - :fold :compaction/fold - :infer :llm/infer - :parse :llm/parse - :provider-error :loop/provider-error - :no-call :loop/no-call - :dispatch :tool/dispatch - :journal :journal/record - :settle :gate/settle - :arbiter :gate/arbiter - :route :loop/route} - - :edges {:start :measure - ;; measure WRITES :compaction/route and the dispatch READS it — - ;; never the other way round. A predicate that computed the tier - ;; here would put the routing back inside code and leave the - ;; manifest showing an opaque box. - :measure {:cap :cap - :none :infer} - :cap {:prune :prune - :none :infer} - :prune :fold - :fold :infer - :infer :parse - :parse {:provider-error :provider-error - :no-call :no-call - :tool :dispatch} - :provider-error :route - :no-call :route - :dispatch :journal - :journal :settle - :settle :arbiter - :arbiter :route - :route {:continue :start - :done :end - :abandoned :end - :exhausted :end}} - - :dispatches {:measure [[:cap {:compaction/route :cap}] - [:none _]] - :cap [[:prune {:compaction/route :prune}] - [:none _]] - ;; ORDER MATTERS here, as in loop.edn: a provider failure passes - ;; the data through with :parsed ABSENT, and a map pattern needs - ;; its key present, so :provider-error is tried first. - :parse [[:provider-error {:call {:ok false}}] - [:no-call {:parsed nil}] - [:no-call {:parsed {:name "__parse_error__"}}] - [:tool _]] - :route [[:continue {:verdict :continue}] - [:done {:verdict :done}] - [:abandoned {:verdict :abandoned}] - [:exhausted {:verdict :exhausted}]]} - - ;; EVERY ordering rule this turn claims, and which of them the compiler - ;; catches. `:constraints` is DERIVED from the `:enforced true` entries - ;; (workflow/enforced-constraints), so the documented set and the checked set - ;; cannot drift apart. The turn-shaped manifests all carry the same four — - ;; they are the shape of a turn, not of this particular workflow. - :invariants - [{:type :must-follow :if :dispatch :then :journal :enforced true - :protects "A dispatched tool call is always recorded."} - - {:type :must-follow :if :journal :then :arbiter :enforced true - :protects "A recorded turn always faces a gate."} - - {:type :must-precede :cell :dispatch :before :arbiter :enforced true - :protects "The arbiter reads what the turn DID; a gate deciding before the - tool ran would steer on the previous turn's state."} - - {:type :must-precede :cell :settle :before :arbiter :enforced true - :protects "A gate cannot be credited with an outcome that preceded it: - :gate/settle closes this turn's open predictions BEFORE the - arbiter chooses, so a resolution closes against the gate that - asked for it and not the one about to."} - - {:type :must-precede :cell :parse :before :dispatch :enforced true - :protects "Nothing is dispatched that was not parsed out of a reply."}]} + :extends "turn"} diff --git a/resources/manifests/turn.edn b/resources/manifests/turn.edn new file mode 100644 index 00000000..e49237aa --- /dev/null +++ b/resources/manifests/turn.edn @@ -0,0 +1,128 @@ +;; The TURN CHAIN, written once. loop, worker, reviewer and supervisor each +;; say `:extends "turn"` and carry only what differs — their tail — so a +;; change to how a turn runs is made here, once, and every role picks it up. +;; They used to carry four copies of this graph, and copies drift. +;; +;; How an extension resolves (manifests/read-definition): the child's :cells, +;; :edges and :dispatches entries replace this file's key by key, and a nil +;; value removes one; :invariants are these plus the child's; any other key is +;; the child's where it has one. Saving this file compiles every manifest that +;; extends it first, and a change that breaks one of them is refused naming it. +;; +;; On its own this is a complete loop that returns every terminal verdict to +;; whatever composed it, which is exactly the supervisor. It is marked a +;; fragment so it is not offered as a run's workflow. +;; +;; Dispatch entries are PATTERNS over the data map — {:verdict :done} matches +;; any data map carrying that key with that value, `_` matches anything, and +;; the first entry that matches wins. The cell computes the decision INTO the +;; data map and the pattern only reads it, so the routing stays visible here +;; where it can be edited, and a table of patterns can be checked for a +;; branch nothing can reach. A (fn [d] ...) form is still accepted where a +;; pattern cannot say it. +{:description "The turn chain every turn-at-a-time role shares: assemble, the context-budget ladder, infer, parse, dispatch, journal, settle, arbiter, route. loop, worker, reviewer and supervisor extend it and add only their tail. Not a workflow to run on its own." + :fragment? true + :input-schema + ;; What a driver hands this manifest to start a run. mycelium checks it in + ;; run-compiled BEFORE any cell runs, so a caller that builds the map wrongly + ;; is told which key, instead of a nil surfacing several cells later. + [:map [:branch :map] [:turn :int]] + + :cells {:start :loop/assemble + ;; THE CONTEXT-BUDGET LADDER, as nodes. Compaction used to be a + ;; side effect of building the request — invisible to every + ;; manifest, unreachable by introspect, tunable only by moving a + ;; character count. It runs between assembling the turn and calling + ;; the model, which is the only place the cheap rungs can act, and + ;; each rung is its own node so a project can rewire or drop one. + :measure :compaction/measure + :cap :compaction/cap + :prune :compaction/prune + :fold :compaction/fold + :infer :llm/infer + :parse :llm/parse + :provider-error :loop/provider-error + :no-call :loop/no-call + :dispatch :tool/dispatch + :journal :journal/record + :settle :gate/settle + :arbiter :gate/arbiter + :route :loop/route} + + :edges {:start :measure + ;; measure WRITES :compaction/route and the dispatch READS it — + ;; never the other way round. A predicate that computed the tier + ;; here would put the routing back inside code and leave the + ;; manifest showing an opaque box. + :measure {:cap :cap + :none :infer} + :cap {:prune :prune + :none :infer} + :prune :fold + :fold :infer + :infer :parse + :parse {:provider-error :provider-error + :no-call :no-call + :tool :dispatch} + :provider-error :route + :no-call :route + :dispatch :journal + :journal :settle + :settle :arbiter + :arbiter :route + ;; The back edge: one turn ends where the next begins. Every terminal + ;; verdict returns to whatever composed this loop; a child that owns + ;; its ending (distil, finish) replaces this edge. + :route {:continue :start + :done :end + :abandoned :end + :exhausted :end}} + + :dispatches {:measure [[:cap {:compaction/route :cap}] + [:none _]] + :cap [[:prune {:compaction/route :prune}] + [:none _]] + ;; ORDER MATTERS in this one table. A provider failure passes + ;; the data through untouched, so :parsed is then ABSENT, and + ;; a map pattern needs its key present: {:parsed nil} would not + ;; match that data map, and the data would fall through to :tool. + ;; The provider-error branch has to be tried first. `manifest show` + ;; reports this table as order-dependent, which is correct. + :parse [[:provider-error {:call {:ok false}}] + [:no-call {:parsed nil}] + [:no-call {:parsed {:name "__parse_error__"}}] + [:tool _]] + :route [[:continue {:verdict :continue}] + [:done {:verdict :done}] + [:abandoned {:verdict :abandoned}] + [:exhausted {:verdict :exhausted}]]} + + ;; EVERY ordering rule a turn claims, and which of them the compiler + ;; catches. `:constraints` is DERIVED from the `:enforced true` entries + ;; (workflow/enforced-constraints), so an editor can read this list and know + ;; what is actually defended rather than having to go and read the cells. + ;; They are the shape of a turn, not of any one workflow, which is why they + ;; live here. + :invariants + [{:type :must-follow :if :dispatch :then :journal :enforced true + :protects "A dispatched tool call is always recorded."} + + {:type :must-follow :if :journal :then :arbiter :enforced true + :protects "A recorded turn always faces a gate."} + + {:type :must-precede :cell :dispatch :before :arbiter :enforced true + :protects "The arbiter reads what the turn DID. A gate deciding before the + tool ran would steer on the previous turn's state. The + provider-error and no-call routes reach neither node, which is + why this is expressible as a precedence rather than as a + must-follow: it binds only the paths where a gate actually + fires."} + + {:type :must-precede :cell :parse :before :dispatch :enforced true + :protects "Nothing is dispatched that was not parsed out of a reply."} + + {:type :must-precede :cell :settle :before :arbiter :enforced true + :protects "A gate cannot be credited with an outcome that preceded it: + :gate/settle closes this turn's open predictions BEFORE the + arbiter chooses, so a resolution closes against the gate that + asked for it and not the one about to."}]} diff --git a/resources/manifests/worker.edn b/resources/manifests/worker.edn index 98149b58..17655327 100644 --- a/resources/manifests/worker.edn +++ b/resources/manifests/worker.edn @@ -1,109 +1,24 @@ ;; The WORKER sub-loop: one branch doing the actual work, iterating a turn at a -;; time until it reaches a terminal verdict. This is the factory loop's per-turn -;; graph with the finish removed — instead of finishing, a terminal verdict -;; routes to :end, returning control (and the branch, carrying :verdict) to -;; whatever composed this sub-loop. A top-level manifest runs it as a cell -;; (mycelium.compose/workflow->cell) and decides what happens next — e.g. hand a -;; :done branch to a critic before finishing. Running it standalone is the same -;; as the flat loop up to the point it would finish. +;; time until it reaches a terminal verdict. This is the turn chain +;; (manifests/turn.edn, which it extends) with task reflection on every ending +;; and no finish — a terminal verdict routes to :end, returning control (and +;; the branch, carrying :verdict) to whatever composed this sub-loop. A +;; top-level manifest runs it as a cell (mycelium.compose/workflow->cell) and +;; decides what happens next — e.g. hand a :done branch to a critic before +;; finishing. Running it standalone is the same as the flat loop up to the +;; point it would finish. {:description "The worker sub-loop: one branch doing the actual per-turn work, returning its verdict to whatever composed it. Not usually run on its own — it is the implementor unit the team, feature, and decompose loops build on." - :input-schema - ;; What a driver hands this manifest to start a run. mycelium checks it in - ;; run-compiled BEFORE any cell runs, so a caller that builds the map wrongly - ;; is told which key, instead of a nil surfacing several cells later. - [:map [:branch :map] [:turn :int]] - - :cells {:start :loop/assemble - ;; THE CONTEXT-BUDGET LADDER, as nodes. Compaction used to be a - ;; side effect of building the request — invisible to every - ;; manifest, unreachable by introspect, tunable only by moving a - ;; character count. It runs between assembling the turn and calling - ;; the model, which is the only place the cheap rungs can act, and - ;; each rung is its own node so a project can rewire or drop one. - :measure :compaction/measure - :cap :compaction/cap - :prune :compaction/prune - :fold :compaction/fold - :infer :llm/infer - :parse :llm/parse - :provider-error :loop/provider-error - :no-call :loop/no-call - :dispatch :tool/dispatch - :journal :journal/record - :settle :gate/settle - :arbiter :gate/arbiter - :route :loop/route - ;; Task reflection on every ending (blt.26): what this attempt owes - ;; the next one is distilled whether the worker runs standalone or - ;; composed — the parent decides finishing, not remembering. - :distil :memory/distil} - - :edges {:start :measure - ;; measure WRITES :compaction/route and the dispatch READS it — - ;; never the other way round. A predicate that computed the tier - ;; here would put the routing back inside code and leave the - ;; manifest showing an opaque box. - :measure {:cap :cap - :none :infer} - :cap {:prune :prune - :none :infer} - :prune :fold - :fold :infer - :infer :parse - :parse {:provider-error :provider-error - :no-call :no-call - :tool :dispatch} - :provider-error :route - :no-call :route - :dispatch :journal - :journal :settle - :settle :arbiter - :arbiter :route - ;; :continue iterates; every terminal verdict returns to the parent, - ;; which reads :verdict to decide finishing vs critique. - :route {:continue :start - :done :distil - :abandoned :distil - :exhausted :distil} - :distil :end} - - :dispatches {:measure [[:cap {:compaction/route :cap}] - [:none _]] - :cap [[:prune {:compaction/route :prune}] - [:none _]] - ;; ORDER MATTERS here, as in loop.edn: a provider failure passes - ;; the data through with :parsed ABSENT, and a map pattern needs - ;; its key present, so :provider-error is tried first. - :parse [[:provider-error {:call {:ok false}}] - [:no-call {:parsed nil}] - [:no-call {:parsed {:name "__parse_error__"}}] - [:tool _]] - :route [[:continue {:verdict :continue}] - [:done {:verdict :done}] - [:abandoned {:verdict :abandoned}] - [:exhausted {:verdict :exhausted}]]} - - ;; EVERY ordering rule this turn claims, and which of them the compiler - ;; catches. `:constraints` is DERIVED from the `:enforced true` entries - ;; (workflow/enforced-constraints), so the documented set and the checked set - ;; cannot drift apart. The turn-shaped manifests all carry the same four — - ;; they are the shape of a turn, not of this particular workflow. - :invariants - [{:type :must-follow :if :dispatch :then :journal :enforced true - :protects "A dispatched tool call is always recorded."} - - {:type :must-follow :if :journal :then :arbiter :enforced true - :protects "A recorded turn always faces a gate."} - - {:type :must-precede :cell :dispatch :before :arbiter :enforced true - :protects "The arbiter reads what the turn DID; a gate deciding before the - tool ran would steer on the previous turn's state."} - - {:type :must-precede :cell :settle :before :arbiter :enforced true - :protects "A gate cannot be credited with an outcome that preceded it: - :gate/settle closes this turn's open predictions BEFORE the - arbiter chooses, so a resolution closes against the gate that - asked for it and not the one about to."} - - {:type :must-precede :cell :parse :before :dispatch :enforced true - :protects "Nothing is dispatched that was not parsed out of a reply."}]} + :extends "turn" + + ;; Task reflection on every ending (blt.26): what this attempt owes the next + ;; one is distilled whether the worker runs standalone or composed — the + ;; parent decides finishing, not remembering. + :cells {:distil :memory/distil} + + ;; :continue iterates; every terminal verdict returns to the parent, which + ;; reads :verdict to decide finishing vs critique. + :edges {:route {:continue :start + :done :distil + :abandoned :distil + :exhausted :distil} + :distil :end}} diff --git a/resources/manual.edn b/resources/manual.edn index f275eba2..ba0384d3 100644 --- a/resources/manual.edn +++ b/resources/manual.edn @@ -85,6 +85,16 @@ :summary "A checked manifest edit as data: ops (rename-cell, add-cell, remove-cell, set-edge, delete-edge, set-cell-field, set-dispatches) applied to the manifest as written, every reference to a renamed node rewritten (edges, dispatches, :invariants), the whole batch compiled once at the end so a node can be added and wired in one call. `manifest patch` is this over the stored text; `render` writes the result back over the original so comments survive; `cell-refs` and `diff-manifests` are behind `manifest refs` and `manifest diff`."} {:name samizdat.workflow/catalog :summary "Every workflow available to select or adapt, with its description."} + {:name samizdat.heldout/check-edit + :summary "The held-out gate, stage 1: replay the project's battery (.samizdat/battery, stored in battery_cases) with a candidate cell, manifest or policy edit in place and without it, each case in a child process on the tree its recorded run started from, and refuse the edit when a target that passed fails. Ties accepted; a case that does not run at baseline is set aside by name. Opt-in by having cases; gates.edn :heldout. Each target is recorded in heldout_checks with its fixture, harness revision, model and scorer."} + {:name samizdat.heldout/draft! + :summary "Freeze a finished run as a battery case: its replay, the expectations battery/draft-case writes, its git baseline (pinned under refs/samizdat/battery/) as the fixture, its loop and model. Behind `battery add`. Cases are written once and never rewritten."} + {:name samizdat.heldout/live-verdict + :summary "The held-out gate, stage 2, over arena rows: per task, floor (candidate median fitness >= S* - delta, delta from the baseline's own spread), cost (a gain pays for its tokens by :fitness :cost-rule; inside the band only a cost cut or a structural change is admissible), guards (no acceptance criterion the baseline always met is lost, no suite goes red). Refusals named per task."} + {:name samizdat.manifests/read-definition + :summary "A manifest's text as the whole graph that runs, its :extends resolved. manifests/turn.edn is the turn chain loop, worker, reviewer and supervisor extend; a child's :cells/:edges/:dispatches entries win key by key and a nil removes one, the base's :invariants hold and the child's are added, a key in :replaces is the child's alone. read-raw is the file's own part, which is what an edit rewrites."} + {:name samizdat.manifests/validate-edit! + :summary "Check a manifest edit, and when other manifests extend it, compile each of them against the edit too — a base change that breaks a role built on it is refused naming that role. Behind manifest save and patch and the file validator."} {:name samizdat.manifests/unguarded-cycles :summary "Every cycle in a manifest that cannot change its own exit — no dispatch on it with an edge out reads a key a cell on it promises in :output — with what its exits read and what its cells write. A compile warning (:unguarded-cycle) and a paragraph on manifest save/patch, never a refusal: the shipped manifests all pass, and termination itself stays with max-turns and the deadlines."} {:name samizdat.manifests/preconditions diff --git a/resources/prompts/battery-tool.md b/resources/prompts/battery-tool.md new file mode 100644 index 00000000..93561f4a --- /dev/null +++ b/resources/prompts/battery-tool.md @@ -0,0 +1,3 @@ +{% if usage %}`battery` holds the held-out cases every cell, manifest and policy edit is replayed against before it may go live: {action: list | add, run_id?}. `list` shows them. `add {run_id}` freezes a finished run of this project as a case — its conversation, the tree it started from, and what it did (how it ended, the tools it finished with, the gates that fired and were met). Add one when a run shows a behaviour a later edit must not break. A case cannot be removed or weakened: editing its file changes nothing that runs.{% endif %}{% if listed %}{% if none %}The battery is empty, so edits go live on the compile and soak alone. `battery add {run_id}` freezes a finished run as the first case.{% else %}{% for c in cases %}{{c.id}} ({{c.subject}}, {{c.targets}} target{{c.targets-s}}{% if c.loop %}, loop {{c.loop}}{% endif %}, fixture {{c.fixture}}) +{% endfor %}{% endif %}{% if altered %} +Edited against the stored battery, and ignored — the stored case is what runs: {{altered}}{% endif %}{% endif %}{% if added %}Added `{{id}}` with {{targets}} target{{targets-s}} ({{path}}). Every cell, manifest and policy edit from now on is replayed against it, and refused if a target that passes today fails.{% endif %}{% if add-failed %}Could not add run {{run-id}}: {{reason}}{% endif %} diff --git a/resources/prompts/heldout-refused.md b/resources/prompts/heldout-refused.md new file mode 100644 index 00000000..f6785d7c --- /dev/null +++ b/resources/prompts/heldout-refused.md @@ -0,0 +1,5 @@ +Refused by the held-out battery: replaying {{cases}} recorded case{{cases-s}} with this edit in place, a target that passes today fails. +{{broke}} +{% if counted %}({{passed-before}} of {{total}} targets passed before, {{passed-after}} after.) {% endif %}Nothing was saved; what is running is unchanged. Read the case (`battery list`) and what the edit changes on that path, and change the edit rather than the case — a case cannot be weakened.{% if unmeasured %} +Not measured, because they do not run at baseline either: {{unmeasured}}.{% endif %}{% if altered %} +Case files edited against the stored battery, and ignored: {{altered}}.{% endif %} diff --git a/resources/prompts/prompt-tool.md b/resources/prompts/prompt-tool.md index 91879fb8..8ee248fe 100644 --- a/resources/prompts/prompt-tool.md +++ b/resources/prompts/prompt-tool.md @@ -1 +1 @@ -{% if saved %}Saved prompt '{{name}}' as v{{version}} in this project. It is what the next render reads; the shipped template is unchanged.{% endif %}{% if unbound %}Prompt '{{name}}' was not stored: no project store is bound, so there is nowhere to put a version.{% endif %}{% if reverted %}Reverted prompt '{{name}}' to the body of v{{from}}, stored as v{{version}}. Reverting is itself an edit, so the version you left behind is still readable.{% endif %}{% if no-prompt %}No prompt '{{name}}'{% if version %} v{{version}}{% endif %}. `prompt list` shows what there is.{% endif %}{% if no-versions %}No stored versions of '{{name}}' in this project.{% if shipped %} It is still the shipped template.{% endif %}{% endif %}{% if no-revert %}No prompt '{{name}}' v{{version}} to revert to.{% endif %}{% if bad-render %}Prompt '{{name}}' was NOT saved. {{complaint}}{% endif %}{% if unknown-action %}Unknown `prompt` action '{{action}}'. {{usage}}{% endif %}{% if needs-action %}`prompt` needs an `action`. {{usage}}{% endif %} +{% if saved %}Saved prompt '{{name}}' as v{{version}} in this project. It is what the next render reads; the shipped template is unchanged. It was not replayed against the held-out battery: the recorded replies are fixed, so a replay cannot show what different words would make the model say — start an `experiment` to measure it.{% endif %}{% if unbound %}Prompt '{{name}}' was not stored: no project store is bound, so there is nowhere to put a version.{% endif %}{% if reverted %}Reverted prompt '{{name}}' to the body of v{{from}}, stored as v{{version}}. Reverting is itself an edit, so the version you left behind is still readable.{% endif %}{% if no-prompt %}No prompt '{{name}}'{% if version %} v{{version}}{% endif %}. `prompt list` shows what there is.{% endif %}{% if no-versions %}No stored versions of '{{name}}' in this project.{% if shipped %} It is still the shipped template.{% endif %}{% endif %}{% if no-revert %}No prompt '{{name}}' v{{version}} to revert to.{% endif %}{% if bad-render %}Prompt '{{name}}' was NOT saved. {{complaint}}{% endif %}{% if unknown-action %}Unknown `prompt` action '{{action}}'. {{usage}}{% endif %}{% if needs-action %}`prompt` needs an `action`. {{usage}}{% endif %} diff --git a/resources/prompts/system-tools.md b/resources/prompts/system-tools.md index b1813219..c295555e 100644 --- a/resources/prompts/system-tools.md +++ b/resources/prompts/system-tools.md @@ -294,6 +294,17 @@ verdict({name}) fitness per turn before and after. `worse` and `unchanged` both mean revert — a change nobody can justify is debt, and "it did not hurt" is not a reason to carry one. +battery({action, run_id?}) + The held-out cases every cell, manifest and policy save is replayed + against first: an edit under which a case target that passes today + fails is refused, naming the target, and nothing is saved. Actions: + list The cases, with their targets and fixtures. + add {run_id} Freeze a finished run of this project as a case: + its conversation, the tree it started from, and + what it did. Add one when a run shows behaviour a + later edit must not break. There is no remove, + and editing a case's file changes nothing that + runs. policy({action, ...}) The numbers and tables behind every decision — gates.edn (every threshold, budget and steer gate), the phase machine, the wordlists, the diff --git a/resources/roles.edn b/resources/roles.edn index 048f564f..e0ebabfb 100644 --- a/resources/roles.edn +++ b/resources/roles.edn @@ -79,7 +79,7 @@ :tools #{"intervene" "read_file" "read_digest" "grep" "lsp" "skill" "doc" "complete" "manual" "introspect" "cells" "cell" "manifest" "prompt" "policy" "adopt" "fetch_turn" "fetch_artifact" "recall" "remember" "forget" "retire" "outcome" - "experiment" "verdict" "message" "eval" "shell" "websearch" "done" "give_up"} + "experiment" "verdict" "battery" "message" "eval" "shell" "websearch" "done" "give_up"} :prompt "roles/supervisor" :sees #{:run-health :gate-health :failures}} diff --git a/resources/userspace.edn b/resources/userspace.edn index 1e8f1ab4..903a81a7 100644 --- a/resources/userspace.edn +++ b/resources/userspace.edn @@ -28,7 +28,10 @@ :board-bt "manifests/board-bt.edn" :feature "manifests/feature.edn" :decompose "manifests/decompose.edn" - :repair "manifests/repair.edn"} + :repair "manifests/repair.edn" + ;; Not a workflow: the turn chain loop, worker, reviewer and supervisor + ;; extend (:extends "turn"). + :turn "manifests/turn.edn"} :policies {:gates "gates.edn" @@ -56,6 +59,8 @@ {:acceptance-failed "prompts/acceptance-failed.md" :acceptance-judge "prompts/acceptance-judge.md" :adopt-tool "prompts/adopt-tool.md" + :battery-tool "prompts/battery-tool.md" + :heldout-refused "prompts/heldout-refused.md" :adoption-offer "prompts/adoption-offer.md" :architect "prompts/architect.md" :ask-tool "prompts/ask-tool.md" diff --git a/src/samizdat/agent/select.clj b/src/samizdat/agent/select.clj index ffe2cf17..b098c1c3 100644 --- a/src/samizdat/agent/select.clj +++ b/src/samizdat/agent/select.clj @@ -51,7 +51,7 @@ ;; about which workflow suits which task; this is a claim about which ;; ones can drive a run at all, and it must not depend on a policy value ;; the agent may widen (karamazov-4sx). - (filterv #(and (contains? allowed (:name %)) (:turn-sliceable? %)) + (filterv #(and (contains? allowed (:name %)) (:turn-sliceable? %) (not (:fragment? %))) (workflow/catalog conn)))) (defn history-lines diff --git a/src/samizdat/agent/tools.clj b/src/samizdat/agent/tools.clj index ce3d3e20..5566e986 100644 --- a/src/samizdat/agent/tools.clj +++ b/src/samizdat/agent/tools.clj @@ -66,6 +66,7 @@ [samizdat.agent.tools.mutate] [samizdat.agent.tools.manifest] [samizdat.agent.tools.experiments] + [samizdat.agent.tools.battery] [samizdat.agent.tools.policy] [samizdat.agent.tools.prompts] [samizdat.agent.tools.skills] diff --git a/src/samizdat/agent/tools/battery.clj b/src/samizdat/agent/tools/battery.clj new file mode 100644 index 00000000..2fcc7918 --- /dev/null +++ b/src/samizdat/agent/tools/battery.clj @@ -0,0 +1,59 @@ +;; samizdat - a self-hosting agentic harness +;; Copyright (C) 2026 Dmitri Sotnikov +;; +;; This program is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. +;; +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; +;; You should have received a copy of the GNU General Public License +;; along with this program. If not, see . +;; +;; SPDX-License-Identifier: GPL-3.0-or-later + +(ns samizdat.agent.tools.battery + "`battery` — the held-out cases every cell, manifest and policy edit is + replayed against before it may go live (samizdat.heldout). `list` shows the + battery; `add {run_id}` freezes a finished run of this project as a case. + There is no remove: the battery may grow and may not be weakened." + (:require [clojure.string :as str] + [samizdat.agent.tools.base :as base] + [samizdat.heldout :as heldout] + [samizdat.prompt :as prompt])) + +(defn- msg [vars] (prompt/render "battery-tool" vars)) + +(defmethod base/run-tool "battery" [{:keys [branch conn] :as ctx}] + (let [action (some-> (base/arg ctx :action) str str/trim str/lower-case not-empty)] + (case action + "list" + (let [{:keys [cases altered]} (heldout/cases conn)] + (base/ok branch + (msg {:listed true + :none (empty? cases) + :cases (for [c cases] + {:id (:id c) :subject (:subject c) + :targets (count (:expect c)) + :targets-s (if (= 1 (count (:expect c))) "" "s") + :loop (or (:loop c) "") + :fixture (str (get-in c [:fixture :sha]))}) + :altered (str/join ", " altered)}))) + + "add" + (if-let [run-id (some-> (base/arg ctx :run_id) str str/trim not-empty)] + (try + (let [{:keys [id path targets]} (heldout/draft! conn run-id)] + (base/ok branch (msg {:added true :id id :path path :targets targets + :targets-s (if (= 1 targets) "" "s")}) + :progress? true)) + (catch Throwable e + (base/rejected branch (msg {:add-failed true :run-id run-id + :reason (or (ex-message e) (str e))})))) + (base/malformed branch (base/missing ctx :run_id))) + + (base/malformed branch (msg {:usage true}))))) diff --git a/src/samizdat/agent/tools/introspect.clj b/src/samizdat/agent/tools/introspect.clj index d94f67ec..9e2e2ade 100644 --- a/src/samizdat/agent/tools/introspect.clj +++ b/src/samizdat/agent/tools/introspect.clj @@ -40,8 +40,7 @@ A separate namespace requiring only base + the read seams, so the tool surface grows by a plug-in file rather than by editing the aggregator. Render fns are exposed (not private) so a test can call them directly." - (:require [clojure.edn :as edn] - [clojure.string :as str] + (:require [clojure.string :as str] [mycelium.cell :as cell] [samizdat.agent.tools.base :as base] [samizdat.cells :as cells] @@ -73,10 +72,10 @@ (select-keys tw [:name :version :definition]) (let [nm (or (get-in ctx [:config :run :loop]) "loop")] {:name nm - :definition (edn/read-string (manifests/manifest-body! nm))})) + :definition (manifests/read-definition (manifests/manifest-body! nm))})) (let [nm (or (get-in ctx [:config :run :loop]) "loop")] {:name nm - :definition (edn/read-string (manifests/manifest-body! nm))}))) + :definition (manifests/read-definition (manifests/manifest-body! nm))}))) (defn loop-def "The active loop's workflow definition — :cells (node -> cell-id), :edges diff --git a/src/samizdat/agent/tools/manifest.clj b/src/samizdat/agent/tools/manifest.clj index 18cc3843..2fa45b4c 100644 --- a/src/samizdat/agent/tools/manifest.clj +++ b/src/samizdat/agent/tools/manifest.clj @@ -41,6 +41,7 @@ [clojure.string :as str] [mycelium.patch :as patch] [samizdat.agent.tools.base :as base] + [samizdat.heldout :as heldout] [samizdat.manifests :as manifests] [samizdat.prompt :as prompt] [samizdat.store.userspace :as us] @@ -54,9 +55,10 @@ on any error. The tool used to run a bare pre-compile that skipped the last two, so a manifest that could not run could still be saved — and then threw out of load-loop! at the next run start (karamazov-blt.6)." - [edn-text] - (manifests/compile-loop (manifests/read-definition edn-text)) - true) + [name edn-text] + ;; Through validate-edit!: an edit to a manifest others extend is compiled + ;; against each of them too (karamazov-xtd3). + (manifests/validate-edit! name edn-text manifests/compile-loop)) (defn- refused "The complaint for a throwable out of validate!. @@ -364,8 +366,13 @@ ;; loop this tool exists to invite — is not billed to the branch's ;; failure counter. The outer catch is left for what happens AFTER ;; this point, notably the store write. - (if-let [complaint (try (validate! edn-text) nil - (catch Throwable e (refused e)))] + (if-let [complaint (or (try (validate! name edn-text) nil + (catch Throwable e (refused e))) + ;; The held-out battery, last: the dearest + ;; check, and only for an edit that compiles + ;; (karamazov-7mo.4). + (heldout/check-edit conn {:kind :manifest :name name + :text edn-text}))] ;; The complaint plus `usage`, which already says a save ;; validates before it stores — no new sentence in src/. (base/rejected branch @@ -396,19 +403,32 @@ ;; not applied to whatever is there now. (base/rejected branch (say :stale :name name :version version :expect expect)) (let [old (manifests/read-definition body) + raw (manifests/read-raw body) + ;; The ops apply to the whole graph; an extending + ;; manifest's file then keeps only what differs from its + ;; base, :extends and all (karamazov-xtd3). + file-text (fn [new] + (if (:extends raw) + (patch/render body raw (manifests/extension-delta raw new)) + (patch/render body old new))) ;; Caught HERE for the reason save's is: a refused op or ;; a result that does not compile is a correctable ;; edit, not evidence about the branch (karamazov-gn64). - outcome (try {:new (patch/apply-ops - old {:ops ops - :validator #(manifests/compile-loop %)})} + outcome (try (let [new (patch/apply-ops + old {:ops ops + :validator #(manifests/compile-loop %)}) + text (file-text new)] + (validate! name text) + (if-let [r (heldout/check-edit conn {:kind :manifest :name name + :text text})] + {:complaint r} + {:new new :text text})) (catch Throwable e {:complaint (refused e)}))] (if-let [complaint (:complaint outcome)] (base/rejected branch (str "`manifest patch` refused: " (deflag complaint) "\n\n" (ops-help))) - (let [new (:new outcome) - text (patch/render body old new) + (let [{:keys [new text]} outcome v (save! conn name text why)] (base/ok branch (str (saved-line name v) diff --git a/src/samizdat/agent/tools/mutate.clj b/src/samizdat/agent/tools/mutate.clj index 0a04ba3d..63e19a55 100644 --- a/src/samizdat/agent/tools/mutate.clj +++ b/src/samizdat/agent/tools/mutate.clj @@ -25,12 +25,12 @@ A separate namespace requiring only base, so it plugs into the tool surface without dragging the mutation machinery into the aggregator." - (:require [clojure.edn :as edn] - [clojure.string :as str] + (:require [clojure.string :as str] [mycelium.cell :as cell] [samizdat.agent.state :as state] [samizdat.agent.tools.base :as base] [samizdat.cells :as cells] + [samizdat.heldout :as heldout] [samizdat.manifests :as manifests] [samizdat.mutation :as mutation] [samizdat.prompt :as prompt] @@ -64,7 +64,7 @@ run was not driving: a bad edit to the evolved loop could commit, and a valid one could be refused (karamazov-blt.2)." [ctx] - (edn/read-string (userspace/body! :manifest (active-name ctx)))) + (manifests/read-definition (userspace/body! :manifest (active-name ctx)))) (defn- extra-defs "Every OTHER manifest this project can run — shipped and stored — as @@ -79,7 +79,7 @@ :when (not= nm active) :let [body (manifests/manifest-body nm)] :when body - :let [d (try (edn/read-string body) (catch Throwable _ nil))] + :let [d (try (manifests/read-definition body) (catch Throwable _ nil))] :when (seq (:cells d))] [nm d]))) @@ -192,6 +192,7 @@ ;; just installed (karamazov-blt.2). :compile-fn manifests/compile-definition :soak-input (soak-input) + :heldout-fn #(heldout/check-edit conn %) :conn conn :run-id run-id})] (case (:status r) :committed diff --git a/src/samizdat/agent/tools/policy.clj b/src/samizdat/agent/tools/policy.clj index 7dc3e8a5..cf127124 100644 --- a/src/samizdat/agent/tools/policy.clj +++ b/src/samizdat/agent/tools/policy.clj @@ -41,6 +41,7 @@ [samizdat.agent.gates :as gates] [samizdat.agent.phases :as phases] [samizdat.agent.tools.base :as base] + [samizdat.heldout :as heldout] [samizdat.lexicon :as lexicon] [samizdat.manual :as manual] [samizdat.agent.roles :as roles] @@ -238,11 +239,20 @@ :else (let [parsed (try {:ok (edn/read-string (str body))} (catch Throwable e {:error (ex-message e)}))] - (if (:error parsed) + (cond + (:error parsed) ;; A body that does not read is a rejected edit, not a branch ;; failure: nothing was stored and the complaint says where. (base/rejected branch (msg {:bad-edn true :name name :complaint (:error parsed)})) + + ;; The held-out battery, before anything is stored: a table + ;; under which a recorded case's target stops passing is not + ;; saved (karamazov-7mo.4). + :else + (if-let [refusal (heldout/check-edit (or (:conn ctx) (userspace/conn)) + {:kind :policy :name name :text (str body)})] + (base/rejected branch refusal) ;; Warm the cache so the seed exists and the version we might ;; roll back to is real, then store and recompile. (do (userspace/body :policy name) @@ -279,7 +289,7 @@ ;; is where it started — a rejected edit. (base/rejected branch (msg {:rolled-back true :name name - :complaint (ex-message e)}))))))))))) + :complaint (ex-message e)})))))))))))) "revert" (let [v (some-> (base/arg ctx :version) str str/trim not-empty parse-long) diff --git a/src/samizdat/capabilities.clj b/src/samizdat/capabilities.clj index 319fd02a..43c4fb36 100644 --- a/src/samizdat/capabilities.clj +++ b/src/samizdat/capabilities.clj @@ -61,6 +61,7 @@ [samizdat.export] [samizdat.hashline] [samizdat.leakage] + [samizdat.heldout] [samizdat.layers] [samizdat.lisp] [samizdat.llm.client] diff --git a/src/samizdat/heldout.clj b/src/samizdat/heldout.clj new file mode 100644 index 00000000..27342839 --- /dev/null +++ b/src/samizdat/heldout.clj @@ -0,0 +1,598 @@ +;; samizdat - a self-hosting agentic harness +;; Copyright (C) 2026 Dmitri Sotnikov +;; +;; This program is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. +;; +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; +;; You should have received a copy of the GNU General Public License +;; along with this program. If not, see . +;; +;; SPDX-License-Identifier: GPL-3.0-or-later + +(ns samizdat.heldout + "The held-out gate, stage 1 (karamazov-7mo.4 / ylte.4): replay the project's + frozen battery against its userspace AS IT IS and AS A CANDIDATE EDIT WOULD + MAKE IT, and refuse the edit when a target that passed before fails after. + MECHANISM ONLY — whether it runs, and its budgets, are gates.edn :heldout; + its words are prompts/heldout-*.md. + + WHY A CHILD PROCESS PER CASE. A replay has to see the candidate as the + project's userspace, everywhere a run reads it: the cells, the manifests, + the policy tables, from every branch fiber. A dynamic binding does not + cross a fiber, and a global override would leak the candidate into the live + run that is proposing it. So the candidate is MATERIALIZED: the recorded + run's fixture (the git baseline it started from) is unpacked into a temp + directory, the project's .samizdat/ is copied beside it with the candidate + file written in, and a fresh process (samizdat.heldout.child) starts the + harness on that directory and replays the case into an in-memory database. + Nothing it does touches the project or the live run. + + WHAT IS COMPARED. Per case, per target (battery/check), the baseline's + verdict against the candidate's. The rule is RRSI's non-compensatory one: + no target that passed before fails after — fixing one thing does not buy + breaking another — and the two sides must cover the same targets + (battery/accept?). Ties are accepted, deliberately: see samizdat.battery. + A case that cannot run at baseline is not this edit's doing and is set + aside, named; a case that ran at baseline and cannot run on the candidate + is a regression. + + THE BATTERY MAY GROW AND MAY NOT BE WEAKENED. The authority is the + battery_cases table, not the files: a case file not yet stored is added, a + stored case whose file was edited runs AS STORED and the edit is named, and + one whose file was deleted still runs. The agent may add a case from an + observed failure (the `battery` tool); nothing here lets it remove one. + + WHAT IT CANNOT JUDGE. The replies are fixed, so a prompt edit cannot change + what the model says under replay; such an edit is not sent here (the prompt + tool says so). Replay measures what the harness DOES with a given + conversation — the arena's live arms (stage 2) measure the rest." + (:require [clojure.edn :as edn] + [clojure.java.io :as io] + [clojure.string :as str] + [clojure.tools.logging :as log] + [samizdat.agent.gates :as gates] + [samizdat.battery :as battery] + [samizdat.engine.proc :as proc] + [samizdat.prompt :as prompt] + [samizdat.security.secrets :as secrets] + [samizdat.store.db :as db] + [samizdat.store.journal :as journal] + [samizdat.store.runs :as runs] + [samizdat.userspace :as userspace] + [samizdat.util :as util])) + +(defn- policy [] (gates/threshold :heldout)) + +;;; ------------------------------------------------------------- the battery + +(defn- sh-run + [timeout-ms cmd] + (proc/run {:timeout-ms timeout-ms :env (secrets/scrubbed-process-env)} "sh" "-c" cmd)) + +(defn case-dir + "Where this project's case files live, or nil unbound." + [] + (some-> (userspace/project-dir) (str "/" (:dir (policy))))) + +(defn- case-id + "A case's id: its own :id, else its file's name without .edn." + [c] + (str (or (:id c) + (some-> (:path c) io/file .getName (str/replace #"\.edn$" ""))))) + +(defn- stored-form + "What the table keeps of a case: the case as data, without where it was + read from (which the row already says)." + [c] + (dissoc c :path :subject)) + +(defn add-case! + "Store case `c` under `subject`, unless a case with its id is stored already + — an id is written once, which is what makes the battery something that can + grow and cannot be rewritten. Returns true when it was added." + [conn subject c] + (let [id (case-id c)] + (when-not (seq (db/fetch conn ["SELECT id FROM battery_cases WHERE id = ?" id])) + (db/execute! conn ["INSERT INTO battery_cases (id, subject, body, added_at) + VALUES (?, ?, ?, ?)" + id (str subject) (pr-str (stored-form (assoc c :id id))) (db/now)]) + true))) + +(defn cases + "The battery: every stored case, after adding each case file not yet stored. + + Returns {:cases [case …] :altered [path …]}. A file whose content differs + from its stored case is not what runs — the stored case is — and its path is + under :altered so whoever reads the verdict can see the attempt. A stored + case whose file is gone runs all the same." + [conn] + (let [files (battery/load-cases (case-dir)) + by-id (fn [] (into {} (map (juxt :id identity)) + (db/fetch conn ["SELECT id, subject, body FROM battery_cases ORDER BY id"]))) + _ (doseq [c files :when (not (contains? (by-id) (case-id c)))] + (add-case! conn (:subject c) c)) + stored (by-id) + altered (vec (for [c files + :let [row (get stored (case-id c))] + :when (and row (not= (edn/read-string (:body row)) + (stored-form (assoc c :id (case-id c)))))] + (:path c)))] + {:cases (vec (for [[id row] (sort-by key stored)] + (assoc (edn/read-string (:body row)) :id id :subject (:subject row)))) + :altered altered})) + +;;; ------------------------------------------------------------- adding one + +(defn draft! + "A case from finished run `run-id` of this project: battery/draft-case's + replay and expectations, plus what the replay needs to reproduce it — the + fixture (the run's :git-baseline, pinned under refs/samizdat/battery/ so + git's collector keeps it), the manifest that drove it, the model that + answered. Written to the battery directory and stored, and returned as + {:id :path :targets}. Throws when the run has no baseline to replay from." + [conn run-id] + (let [c (battery/draft-case conn run-id) + sha (some :ref (journal/notes conn run-id :git-baseline)) + loop-nm (some :name (journal/notes conn run-id :loop-workflow)) + ;; The ids the run's tasks were given, in the order it made them: a + ;; recorded reply that names one (`task show sz-ffd83f`) only means + ;; the same thing under replay if the replay hands out the same ids. + task-ids (mapv :id (db/fetch conn ["SELECT id FROM tasks WHERE run_id = ? + ORDER BY created_at, rowid" run-id])) + root (userspace/project-root)] + (when (str/blank? (str sha)) + (throw (ex-info (str "run " run-id " recorded no git baseline to replay from") + {:error :no-baseline :run-id run-id}))) + (sh-run (:stage-timeout-ms (policy)) + (str "git -C " (util/sh-quote root) " update-ref " + (util/sh-quote (str "refs/samizdat/battery/" (:id c))) " " (util/sh-quote sha))) + (let [c (cond-> (assoc c :fixture {:sha sha} + :model (:model (runs/get-run conn run-id)) + :task-ids task-ids) + loop-nm (assoc :loop loop-nm)) + subject (.getName (io/file root)) + f (io/file (case-dir) subject (str (:id c) ".edn"))] + (.mkdirs (.getParentFile f)) + (spit f (pr-str c)) + (add-case! conn subject c) + {:id (:id c) :path (str f) :targets (count (:expect c))}))) + +;;; ------------------------------------------------------------- staging + +(defn- delete-tree! + [f] + (let [f (io/file f)] + (when (.isDirectory f) + (doseq [c (.listFiles f)] (delete-tree! c))) + (.delete f))) + +(defn- relative + [root f] + (let [r (str (.getCanonicalPath (io/file root)) "/") + p (.getCanonicalPath (io/file f))] + (when (str/starts-with? p r) (subs p (count r))))) + +(defn- userspace-files + "The project's .samizdat files a replay reads, as [relative-path file]: + everything but its databases and the battery itself." + [] + (let [dir (userspace/project-dir) + battery (:dir (policy))] + (vec (for [f (file-seq (io/file dir)) + :when (.isFile f) + :let [rel (relative dir f)] + :when (and rel + (not (re-find (re-pattern (:skip (policy))) rel)) + (not (str/starts-with? rel (str battery "/"))))] + [rel f])))) + +(defn candidate-path + "The path, relative to .samizdat/, that `kind`/`name` is served from, or + nil when the project's map has no such role." + [kind name] + (some->> (userspace/project-path kind name) (relative (userspace/project-dir)))) + +(defn stage! + "Lay case `c` out under `dest` as a project: its fixture (the tree the + recorded run started from, out of the project's git) with this project's + .samizdat/ over it, and `candidate` ({:kind :name :text}) written in when + given. Returns nil, or {:error …} saying what could not be staged." + [c dest candidate] + (let [root (userspace/project-root) + sha (get-in c [:fixture :sha])] + (.mkdirs (io/file dest)) + (cond + (str/blank? (str sha)) + {:error :no-fixture} + + :else + (let [r (sh-run (:stage-timeout-ms (policy)) + (str "git -C " (util/sh-quote root) " archive " (util/sh-quote sha) + " | tar -x -C " (util/sh-quote dest)))] + (if (or (:timeout r) (not (zero? (long (or (:exit r) 1))))) + {:error :fixture :detail (str/trim (str (:err r)))} + (let [us (io/file dest ".samizdat")] + ;; The fixture's own .samizdat, if it committed one, is not this + ;; project's workflow: the userspace under test is. + (delete-tree! us) + ;; A REPOSITORY, as the recorded run had: the write ledger, the + ;; critic's diff and the run's own git calls all read the tree + ;; through git, and an unpacked archive has none — so a replay's + ;; edits went unseen and `done` was withheld on files it had + ;; written. Committed before the userspace goes in, which is kept + ;; out of it as the project keeps it out of its own. + (sh-run (:stage-timeout-ms (policy)) + (str "cd " (util/sh-quote dest) + " && git init -q && git add -A" + " && git -c user.name=heldout -c user.email=heldout@localhost" + " commit -q --no-verify -m fixture" + " && echo .samizdat/ >> .git/info/exclude")) + (doseq [[rel f] (userspace-files) + :let [to (io/file us rel)]] + (.mkdirs (.getParentFile to)) + (io/copy f to)) + (when-let [rel (and candidate (candidate-path (:kind candidate) (:name candidate)))] + (let [to (io/file us rel)] + (.mkdirs (.getParentFile to)) + (spit to (:text candidate)))) + nil)))))) + +;;; ------------------------------------------------------------- one replay + +(defn- sh-lines [out] (some->> out str/split-lines (map str/trim) (remove str/blank?))) + +(def ^:private classpath + (memoize + (fn [dir] + ;; The last line that is a classpath: dependency chatter shares the + ;; stream on a cold cache. Relative roots are made absolute, since the + ;; child runs in the fixture and ./src would resolve against it. + (let [r (sh-run (:stage-timeout-ms (policy)) + (str "cd " (util/sh-quote dir) " && " (util/sh-quote (:command (policy))) " -Spath")) + raw (->> (sh-lines (:out r)) + (filter #(and (str/includes? % ":") + (or (str/starts-with? % "/") (str/starts-with? % "./")))) + last)] + (when raw + (->> (str/split raw #":") + (map (fn [e] (if (str/starts-with? e "./") (str dir (subs e 1)) e))) + (str/join ":"))))))) + +(defn harness-dir + "The samizdat checkout a child is started from: policy's :harness-dir, else + this process's working directory." + [] + (or (:harness-dir (policy)) (System/getProperty "user.dir"))) + +(defn harness-revision + "The commit of the harness a verdict was measured on, or nil." + [] + (some-> (sh-run 15000 (str "git -C " (util/sh-quote (harness-dir)) " rev-parse HEAD")) + :out str/trim not-empty)) + +(defn- free-port [] + (let [s (java.net.ServerSocket. 0)] + (try (.getLocalPort s) (finally (.close s))))) + +(defn- temp-dir [] + (let [f (java.io.File/createTempFile "samizdat-heldout" "")] + (.delete f) + (.mkdirs f) + f)) + +(defn run-case! + "Replay case `c` against this project's userspace with `candidate` written + in (nil for the userspace as it is), in a child process. Returns the + child's {:run-id :status :result} (:result is battery/check's) or + {:error …}. The staging directory is removed either way." + [c candidate] + (let [tmp (temp-dir) + root (str tmp "/root") + in-f (str tmp "/in.edn") + out-f (str tmp "/out.edn") + log-f (str tmp "/child.log")] + (try + (or (stage! c root candidate) + (if-let [cp (classpath (harness-dir))] + (let [_ (spit in-f (pr-str {:case c :root root :http-port (free-port) + :slack (:turns-slack (policy)) + :trail-turns (:trail-turns (policy)) + :trail-chars (:trail-chars (policy))})) + form (pr-str (list 'do (list 'require (list 'quote 'samizdat.heldout.child)) + (list 'samizdat.heldout.child/-main))) + r (proc/run {:timeout-ms (:case-timeout-ms (policy)) + :env (assoc (secrets/scrubbed-process-env) + "HELDOUT_IN" in-f + "HELDOUT_OUT" out-f + "HARNESS_ROOT" root)} + "sh" "-c" (str "cd " (util/sh-quote root) + " && exec " (util/sh-quote (:command (policy))) + " -Scp " (util/sh-quote cp) + " -e " (util/sh-quote form) + " > " (util/sh-quote log-f) " 2>&1"))] + (cond + (:timeout r) {:error :timeout :ms (:ms r)} + (.exists (io/file out-f)) (edn/read-string (slurp out-f)) + :else {:error :no-result + :detail (when (.exists (io/file log-f)) + (->> (slurp log-f) str/split-lines (take-last 12) + (str/join "\n")))})) + {:error :no-classpath :detail (harness-dir)})) + (catch Throwable e {:error :rig :detail (or (ex-message e) (str e))}) + (finally (delete-tree! tmp))))) + +;;; ------------------------------------------------------------- the gate + +(defn- fingerprint + "A key for the userspace a measurement was taken on: the project, and every + file's path and CONTENT with `candidate` in place of its file. By content rather than + mtime, so the candidate's own measurement is the baseline of the state a + commit of it produces — one replay per edit, not two." + [candidate] + (let [rel (when candidate (candidate-path (:kind candidate) (:name candidate))) + files (into {} (map (fn [[r f]] [r (slurp f)])) (userspace-files)) + files (cond-> files rel (assoc rel (:text candidate)))] + (hash [(userspace/project-root) (vec (sort-by key files))]))) + +(defonce ^:private measured (atom {})) + +(defn- measure + "{case-id result} for `cases` on the userspace with `candidate`, from the + cache when this exact userspace was measured before — unless `fresh?`." + [cases candidate run-case & [fresh?]] + (let [fp (fingerprint candidate)] + (into {} + (for [c cases] + (let [k [fp (:id c) (hash c)]] + [(:id c) + (or (when-not fresh? (get @measured k)) + (let [r (run-case c candidate)] + ;; Only a result is kept. A failure may be the machine + ;; (a killed child, a cold cache), and a kept failure + ;; would stand for this userspace until it changed. + (when-not (:error r) + (swap! measured assoc k r)) + r))]))))) + +(defn- named + "`result`'s targets, each named with its case so two cases' targets of the + same name stay two targets." + [id result] + (update result :targets + (fn [ts] (mapv #(update % :name (fn [n] (str id " — " n))) ts)))) + +(defn compare-results + "The verdict on `before` and `after` ({case-id child-result}): {:ok? …}. + + Per case: one that did not run at baseline is set aside under :unmeasured; + one that ran at baseline and did not run on the candidate is a regression + by itself; otherwise its targets are compared — every target that passed + before must pass after, over the same set of targets." + [cases before after] + (let [per (for [c cases + :let [id (:id c) + b (get before id) + a (get after id)]] + (cond + (:error b) {:id id :unmeasured (:error b) :detail (:detail b)} + (:error a) {:id id :broke [(str id " — no longer runs (" (name (:error a)) ")")]} + :else + (let [b (named id (:result b)) + a (named id (:result a))] + {:id id + :before b :after a + :broke (vec (concat (battery/regressions b a) + (when-not (battery/accept? b a) + (when (empty? (battery/regressions b a)) + [(str id " — measured different targets")]))))}))) + broke (vec (mapcat :broke per))] + {:ok? (empty? broke) + :regressions broke + :cases (count cases) + :unmeasured (vec (for [p per :when (:unmeasured p)] (:id p))) + ;; Why each of those did not run, so a battery that measured nothing + ;; says what to fix rather than only that it measured nothing. + :unmeasured-why (into {} (for [p per :when (:unmeasured p)] + [(:id p) (str (name (:unmeasured p)) + (when (:detail p) (str ": " (:detail p))))])) + :targets (vec (for [p per :when (:after p) + t (:targets (:after p)) + :let [was (some #(when (= (:name %) (:name t)) %) (:targets (:before p)))]] + {:case (:id p) :name (:name t) + :before (boolean (:ok? was)) :after (boolean (:ok? t))})) + :passed-before (reduce + 0 (keep #(get-in % [:before :passed]) per)) + :passed-after (reduce + 0 (keep #(get-in % [:after :passed]) per)) + :total (reduce + 0 (keep #(get-in % [:after :total]) per))})) + +(defn record! + "One heldout_checks row per target measured, with what the number was + measured ON beside it: the case's fixture, the harness revision, the + recorded model, the scorer. A score without those is only a number." + [conn candidate cases verdict] + (let [rev (harness-revision) + by-id (into {} (map (juxt :id identity)) cases)] + (doseq [{target :name :keys [case before after]} (:targets verdict) + :let [c (get by-id case)]] + (db/execute! conn ["INSERT INTO heldout_checks + (edit_kind, edit_name, case_id, target, ok_before, ok_after, + accepted, fixture, revision, model, scorer, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" + (some-> (:kind candidate) name) (str (:name candidate)) + case target (if before 1 0) (if after 1 0) (if (:ok? verdict) 1 0) + (str (get-in c [:fixture :sha])) (str rev) (str (:model c)) + "battery/check" (db/now)])))) + +(defn checks + "The recorded per-target measurements for edits of `kind`/`name`, newest + first." + [conn kind name] + (db/fetch conn ["SELECT * FROM heldout_checks WHERE edit_kind = ? AND edit_name = ? + ORDER BY id DESC" + (clojure.core/name kind) (str name)])) + +(defn gate! + "Measure `candidate` ({:kind :name :text}) against the battery. nil when the + gate does not apply — disabled in policy, no project files, or no cases (a + project with no battery tunes itself as before; RFC-014's opt-in). Else + {:ok? …} per compare-results, plus :altered (case files edited against the + stored battery), recorded per target. + + `opts` may carry :run-case (fn [case candidate] -> child result), which a + test injects in place of the child process." + ([conn candidate] (gate! conn candidate nil)) + ([conn candidate {:keys [run-case]}] + (let [p (policy)] + (when (and (:enabled? p) (userspace/files?) conn) + (let [{:keys [cases altered]} (cases conn) + cases (vec (take (:max-cases p) cases))] + (when (seq cases) + (let [run-case (or run-case run-case!) + before (measure cases nil run-case) + after (measure cases candidate run-case) + first-read (compare-results cases before after) + ;; A REPLAY IS NOT EXACT. A recording that forked replays + ;; its branches in whatever order their turns land, and the + ;; gates that read time follow, so one measurement can show + ;; a flip that is noise (measured on endless-flight: the + ;; same baseline read 11/14 and 13/14 minutes apart). A + ;; refusal is therefore confirmed: both sides measured once + ;; more, fresh, and only a flip seen both times refuses. + ;; Accepting costs nothing extra. + verdict (if (:ok? first-read) + first-read + (let [again (compare-results cases + (measure cases nil run-case true) + (measure cases candidate run-case true)) + seen (set (:regressions again)) + held (filterv seen (:regressions first-read))] + (assoc first-read + :ok? (empty? held) + :regressions held + :unconfirmed (filterv (complement seen) + (:regressions first-read))))) + verdict (assoc verdict :altered altered)] + (try (record! conn candidate cases verdict) + (catch Throwable e (log/warn "heldout: recording the verdict failed:" (ex-message e)))) + verdict))))))) + +(defn refusal + "What an edit's author reads when the battery refused it: the targets that + broke by name, so the next attempt does not re-derive the same edit." + [verdict] + (prompt/render "heldout-refused" + {:broke (str/join "\n" (map #(str " " %) (:regressions verdict))) + :cases (:cases verdict) + :cases-s (if (= 1 (:cases verdict)) "" "s") + :passed-before (:passed-before verdict) + :passed-after (:passed-after verdict) + :total (:total verdict) + :counted (pos? (long (or (:total verdict) 0))) + :unmeasured (str/join ", " (:unmeasured verdict)) + :altered (str/join ", " (:altered verdict))})) + +(defn check-edit + "The refusal an edit's author reads, or nil when the edit may go live — the + battery passed it, or there is no battery to consult. The one call a tool + makes before it saves a cell, manifest or policy table." + ([conn candidate] (check-edit conn candidate nil)) + ([conn candidate opts] + (let [v (gate! conn candidate opts)] + (when (and v (seq (:unmeasured v)) (= (count (:unmeasured v)) (:cases v))) + (log/warn "heldout: no case ran at baseline, so" (:kind candidate) (:name candidate) + "went unmeasured:" (pr-str (:unmeasured-why v)))) + (when (and v (not (:ok? v))) + (refusal v))))) + +;;; ------------------------------------------------------------- stage 2: live arms + +(defn- median [xs] + (let [v (vec (sort (remove nil? xs)))] + (when (seq v) (nth v (quot (count v) 2))))) + +(defn- sd [xs] + (let [v (vec (remove nil? xs)) n (count v)] + (when (> n 1) + (let [m (/ (reduce + v) n)] + (Math/sqrt (/ (reduce + (map #(let [d (- % m)] (* d d)) v)) (dec n))))))) + +(defn- criteria-held + "Criterion names that passed in every row of `rows` that decided them." + [rows] + (let [decided (for [r rows, res (get-in r [:acceptance :results]) + :when (some? (:passed? res))] + [(:name res) (:passed? res)])] + (set (for [[nm vs] (group-by first decided) + :when (every? second vs)] + nm)))) + +(defn- criteria-failed [rows] + (set (for [r rows, res (get-in r [:acceptance :results]) + :when (false? (:passed? res))] + (:name res)))) + +(defn live-verdict + "Stage 2 of the held-out gate: the arena's live arms read against each other + (karamazov-7mo.4's design, from RRSI 2609.24972 and GEPA). Replay cannot + judge what a changed prompt makes the model SAY; interleaved live runs of a + baseline arm and a candidate arm on the same pinned tree can. + + `rows` are arena rows ({:arm :task :fitness :tokens :green? :acceptance}). + Per task, RRSI's non-compensatory order, each a refusal on its own: + + 1. floor — median fitness(candidate) >= S* - delta. S* is the best any + kept version reached on the task (`best`, a {task score} map, and the + baseline's own median if higher); delta is z standard deviations of the + baseline's fitness on that task, never under min-band. + 2. cost — a gain beyond delta must pay for its tokens: relative token + change dC <= base + per-fitness * dS. Inside the band only a cost cut + (dC < -base) or a change declared `structural?` (a new component) is + admissible — a neutral edit that adds tokens is refused. + 3. guards — no acceptance criterion that passed in every baseline row + fails in a candidate row, and no candidate row loses the suite where + every baseline row had it green. + + Returns {:accept? bool :tasks {task {:accept? :refused [kw …] :dS :dC + :delta :floor}}}. A task either arm has no rows for is not judged." + [rows {:keys [baseline candidate best structural? cost-rule noise]}] + (let [{:keys [z min-band]} noise + {:keys [base per-fitness]} cost-rule + tasks (into {} + (for [[task rs] (group-by :task rows) + :let [b (filter #(= baseline (:arm %)) rs) + c (filter #(= candidate (:arm %)) rs)] + :when (and (seq b) (seq c))] + (let [sb (median (map :fitness b)) + sc (median (map :fitness c)) + tb (median (map :tokens b)) + tc (median (map :tokens c)) + delta (max (or min-band 0.0) + (* (or z 0.0) (or (sd (map :fitness b)) 0.0))) + s* (max (or sb 0.0) (or (get best task) Double/NEGATIVE_INFINITY)) + ds (when (and sb sc) (- sc sb)) + dc (when (and tb tc (pos? tb)) (/ (- tc tb) (double tb))) + floor? (and sc (< sc (- s* delta))) + cost? (and ds dc base + (if (> ds delta) + (> dc (+ base (* (or per-fitness 0.0) ds))) + (and (<= (Math/abs (double ds)) delta) + (not structural?) + (>= dc (- base))))) + lost-criteria (seq (filter (criteria-failed c) (criteria-held b))) + lost-suite? (and (every? :green? b) (some #(false? (:green? %)) c)) + refused (cond-> [] + floor? (conj :floor) + cost? (conj :cost) + lost-criteria (conj :criterion) + lost-suite? (conj :suite))] + [task {:accept? (empty? refused) :refused refused + :dS ds :dC dc :delta delta :floor (- s* delta) + :lost-criteria (vec (sort lost-criteria))}])))] + {:accept? (boolean (and (seq tasks) (every? :accept? (vals tasks)))) + :tasks tasks})) diff --git a/src/samizdat/heldout/child.clj b/src/samizdat/heldout/child.clj new file mode 100644 index 00000000..2873ca70 --- /dev/null +++ b/src/samizdat/heldout/child.clj @@ -0,0 +1,114 @@ +;; samizdat - a self-hosting agentic harness +;; Copyright (C) 2026 Dmitri Sotnikov +;; +;; This program is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. +;; +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; +;; You should have received a copy of the GNU General Public License +;; along with this program. If not, see . +;; +;; SPDX-License-Identifier: GPL-3.0-or-later + +(ns samizdat.heldout.child + "The process one held-out case replays in (samizdat.heldout). Started in a + staged project directory — the case's fixture with the userspace under test + in its .samizdat/ — it brings the harness up on that directory with an + in-memory database, replays the recorded conversation through the real + driver, checks the case's expectations against the run it produced, and + writes the result to $HELDOUT_OUT. + + Required by nothing: it is only ever the entry point of a child process, + and it needs the driver, which the tool layer that starts a gate cannot + require without a cycle. + + NO PROVIDER. Every model turn is served from the recording. A side call a + cell makes on its own (a critic, a judge) was never recorded, so it is + refused the same way on both sides of a comparison — the verdict compares + like with like, and nothing is spent." + (:require [clojure.edn :as edn] + [samizdat.agent.beam :as beam] + [samizdat.battery :as battery] + [samizdat.llm.client :as llm] + [samizdat.replay :as replay] + [samizdat.server :as server] + [samizdat.store.db :as db] + [samizdat.store.tasks :as tasks] + [samizdat.system :as system])) + +(defn- trail + "The end of what the replayed run did — branch, turn, tool, category and + the head of each result — so a target that fails can be read without the + database, which dies with this process." + [conn run-id n width] + (vec (reverse + (for [r (db/fetch conn ["SELECT branch_id, turn, tool_name, category, result FROM turns + WHERE run_id = ? ORDER BY id DESC LIMIT ?" run-id n])] + [(:branch_id r) (:turn r) (:tool_name r) (:category r) + (let [s (str (:result r))] (subs s 0 (min width (count s))))])))) + +(defn- turns-cap + "The turn cap a replay runs under: the recording's longest branch plus the + policy's slack, so a candidate that takes more turns exhausts the recording + (a named result) rather than being cut off first." + [c slack] + (+ (long (or slack 0)) + (reduce max 0 (map count (vals (get-in c [:replay :replies])))))) + +(defn- recorded-ids + "A new-id that hands out the recorded run's task ids, in the order it made + them, then fresh ones: a reply naming a task means the same task under + replay (heldout/draft!)." + [ids fresh] + (let [ids (atom (seq ids))] + (fn [] + (if-let [id (first @ids)] + (do (swap! ids next) id) + (fresh))))) + +(defn run + "Replay case `c` in the harness started on `root`. Returns + {:run-id :status :result :trail}." + [{:keys [case root http-port slack trail-turns trail-chars]}] + (system/start! #'server/handler + {:run (cond-> {:root root} + (:loop case) (assoc :loop (:loop case))) + :db {:path ":memory:"} + :http {:port http-port}}) + (try + ;; new-id is private, hence the -fn form. + (with-redefs-fn {#'llm/chat (fn [& _] + (throw (ex-info "held-out replay: no provider" {:heldout true}))) + #'tasks/new-id (recorded-ids (:task-ids case) @#'tasks/new-id)} + (fn [] + (let [cfg (system/config) + conn (system/conn) + rep (:replay case) + result (beam/run! {:conn conn :config cfg + :llm-adapter (system/adapter) + :llm-config (:llm cfg) + :root root + :problem (:problem rep) + :max-turns (turns-cap case slack) + :beam-width 1 + :complete (replay/case-complete-fn rep)}) + run-id (:run-id result)] + {:run-id run-id + :status (:status result) + :result (battery/check conn run-id (:expect case)) + :trail (trail conn run-id (or trail-turns 0) (or trail-chars 0))}))) + (finally (try (system/stop!) (catch Throwable _ nil))))) + +(defn -main [& _] + (let [in (edn/read-string (slurp (System/getenv "HELDOUT_IN"))) + out (System/getenv "HELDOUT_OUT") + row (try (run in) + (catch Throwable e {:error :child :detail (or (ex-message e) (str e))}))] + (spit out (pr-str row)) + (System/exit 0))) diff --git a/src/samizdat/llm/fence.clj b/src/samizdat/llm/fence.clj index bee09c6c..53b81d6b 100644 --- a/src/samizdat/llm/fence.clj +++ b/src/samizdat/llm/fence.clj @@ -602,6 +602,31 @@ {:name (:name value) :args (let [a (:args value)] (if (map? a) a {}))})))) +(defn- unsmuggle + "`args` with the rest of the call recovered, when the model put every + argument after the first INSIDE the first one's string, escaped: GLM-5.3 + sent {\"action\": \"decline\\\",\\\"kind\\\":\\\"policy\\\",…\"} in run bcd61b39 + and the tool read the whole tail as the action (karamazov-q9v1). Decoded, + that string is the object's own continuation, so wrapping it back as + {\"\":\"\"} reads as the object that was meant. Taken only + when that reads as an object with MORE keys than one; anything else — a + string that merely contains quotes — is returned untouched. Returns + [args repaired?]." + [args] + (if-let [[k v] (and (map? args) (= 1 (count args)) (first args))] + (if (and (string? v) (str/includes? v "\":")) + ;; The smuggled tail usually carries the object's own closing brace + ;; (…edit.\"}), and sometimes stops at the last value; try both. + (let [head (str "{\"" (name k) "\":\"" v) + r (some (fn [t] + (let [r (read-json t)] + (when (and (:ok r) (map? (:value r)) (< 1 (count (:value r)))) + r))) + [head (str head "\"}")])] + (if r [(:value r) true] [args false])) + [args false]) + [args false])) + (defn reattach "The complete assistant turn, given what the request was prefilled with. @@ -792,9 +817,9 @@ (parse-error "tool-call `name` must not be empty" base) :else - (merge base - {:name (:name parsed) - :args (let [a (:args parsed)] (if (map? a) a {}))}))) + (let [[args smuggled?] (unsmuggle (let [a (:args parsed)] (if (map? a) a {})))] + (merge (cond-> base smuggled? (assoc :auto-repaired? true)) + {:name (:name parsed) :args args})))) ;; One repair pass. If the repair changed nothing there is no point ;; re-parsing, and the error message should name the causes the @@ -839,9 +864,8 @@ (if (and (map? parsed) (string? (:name parsed)) (not (str/blank? (:name parsed)))) - (merge base - {:name (:name parsed) - :args (let [a (:args parsed)] (if (map? a) a {}))}) + (let [[args _] (unsmuggle (let [a (:args parsed)] (if (map? a) a {})))] + (merge base {:name (:name parsed) :args args})) (parse-error "tool-call body must be a JSON object with a non-empty `name` string" base))))))))))))) diff --git a/src/samizdat/manifests.clj b/src/samizdat/manifests.clj index 67a8a660..2d46484f 100644 --- a/src/samizdat/manifests.clj +++ b/src/samizdat/manifests.clj @@ -81,12 +81,125 @@ " at " (manifest-resource name) " and nothing stored") {:manifest name})))) -(defn read-definition - "Parse a workflow definition from EDN text. Dispatch predicates stay as - forms here; maestro evaluates them at compile time." +(defn read-raw + "A manifest file's own map, `:extends` unresolved: what an edit rewrites. + Dispatch predicates stay as forms here; maestro evaluates them at compile + time." [edn-text] (edn/read-string edn-text)) +;; --- extension --------------------------------------------------------------- +;; +;; A manifest may say `:extends "turn"` and carry only what differs from that +;; manifest (karamazov-xtd3): loop, worker, reviewer and supervisor used to +;; carry four copies of one turn chain, and copies drift. The link is resolved +;; HERE, at read, so every reader — the drivers, the compile checks, the +;; catalogue, introspect — sees the whole graph and none of them knows the +;; link exists. Only an EDIT needs the file's own part (`read-raw`). + +(def ^:private merged-sections + "The sections an extension merges key by key into its base's." + #{:cells :edges :dispatches}) + +(defn- merge-section + "`child`'s entries over `base`'s, a nil value removing the base's entry." + [base child] + (reduce-kv (fn [m k v] (if (nil? v) (dissoc m k) (assoc m k v))) + (or base {}) child)) + +(defn- extend-definition + "`child` (a raw map carrying :extends) laid over its resolved `base`. + + :cells, :edges and :dispatches merge key by key and a nil removes; the + base's :invariants hold in the child and the child's are added; any other + key is the child's where it has one. A key the child lists in :replaces is + the child's alone — how a child that renames a node drops the base's + invariants naming the old one. :fragment? says what the BASE is and is not + inherited." + [base child] + (let [replaces (set (:replaces child)) + own (dissoc child :extends :replaces) + d (merge (dissoc base :fragment?) own) + d (reduce (fn [d k] + (if (and (contains? own k) (not (replaces k))) + (assoc d k (merge-section (get base k) (get own k))) + d)) + d merged-sections) + d (if (and (contains? own :invariants) (not (replaces :invariants))) + (let [bi (vec (:invariants base))] + (assoc d :invariants (into bi (remove (set bi)) (:invariants own)))) + d)] + (reduce (fn [d k] (if (and (replaces k) (nil? (get own k))) (dissoc d k) d)) + d replaces))) + +(defn- resolve-extends + "`raw` with its :extends chain resolved, refusing a base that resolves to + nothing and a chain that returns to itself — each by name, since the + author of the edit is the one reading the refusal." + [raw seen] + (if-let [base-name (some-> (:extends raw) name)] + (do + (when (some #{base-name} seen) + (throw (ex-info (str "manifest :extends chain returns to " base-name + " (" (str/join " -> " (conj seen base-name)) ")." + " A manifest cannot extend itself, directly or through another.") + {:extends base-name :chain seen}))) + (let [text (manifest-body base-name)] + (when-not text + (throw (ex-info (str "manifest extends \"" base-name "\", and there is no" + " manifest named " base-name " — add that role to the" + " project's userspace.edn or change :extends.") + {:extends base-name}))) + (extend-definition (resolve-extends (read-raw text) (conj seen base-name)) raw))) + raw)) + +(defn read-definition + "Parse a workflow definition from EDN text, with any :extends resolved: + the whole graph that runs. Dispatch predicates stay as forms here; maestro + evaluates them at compile time." + [edn-text] + (let [raw (read-raw edn-text)] + (if (map? raw) (resolve-extends raw []) raw))) + +(defn extension-delta + "The file map an extending manifest should hold so that it resolves to + `new`: `raw-old` (the file as it stands, :extends and all) with each merged + section cut down to what differs from the base — a nil for an entry the + base has and `new` does not — and the invariants to the ones the base does + not already carry. When `new` has dropped one of the base's invariants the + child lists them all and names :invariants in :replaces. What `manifest + patch` writes back, so a patch applied to the resolved graph does not + inline the chain into the file." + [raw-old new] + (let [base (resolve-extends {:extends (:extends raw-old)} []) + base (dissoc base :extends) + section (fn [k] + (let [b (get base k) n (get new k)] + (into {} + (concat (for [[kk v] n :when (not= v (get b kk))] [kk v]) + (for [kk (keys b) :when (not (contains? n kk))] [kk nil]))))) + bi (vec (:invariants base)) + ni (vec (:invariants new)) + replace-inv? (not (every? (set ni) bi)) + own-inv (if replace-inv? ni (vec (remove (set bi) ni))) + others (remove (into merged-sections #{:invariants :extends :replaces}) + (distinct (concat (keys new) (keys raw-old)))) + d (reduce (fn [d k] + (cond + (not (contains? new k)) (dissoc d k) + (and (= (get new k) (get base k)) (not (contains? raw-old k))) d + :else (assoc d k (get new k)))) + raw-old others) + kept (set (:replaces raw-old)) + d (reduce (fn [d k] + (let [s (if (kept k) (get new k) (section k))] + (if (seq s) (assoc d k s) (dissoc d k)))) + d merged-sections) + replaces (cond-> (set (remove #{:invariants} kept)) + replace-inv? (conj :invariants)) + d (if (seq own-inv) (assoc d :invariants own-inv) (dissoc d :invariants))] + (if (seq replaces) (assoc d :replaces (vec (sort replaces))) (dissoc d :replaces)))) + (defn- cell-ref-id "The cell id out of a manifest's `:cells` value. @@ -606,6 +719,65 @@ (cells/load-cells!) (compile-definition definition opts))) +;; --- an edit to a base ------------------------------------------------------- + +(defn- manifest-names + "Every manifest the project has: its map's roles and whatever the store + holds that the map does not name." + [] + (distinct (concat (userspace/roles :manifest) + (map :name (userspace/names :manifest))))) + +(defn dependents + "The manifests that extend `base-name`, directly or through another, read + from their text as it stands (not through the validator, which is the + thing an edit to the base is about to change under them)." + [base-name] + (let [links (into {} + (keep (fn [nm] + (when-let [ext (some-> (userspace/unchecked-body :manifest nm) + (as-> t (try (read-raw t) (catch Throwable _ nil))) + (as-> m (when (map? m) (:extends m))))] + [nm (name ext)]))) + (manifest-names))] + (loop [found [] frontier #{(str base-name)}] + (let [next (sort (for [[nm ext] links + :when (and (frontier ext) (not (some #{nm} found)))] + nm))] + (if (seq next) + (recur (into found next) (set next)) + found))))) + +(def ^:dynamic ^:private *sweeping* + "True while an edit to a base is compiling what extends it, so a dependent + that is itself a base does not sweep again from inside the sweep." + false) + +(defn validate-edit! + "Check `text` as the manifest `name` with `compile-fn` (compile-loop, or + compile-definition where the registry must not be reloaded), throwing on + the first failure. When other manifests extend `name`, each is compiled + against `text` too, and a change that breaks one is refused naming it — a + base that compiles alone and breaks every role built on it is the edit + this exists to stop. A dependent the project already has refused on its + own is skipped: that is not this edit's doing." + [name text compile-fn] + (compile-fn (read-definition text)) + (when-not *sweeping* + (binding [*sweeping* true + userspace/*candidate* (assoc userspace/*candidate* [:manifest (str name)] text)] + (doseq [dep (dependents name) + :when (not (userspace/rejection :manifest dep)) + :let [dep-text (userspace/unchecked-body :manifest dep)] + :when dep-text] + (try (compile-fn (read-definition dep-text)) + (catch Throwable e + (throw (ex-info (str "this change to " name " breaks manifest " dep + ", which extends it: " (or (ex-message e) (str e))) + {:manifest (str name) :dependent dep} + e))))))) + true) + ;; --- the per-turn slice ------------------------------------------------------ (def start-node @@ -790,6 +962,9 @@ (when edn (let [d (try (read-definition edn) (catch Throwable _ nil))] {:name nm :description (str (:description d)) + ;; A base others extend (turn.edn), not a workflow + ;; to run: off the switch menu and selection. + :fragment? (boolean (:fragment? (try (read-raw edn) (catch Throwable _ nil)))) ;; Carried, not filtered here: the catalogue is the ;; full inventory (the mutation tools and the ;; selectability test read it), and it is the SWITCH @@ -806,10 +981,12 @@ ;; last. (userspace/register-validator! :manifest - (fn [_ text] - (let [{:keys [value problem]} (userspace/read-edn text)] + (fn [nm text] + (let [{:keys [problem]} (userspace/read-edn text)] (or problem (try (cells/load-cells!) - (compile-definition value) + ;; Through validate-edit!, so an edit to a base is checked + ;; against every manifest that extends it (karamazov-xtd3). + (validate-edit! nm text compile-definition) nil (catch Throwable e (userspace/problem :compile e))))))) diff --git a/src/samizdat/mutation.clj b/src/samizdat/mutation.clj index 70815167..426bec20 100644 --- a/src/samizdat/mutation.clj +++ b/src/samizdat/mutation.clj @@ -386,12 +386,15 @@ :soak-input — the initial data map the soak dry-run starts from :compile-fn — how to compile+validate (default mycelium pre-compile) :rationale — why, stored with the committed version (karamazov-c58) + :heldout-fn — (fn [candidate] -> refusal or nil), the held-out battery, + consulted after the soak and before the commit + (samizdat.heldout/check-edit; karamazov-7mo.4) :conn :run-id — to journal the outcome (optional) Returns {:status :committed :version n} or {:status :rolled-back :reason ...} with the registry restored and nothing written to the store." - [{:keys [name body loop-def extra-defs soak-input compile-fn rationale conn run-id]}] + [{:keys [name body loop-def extra-defs soak-input compile-fn rationale conn run-id heldout-fn]}] (let [compile-fn (or compile-fn myc/pre-compile) shadowing (shadowed-cells name body) unearned (unearned-marks body) @@ -440,7 +443,13 @@ (str "manifest '" nm "': " r))) extra-defs))] (fail reason) - (if-let [reason (soak compile-fn loop-def soak-input)] + (if-let [reason (or (soak compile-fn loop-def soak-input) + ;; BATTERY — the dearest check, last: does the + ;; candidate make a recorded case regress? It reads + ;; the candidate from its text, in a child process, + ;; so what is installed here does not matter to it. + (when heldout-fn + (heldout-fn {:kind :cell :name name :text body})))] (fail reason) ;; COMMIT. The candidate is already live; this is what makes it ;; survive a restart and what another run will load. diff --git a/src/samizdat/store/migrations.clj b/src/samizdat/store/migrations.clj index 4bba1895..ff501e46 100644 --- a/src/samizdat/store/migrations.clj +++ b/src/samizdat/store/migrations.clj @@ -884,7 +884,41 @@ created_at TEXT NOT NULL)" "CREATE INDEX IF NOT EXISTS idx_gate_checks_run ON gate_checks(run_id, gate)"]) +(def v37 + "THE HELD-OUT BATTERY and what it measured (karamazov-7mo.4 / ylte.4). + + battery_cases is the battery's authority: a case is written once under its + id and never rewritten, so the running agent may add a case and may not + weaken or delete one — a case file edited or removed under .samizdat/battery + changes nothing that runs (samizdat.heldout/cases). body is the case as EDN: + its replay, its expectations, its fixture sha, its loop and model. + + heldout_checks is one row per target per edit measured, with what the + number was measured ON beside it — the case's fixture, the harness + revision, the recorded model, the scorer — because a score without those + is only a number. accepted is the verdict on the whole edit." + ["CREATE TABLE IF NOT EXISTS battery_cases ( + id TEXT PRIMARY KEY, + subject TEXT NOT NULL, + body TEXT NOT NULL, + added_at TEXT NOT NULL)" + "CREATE TABLE IF NOT EXISTS heldout_checks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + edit_kind TEXT, + edit_name TEXT, + case_id TEXT NOT NULL, + target TEXT NOT NULL, + ok_before INTEGER NOT NULL, + ok_after INTEGER NOT NULL, + accepted INTEGER NOT NULL, + fixture TEXT, + revision TEXT, + model TEXT, + scorer TEXT, + created_at TEXT NOT NULL)" + "CREATE INDEX IF NOT EXISTS idx_heldout_checks_edit ON heldout_checks(edit_kind, edit_name)"]) + (def migrations "Ordered. Index 0 is migration 1; PRAGMA user_version holds the count applied." [v1 v2 v3 v4 v5 v6 v7 v8 v9 v10 v11 v12 v13 v14 v15 v16 v17 v18 v19 v20 v21 v22 v23 v24 - v25 v26 v27 v28 v29 v30 v31 v32 v33 v34 v35 v36]) + v25 v26 v27 v28 v29 v30 v31 v32 v33 v34 v35 v36 v37]) diff --git a/src/samizdat/userspace.clj b/src/samizdat/userspace.clj index 00e98c97..35c6c98f 100644 --- a/src/samizdat/userspace.clj +++ b/src/samizdat/userspace.clj @@ -1028,6 +1028,19 @@ v) hit))) +(defn unchecked-body + "The text of `kind`/`name` as it stands, NOT passed through its validator: + the project's file in file mode (a candidate in `*candidate*` first), `body` + otherwise. For a check that must read a role while another role's edit is + being judged — an edit to a manifest base compiles what extends it — where + reading through the validator would cache a verdict made against a text + that may yet be refused." + [kind name] + (or (get *candidate* [kind (str name)]) + (if (files?) + (some-> (project-path kind name) file-text) + (body kind name)))) + (defn body! "`body`, failing loud when it is absent. For a caller whose whole operation is meaningless without it — a manifest node's cell, the system prompt." diff --git a/src/samizdat/workflow.clj b/src/samizdat/workflow.clj index 7a026808..fe8d3a7e 100644 --- a/src/samizdat/workflow.clj +++ b/src/samizdat/workflow.clj @@ -209,12 +209,12 @@ "The workflow catalog as a text menu — one `- name — description` line each — for injecting into the supervisor's context." [conn] - (str/join "\n" (for [{:keys [name description turn-sliceable?]} (catalog conn) + (str/join "\n" (for [{:keys [name description turn-sliceable? fragment?]} (catalog conn) ;; A workflow a run cannot be pointed at is not an ;; option, and offering it is worse than omitting it: ;; the supervisor is told it may switch, and the switch ;; fails at run start (karamazov-4sx). - :when turn-sliceable?] + :when (and turn-sliceable? (not fragment?))] (str "- " name (when (seq description) (str " — " description)))))) (defn workflow-prompt diff --git a/test/mycelium/patch_workflow_test.clj b/test/mycelium/patch_workflow_test.clj index a4ed0c6d..5eb0aa4d 100644 --- a/test/mycelium/patch_workflow_test.clj +++ b/test/mycelium/patch_workflow_test.clj @@ -152,10 +152,11 @@ ;; ===== render on a samizdat manifest ===== (deftest render-keeps-a-shipped-manifests-prose - ;; loop.edn is the reason render is worth vendoring: forty comment runs - ;; explaining why each edge is where it is. A rename that reprinted the - ;; file would erase every one of them from the stored body. - (let [text (slurp "resources/manifests/loop.edn") + ;; turn.edn (the chain loop.edn and the role loops extend) is the reason + ;; render is worth vendoring: comment runs explaining why each edge is + ;; where it is. A rename that reprinted the file would erase every one of + ;; them from the stored body. + (let [text (slurp "resources/manifests/turn.edn") old (edn/read-string text) new (patch/apply-ops old {:ops [{:op "rename-cell" :from :journal :to :record}] :validator identity}) diff --git a/test/samizdat/boundary_test.clj b/test/samizdat/boundary_test.clj index 30ed6607..94e8bb69 100644 --- a/test/samizdat/boundary_test.clj +++ b/test/samizdat/boundary_test.clj @@ -232,13 +232,14 @@ "ask_human" {:reach :harness-only} "experiment" {:reach :harness-only} "verdict" {:reach :harness-only} + "battery" {:reach :spawns-process :also "heldout/draft! git update-ref under secrets/scrubbed-process-env"} "fetch_turn" {:reach :harness-only} "fetch_artifact" {:reach :harness-only} "cells" {:reach :harness-only} - "cell" {:reach :harness-only} + "cell" {:reach :spawns-process :also "heldout/check-edit: git archive + a jolt child, both under secrets/scrubbed-process-env"} "reload_cells" {:reach :harness-only} - "manifest" {:reach :harness-only} - "policy" {:reach :harness-only} + "manifest" {:reach :spawns-process :also "heldout/check-edit: git archive + a jolt child, both under secrets/scrubbed-process-env"} + "policy" {:reach :spawns-process :also "heldout/check-edit: git archive + a jolt child, both under secrets/scrubbed-process-env"} ;; Writes only inside the project's .samizdat/, and only a text the ;; harness shipped or already stored — never one the model composed. "adopt" {:reach :harness-only} diff --git a/test/samizdat/cell_schema_test.clj b/test/samizdat/cell_schema_test.clj index f1699c97..fa180e3c 100644 --- a/test/samizdat/cell_schema_test.clj +++ b/test/samizdat/cell_schema_test.clj @@ -49,7 +49,7 @@ [samizdat.store.userspace :as us])) (defn- loop-def [] - (edn/read-string (slurp (io/resource "manifests/loop.edn")))) + (manifests/read-definition (slurp (io/resource "manifests/loop.edn")))) (deftest every-cell-a-shipped-manifest-reaches-declares-its-shape ;; The general form. This began as a list of one manifest and grew a @@ -62,7 +62,7 @@ ;; same defcell, so the rule holds for cells this suite never sees. (cells/load-cells!) (doseq [nm (manifests/shipped-manifests)] - (let [d (edn/read-string (slurp (io/resource (manifests/manifest-resource nm)))) + (let [d (manifests/read-definition (slurp (io/resource (manifests/manifest-resource nm)))) ;; What a compile does. A composed sub-workflow cell (orchestrator's ;; :loop/worker) exists only once its child has been registered, so ;; without this it is absent from the registry rather than @@ -290,7 +290,7 @@ (deftest every-shipped-manifest-declares-what-it-starts-from (doseq [nm (manifests/shipped-manifests)] (testing nm - (let [d (edn/read-string (slurp (io/resource (manifests/manifest-resource nm))))] + (let [d (manifests/read-definition (slurp (io/resource (manifests/manifest-resource nm))))] (is (some? (:input-schema d)) (str nm " declares no :input-schema, so the map a run starts from" " is never checked — myc/pre-compile compiles nil and" @@ -303,7 +303,7 @@ ;; notice. Same reasoning manifests/ctx-keys is checked from both ends. (doseq [nm (manifests/shipped-manifests)] (testing nm - (let [d (edn/read-string (slurp (io/resource (manifests/manifest-resource nm))))] + (let [d (manifests/read-definition (slurp (io/resource (manifests/manifest-resource nm))))] (is (m/validate (:input-schema d) (entry-for nm)) (str nm " refuses the map its own driver hands it: " (pr-str (m/explain (:input-schema d) (entry-for nm))))))))) @@ -330,7 +330,7 @@ ;; transition. Both halves are what make it safe to require :verdict at ;; :loop/finish rather than merely plausible. (cells/load-cells!) - (let [d (edn/read-string (slurp (io/resource "manifests/orchestrator.edn")))] + (let [d (manifests/read-definition (slurp (io/resource "manifests/orchestrator.edn")))] (manifests/register-subworkflows! d) (let [out (get-in (cell/get-cell :loop/worker) [:schema :output]) ;; mycelium wraps a composed cell's output as @@ -404,7 +404,7 @@ ;; --- Tier 1 satisfiability: one precondition pass (karamazov-41a.6) --------- (defn- shipped-definition [n] - (edn/read-string (slurp (io/resource (manifests/manifest-resource n))))) + (manifests/read-definition (slurp (io/resource (manifests/manifest-resource n))))) (deftest preconditions-are-one-report-over-ctx-and-data ;; check-requires! answers for ctx keys and the schema chain for data keys, diff --git a/test/samizdat/heldout_test.clj b/test/samizdat/heldout_test.clj new file mode 100644 index 00000000..6a5e5bc9 --- /dev/null +++ b/test/samizdat/heldout_test.clj @@ -0,0 +1,264 @@ +;; samizdat - a self-hosting agentic harness +;; Copyright (C) 2026 Dmitri Sotnikov +;; +;; This program is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. +;; +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; +;; You should have received a copy of the GNU General Public License +;; along with this program. If not, see . +;; +;; SPDX-License-Identifier: GPL-3.0-or-later + +(ns samizdat.heldout-test + "The held-out gate, stage 1 (karamazov-7mo.4 / ylte.4): the battery as the + project stores it, the verdict rule, and the tools that consult it. The + child process itself is exercised against a real project, not here; these + inject the replay." + (:require [clojure.edn :as edn] + [clojure.java.io :as io] + [clojure.string :as str] + [clojure.test :refer [deftest testing is]] + [samizdat.agent.tools.base :as base] + [samizdat.agent.tools] + [samizdat.engine.proc :as proc] + [samizdat.heldout :as heldout] + [samizdat.store.db :as db] + [samizdat.store.userspace :as store] + [samizdat.userspace :as us] + [samizdat.manifests] + [samizdat.prompt] + [samizdat.cells] + [samizdat.agent.tools.policy])) + +(defn- temp-dir [] + (str (java.nio.file.Files/createTempDirectory + "samizdat-heldout" (make-array java.nio.file.attribute.FileAttribute 0)))) + +(defn- delete-recursively [^java.io.File f] + (when (.isDirectory f) + (doseq [c (.listFiles f)] (delete-recursively c))) + (.delete f)) + +(defmacro ^:private with-project + [[root conn] & body] + `(let [~root (temp-dir) + ~conn (db/open! ":memory:") + prev-root# (us/bind-root! ~root)] + (us/bind! ~conn) + (try (us/seed-project!) + ~@body + (finally (us/unbind!) + (us/bind-root! prev-root#) + (db/close ~conn) + (delete-recursively (io/file ~root)))))) + +(defn- write-case! [root id c] + (let [f (io/file root ".samizdat" "battery" "subj" (str id ".edn"))] + (.mkdirs (.getParentFile f)) + (spit f (pr-str c)) + f)) + +(def ^:private a-case + {:id "c1" + :fixture {:sha "abc123"} + :model "m" + :replay {:problem "p" :replies {"T0" ["x"]}} + :expect [{:name "run reaches completed" :assert [:status :completed]} + {:name "calls done" :assert [:tool-called "done"]}]}) + +(defn- result [& oks] + {:run-id "r" :status :completed + :result {:ok? (every? true? oks) :passed (count (filter true? oks)) :total (count oks) + :targets (mapv (fn [n ok] {:name n :ok? ok}) + ["run reaches completed" "calls done"] oks)}}) + +(defn- replay-by-text + "A stand-in for the child: the candidate text decides which targets pass." + [_case candidate] + (let [t (str (:text candidate))] + (cond + (str/includes? t "CRASH") {:error :no-result} + (str/includes? t "BREAK") (result true false) + :else (result true true)))) + +(deftest the-stored-battery-is-the-authority + (with-project [root conn] + (let [f (write-case! root "c1" a-case)] + (testing "a new case file is added" + (let [{:keys [cases altered]} (heldout/cases conn)] + (is (= ["c1"] (map :id cases))) + (is (= "subj" (:subject (first cases)))) + (is (empty? altered)))) + (testing "a case whose file was weakened runs as stored, and the edit is named" + (spit f (pr-str (assoc a-case :expect []))) + (let [{:keys [cases altered]} (heldout/cases conn)] + (is (= 2 (count (:expect (first cases))))) + (is (= [(str f)] altered)))) + (testing "a case whose file was deleted still runs" + (.delete f) + (is (= ["c1"] (map :id (:cases (heldout/cases conn))))))))) + +(deftest the-verdict-is-per-target-and-non-compensatory + (let [cases [{:id "c1"} {:id "c2"}] + ok (result true true)] + (testing "nothing that passed fails: accepted, ties included" + (is (:ok? (heldout/compare-results cases {"c1" ok "c2" ok} {"c1" ok "c2" ok})))) + (testing "a target that passed and now fails is named with its case" + (let [v (heldout/compare-results cases {"c1" ok "c2" ok} + {"c1" ok "c2" (result true false)})] + (is (false? (:ok? v))) + (is (= ["c2 — calls done"] (:regressions v))))) + (testing "fixing one target does not pay for breaking another" + (let [v (heldout/compare-results [{:id "c1"}] {"c1" (result false true)} + {"c1" (result true false)})] + (is (false? (:ok? v))) + (is (= ["c1 — calls done"] (:regressions v))))) + (testing "a case that does not run at baseline is set aside, not blamed" + (let [v (heldout/compare-results cases {"c1" {:error :no-result} "c2" ok} + {"c1" {:error :no-result} "c2" ok})] + (is (:ok? v)) + (is (= ["c1"] (:unmeasured v))))) + (testing "a case that ran and no longer runs is a regression" + (let [v (heldout/compare-results cases {"c1" ok "c2" ok} + {"c1" {:error :timeout} "c2" ok})] + (is (false? (:ok? v))) + (is (re-find #"c1 — no longer runs" (first (:regressions v)))))))) + +(deftest the-gate-measures-a-candidate-and-records-what-it-measured-on + (with-project [root conn] + (testing "no battery, no gate: a project without cases tunes itself as before" + (is (nil? (heldout/gate! conn {:kind :manifest :name "loop" :text "x"} + {:run-case replay-by-text})))) + (write-case! root "c1" a-case) + (let [bad (heldout/gate! conn {:kind :manifest :name "loop" :text "BREAK"} + {:run-case replay-by-text})] + (is (false? (:ok? bad))) + (is (= ["c1 — calls done"] (:regressions bad))) + (is (str/includes? (heldout/refusal bad) "c1 — calls done") "the refusal names what broke")) + (is (:ok? (heldout/gate! conn {:kind :manifest :name "loop" :text "fine"} + {:run-case replay-by-text}))) + (testing "each target is recorded with the fixture, revision, model and scorer" + (let [rows (heldout/checks conn :manifest "loop")] + (is (= 4 (count rows))) + (is (every? #(= "abc123" (:fixture %)) rows)) + (is (every? #(= "m" (:model %)) rows)) + (is (every? #(= "battery/check" (:scorer %)) rows)) + (is (some #(and (= "c1 — calls done" (:target %)) (= 1 (:ok_before %)) (= 0 (:ok_after %))) rows)))))) + +(deftest a-flip-that-does-not-reproduce-does-not-refuse + (with-project [root conn] + (write-case! root "c1" a-case) + (let [n (atom 0) + ;; The candidate's first reading breaks a target; its second does not. + run (fn [_ cand] + (if (and cand (= 1 (swap! n inc))) + (result true false) + (result true true))) + v (heldout/gate! conn {:kind :manifest :name "loop" :text "flaky"} {:run-case run})] + (is (:ok? v) "noise is not a regression") + (is (= ["c1 — calls done"] (:unconfirmed v)) "and it is named as unconfirmed")))) + +(deftest a-baseline-is-measured-once-per-userspace + (with-project [root conn] + (write-case! root "c1" a-case) + (let [calls (atom []) + run (fn [c cand] (swap! calls conj (:text cand)) (result true true))] + (heldout/gate! conn {:kind :manifest :name "loop" :text "one"} {:run-case run}) + (heldout/gate! conn {:kind :manifest :name "loop" :text "two"} {:run-case run}) + (is (= [nil "one" "two"] @calls) "the second edit reuses the baseline")))) + +(deftest a-case-is-staged-as-its-fixture-with-the-candidate-written-in + (with-project [root conn] + (let [git (fn [& args] (apply proc/run {:timeout-ms 15000} "git" "-C" root args)) + _ (git "init" "-q") + _ (spit (io/file root "game.clj") "(ns game)") + _ (git "add" "game.clj") + _ (git "-c" "user.email=t@t" "-c" "user.name=t" "commit" "-q" "-m" "fixture") + sha (str/trim (:out (git "rev-parse" "HEAD"))) + _ (spit (io/file root ".samizdat" "samizdat.sqlite3") "not copied") + dest (str (temp-dir) "/root")] + (try + (is (nil? (heldout/stage! {:fixture {:sha sha}} dest + {:kind :manifest :name "loop" :text "{:candidate true}"}))) + (is (= "(ns game)" (slurp (io/file dest "game.clj"))) "the fixture's tree") + (is (= "{:candidate true}" (slurp (io/file dest ".samizdat" "manifests" "loop.edn"))) + "the candidate in place of the project's file") + (is (.exists (io/file dest ".samizdat" "manifests" "worker.edn")) "the rest of the userspace") + (is (not (.exists (io/file dest ".samizdat" "samizdat.sqlite3"))) "and not the database") + (is (= {:error :no-fixture} (heldout/stage! {} (str dest "2") nil))) + (finally (delete-recursively (.getParentFile (io/file dest)))))))) + +(deftest a-refused-edit-is-not-saved + (with-project [root conn] + (write-case! root "c1" a-case) + (with-redefs [heldout/run-case! replay-by-text] + (let [before (slurp (io/file root ".samizdat" "manifests" "worker.edn")) + ;; A manifest that compiles and that the replay says breaks a target. + text (str (slurp (io/resource "manifests/worker.edn")) "\n;; BREAK\n") + r (base/run-tool {:branch {:id "S1"} :conn conn :tool-name "manifest" + :args {:action "save" :name "worker" :edn text + :rationale "try it"}})] + (is (= :mechanics (:category r)) (:result r)) + (is (str/includes? (:result r) "c1 — calls done")) + (is (= before (slurp (io/file root ".samizdat" "manifests" "worker.edn"))) + "the file that runs is unchanged"))) + (testing "the same through policy save" + (with-redefs [heldout/run-case! replay-by-text] + (let [gates (slurp (io/file root ".samizdat" "gates.edn")) + r (base/run-tool {:branch {:id "S1"} :conn conn :tool-name "policy" + :args {:action "save" :name "gates" + :edn (str gates "\n;; BREAK\n") :rationale "try it"}})] + (is (= :mechanics (:category r)) (:result r)) + ;; gates.edn's own prose says CRASH, which the stand-in reads as + ;; a case that no longer runs: refused either way, and by name. + (is (str/includes? (:result r) "c1 — ") (:result r)) + (is (= gates (slurp (io/file root ".samizdat" "gates.edn"))))))))) + +(deftest the-battery-tool-lists-and-cannot-remove + (with-project [root conn] + (write-case! root "c1" a-case) + (let [r (base/run-tool {:branch {:id "S1"} :conn conn :tool-name "battery" + :args {:action "list"}})] + (is (str/includes? (:result r) "c1")) + (is (str/includes? (:result r) "2 targets"))) + (let [r (base/run-tool {:branch {:id "S1"} :conn conn :tool-name "battery" + :args {:action "remove" :run_id "c1"}})] + (is (= :mechanics (:category r)) "there is no remove")))) + +(defn- row [arm fit tok & {:keys [green? crit] :or {green? true}}] + {:arm arm :task :t :fitness fit :tokens tok :green? green? + :acceptance (when (some? crit) {:results [{:name "horizon fades" :passed? crit}]})}) + +(def ^:private rules {:baseline :base :candidate :cand + :cost-rule {:base 0.10 :per-fitness 1.0} + :noise {:z 2.0 :min-band 0.02}}) + +(deftest stage-two-reads-the-live-arms-in-rrsi-order + (let [base [(row :base 0.50 1000) (row :base 0.52 1000) (row :base 0.48 1000)] + verdict (fn [cand & [opts]] (heldout/live-verdict (concat base cand) (merge rules opts)))] + (testing "a real gain that pays for its tokens is accepted" + (is (:accept? (verdict [(row :cand 0.70 1100) (row :cand 0.72 1100)])))) + (testing "floor: under the best a kept version reached, by more than the noise" + (let [v (verdict [(row :cand 0.50 800)] {:best {:t 0.80}})] + (is (false? (:accept? v))) + (is (= [:floor] (get-in v [:tasks :t :refused]))))) + (testing "cost: a gain that more than doubles the tokens does not pay" + (is (= [:cost] (get-in (verdict [(row :cand 0.70 2500)]) [:tasks :t :refused])))) + (testing "within the band only a cost cut, or a new component, is admissible" + (is (= [:cost] (get-in (verdict [(row :cand 0.50 1000)]) [:tasks :t :refused]))) + (is (:accept? (verdict [(row :cand 0.50 800)])) "cheaper") + (is (:accept? (verdict [(row :cand 0.50 1000)] {:structural? true})) "structural")) + (testing "guards: a criterion the baseline always met, and the suite, may not be lost" + (let [base [(row :base 0.5 1000 :crit true) (row :base 0.5 1000 :crit true)] + v (heldout/live-verdict (concat base [(row :cand 0.9 1000 :crit false :green? false)]) rules)] + (is (= [:criterion :suite] (get-in v [:tasks :t :refused]))) + (is (= ["horizon fades"] (get-in v [:tasks :t :lost-criteria]))))) + (testing "no rows on one side judges nothing, which is not an accept" + (is (false? (:accept? (heldout/live-verdict base rules))))))) diff --git a/test/samizdat/llm_test.clj b/test/samizdat/llm_test.clj index 4279ff91..dcd3f476 100644 --- a/test/samizdat/llm_test.clj +++ b/test/samizdat/llm_test.clj @@ -2110,3 +2110,22 @@ {:function {:name "read_file" :arguments "{\"path\":\"b\"}"}}]})) parsed (fence/parse-tool-call (:content p) {})] (is (= 2 (:fences parsed))))))) + +(deftest arguments-smuggled-into-the-first-string-are-recovered + ;; Run bcd61b39 (karamazov-q9v1): GLM-5.3 put every argument after the first + ;; INSIDE the first one's string, escaped, and the tool read the whole tail + ;; as the action — refused twice on a decision that was right. + (let [raw (str "```tool-call\n" + "{\"name\":\"adopt\",\"args\":{\"action\":\"decline\\\",\\\"kind\\\":\\\"policy\\\"," + "\\\"name\\\":\\\"gates\\\",\\\"rationale\\\":\\\"keep the local edit\\\"}\"}}\n" + "```") + p (fence/parse-tool-call raw)] + (is (= "adopt" (:name p))) + (is (= {:action "decline" :kind "policy" :name "gates" :rationale "keep the local edit"} + (:args p))) + (is (:auto-repaired? p) "recorded as a repair, not silently normalised")) + (testing "a string argument that merely contains quotes is left alone" + (let [p (fence/parse-tool-call + "```tool-call\n{\"name\":\"shell\",\"args\":{\"command\":\"echo \\\"a\\\",\\\"b\\\"\"}}\n```")] + (is (= {:command "echo \"a\",\"b\""} (:args p))) + (is (not (:auto-repaired? p)))))) diff --git a/test/samizdat/manifest_extends_test.clj b/test/samizdat/manifest_extends_test.clj new file mode 100644 index 00000000..57a8e461 --- /dev/null +++ b/test/samizdat/manifest_extends_test.clj @@ -0,0 +1,145 @@ +;; samizdat - a self-hosting agentic harness +;; Copyright (C) 2026 Dmitri Sotnikov +;; +;; This program is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. +;; +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; +;; You should have received a copy of the GNU General Public License +;; along with this program. If not, see . +;; +;; SPDX-License-Identifier: GPL-3.0-or-later + +(ns samizdat.manifest-extends-test + "One turn chain, extended by every turn-at-a-time role (karamazov-xtd3). + loop, worker, reviewer and supervisor carried four copies of the same + graph; now turn.edn holds it and each says :extends \"turn\"." + (:require [clojure.edn :as edn] + [clojure.java.io :as io] + [clojure.string :as str] + [clojure.test :refer [deftest testing is]] + [samizdat.agent.tools.base :as base] + [samizdat.agent.tools.manifest] + [samizdat.manifests :as manifests] + [samizdat.store.db :as db] + [samizdat.store.userspace :as us] + [samizdat.userspace :as userspace] + [samizdat.workflow :as wf])) + +(def ^:private roles ["loop" "worker" "reviewer" "supervisor"]) + +(defn- resolved [nm] (manifests/read-definition (manifests/manifest-body! nm))) + +(defn- raw [nm] (edn/read-string (slurp (io/resource (str "manifests/" nm ".edn"))))) + +(deftest the-four-turn-roles-share-one-chain + (let [turn (resolved "turn")] + (doseq [nm roles + :let [d (resolved nm)]] + (testing nm + (is (= "turn" (:extends (raw nm))) "the file extends turn rather than copying it") + (is (= (:cells turn) (select-keys (:cells d) (keys (:cells turn)))) + "every turn node, with the same cell") + (is (= (dissoc (:edges turn) :route) (select-keys (:edges d) (keys (dissoc (:edges turn) :route)))) + "every turn edge but the tail's") + (is (= (:dispatches turn) (:dispatches d))) + (is (every? (set (:invariants d)) (:invariants turn)) "the turn's invariants hold in every role") + (is (= (:input-schema turn) (:input-schema d))) + (is (not (:fragment? d)) "being a fragment is not inherited") + (is (not (str/blank? (:description d)))) + (is (some? (manifests/compile-loop d)) "it compiles"))))) + +(deftest turn-is-a-fragment-and-not-on-the-menu + (let [with-conn (db/open! ":memory:") + entry (some #(when (= "turn" (:name %)) %) (manifests/catalog with-conn))] + (is (:fragment? entry) "the catalogue says what it is") + (is (not (str/includes? (wf/render-catalog with-conn) "- turn")) + "and the switch menu does not offer it"))) + +(deftest how-an-extension-resolves + (binding [userspace/*candidate* + {[:manifest "b"] (pr-str {:description "base" :fragment? true + :cells {:start :x/a :mid :x/b} + :edges {:start :mid :mid :end} + :invariants [{:type :must-follow :if :start :then :mid}]})}] + (let [d (manifests/read-definition + (pr-str {:extends "b" + :cells {:mid nil :other :x/c} + :edges {:mid nil :start :other :other :end} + :invariants [{:type :must-follow :if :start :then :other}]}))] + (is (= {:start :x/a :other :x/c} (:cells d)) "a child entry wins, a nil removes one") + (is (= {:start :other :other :end} (:edges d))) + (is (= 2 (count (:invariants d))) "invariants are the base's plus the child's") + (is (= "base" (:description d)) "a key the child lacks is the base's") + (is (not (contains? d :fragment?))) + (is (not (contains? d :extends)) "the resolved definition carries no link")) + (testing ":replaces makes a key the child's alone" + (let [d (manifests/read-definition + (pr-str {:extends "b" :replaces [:invariants] + :invariants [{:type :must-follow :if :start :then :other}]}))] + (is (= [{:type :must-follow :if :start :then :other}] (:invariants d))))))) + +(deftest a-missing-base-or-a-cycle-is-refused-by-name + (let [e (try (manifests/read-definition (pr-str {:extends "no-such-base" :cells {}})) + nil (catch Throwable e e))] + (is (some? e)) + (is (str/includes? (ex-message e) "no-such-base"))) + (binding [userspace/*candidate* {[:manifest "c1"] (pr-str {:extends "c2"}) + [:manifest "c2"] (pr-str {:extends "c1"})}] + (let [e (try (manifests/read-definition (pr-str {:extends "c1"})) + nil (catch Throwable e e))] + (is (some? e)) + (is (str/includes? (ex-message e) "c1"))))) + +(defn- run-manifest [conn args] + (base/run-tool {:branch {:id "B1"} :conn conn :tool-name "manifest" :args args})) + +(defn- rename-node [d from to] + (let [kw #(if (= from %) to %) + edge (fn [e] (if (map? e) (update-vals e kw) (kw e)))] + (-> d + (update :cells update-keys kw) + (update :edges #(into {} (map (fn [[k v]] [(kw k) (edge v)])) %)) + (update :dispatches update-keys kw)))) + +(deftest saving-a-base-that-breaks-an-extension-is-refused + (let [conn (db/open! ":memory:") + ;; Compiles on its own; loop's tail still names :route, so loop does not. + broken (pr-str (rename-node (raw "turn") :route :router))] + (is (some? (manifests/compile-loop (manifests/read-definition broken))) "the base alone is fine") + (let [r (run-manifest conn {:action "save" :name "turn" :edn broken + :rationale "rename the router"})] + (is (= :mechanics (:category r)) (:result r)) + (is (re-find #"loop|worker|reviewer" (:result r)) "the refusal names what it broke") + (is (not (str/includes? (str (:body (us/load-latest conn :manifest "turn"))) ":router")) + "nothing was stored")))) + +(deftest a-patch-to-an-extension-writes-only-its-own-part + (let [conn (db/open! ":memory:")] + (run-manifest conn {:action "save" :name "w2" :rationale "a worker to tune" + :edn (slurp (io/resource "manifests/worker.edn"))}) + (let [r (run-manifest conn {:action "patch" :name "w2" :rationale "end without distilling" + :ops [{:op "set-edge" :from "route" :label "abandoned" :to "end"}]}) + body (:body (us/load-latest conn :manifest "w2")) + own (edn/read-string body)] + (is (= :neutral (:category r)) (:result r)) + (is (= "turn" (:extends own)) "the file still extends the chain") + (is (not (contains? (:cells own) :infer)) "and does not inline it") + (is (= :end (get-in own [:edges :route :abandoned]))) + (is (= :end (get-in (manifests/read-definition body) [:edges :route :abandoned])))) + (testing "a rename reaches the base's invariants through :replaces" + (let [r (run-manifest conn {:action "patch" :name "w2" :rationale "name the node for what it does" + :ops [{:op "rename-cell" :from "journal" :to "record"}]}) + body (:body (us/load-latest conn :manifest "w2")) + d (manifests/read-definition body)] + (is (= :neutral (:category r)) (:result r)) + (is (= "turn" (:extends (edn/read-string body)))) + (is (contains? (:cells d) :record)) + (is (not (contains? (:cells d) :journal))) + (is (some? (manifests/compile-loop d))))))) diff --git a/test/samizdat/manifest_test.clj b/test/samizdat/manifest_test.clj index 5db7efac..77ea28cc 100644 --- a/test/samizdat/manifest_test.clj +++ b/test/samizdat/manifest_test.clj @@ -618,7 +618,9 @@ ;; as patterns prints nothing of the kind. (with-db (fn [conn] - (let [patterns (slurp (io/resource "manifests/loop.edn")) + ;; turn.edn: the chain, where the :parse table is written out (loop.edn + ;; extends it and carries only its tail). + (let [patterns (slurp (io/resource "manifests/turn.edn")) _ (is (str/includes? patterns "[:tool _]") "the fixture's target is present") forms (str/replace patterns "[:tool _]" "[:tool (fn [d] true)]") run (fn [args] (base/run-tool {:branch {:id "B1"} :conn conn @@ -721,14 +723,16 @@ (remove (set (str/split-lines before)) (str/split-lines after))) (deftest patch-rewires-the-loop-without-re-emitting-it + ;; On turn.edn, the chain every turn-shaped role extends and the file whose + ;; comments the render exists to keep. (with-db (fn [conn] - (wf/load-loop! conn) ; seed "loop" v1 - (let [r (run-manifest conn {:action "patch" :name "loop" + (wf/load-loop! conn "turn") ; seed "turn" v1 + (let [r (run-manifest conn {:action "patch" :name "turn" :rationale "record says what the node does" :ops [{:op "rename-cell" :from "journal" :to "record"}]}) - v1 (us/load-version conn :manifest "loop" 1) - row (us/load-latest conn :manifest "loop")] + v1 (us/load-version conn :manifest "turn" 1) + row (us/load-latest conn :manifest "turn")] (is (= :neutral (:category r)) (:result r)) (is (:progress? r)) (is (= 2 (:version row))) @@ -740,7 +744,7 @@ (is (every? #(str/includes? % ":record") changed)))) (testing "the :invariants followed the rename, so the next compile holds" (is (str/includes? (:body row) ":if :dispatch :then :record")) - (is (= "loop" (:name (wf/load-loop! conn "loop"))))) + (is (= "turn" (:name (wf/load-loop! conn "turn"))))) (testing "the reply names the version and what changed" (is (str/includes? (:result r) "v2")) (is (re-find #"- :journal" (:result r))) diff --git a/test/samizdat/mutation_test.clj b/test/samizdat/mutation_test.clj index 3c7baf6f..8dedda13 100644 --- a/test/samizdat/mutation_test.clj +++ b/test/samizdat/mutation_test.clj @@ -217,6 +217,31 @@ "and durable in the project's store")) (finally (us/unbind!) (db/close c))))) +(deftest a-proposal-the-held-out-battery-refuses-is-not-saved + ;; karamazov-7mo.4: validate and soak pass, the battery says a recorded case + ;; regresses, and the candidate is refused with the target named — the + ;; registry restored and nothing in the project's history. + (write-cells! (str @root "/cells") "(fn [_ d] (update d :n inc))") + (cells/load-cells! (:dirs (opts))) + (let [c (db/open! ":memory:")] + (try + (us/bind! c) + (let [body (str "(ns cells.mini (:require [mycelium.cell :as cell]))\n" + "(cell/defcell :mini/start {:doc \"s\" :pure true :requires []}\n" + " (fn [_ d] (update d :n + 100)))\n") + seen (atom nil) + r (mut/propose-cell! (assoc (opts) :name "mini" :body body + :heldout-fn (fn [cand] + (reset! seen cand) + "battery: c1 — calls done")))] + (is (= :rolled-back (:status r))) + (is (str/includes? (str (:reason r)) "c1 — calls done")) + (is (= {:kind :cell :name "mini" :text body} @seen) "the gate read the candidate's text") + (is (nil? (us/body :cell "mini")) "nothing entered the project's history") + (is (= 1 (:n ((:handler (cell/get-cell :mini/start)) {} {:n 0}))) + "the registry is back on the running cell")) + (finally (us/unbind!) (db/close c))))) + (deftest a-proposal-whose-requires-is-not-true-is-refused-with-the-fix ;; The `cell` tool commits through here, and here never ran the userspace ;; validator — so a cell reading ctx keys it does not declare was refused diff --git a/test/samizdat/oversight_test.clj b/test/samizdat/oversight_test.clj index 9e60b313..815d1a31 100644 --- a/test/samizdat/oversight_test.clj +++ b/test/samizdat/oversight_test.clj @@ -470,13 +470,28 @@ (let [resume (do (cells/load-cells!) @(ns-resolve 'cells.oversight 'resume-branch)) finished {:id "S0" :messages [{:role "user" :content "hello"} {:role "assistant" :content "a conclusion"}] - :final-answer "done for now" :verdict :done :advisory? true} - next-pass (resume finished)] - (is (= 2 (count (:messages next-pass))) + :final-answer "done for now" :verdict :done :advisory? true + :status :abandoned :inactive-reason "no call three times" + :consecutive-mechanics-failures 3 :consecutive-provider-errors 2} + next-pass (resume finished "the brief for this pass")] + (is (= "hello" (:content (first (:messages next-pass)))) "the conversation so far is kept — that is the whole point of a stream") (is (nil? (:final-answer next-pass)) "not already answered") (is (nil? (:verdict next-pass)) "not already finished") - (is (:advisory? next-pass) "still an advisory branch, not shippable work"))) + (is (:advisory? next-pass) "still an advisory branch, not shippable work") + ;; Run bcd61b39 (karamazov-3keg): the resumed branch was never handed the + ;; new pass's brief, and with :status dissoc'd it was not ACTIVE, so every + ;; pass after the first ran one turn and ended :abandoned. + (is (= {:role "user" :content "the brief for this pass"} (last (:messages next-pass))) + "this pass's brief is what the supervisor reads next") + (is (= 3 (count (:messages next-pass)))) + (is (samizdat.agent.state/active? next-pass) "a resumed pass can take more than one turn") + (is (nil? (:inactive-reason next-pass))) + (is (nil? (:consecutive-mechanics-failures next-pass)) "not charged for the last pass's ending") + (is (nil? (:consecutive-provider-errors next-pass))) + (testing "and route lets it go on" + (let [route (:handler (cell/get-cell! :loop/route))] + (is (= :continue (:verdict (route {:max-turns 10} {:branch next-pass :turn 2})))))))) (defn- event-count [conn run-id kind] (:n (first (db/fetch conn ["SELECT count(*) AS n FROM events diff --git a/test/samizdat/symbolic/dispatch_test.clj b/test/samizdat/symbolic/dispatch_test.clj index a463eaeb..f9075c38 100644 --- a/test/samizdat/symbolic/dispatch_test.clj +++ b/test/samizdat/symbolic/dispatch_test.clj @@ -222,7 +222,7 @@ ;; load-bearing is :parse: a provider failure leaves :parsed unset, so the ;; provider-error branch has to be tried first. That is intentional, and ;; this pins it as the whole of the manifest's order-dependence. - (let [def (edn/read-string (slurp (io/resource "manifests/loop.edn"))) + (let [def (manifests/read-definition (slurp (io/resource "manifests/loop.edn"))) tables (:dispatches def)] (is (= #{:measure :cap :parse :route} (set (keys tables)))) (doseq [[cell table] tables diff --git a/test/samizdat/test_runner.clj b/test/samizdat/test_runner.clj index e68378ef..1421c622 100644 --- a/test/samizdat/test_runner.clj +++ b/test/samizdat/test_runner.clj @@ -97,6 +97,7 @@ [samizdat.prompt-test] [samizdat.workflow-test] [samizdat.manifest-test] + [samizdat.manifest-extends-test] [samizdat.judge-test] [samizdat.team-test] [samizdat.claims-test] @@ -187,6 +188,7 @@ [samizdat.replroots-test] [samizdat.oversight-test] [samizdat.battery-test] + [samizdat.heldout-test] [samizdat.procedure-test] [samizdat.replay-test] [samizdat.mechanics-test] @@ -260,6 +262,7 @@ samizdat.replroots-test samizdat.oversight-test samizdat.battery-test + samizdat.heldout-test samizdat.procedure-test samizdat.replay-test samizdat.mechanics-test @@ -309,6 +312,7 @@ samizdat.prompt-test samizdat.workflow-test samizdat.manifest-test + samizdat.manifest-extends-test samizdat.judge-test samizdat.team-test samizdat.claims-test