diff --git a/docs/RFCS/RFC-014-validation-and-procedure.md b/docs/RFCS/RFC-014-validation-and-procedure.md index 3e34b18b..a0b6e9cb 100644 --- a/docs/RFCS/RFC-014-validation-and-procedure.md +++ b/docs/RFCS/RFC-014-validation-and-procedure.md @@ -185,10 +185,17 @@ decides. ### The gate in production (karamazov-7mo.4 / ylte.4) `samizdat.heldout` is what runs the battery. Every `cell save`, `manifest -save`/`patch` and `policy save` calls `heldout/check-edit` before it stores: -the battery is replayed with the candidate in place and without it, and an -edit under which a target that passed fails is refused with the targets named -(`prompts/heldout-refused.md`). Nothing is saved; what runs is unchanged. +save`/`patch` and `policy save` validates in the turn as before, then hands +the edit to `heldout/defer!`: the battery is replayed with the candidate in +place and without it on a background lane, and an edit under which a target +that passed fails is refused with the targets named +(`prompts/heldout-refused.md`). **Off the turn** (karamazov-nha1): a refusal is +confirmed by a second replay, so one case's refusal took 733 s against a +900 s turn deadline while the gate ran inside the tool call. The tool now +answers at once that the edit is pending and not live; the verdict reaches +the branch that made it as a `message` directive on a later turn +(`prompts/heldout-decided.md`) and is journalled as `:heldout-verdict`. Edits +are measured and committed one at a time, in the order they were made. **One child process per case.** A replay must read the candidate as the project's userspace from every branch fiber, and a dynamic binding does not @@ -201,6 +208,13 @@ replays the recording through `beam/run!`, and runs `battery/check`. Side calls a cell makes on its own (critic, judges) were never recorded and are refused identically on both sides. +**The same computation every time** (karamazov-x0dx). A recording that forked +replayed its branches concurrently, and which turn landed first was timing: +one baseline read 11/14 and 13/14 minutes apart. The child runs +`beam/run!` with `:serial-turns? true` — each round's turns run one after +another in branch order — and `:oversight? false`, since the supervisor's +passes fire on a clock and the recording cannot answer them. + **The rule is non-compensatory, per target.** No target that passed at baseline may fail on the candidate, and both sides must cover the same targets. Ties are accepted. A case that does not run at baseline is set aside diff --git a/resources/cells/feature.clj b/resources/cells/feature.clj index 975afcd6..8af1e018 100644 --- a/resources/cells/feature.clj +++ b/resources/cells/feature.clj @@ -537,9 +537,16 @@ (let [r (when-not (str/blank? (str cmd)) (proc/run {:timeout-ms (or (get-in config [:run :verify-timeout-ms]) 600000)} "sh" "-c" (str "cd " root " && " cmd))) - tests-passed? (if r - (and (not (:timeout r)) (zero? (or (:exit r) 1))) - true) + green? (if r + (and (not (:timeout r)) (zero? (or (:exit r) 1))) + true) + ;; Green on a suite that never ran the tests this run added or + ;; changed is not green about them (karamazov-khzy). + unrun (when (and r green?) + (verify/unrun-tests root + (gitdiff/changed-files root (:git-baseline ctx)) + (str (:out r) "\n" (:err r)))) + tests-passed? (and green? (empty? unrun)) ;; The criteria are checked whether or not the suite is green: ;; a run whose suite is red AND whose criteria fail should hear ;; about both at once rather than one per round. Fail-open on a @@ -556,6 +563,8 @@ (remove str/blank? [(cond (nil? r) "no :verify-cmd configured" (:timeout r) "tests TIMED OUT" + (seq unrun) (prompt/render "tests-not-run" + {:nses (str/join ", " unrun)}) tests-passed? "tests passed" :else (str "tests FAILED (exit " (:exit r) ")\n" (tail (str (:out r) "\n" (:err r)) 25))) diff --git a/resources/gates.edn b/resources/gates.edn index becc31c2..a3b30127 100644 --- a/resources/gates.edn +++ b/resources/gates.edn @@ -708,6 +708,22 @@ the tracked set narrows to the intersection so divergent touches break the streak on their own; a no-file call resets it."} + :refused-repeat-threshold + {:value 2 :kind :threshold :capability-tunable? true + :provenance ["karamazov-q1tt"] + :doc "How many refusals of one command head (by the shell policy: denied, + or needing an approval nobody gives) before the refused-again gate + says so. Two: the first refusal already explained itself; the second + is the branch trying the same thing another way. Run bcd61b39 retried + `rm`, `nohup jolt … &` and `true` ~15 times over ~50 turns."} + + :max-refused-nudges + {:value 3 :kind :re-fire-guard :capability-tunable? true + :provenance ["karamazov-q1tt"] + :doc "How often the refused-again gate may fire on one branch. Each firing + names the head it is about, so a branch that moves on from `rm` to + `nohup` hears about the second too; past this it is the supervisor's."} + :max-file-thrash-nudges {:value 1 :kind :re-fire-guard :capability-tunable? true :doc "How often the file-thrash gate may fire on one branch. One-shot per @@ -1978,7 +1994,8 @@ :trail-turns 12 :trail-chars 300 :skip "\\.sqlite3(-wal|-shm|-journal)?$"} - :provenance ["karamazov-7mo.4" "karamazov-ylte.4" "2609.24972v2" "2609.09153v1"] + :provenance ["karamazov-7mo.4" "karamazov-ylte.4" "karamazov-nha1" "karamazov-x0dx" + "2609.24972v2" "2609.09153v1"] :kind :policy :capability-tunable? false :doc "THE HELD-OUT GATE, STAGE 1 (samizdat.heldout). Every cell, manifest and policy edit a tool saves is first replayed against the project's @@ -1986,7 +2003,13 @@ battery_cases the first time it is read — with the edit in place and without it, each case in a child process on a copy of the tree the recorded run started from. An edit under which a target that passed - fails is refused and nothing is saved. Ties are accepted. + fails is refused and nothing is saved. Ties are accepted. The replay + runs OFF THE TURN (karamazov-nha1): the tool answers at once that + the edit is pending, and the verdict — live, or refused with the + targets named — reaches the branch that made it as a message on a + later turn. Replays run one turn at a time with no supervisor + stream, so two replays of one case are the same computation + (karamazov-x0dx). A project with no cases is not gated: it tunes itself on the compile and the soak, as before. :enabled? false turns the gate off where @@ -3221,7 +3244,12 @@ :root-strip-regex "^(test|gui|src)/" :ext-strip-regex "\\.cljc?$" :ns-whitelist-regex "[a-zA-Z0-9.*-]+" - :cmd-prefix "jolt -A:test -e '"} + :cmd-prefix "jolt -A:test -e '" + ;; What the suite prints for each namespace it ran, {{ns}} standing for + ;; the namespace (clojure.test's run-tests). A changed test namespace + ;; the full suite's output does not show is a test the suite never ran + ;; (karamazov-khzy); an output with no marker at all is not judged. + :ran-marker "Testing {{ns}}"} :kind :threshold :capability-tunable? false :doc "The focused-verify conventions as data (see the comment above)."} @@ -3944,6 +3972,30 @@ :prediction "the branch retracts, decomposes, or changes technique" :window 3} + {:gate :refused-again + :provenance ["karamazov-q1tt"] + :priority 9.4 + :budget :max-refused-nudges + :doc "A command head the shell policy refused was just refused again. The + refusal text explains itself once; the second attempt — usually the + same head in a different command — means the branch has not taken + it in, and in an unattended run nobody will approve it. Names the + head and the count and asks for a different route. Above + file-thrash: a branch spending turns on refusals is not working at + all." + ;; the quantity the :when compares, recorded at every check (na2k.7) + :measure (get-in branch [:refused-heads (:last-refused branch)] 0) + :when (when-let [h (:last-refused branch)] + (>= (get-in branch [:refused-heads h] 0) + (threshold :refused-repeat-threshold))) + :message-form (sp/render-str (prompt "refused-again") + {:head (:last-refused branch) + :count (get-in branch [:refused-heads (:last-refused branch)]) + :total (:refusals branch) + :goal (state/stated-goal branch)}) + :prediction "the branch stops issuing the refused head" + :window 3} + {:gate :file-thrash :provenance ["karamazov-g86"] :priority 9.5 diff --git a/resources/manual.edn b/resources/manual.edn index ba0384d3..d2adae26 100644 --- a/resources/manual.edn +++ b/resources/manual.edn @@ -85,6 +85,8 @@ :summary "A checked manifest edit as data: ops (rename-cell, add-cell, remove-cell, set-edge, delete-edge, set-cell-field, set-dispatches) applied to the manifest as written, every reference to a renamed node rewritten (edges, dispatches, :invariants), the whole batch compiled once at the end so a node can be added and wired in one call. `manifest patch` is this over the stored text; `render` writes the result back over the original so comments survive; `cell-refs` and `diff-manifests` are behind `manifest refs` and `manifest diff`."} {:name samizdat.workflow/catalog :summary "Every workflow available to select or adapt, with its description."} + {:name samizdat.heldout/defer! + :summary "The held-out gate off the turn: after a save validates, the battery replays on a background lane and the edit is committed or refused there; the branch that made it is told by a message on a later turn. nil when the project has no battery, and the tool saves as before."} {:name samizdat.heldout/check-edit :summary "The held-out gate, stage 1: replay the project's battery (.samizdat/battery, stored in battery_cases) with a candidate cell, manifest or policy edit in place and without it, each case in a child process on the tree its recorded run started from, and refuse the edit when a target that passed fails. Ties accepted; a case that does not run at baseline is set aside by name. Opt-in by having cases; gates.edn :heldout. Each target is recorded in heldout_checks with its fixture, harness revision, model and scorer."} {:name samizdat.heldout/draft! diff --git a/resources/prompts/heldout-decided.md b/resources/prompts/heldout-decided.md new file mode 100644 index 00000000..29ead4fe --- /dev/null +++ b/resources/prompts/heldout-decided.md @@ -0,0 +1,3 @@ +{% if accepted %}Your edit to {{kind}} `{{name}}` passed the held-out battery{% if total %} ({{passed}} of {{total}} targets, nothing that passed before fails){% endif %} and is now live as v{{version}}.{% endif %}{% if refused %}Your edit to {{kind}} `{{name}}` was NOT saved — the held-out battery refused it. + +{{refusal}}{% endif %}{% if save-failed %}Your edit to {{kind}} `{{name}}` passed the held-out battery but did not go live: {{reason}}. What runs is unchanged.{% endif %} diff --git a/resources/prompts/heldout-pending.md b/resources/prompts/heldout-pending.md new file mode 100644 index 00000000..ec93b389 --- /dev/null +++ b/resources/prompts/heldout-pending.md @@ -0,0 +1 @@ +{{kind}} `{{name}}` compiled{% if soaked %} and dry-ran{% endif %}, and is now being replayed against this project's held-out battery, off your turn. It is NOT live yet: what runs is what ran before. You will get a message on a later turn saying whether it went live or which recorded behaviour it broke. Carry on meanwhile — and do not stack another edit to the same file on top of this one before you hear back. diff --git a/resources/prompts/refused-again.md b/resources/prompts/refused-again.md new file mode 100644 index 00000000..562430eb --- /dev/null +++ b/resources/prompts/refused-again.md @@ -0,0 +1,5 @@ +`{{head}}` has now been refused {{count}} times in this run ({{total}} refused commands in all). Rewording the command does not change the answer: the policy judges `{{head}}` itself, and in a run nobody is watching, a command that needs approval will not get it. + +Stop reaching for `{{head}}`. Take a route that does not need it — a command that has already run in this run, a tool made for the job (`write_file`/`edit_file`/`patch` to change files, `read_file` to read them, `eval` to run code), or leave out the step if it was only tidying up. If the step truly cannot be done without it, say so in your answer and carry on with the rest.{% if goal %} + +You are working on: {{goal}}{% endif %} diff --git a/resources/prompts/tests-not-run.md b/resources/prompts/tests-not-run.md new file mode 100644 index 00000000..ff584d09 --- /dev/null +++ b/resources/prompts/tests-not-run.md @@ -0,0 +1 @@ +tests passed, but the suite never ran {{nses}} — test code this run added or changed that the project's test runner does not include. A green suite says nothing about tests it did not run. Register them where the runner lists its test namespaces, then run the suite again and check they appear in its output. diff --git a/resources/userspace.edn b/resources/userspace.edn index 903a81a7..61002165 100644 --- a/resources/userspace.edn +++ b/resources/userspace.edn @@ -60,7 +60,11 @@ :acceptance-judge "prompts/acceptance-judge.md" :adopt-tool "prompts/adopt-tool.md" :battery-tool "prompts/battery-tool.md" + :heldout-decided "prompts/heldout-decided.md" + :heldout-pending "prompts/heldout-pending.md" :heldout-refused "prompts/heldout-refused.md" + :refused-again "prompts/refused-again.md" + :tests-not-run "prompts/tests-not-run.md" :adoption-offer "prompts/adoption-offer.md" :architect "prompts/architect.md" :ask-tool "prompts/ask-tool.md" diff --git a/src/samizdat/agent/arbiter.clj b/src/samizdat/agent/arbiter.clj index aabc0f44..bae135ab 100644 --- a/src/samizdat/agent/arbiter.clj +++ b/src/samizdat/agent/arbiter.clj @@ -190,7 +190,7 @@ :settle-called vocabulary. Kept beside the `case` below and asserted against it by the coherence test, so a gate that falls out of both is caught at test time rather than by settling :unmet forever." - #{:stuck :prologue-cap :progress-stalled :file-thrash :human-directive}) + #{:stuck :prologue-cap :progress-stalled :file-thrash :human-directive :refused-again}) (defn settle "Decide whether an open prediction came true, given what the branch did. @@ -271,6 +271,11 @@ ;; counter measures (karamazov-g86). :file-thrash (< (get-in branch-after [:file-touch :streak] 0) (gates/threshold :file-thrash-threshold)) + ;; Met when a turn after it was NOT refused: the branch took another + ;; route. Read off the refusal record rather than a tool name, since + ;; the refused call and the one that replaces it are both `shell` + ;; (karamazov-q1tt). + :refused-again (nil? (:last-refused branch-after)) :human-directive true (tools-met? gate)) (if late? :met-late :met) diff --git a/src/samizdat/agent/beam.clj b/src/samizdat/agent/beam.clj index fd71c2d1..0df5b267 100644 --- a/src/samizdat/agent/beam.clj +++ b/src/samizdat/agent/beam.clj @@ -629,63 +629,74 @@ (handoff/forfeit! (:conn ctx) (:run-id ctx) b turn (quot (or deadline 0) 1000))) (update :timeouts (fnil inc 0)))) - pending (reduce (fn [acc b] - (let [prev (when cancelling (get @cancelling (:id b)))] - (conj acc - (if (and prev (not (realized? prev))) - [b ::still-cancelling] - (do (when prev (swap! cancelling dissoc (:id b))) - ;; Each turn is a task the beam holds the - ;; canceller of. Started with a yield, so all - ;; five start now rather than each after the - ;; previous one's prefix. - [b (cancel/start! (cancel/spawn #(advance-branch ctx b turn)))]))))) - [] - branches)] + ;; Start one branch's turn, or mark it still winding down. + begin (fn [b] + (let [prev (when cancelling (get @cancelling (:id b)))] + (if (and prev (not (realized? prev))) + [b ::still-cancelling] + (do (when prev (swap! cancelling dissoc (:id b))) + ;; Each turn is a task the beam holds the canceller + ;; of. Started with a yield, so all five start now + ;; rather than each after the previous one's prefix. + [b (cancel/start! (cancel/spawn #(advance-branch ctx b turn)))])))) + ;; Wait for one started turn and read how it ended. + settle (fn [[b t]] + (if (= ::still-cancelling t) + (do (log/warn "branch" (:id b) "is still winding down a cancelled turn;" + "skipping turn" turn "to keep its turns serial") + (forfeit b)) + ;; Parks on the turn's SIGNAL, never on the turn task itself: + ;; ebb's timeout would wait for the cancelled child, and the + ;; barrier must not stall on a read the cancel cannot reach. + (let [[tag r] (if stall-ms + (cancel/await-unless + t (max 1 (quot stall-ms 10)) + #(runs/stalled? (:conn ctx) (:run-id ctx) stall-ms)) + (cancel/await-or-cancel t deadline))] + (case tag + :ok r + :stalled + (do (log/warn "branch" (:id b) "has journalled nothing for" + stall-ms "ms on turn" turn "— cancelled") + (when cancelling (swap! cancelling assoc (:id b) (:done t))) + (assoc b :status :abandoned + :inactive-reason (str "stalled: silent " + (quot stall-ms 1000) " s"))) + :timeout + (do (log/warn "branch" (:id b) "exceeded the turn deadline on turn" turn + "— cancelled") + ;; Not a verification failure: the branch did not get an + ;; answer to be wrong about. It loses the turn and is told + ;; so; the cancelled turn is REMEMBERED until it terminates + ;; so the next round does not run beside it. + (when cancelling (swap! cancelling assoc (:id b) (:done t))) + (forfeit b)) + :err + (do (log/warn "branch" (:id b) "died on turn" turn ":" (ex-message r)) + (assoc b :status :abandoned + :inactive-reason (str "branch error: " (ex-message r)))))))) + ;; SERIAL, for a replay (karamazov-x0dx): each turn runs to its end + ;; before the next starts, in the order the branches are listed. A + ;; held-out replay of a run that forked read 11/14 and 13/14 on the + ;; same baseline minutes apart, because which concurrent turn landed + ;; first was timing; a verdict that compares two replays needs them to + ;; be the same computation. Live runs stay concurrent. + started (atom [])] (try - (reduce (fn [acc [b t]] - (conj acc - (if (= ::still-cancelling t) - (do (log/warn "branch" (:id b) "is still winding down a cancelled turn;" - "skipping turn" turn "to keep its turns serial") - (forfeit b)) - ;; Parks on the turn's SIGNAL, never on the turn task itself: - ;; ebb's timeout would wait for the cancelled child, and the - ;; barrier must not stall on a read the cancel cannot reach. - (let [[tag r] (if stall-ms - (cancel/await-unless - t (max 1 (quot stall-ms 10)) - #(runs/stalled? (:conn ctx) (:run-id ctx) stall-ms)) - (cancel/await-or-cancel t deadline))] - (case tag - :ok r - :stalled - (do (log/warn "branch" (:id b) "has journalled nothing for" - stall-ms "ms on turn" turn "— cancelled") - (when cancelling (swap! cancelling assoc (:id b) (:done t))) - (assoc b :status :abandoned - :inactive-reason (str "stalled: silent " - (quot stall-ms 1000) " s"))) - :timeout - (do (log/warn "branch" (:id b) "exceeded the turn deadline on turn" turn - "— cancelled") - ;; Not a verification failure: the branch did not get an - ;; answer to be wrong about. It loses the turn and is told - ;; so; the cancelled turn is REMEMBERED until it terminates - ;; so the next round does not run beside it. - (when cancelling (swap! cancelling assoc (:id b) (:done t))) - (forfeit b)) - :err - (do (log/warn "branch" (:id b) "died on turn" turn ":" (ex-message r)) - (assoc b :status :abandoned - :inactive-reason (str "branch error: " (ex-message r))))))))) - [] - pending) + (if (:serial-turns? ctx) + (reduce (fn [acc b] + (let [p (begin b)] + (swap! started conj p) + (conj acc (settle p)))) + [] branches) + (let [pending (reduce (fn [acc b] (conj acc (begin b))) [] branches)] + (reset! started pending) + (reduce (fn [acc p] (conj acc (settle p))) [] pending))) (catch Throwable e ;; The round itself was cancelled (an abort) with turns in flight: ;; every turn goes down with it before the signal travels on. (when (cancel/control-signal? e) - (doseq [[_ t] pending :when (map? t)] ((:cancel t)))) + (doseq [[_ t] @started :when (map? t)] ((:cancel t)))) (throw e))))) (defn dispose-branch-engines! @@ -909,7 +920,12 @@ ;; stage returned. Runs fps5 and fps6 both ended having never reached ;; it, because the implementer stalled and never returned — the ;; watchdog was downstream of the thing it watches for. - ctx (assoc ctx :stop-oversight (oversight-stream ctx))] + ;; A replay asks for none (:oversight? false): its passes fire on a + ;; clock, so where their turns land would be timing again, and the + ;; recording has nothing to answer them with (karamazov-x0dx). + ctx (assoc ctx :stop-oversight (if (false? (:oversight? ctx)) + (constantly nil) + (oversight-stream ctx)))] (try (let [data (myc/run-compiled (beam-manifest conn config) ctx {:branches branches :turn start-turn})] @@ -1031,7 +1047,8 @@ land a `done` wins and the rest are abandoned, since paying for four more provider calls after the answer exists is pure waste." [{:keys [conn config llm-adapter llm-config problem max-turns beam-width - token-budget abort on-start seed-run quarantine complete] :as opts}] + token-budget abort on-start seed-run quarantine complete + serial-turns? oversight?] :as opts}] (let [;; What the CALLER asked for, kept apart from the config's numbers: ;; triage below may size a run below the config, never over an ;; explicit request (karamazov-1wv9). @@ -1195,6 +1212,11 @@ ;; real tokens on every branch of a real run. nil when absent, and ;; call-model falls back to infer/complete-fn exactly as before. :complete complete + ;; A replay's switches (samizdat.heldout.child, karamazov-x0dx): + ;; one turn at a time, and no supervisor stream. Both nil — the + ;; live behaviour — unless a caller names them. + :serial-turns? serial-turns? + :oversight? oversight? :max-turns max-turns :beam? (> width 1) :beam-width width :token-budget token-budget :root root diff --git a/src/samizdat/agent/state.clj b/src/samizdat/agent/state.clj index 6e6ca6c4..4115ae22 100644 --- a/src/samizdat/agent/state.clj +++ b/src/samizdat/agent/state.clj @@ -561,10 +561,23 @@ emitting nothing but garbage would hold a beam slot to the turn budget. Any well-formed call clears the tally — the branch has demonstrated it can work the protocol, whatever the call then did." - [branch {:keys [category progress? claim tool policy-refusal? weight]}] + [branch {:keys [category progress? claim tool policy-refusal? weight] :as outcome}] (let [real-progress? (and progress? - (or (nil? claim) (advances-thesis? branch claim)))] + (or (nil? claim) (advances-thesis? branch claim))) + ;; What a policy refused, by command head — whatever the category. An + ;; approval refusal is :neutral (the call waits on a human), so the + ;; :mechanics counters below never saw one, and run bcd61b39 retried + ;; `rm`, `nohup jolt … &` and `true` ~15 times over ~50 turns with no + ;; counter moving (karamazov-q1tt). Counted for the run, never reset: + ;; a head refused twice is refused whatever came between. + refused (when policy-refusal? (get-in outcome [:policy :refused]))] (cond-> branch + policy-refusal? (update :refusals (fnil inc 0)) + refused (update-in [:refused-heads refused] (fnil inc 0)) + ;; The head THIS turn was refused on, so the gate fires on the repeat + ;; itself and not on every turn after it. + refused (assoc :last-refused refused) + (not refused) (dissoc :last-refused) ;; :weight is the failure's cost, default 1. A timeout carries ;; gates.edn :timeout-failure-weight (2): it burned a whole time budget, ;; and counting it like a millisecond failure made a branch stuck diff --git a/src/samizdat/agent/tools/manifest.clj b/src/samizdat/agent/tools/manifest.clj index 2fa45b4c..96f1b261 100644 --- a/src/samizdat/agent/tools/manifest.clj +++ b/src/samizdat/agent/tools/manifest.clj @@ -366,23 +366,28 @@ ;; loop this tool exists to invite — is not billed to the branch's ;; failure counter. The outer catch is left for what happens AFTER ;; this point, notably the store write. - (if-let [complaint (or (try (validate! name edn-text) nil - (catch Throwable e (refused e))) - ;; The held-out battery, last: the dearest - ;; check, and only for an edit that compiles - ;; (karamazov-7mo.4). - (heldout/check-edit conn {:kind :manifest :name name - :text edn-text}))] + (if-let [complaint (try (validate! name edn-text) nil + (catch Throwable e (refused e)))] ;; The complaint plus `usage`, which already says a save ;; validates before it stores — no new sentence in src/. (base/rejected branch (str "`manifest save` refused: " complaint "\n\n" usage)) - (let [v (save! conn name edn-text why)] + ;; The held-out battery, off the turn (karamazov-nha1): when the + ;; project has one, the edit is replayed in the background and + ;; saved only if it passes; the branch is told either way. + (if (heldout/defer! conn {:kind :manifest :name name :text edn-text} + {:commit! #(save! conn name edn-text why) + :run-id (:run-id ctx) :branch-id (:id branch)}) (base/ok branch - (str (saved-line name v) (dispatch-report edn-text) - (cycle-report edn-text)) - :progress? true))))) + (str (prompt/render "heldout-pending" {:kind "manifest" :name name}) + (dispatch-report edn-text) (cycle-report edn-text)) + :progress? true) + (let [v (save! conn name edn-text why)] + (base/ok branch + (str (saved-line name v) (dispatch-report edn-text) + (cycle-report edn-text)) + :progress? true)))))) "patch" (let [name (base/arg ctx :name) @@ -419,19 +424,21 @@ :validator #(manifests/compile-loop %)}) text (file-text new)] (validate! name text) - (if-let [r (heldout/check-edit conn {:kind :manifest :name name - :text text})] - {:complaint r} - {:new new :text text})) + {:new new :text text}) (catch Throwable e {:complaint (refused e)}))] (if-let [complaint (:complaint outcome)] (base/rejected branch (str "`manifest patch` refused: " (deflag complaint) "\n\n" (ops-help))) (let [{:keys [new text]} outcome - v (save! conn name text why)] + deferred (heldout/defer! conn {:kind :manifest :name name :text text} + {:commit! #(save! conn name text why) + :run-id (:run-id ctx) :branch-id (:id branch)}) + v (when-not deferred (save! conn name text why))] (base/ok branch - (str (saved-line name v) + (str (if deferred + (prompt/render "heldout-pending" {:kind "manifest" :name name}) + (saved-line name v)) "\n\n" (format-diff (patch/diff-manifests old new) old new) (dispatch-report text) (cycle-report text)) diff --git a/src/samizdat/agent/tools/mutate.clj b/src/samizdat/agent/tools/mutate.clj index 63e19a55..9fe15139 100644 --- a/src/samizdat/agent/tools/mutate.clj +++ b/src/samizdat/agent/tools/mutate.clj @@ -192,9 +192,17 @@ ;; just installed (karamazov-blt.2). :compile-fn manifests/compile-definition :soak-input (soak-input) - :heldout-fn #(heldout/check-edit conn %) + :defer-fn (fn [cand commit!] + (heldout/defer! conn cand + {:commit! commit! + :run-id run-id :branch-id (:id branch)})) :conn conn :run-id run-id})] (case (:status r) + :pending + (base/ok branch (prompt/render "heldout-pending" + {:kind "cell" :name name :soaked true}) + :progress? true) + :committed (base/ok branch (str "Saved cell '" name "' as v" (:version r) diff --git a/src/samizdat/agent/tools/policy.clj b/src/samizdat/agent/tools/policy.clj index cf127124..438bd8d9 100644 --- a/src/samizdat/agent/tools/policy.clj +++ b/src/samizdat/agent/tools/policy.clj @@ -238,7 +238,20 @@ :names (str/join ", " (policy-names))})) :else (let [parsed (try {:ok (edn/read-string (str body))} - (catch Throwable e {:error (ex-message e)}))] + (catch Throwable e {:error (ex-message e)})) + ;; SELF-GRADING IS NAMED, on both paths below (M10). + self-graded! (fn [] + (when (= "gates" name) + ;; Against the STORED current body rather than + ;; gates/config: the cache is a process global and can + ;; lag what this project has in force, which would name + ;; an edit that did not happen (or miss one that did). + (let [changed (gates/self-graded-changes + (try (userspace/edn-body :policy "gates") + (catch Throwable _ nil)) + (:ok parsed))] + (when (seq changed) + (safely-note ctx changed why)))))] (cond (:error parsed) ;; A body that does not read is a rejected edit, not a branch @@ -246,13 +259,32 @@ (base/rejected branch (msg {:bad-edn true :name name :complaint (:error parsed)})) - ;; The held-out battery, before anything is stored: a table - ;; under which a recorded case's target stops passing is not - ;; saved (karamazov-7mo.4). + ;; A table that will not load is refused here, in the turn — + ;; the held-out battery below is only for one that would. + (some-> (userspace/validator :policy) (as-> f (f name (str body)))) + (base/rejected branch (msg {:rolled-back true :name name + :complaint (:message ((userspace/validator :policy) + name (str body)))})) + + ;; The held-out battery, off the turn (karamazov-nha1): when + ;; the project has one the table is replayed in the background + ;; and saved only if it passes, and the branch is told. + (heldout/defer! (or (:conn ctx) (userspace/conn)) + {:kind :policy :name name :text (str body)} + {:commit! (fn [] + (self-graded!) + (let [v (userspace/save! :policy name (str body) why)] + (when (nil? v) + (throw (ex-info (msg {:unbound true :name name}) {}))) + (try (reload-and-verify! name) v + (catch Throwable e + (rollback! name v) + (throw e))))) + :run-id (:run-id ctx) :branch-id (:id branch)}) + (base/ok branch (prompt/render "heldout-pending" {:kind "policy" :name name}) + :progress? true) + :else - (if-let [refusal (heldout/check-edit (or (:conn ctx) (userspace/conn)) - {:kind :policy :name name :text (str body)})] - (base/rejected branch refusal) ;; Warm the cache so the seed exists and the version we might ;; roll back to is real, then store and recompile. (do (userspace/body :policy name) @@ -264,17 +296,7 @@ ;; record where a reader of the run can see it. Journalled ;; before the save so a rollback still leaves the attempt ;; visible. - (when (= "gates" name) - ;; Against the STORED current body rather than - ;; gates/config: the cache is a process global and can - ;; lag what this project has in force, which would name - ;; an edit that did not happen (or miss one that did). - (let [changed (gates/self-graded-changes - (try (userspace/edn-body :policy "gates") - (catch Throwable _ nil)) - (:ok parsed))] - (when (seq changed) - (safely-note ctx changed why)))) + (self-graded!) (let [v (userspace/save! :policy name (str body) why)] (if (nil? v) (base/fail branch (msg {:unbound true :name name})) @@ -289,7 +311,7 @@ ;; is where it started — a rejected edit. (base/rejected branch (msg {:rolled-back true :name name - :complaint (ex-message e)})))))))))))) + :complaint (ex-message e)}))))))))))) "revert" (let [v (some-> (base/arg ctx :version) str str/trim not-empty parse-long) diff --git a/src/samizdat/agent/verify.clj b/src/samizdat/agent/verify.clj index 94d0169c..e3c5e538 100644 --- a/src/samizdat/agent/verify.clj +++ b/src/samizdat/agent/verify.clj @@ -16,7 +16,8 @@ The runner is a thin effect; the DECISION (`verify-block`) and the command derivation (`focused-cmd`) are pure, so the gate is testable without spawning a process. Same split as planner.clj vs cells/team.clj." - (:require [samizdat.prompt :as prompt] + (:require [clojure.java.io :as io] + [samizdat.prompt :as prompt] [clojure.string :as str] [samizdat.agent.gates :as gates] [samizdat.engine.proc :as proc] @@ -61,6 +62,36 @@ (when (and ns (re-matches (re-pattern (:ns-whitelist-regex c)) ns)) ns))))) +(defn unrun-tests + "The test namespaces among `changed` (paths relative to `root`, as + gitdiff/changed-files gives them) that the suite's `output` shows no sign of + having run, or nil. + + A GREEN SUITE THAT NEVER RAN THE NEW TESTS IS NOT EVIDENCE ABOUT THEM. Run + bcd61b39 added test/flight/horizon_test.clj and never registered it in the + project's runner; the suite reported the baseline's 94 tests green, and two + supervisor passes called the work verified from that (karamazov-khzy). The + focused ship check could not see it — it requires the changed namespaces + directly, which is exactly what the runner did not. + + What counts as having run is the runner's own marker, project data + (:ran-marker, `Testing {{ns}}` for clojure.test). An output that carries no + marker at all is a runner this cannot read, and is not judged: nil. Paths + no longer on disk (a deleted test) are not asked about." + [root changed output] + (let [marker (:ran-marker (conventions)) + out (str output) + mark (fn [ns] (str/replace (str marker) "{{ns}}" ns))] + (when (and (seq marker) (str/includes? out (mark ""))) + (->> changed + (filter test-file?) + (filter #(.isFile (io/file (str root) (str %)))) + (keep ns-from-test-path) + distinct + (remove #(str/includes? out (mark %))) + vec + not-empty)))) + (defn focused-cmd "A test command that runs ONLY the test namespaces among `changed`, with an exit code that reflects pass/fail (jolt's bare -e does not exit non-zero on a diff --git a/src/samizdat/heldout.clj b/src/samizdat/heldout.clj index 27342839..2d509684 100644 --- a/src/samizdat/heldout.clj +++ b/src/samizdat/heldout.clj @@ -63,6 +63,7 @@ [samizdat.prompt :as prompt] [samizdat.security.secrets :as secrets] [samizdat.store.db :as db] + [samizdat.store.interventions :as interventions] [samizdat.store.journal :as journal] [samizdat.store.runs :as runs] [samizdat.userspace :as userspace] @@ -510,6 +511,87 @@ (when (and v (not (:ok? v))) (refusal v))))) +;;; ------------------------------------------------------------- off the turn + +;; ONE LANE. Edits are measured and committed in the order they were made, so +;; each one's baseline is the userspace every earlier one left. +(defonce ^:private lane (Object.)) + +;; The verdicts not yet reached, so a caller that must not leave one behind — +;; a test, a shutdown — can wait for them. +(defonce ^:private outstanding (atom #{})) + +(defn await-pending! + "Wait for every deferred edit to reach its verdict." + [] + (doseq [f @outstanding] (deref f))) + +(defn defer! + "Take `candidate` ({:kind :name :text}) off the turn: when the battery + applies, measure it on a background thread and then either run `commit!` or + refuse, and tell the branch that made the edit either way. Returns + {:pending true :done }, or nil when the gate does not apply — + disabled, no project files, no cases — and the caller commits as before. + + WHY NOT IN THE TURN. A replay costs minutes, a refusal is confirmed by a + second one, and the turn deadline is 900 s: proving this gate on + endless-flight, one case's refusal took 733 s (karamazov-nha1). An edit + checked inside the tool call would be cancelled mid-gate once a battery had + a few cases. The edit that is pending is not live; what runs is what ran + before, until the verdict. + + `opts`: :commit! (no-arg fn, returns the saved version, throws to refuse), + :run-id and :branch-id (who is told, via a `message` for the branch's next + turn, and where the verdict is journalled), :run-case (a test's stand-in for + the child)." + [conn candidate {:keys [commit! run-id branch-id run-case]}] + (let [p (policy)] + (when (and (:enabled? p) (userspace/files?) conn + (seq (:cases (cases conn)))) + (let [what {:kind (some-> (:kind candidate) name) :name (str (:name candidate))} + note! (fn [kind data] + (when run-id + (try (journal/note! conn run-id kind {:branch-id branch-id :data (merge what data)}) + (catch Throwable _ nil)))) + tell! (fn [vars] + (when (and run-id branch-id) + (try (interventions/submit! conn run-id + {:branch-id branch-id :kind "message" + :issued-by "heldout" + :payload (prompt/render "heldout-decided" + (merge what vars))}) + (catch Throwable e + (log/warn "heldout: telling" branch-id "failed:" (ex-message e))))))] + (note! :heldout-pending {}) + (let [done (promise) + fut + (future + (try + (locking lane + (try + (let [v (gate! conn candidate {:run-case run-case})] + (if (or (nil? v) (:ok? v)) + (let [saved (try {:version (commit!)} + (catch Throwable e {:error (or (ex-message e) (str e))}))] + (note! :heldout-verdict (merge {:accepted (nil? (:error saved))} + saved + (select-keys v [:passed-before :passed-after :total]))) + (tell! (if (:error saved) + {:save-failed true :reason (:error saved)} + {:accepted true :version (:version saved) + :passed (:passed-after v) :total (:total v)}))) + (let [r (refusal v)] + (note! :heldout-verdict {:accepted false :regressions (:regressions v)}) + (tell! {:refused true :refusal r})))) + (catch Throwable e + (log/warn "heldout: measuring" (:name what) "failed:" (ex-message e)) + (note! :heldout-verdict {:accepted false :error (ex-message e)}) + (tell! {:save-failed true :reason (ex-message e)})))) + (finally (swap! outstanding disj @done))))] + (swap! outstanding conj fut) + (deliver done fut) + {:pending true :done fut}))))) + ;;; ------------------------------------------------------------- stage 2: live arms (defn- median [xs] diff --git a/src/samizdat/heldout/child.clj b/src/samizdat/heldout/child.clj index 2873ca70..39d4b99f 100644 --- a/src/samizdat/heldout/child.clj +++ b/src/samizdat/heldout/child.clj @@ -97,7 +97,12 @@ :problem (:problem rep) :max-turns (turns-cap case slack) :beam-width 1 - :complete (replay/case-complete-fn rep)}) + :complete (replay/case-complete-fn rep) + ;; The same computation every time: one turn + ;; at a time, no clock-driven supervisor + ;; (karamazov-x0dx). + :serial-turns? true + :oversight? false}) run-id (:run-id result)] {:run-id run-id :status (:status result) diff --git a/src/samizdat/manifests.clj b/src/samizdat/manifests.clj index 2d46484f..28f787e5 100644 --- a/src/samizdat/manifests.clj +++ b/src/samizdat/manifests.clj @@ -373,7 +373,9 @@ :token-budget ;; What the beam driver adds :problem :beam? :beam-width :turn-workflow :iterating-loop? :git-baseline - :repl-session :live-branches :cancelling}) + :repl-session :live-branches :cancelling + ;; A replay's switches (samizdat.heldout.child, karamazov-x0dx) + :serial-turns? :oversight?}) (defn cell-requires "The ctx keys `cell-id` declares it reads. `:requires` is mycelium's own diff --git a/src/samizdat/mutation.clj b/src/samizdat/mutation.clj index 426bec20..742bc6fd 100644 --- a/src/samizdat/mutation.clj +++ b/src/samizdat/mutation.clj @@ -386,15 +386,17 @@ :soak-input — the initial data map the soak dry-run starts from :compile-fn — how to compile+validate (default mycelium pre-compile) :rationale — why, stored with the committed version (karamazov-c58) - :heldout-fn — (fn [candidate] -> refusal or nil), the held-out battery, - consulted after the soak and before the commit - (samizdat.heldout/check-edit; karamazov-7mo.4) + :defer-fn — (fn [candidate commit!] -> truthy when it took the commit), + the held-out battery off the turn (samizdat.heldout/defer!, + karamazov-nha1): called after a clean soak; when it takes + the edit, the registry is restored and the result is + {:status :pending} — `commit!` saves and loads it later :conn :run-id — to journal the outcome (optional) Returns {:status :committed :version n} or {:status :rolled-back :reason ...} with the registry restored and nothing written to the store." - [{:keys [name body loop-def extra-defs soak-input compile-fn rationale conn run-id heldout-fn]}] + [{:keys [name body loop-def extra-defs soak-input compile-fn rationale conn run-id defer-fn]}] (let [compile-fn (or compile-fn myc/pre-compile) shadowing (shadowed-cells name body) unearned (unearned-marks body) @@ -443,14 +445,25 @@ (str "manifest '" nm "': " r))) extra-defs))] (fail reason) - (if-let [reason (or (soak compile-fn loop-def soak-input) - ;; BATTERY — the dearest check, last: does the - ;; candidate make a recorded case regress? It reads - ;; the candidate from its text, in a child process, - ;; so what is installed here does not matter to it. - (when heldout-fn - (heldout-fn {:kind :cell :name name :text body})))] + (if-let [reason (soak compile-fn loop-def soak-input)] (fail reason) + (if (and defer-fn + ;; BATTERY, off the turn: the candidate is replayed from its + ;; text in a child process, and goes live only if nothing + ;; that passed before fails. Until then it is not live, so + ;; the registry goes back to the running cells. + (defer-fn {:kind :cell :name name :text body} + (fn [] + (let [v (userspace/save! :cell name body rationale)] + (when (nil? v) + (throw (ex-info "no project store is bound" {}))) + (reload-cells! nil) + (when (and conn run-id) + (journal/note! conn run-id :mutation-committed + {:data {:cell name :version v :heldout true}})) + v)))) + (do (cell/registry-restore! snapshot) + {:status :pending}) ;; COMMIT. The candidate is already live; this is what makes it ;; survive a restart and what another run will load. ;; @@ -471,6 +484,6 @@ (journal/note! conn run-id :mutation-committed {:data {:cell name :version v}})) (log/info "cell" name "committed as version" v) - {:status :committed :version v}))))) + {:status :committed :version v})))))) (catch Throwable e (fail (str "the candidate did not load — " (or (ex-message e) (str e))))))))) diff --git a/src/samizdat/security/policy.clj b/src/samizdat/security/policy.clj index 05e18a6f..e504277b 100644 --- a/src/samizdat/security/policy.clj +++ b/src/samizdat/security/policy.clj @@ -662,7 +662,9 @@ (if protected-path {:protected true :path protected-path :rule rule-text} {:denied true :head head :rule rule-text})) - :policy {:effect :deny}} + ;; :refused names what was refused, so a branch can count a head it + ;; keeps being refused across varied commands (karamazov-q1tt). + :policy {:effect :deny :refused (or (some-> blocked-segment command-head) head)}} :ask ;; The refusal has to teach the fix, or it is just a wall. Observed live @@ -694,6 +696,7 @@ :note note :unanswered timed-out}) :policy {:effect :ask + :refused (or (some-> blocked-segment command-head) head) ;; No grant unlocks a COMPLEX command (invariant 5 downgrades ;; it to :ask even over a grant), so suggesting `head *` for ;; one taught a fix that could not work — the observed diff --git a/test/samizdat/beam_cancel_test.clj b/test/samizdat/beam_cancel_test.clj index 2a4fd268..06faeb98 100644 --- a/test/samizdat/beam_cancel_test.clj +++ b/test/samizdat/beam_cancel_test.clj @@ -245,3 +245,28 @@ (is (str/includes? (:content (first msgs)) "deadline")) (is (nil? (:tool (samizdat.store.journal/last-note c rid :forfeit))))))) (finally (db/close c))))) + +;; --- serial turns, for a replay (karamazov-x0dx) ----------------------------- +;; +;; A held-out replay of a run that forked read 11/14 and 13/14 on the same +;; baseline minutes apart: every branch's turn starts at once, and which lands +;; first is timing. Under :serial-turns? each turn runs to its end before the +;; next starts, in the order the branches are listed. + +(deftest serial-turns-run-one-at-a-time-in-list-order + (let [log (atom []) + bs (mapv #(state/new-branch {:id % :problem "p"}) ["B1" "B1.2" "B1.3"])] + (with-redefs [beam/advance-branch (fn [_ b _] + (swap! log conj [:start (:id b)]) + ;; the first listed is the slowest, so + ;; a concurrent round would finish it last + (ebb/? (ebb/sleep (if (= "B1" (:id b)) 150 10))) + (swap! log conj [:end (:id b)]) + (assoc b :ran true))] + (let [out (beam/advance-all (assoc (ctx 5000) :serial-turns? true) bs 1)] + (is (= [[:start "B1"] [:end "B1"] [:start "B1.2"] [:end "B1.2"] + [:start "B1.3"] [:end "B1.3"]] + @log) + "each turn ends before the next begins, in list order") + (is (= ["B1" "B1.2" "B1.3"] (mapv :id out)) "results in the branches' order") + (is (every? :ran out)))))) diff --git a/test/samizdat/feature_test.clj b/test/samizdat/feature_test.clj index 8b1c60a0..3549a95a 100644 --- a/test/samizdat/feature_test.clj +++ b/test/samizdat/feature_test.clj @@ -472,6 +472,29 @@ (let [{:keys [result]} (run "YES")] (is (= :completed (:status result))))))) +(deftest a-green-suite-that-never-ran-the-new-tests-is-not-green + ;; karamazov-khzy, run bcd61b39: the suite reported the baseline's 94 tests + ;; green while the run's new horizon test was not in the project's runner, + ;; and two supervisor passes judged it verified green from that. + (with-redefs [judge/deterministic-block (constantly nil) + judge/parse-verdict (constantly :complete) + judge/blocking-findings (constantly nil) + gitdiff/changed-files (constantly ["src/x.clj" "test/x_test.clj"]) + proc/run (constantly {:exit 0 :out "Testing old-test\nRan 94 tests"}) + verify/unrun-tests (fn [_ _ out] + (when (str/includes? (str out) "Ran 94") ["x-test"])) + llm/chat (roles-answering-acceptance {:review :pass} "YES")] + (let [conn (db/open! ":memory:") + r (run-feature conn {:config {:run {:loop "feature" :subtasks ["alpha"] + :verify-cmd "run-tests" + :max-revisions 1 :max-revisions-hard 1}}}) + route (journal/notes conn (:run-id r) :route) + verify (journal/notes conn (:run-id r) :verify)] + (is (not= :completed (:status r)) "a suite that skipped the new tests does not ship") + (is (some #(false? (:tests-passed %)) route) "the tests half reads not passed") + (is (some #(str/includes? (str (:note %)) "x-test") verify) + "and the note names the namespace the suite never ran")))) + (deftest a-switch-directive-changes-the-implement-approach-mid-run ;; self-healing: the supervisor decides the board isn't working and switches ;; this run's implement stage to the decompose loop with a `switch` diff --git a/test/samizdat/finalization_test.clj b/test/samizdat/finalization_test.clj index 2dd3432e..f89182c8 100644 --- a/test/samizdat/finalization_test.clj +++ b/test/samizdat/finalization_test.clj @@ -307,6 +307,36 @@ :branch-before hot :branch-after hot})))))))) +;; --- a refused command tried again ------------------------------------------ + +(deftest a-refused-command-that-is-tried-again-is-named-and-steered + ;; Run bcd61b39 (karamazov-q1tt): ~15 shell calls came back "needs approval" + ;; — `rm -f`, `nohup jolt ... &`, even `true` — over ~50 turns, the same + ;; heads in varied commands, with nobody there to approve. An approval + ;; refusal is :neutral, so no counter moved and no gate fired. + (let [g (gates/by-name :refused-again) + refuse (fn [b head] + (state/record-outcome b {:category :neutral :progress? false + :policy-refusal? true + :policy {:effect :ask :refused head}})) + b0 (state/new-branch {:id "B1" :problem "p"}) + once (refuse b0 "rm") + twice (refuse once "rm") + other (refuse once "jolt") + then-ok (state/record-outcome twice {:category :success :progress? true})] + (is (some? g) "the :refused-again gate entry exists in gates.edn") + (is (= 1 (get-in once [:refused-heads "rm"])) + "an approval refusal counts, though it is :neutral") + (is (= 2 (:refusals twice))) + (is (not ((:when g) {:branch once})) "one refusal is information, not a loop") + (is ((:when g) {:branch twice}) "the same head refused again is") + (is (not ((:when g) {:branch other})) "a different head is not a repeat") + (is (not ((:when g) {:branch then-ok})) "it fires on the repeat, not on every later turn") + (is (= 2 (get-in then-ok [:refused-heads "rm"])) "and the count lasts the run") + (let [msg ((:message g) {:branch twice})] + (is (str/includes? msg "`rm`") "the message names what was refused") + (is (str/includes? msg "2") "and how often")))) + ;; --- retry carries the diagnosis -------------------------------------------- (deftest a-different-failure-of-the-same-call-inherits-the-diagnosis diff --git a/test/samizdat/heldout_test.clj b/test/samizdat/heldout_test.clj index 6a5e5bc9..ffec4896 100644 --- a/test/samizdat/heldout_test.clj +++ b/test/samizdat/heldout_test.clj @@ -30,6 +30,8 @@ [samizdat.engine.proc :as proc] [samizdat.heldout :as heldout] [samizdat.store.db :as db] + [samizdat.store.interventions :as interventions] + [samizdat.store.runs :as runs] [samizdat.store.userspace :as store] [samizdat.userspace :as us] [samizdat.manifests] @@ -195,31 +197,64 @@ (is (= {:error :no-fixture} (heldout/stage! {} (str dest "2") nil))) (finally (delete-recursively (.getParentFile (io/file dest)))))))) -(deftest a-refused-edit-is-not-saved +(defn- messages-to [conn rid bid] + (mapv interventions/text-of (interventions/pending conn rid bid))) + +(deftest an-edit-is-measured-off-the-turn-and-its-author-is-told + ;; karamazov-nha1: the battery costs minutes, a refusal a second measurement, + ;; and a turn has 900 s. The tool call answers at once that the edit is + ;; pending; the verdict arrives as a message on the author's next turn. + (with-project [root conn] + (write-case! root "c1" a-case) + (let [rid (runs/start-run! conn {:problem "p"}) + worker (io/file root ".samizdat" "manifests" "worker.edn") + before (slurp worker) + save (fn [text] + (base/run-tool {:branch {:id "S1"} :conn conn :run-id rid :tool-name "manifest" + :args {:action "save" :name "worker" :edn text + :rationale "try it"}}))] + (with-redefs [heldout/run-case! replay-by-text] + (testing "a bad edit: pending now, refused later, never saved" + (let [r (save (str before "\n;; BREAK\n"))] + (is (= :neutral (:category r)) (:result r)) + (is (str/includes? (:result r) "NOT live yet")) + (heldout/await-pending!) + (is (= before (slurp worker)) "the file that runs is unchanged") + (is (some #(str/includes? (str %) "c1 — calls done") (messages-to conn rid "S1")) + "the refusal reaches the author, naming what broke"))) + (testing "a good edit: pending now, live once it passes" + (let [text (str before "\n;; fine\n") + r (save text)] + (is (str/includes? (:result r) "NOT live yet")) + (is (= before (slurp worker)) "not live before the verdict") + (heldout/await-pending!) + (is (= text (slurp worker)) "saved once the battery passed it") + (is (some #(str/includes? (str %) "is now live") (messages-to conn rid "S1"))))))))) + +(deftest a-policy-edit-is-deferred-the-same-way (with-project [root conn] (write-case! root "c1" a-case) - (with-redefs [heldout/run-case! replay-by-text] - (let [before (slurp (io/file root ".samizdat" "manifests" "worker.edn")) - ;; A manifest that compiles and that the replay says breaks a target. - text (str (slurp (io/resource "manifests/worker.edn")) "\n;; BREAK\n") - r (base/run-tool {:branch {:id "S1"} :conn conn :tool-name "manifest" - :args {:action "save" :name "worker" :edn text - :rationale "try it"}})] - (is (= :mechanics (:category r)) (:result r)) - (is (str/includes? (:result r) "c1 — calls done")) - (is (= before (slurp (io/file root ".samizdat" "manifests" "worker.edn"))) - "the file that runs is unchanged"))) - (testing "the same through policy save" + (let [rid (runs/start-run! conn {:problem "p"}) + gates (slurp (io/file root ".samizdat" "gates.edn"))] (with-redefs [heldout/run-case! replay-by-text] - (let [gates (slurp (io/file root ".samizdat" "gates.edn")) - r (base/run-tool {:branch {:id "S1"} :conn conn :tool-name "policy" + (let [r (base/run-tool {:branch {:id "S1"} :conn conn :run-id rid :tool-name "policy" :args {:action "save" :name "gates" :edn (str gates "\n;; BREAK\n") :rationale "try it"}})] - (is (= :mechanics (:category r)) (:result r)) - ;; gates.edn's own prose says CRASH, which the stand-in reads as - ;; a case that no longer runs: refused either way, and by name. - (is (str/includes? (:result r) "c1 — ") (:result r)) - (is (= gates (slurp (io/file root ".samizdat" "gates.edn"))))))))) + (is (str/includes? (:result r) "NOT live yet") (:result r)) + (heldout/await-pending!) + ;; gates.edn's own prose says CRASH, which the stand-in reads as a + ;; case that no longer runs: refused either way, and by name. + (is (= gates (slurp (io/file root ".samizdat" "gates.edn")))) + (is (some #(str/includes? (str %) "c1 — ") (messages-to conn rid "S1")))))))) + +(deftest a-policy-table-that-will-not-load-is-refused-in-the-turn + (with-project [root conn] + (write-case! root "c1" a-case) + (let [r (base/run-tool {:branch {:id "S1"} :conn conn :tool-name "policy" + :args {:action "save" :name "gates" :edn "[:not :a-map]" + :rationale "break it"}})] + (is (= :mechanics (:category r)) (:result r)) + (is (not (str/includes? (:result r) "NOT live yet")) "no replay for a table that cannot load")))) (deftest the-battery-tool-lists-and-cannot-remove (with-project [root conn] diff --git a/test/samizdat/mutation_test.clj b/test/samizdat/mutation_test.clj index 8dedda13..8f0f2ad3 100644 --- a/test/samizdat/mutation_test.clj +++ b/test/samizdat/mutation_test.clj @@ -217,10 +217,9 @@ "and durable in the project's store")) (finally (us/unbind!) (db/close c))))) -(deftest a-proposal-the-held-out-battery-refuses-is-not-saved - ;; karamazov-7mo.4: validate and soak pass, the battery says a recorded case - ;; regresses, and the candidate is refused with the target named — the - ;; registry restored and nothing in the project's history. +(deftest a-proposal-the-held-out-battery-takes-is-not-live-until-it-passes + ;; karamazov-nha1: validate and soak pass in the turn; the battery runs + ;; off it, so the candidate is not live and not saved until it answers. (write-cells! (str @root "/cells") "(fn [_ d] (update d :n inc))") (cells/load-cells! (:dirs (opts))) (let [c (db/open! ":memory:")] @@ -230,16 +229,20 @@ "(cell/defcell :mini/start {:doc \"s\" :pure true :requires []}\n" " (fn [_ d] (update d :n + 100)))\n") seen (atom nil) + commit (atom nil) r (mut/propose-cell! (assoc (opts) :name "mini" :body body - :heldout-fn (fn [cand] - (reset! seen cand) - "battery: c1 — calls done")))] - (is (= :rolled-back (:status r))) - (is (str/includes? (str (:reason r)) "c1 — calls done")) - (is (= {:kind :cell :name "mini" :text body} @seen) "the gate read the candidate's text") - (is (nil? (us/body :cell "mini")) "nothing entered the project's history") + :defer-fn (fn [cand commit!] + (reset! seen cand) + (reset! commit commit!) + true)))] + (is (= :pending (:status r)) "validated and soaked, then handed to the battery") + (is (= {:kind :cell :name "mini" :text body} @seen) "the gate reads the candidate's text") + (is (nil? (us/body :cell "mini")) "nothing in the project's history yet") (is (= 1 (:n ((:handler (cell/get-cell :mini/start)) {} {:n 0}))) - "the registry is back on the running cell")) + "and the registry is back on the running cell until the verdict") + (testing "the deferred commit saves the version" + (is (= 1 (@commit))) + (is (= body (us/body :cell "mini"))))) (finally (us/unbind!) (db/close c))))) (deftest a-proposal-whose-requires-is-not-true-is-refused-with-the-fix diff --git a/test/samizdat/security/policy_test.clj b/test/samizdat/security/policy_test.clj index c264cd11..84ef78ae 100644 --- a/test/samizdat/security/policy_test.clj +++ b/test/samizdat/security/policy_test.clj @@ -322,7 +322,9 @@ (is (str/includes? (:result r) "python3 evil.py") "the refusal quotes the statement that was not allowed") (is (not (str/includes? (:result r) "Split it up")) - "and does not tell it to split a command that already decomposed")) + "and does not tell it to split a command that already decomposed") + (is (= "python3" (get-in r [:policy :refused])) + "and names the head it refused, so a branch can count it (karamazov-q1tt)")) (testing "a genuinely opaque command still gets the compound lesson" (let [r (policy/run-shell {:args {:command "echo $(rm -rf ~)"}})] (is (:needs-approval r)) diff --git a/test/samizdat/verify_test.clj b/test/samizdat/verify_test.clj index 2cbe983f..1589fc6e 100644 --- a/test/samizdat/verify_test.clj +++ b/test/samizdat/verify_test.clj @@ -201,3 +201,23 @@ ;; fire-time read: the command prefix is data, swapped without touching src (with-redefs [gates/threshold (fn [_] (assoc cfg :cmd-prefix "pytest {{expr}}"))] (is (str/starts-with? (verify/focused-cmd ["test/x_test.clj"]) "pytest "))))) + +(deftest a-changed-test-the-suite-did-not-run-is-named + ;; karamazov-khzy: the run added test/flight/horizon_test.clj and the + ;; project's runner never required it, so the suite's green said nothing + ;; about the new tests. + (let [root (str (java.nio.file.Files/createTempDirectory + "samizdat-unrun" (make-array java.nio.file.attribute.FileAttribute 0))) + write (fn [rel] (let [f (clojure.java.io/file root rel)] + (.mkdirs (.getParentFile f)) (spit f "(ns x)") f))] + (write "test/flight/horizon_test.clj") + (write "test/flight/game_test.clj") + (let [changed ["src/flight/horizon.clj" "test/flight/horizon_test.clj" + "test/flight/game_test.clj" "test/flight/deleted_test.clj"]] + (is (= ["flight.horizon-test"] + (verify/unrun-tests root changed "Testing flight.game-test\nRan 94 tests"))) + (is (empty? (verify/unrun-tests root changed + "Testing flight.game-test\nTesting flight.horizon-test\n")) + "both ran") + (is (empty? (verify/unrun-tests root changed "94 passed")) + "a runner that names no namespace cannot be judged, so it is not"))))