Keep tests and acceptance apart, ask RFC designs for the RFC, excuse blocked checks

resources/cells/board.clj

@@ -682,7 +682,8 @@
    :requires [:conn :run-id]
    :input  [:map [:board/task {:optional true} :any]
             [:board/plan-text {:optional true} :any]
-            [:board/plan-attempts {:optional true} :int]]
+            [:board/plan-attempts {:optional true} :int]
+            [:board/plan-decision {:optional true} :any]]
    :output [:map [:board/design-decision :keyword]
             [:board/design-findings {:optional true} :any]]}
   (fn [{:keys [conn run-id] :as ctx} {:keys [board/task board/plan-text] :as data}]
@@ -729,10 +730,12 @@
                          :else :revise)
               findings (cond
                          (and no-plan? (= :revise decision))
-                         (str "You did not produce a plan. End your planning step "
-                              "with a `plan` call that names the files this change "
-                              "touches, the tests that pin it, and a one- or "
-                              "two-sentence goal stating how it meets the WHOLE ask.")
+                         ;; Named by the path: an RFC task's plan-text is the
+                         ;; plan's `rfc`, and asking it for files, tests and a
+                         ;; goal it had already given told it nothing
+                         ;; (karamazov-na2k.22).
+                         (prompt/render "design-no-plan"
+                                        {:rfc (= :rfc (:board/plan-decision data))})
                          (= :revise decision) (:findings reviewed)
                          :else nil)]
           (journal/note! conn run-id :design-review

resources/cells/feature.clj

@@ -517,7 +517,9 @@
    :input  [:map [:branch :map] [:review/decision :keyword] [:critic/decision :keyword]]
    ;; Both keys on every branch of the cond, the note carrying WHY on the
    ;; paths where nothing ran.
-   :output [:map [:verify/passed? :boolean] [:verify/note :any]]}
+   :output [:map [:verify/passed? :boolean] [:verify/note :any]
+            [:verify/tests-passed? {:optional true} :any]
+            [:verify/accepted? {:optional true} :any]]}
   (fn [{:keys [conn run-id root config] :as ctx} data]
     (let [cmd (get-in config [:run :verify-cmd])
           criteria? (seq (get-in config [:run :acceptance]))]
@@ -565,7 +567,10 @@
                          {:data {:passed passed? :exit (:exit r) :timeout (:timeout r)
                                  :tests-passed tests-passed? :accepted accepted?
                                  :note (judge/for-the-record :reply-chars note)}})
-          (assoc data :verify/passed? passed? :verify/note note))))))
+          ;; The two halves as well as the verdict, so the route note can
+          ;; say which one refused (karamazov-na2k.23).
+          (assoc data :verify/passed? passed? :verify/note note
+                 :verify/tests-passed? tests-passed? :verify/accepted? accepted?))))))
 
 (cell/defcell :feature/supervise
   {:doc "Where the supervisor's directives about the OUTER loop land.
@@ -687,7 +692,16 @@
                      {:data {:decision decision :revision rev :soft-cap soft-cap
                              :hard-cap hard-cap :hollow hollow :strategy strategy
                              :auto-switch auto-next
-                             :tests-passed (:verify/passed? data)
+                             ;; THE TESTS, not gate 2 as a whole: a green
+                             ;; suite refused on an acceptance criterion
+                             ;; read as a red one, so the supervisor's
+                             ;; green-but-sent-back trigger never fired
+                             ;; (karamazov-na2k.23). Where verify did not run
+                             ;; the halves, the verdict stands in.
+                             :tests-passed (if (contains? data :verify/tests-passed?)
+                                             (:verify/tests-passed? data)
+                                             (:verify/passed? data))
+                             :accepted (:verify/accepted? data)
                              :gave-up (boolean give-up?) :runaway runaway?}})
       (case decision
         :ship

resources/cells/oversight.clj

@@ -424,6 +424,11 @@
                                     :at-cap? (at-cap? round)
                                     :hollow? (:hollow round)
                                     :tests-passed? (:tests-passed round)
+                                    ;; Whether the operator's criteria held,
+                                    ;; and gate 2's own words for why not
+                                    ;; (karamazov-na2k.23).
+                                    :accepted? (:accepted round)
+                                    :verify-note (:note (journal/last-note conn run-id :verify))
                                     :review (some-> (journal/last-note conn run-id :review)
                                                     :decision keyword)
                                     :critic (some-> (journal/last-note conn run-id :critique)

resources/gates.edn

@@ -2228,6 +2228,7 @@
     :nobody-shipped "NO IMPLEMENTOR SHIPPED — the implement round produced nothing verified"
     :tests-failing "TESTS FAILING — {{detail}}"
     :tests-failing-fallback "the tests did not pass"
+    :acceptance-failing "ACCEPTANCE NOT MET, TESTS GREEN — the round was sent back on the operator's criteria, not on its tests: {{detail}}"
     :revision-cap "REVISION CAP REACHED — {{revision}} revisions (soft cap {{soft-cap}}) with no verified solution. This is a soft stop for YOU to decide: keep solving with a new approach, or STOP if it is a genuine dead end."
     :thrash "THRASH — a branch spent a third+ of its turns on empty/mis-parsed calls"
     :reviewer-bounced "REVIEWER BOUNCED — the reviewer sent the work back"

resources/prompts/design-no-plan.md

@@ -0,0 +1 @@
+{% if rfc %}Your plan call did not carry the RFC. This change is designed as an RFC, and the `rfc` field of `plan` is what the reviewer reads: call `plan` again with the whole document in `rfc`, beside the files, the tests and the goal.{% else %}You did not produce a plan. End your planning step with a `plan` call that names the files this change touches, the tests that pin it, and a one- or two-sentence goal stating how it meets the WHOLE ask.{% endif %}

resources/prompts/system-tools.md

@@ -23,9 +23,10 @@ give_up({reason})
 ### Developing at the REPL
 
 ```
-plan({files, tests?, goal?})
+plan({files, tests?, goal?, rfc?})
     Say which files you are about to create or edit, which tests you will
-    write, and why — one line. Every entry in files and tests is a bare
+    write, and why — one line. When the design step asks for an RFC, `rfc`
+    carries the whole document; it is what the reviewer reads. Every entry in files and tests is a bare
     relative path such as test/flight/ghost_test.clj, nothing else: a path
     with a description after it is refused, because a declared file is what
     you are held to and a sentence can never be written. REQUIRED before eval: the REPL stays closed

resources/userspace.edn

@@ -191,6 +191,7 @@
    :system-structure "prompts/system-structure.md"
    :system-tools "prompts/system-tools.md"
    :system-turn "prompts/system-turn.md"
+   :design-no-plan "prompts/design-no-plan.md"
    :experiment-reviews "prompts/experiment-reviews.md"
    :pruning "prompts/pruning.md"
    :reply-format-dropped "prompts/reply-format-dropped.md"

resources/wordlists.edn

@@ -378,6 +378,14 @@
  ;; refuses refinements is worse than no detector.
  #{"not" "never" "fails" "failed" "cannot" "can't" "no" "without" "refuses"}
 
+ :environment-blocked
+ ;; Output that says the HOST refused an acceptance :check, not the work
+ ;; (karamazov-na2k.24): raylib with no window server. A check that prints one
+ ;; of these is tried once more and then counted as undecided rather than
+ ;; failed. Narrow on purpose: every phrase here is a way for a real failure
+ ;; to be excused, so add one only for an output that measured as the host's.
+ #{"Failed to initialize platform"}
+
  :completeness-blocked
  ;; A stated LIMIT, which is a result rather than a plan. Same first-person
  ;; voice and the same work verb as an abandoned intention — "I have not been

src/samizdat/agent/acceptance.clj

@@ -40,6 +40,7 @@
   policy already refuse to let a run write (karamazov-kvw, files/run-config?).
   A spec the run could edit would be Gate 2 all over again."
   (:require [clojure.string :as str]
+            [samizdat.lexicon :as lexicon]
             [samizdat.prompt :as prompt]))
 
 ;; --- the spec ---------------------------------------------------------------
@@ -91,13 +92,27 @@
       (assoc base :passed? nil :output "not run")
 
       (= :check kind)
-      (let [{:keys [green? timeout? output]}
-            (try (run-check text)
-                 (catch Throwable e {:green? false :output (str "check failed to run: " (ex-message e))}))]
-        (assoc base :passed? (boolean green?)
-               :output (if timeout?
-                         (str "timed out" (when-not (str/blank? (str output)) (str "\n" output)))
-                         (str output))))
+      (let [attempt (fn [] (try (run-check text)
+                                (catch Throwable e {:green? false :output (str "check failed to run: " (ex-message e))})))
+            blocked? (fn [{:keys [green? output]}]
+                       (and (not green?)
+                            (some #(str/includes? (str output) %)
+                                  (lexicon/wordlist :environment-blocked))))
+            ;; A check the ENVIRONMENT refused — no window server, say — is
+            ;; tried once more, and if it is still refused it is undecided,
+            ;; not failed: the work was not what stopped it, and sending the
+            ;; round back to fix it asks an owner to repair the host
+            ;; (run eb64f59d, karamazov-na2k.24).
+            first-try (attempt)
+            r (if (blocked? first-try) (attempt) first-try)
+            {:keys [green? timeout? output]} r]
+        (if (blocked? r)
+          (assoc base :passed? nil :blocked true
+                 :output (str "blocked by the environment\n" output))
+          (assoc base :passed? (boolean green?)
+                 :output (if timeout?
+                           (str "timed out" (when-not (str/blank? (str output)) (str "\n" output)))
+                           (str output)))))
 
       :else
       (let [{:keys [yes? reply]}

src/samizdat/agent/telemetry.clj

@@ -282,7 +282,7 @@
   deserve the supervisor's attention is mechanism — it is the same question
   whatever the project builds — while the words that carry them, and the two
   numbers behind the thrash judgement, are `gates.edn :run-health`."
-  [{:keys [results review revision errors hollow? tests-passed? verify-note
+  [{:keys [results review revision errors hollow? tests-passed? accepted? verify-note
            at-cap? soft-cap self-graded]} health]
   (let [total (count results)
         shipped (count (filter #(= :done (:status %)) results))
@@ -316,6 +316,12 @@
       (conj (signal :tests-failing
                     {:detail (or verify-note (signal :tests-failing-fallback))}))
 
+      ;; Green, and refused on the operator's criteria: its own signal, with
+      ;; gate 2's words, because 'the tests did not pass' is the wrong story
+      ;; and the supervisor acted on it (karamazov-na2k.23).
+      (and (false? accepted?) (not (false? tests-passed?)))
+      (conj (signal :acceptance-failing {:detail (or verify-note "")}))
+
       at-cap?
       (conj (signal :revision-cap {:revision revision :soft-cap soft-cap}))
 

test/samizdat/acceptance_test.clj

@@ -261,3 +261,23 @@
                        :roles {:default :local}
                        :run {:acceptance [{:check "no name"}]}})))
   (is (not (system/started?))))
+
+(deftest a-check-the-environment-blocked-is-undecided-not-failed
+  ;; karamazov-na2k.24, run eb64f59d: three hours in, the game-run check
+  ;; failed with raylib's "Failed to initialize platform" — no window server
+  ;; at that moment; the same check passed earlier in the run — and the round
+  ;; was sent back to fix work that was not broken.
+  (let [calls (atom 0)
+        blocked {:green? false :output "WARNING: SYSTEM: Failed to initialize platform"}
+        [r] (acceptance/check [{:name "the game runs" :kind :check :text "run-game"}]
+                              {:kinds #{:check}
+                               :run-check (fn [_] (swap! calls inc) blocked)})]
+    (is (nil? (:passed? r)) "undecided, like a judge that could not answer")
+    (is (true? (:blocked r)))
+    (is (= 2 @calls) "it was tried again once before being called blocked")
+    (is (acceptance/all-passed? [r]) "so it does not send the round back on its own"))
+  (testing "a check that failed for any other reason still fails"
+    (let [[r] (acceptance/check [{:name "suite" :kind :check :text "t"}]
+                                {:kinds #{:check}
+                                 :run-check (fn [_] {:green? false :output "2 failures"})})]
+      (is (false? (:passed? r))))))

test/samizdat/board_test.clj

@@ -18,6 +18,7 @@
             [samizdat.agent.gitdiff :as gitdiff]
             [samizdat.agent.judge :as judge]
             [samizdat.cells :as cells]
+            [mycelium.cell :as cell]
             [samizdat.llm.client :as llm]
             [samizdat.store.db :as db]
             [samizdat.store.journal :as journal]
@@ -332,6 +333,36 @@
         (testing "fail-open held: construction still ran and the task closed"
           (is (= "done" (:status (first (db/fetch conn ["SELECT status FROM tasks"]))))))))))
 
+(deftest an-rfc-design-with-no-rfc-is-asked-for-the-rfc
+  ;; karamazov-na2k.22, runs 390dcd17 and eb64f59d: an RFC-triaged design
+  ;; called plan with files, tests and a goal but no rfc — plan's signature
+  ;; did not declare one — and was sent back with "you did not produce a
+  ;; plan ... name the files, the tests and a goal", which it had just done.
+  (cells/load-cells!)
+  (let [conn (db/open! ":memory:")
+        rid (runs/start-run! conn {:problem "p"})
+        t (tasks/create! conn {:title "fade the terrain" :body "b"})
+        review (fn [decision]
+                 ((:handler (cell/get-cell! :board/design-review))
+                  {:conn conn :run-id rid :config {}}
+                  {:board/task t :board/plan-text nil :board/plan-attempts 1
+                   :board/plan-decision decision}))]
+    (let [r (review :rfc)]
+      (is (= :revise (:board/design-decision r)))
+      (is (str/includes? (str (:board/design-findings r)) "`rfc`")
+          "the re-prompt names the field the RFC path reads"))
+    (let [r (review :plan)]
+      (is (not (str/includes? (str (:board/design-findings r)) "`rfc`"))
+          "and a plain plan is not asked for one"))))
+
+(deftest the-plan-tool-declares-its-rfc-argument
+  ;; Native tool specs are derived from the signature lines; an argument the
+  ;; signature leaves out is not in the schema a native-tool provider sends.
+  (let [specs (into {} (map (juxt :name identity))
+                    ((requiring-resolve 'samizdat.llm.toolspec/signatures)
+                     (slurp (clojure.java.io/resource "prompts/system-tools.md"))))]
+    (is (contains? (get-in specs ["plan" :parameters :properties]) :rfc))))
+
 (deftest triage-is-three-way-by-size
   ;; karamazov-dq1r: trivial skips, a multi-part task gets an RFC, an ordinary
   ;; single-change task gets the lightweight plan in between.

test/samizdat/feature_test.clj

@@ -460,7 +460,12 @@
           (is (seq at-verify) "the verdicts are on the journal from the verify stage")
           (is (= [true false] (mapv :passed? (:results (first at-verify))))
               "the check passed and the judge failed, each on its own line"))
-        (is (some #(false? (:tests-passed %)) route) "route saw gate 2 red")
+        ;; karamazov-na2k.23: the two halves stay apart on the route note. It
+        ;; used to say :tests-passed false here — the suite was green — and so
+        ;; the supervisor's green-but-sent-back trigger never fired and its
+        ;; brief said the tests were failing.
+        (is (some #(and (true? (:tests-passed %)) (false? (:accepted %))) route)
+            "route saw gate 2 red on acceptance, with the tests green")
         (is (some #(str/includes? (str (:note %)) "says what it saw") verify)
             "and the verify note names the criterion, in the failure's own words")))
     (testing "the same run with a YES completes"

test/samizdat/oversight_test.clj

@@ -938,6 +938,25 @@
             "the finding reaches the brief beside the change it was measured under")))
     (finally (session/reset!))))
 
+(deftest a-green-round-refused-on-acceptance-wakes-the-supervisor-and-says-why
+  ;; karamazov-na2k.23, run eb64f59d: every task shipped, the suite was green,
+  ;; one acceptance judge said no, and three supervisor passes concluded
+  ;; 'shipped clean, nothing wrong'. The brief has to say which criterion.
+  (let [conn (db/open! ":memory:")
+        rid (runs/start-run! conn {:problem "p"})]
+    (journal/note! conn rid :verify
+                   {:data {:passed false :tests-passed true :accepted false
+                           :note "tests passed\nacceptance criteria not met:\nFAIL  the game was exercised, not only unit-tested"}})
+    (journal/note! conn rid :route
+                   {:data {:decision "revise" :revision 0 :soft-cap 6 :strategy "board"
+                           :tests-passed true :accepted false}})
+    (let [{:keys [gather prob]} (reasoning-over conn rid)]
+      (is (true? (:oversight/worth-a-look? gather)) "green work sent back is worth a look")
+      (is (str/includes? (str prob) "the game was exercised, not only unit-tested")
+          "and the brief names the criterion that refused it")
+      (is (not (str/includes? (str prob) "the tests did not pass"))
+          "rather than saying the tests failed"))))
+
 (deftest a-refused-mutation-is-worth-a-model-call
   ;; A refusal means the supervisor tried to change the harness and the
   ;; protocol said no. That is precisely a moment where the next pass will