Refuse runs on an unreachable endpoint, TUI input history, selection and notifications

resources/gates.edn

@@ -286,6 +286,17 @@
         images alive through a slow foreign call, at the cost of a branch
         waiting that much longer for a REPL that may never answer."}
 
+ :endpoint-preflight
+ {:value true :kind :threshold :capability-tunable? false
+  :provenance ["karamazov-rhrf"]
+  :doc "Whether starting or resuming a run is refused, with a 503 naming the
+        endpoint, when the run's model endpoint does not answer at all
+        (refused, unresolvable, timed out) — the same probe that learns what
+        the endpoint is. Run 0f4b9b55 started with the local model server
+        down and spent six turns of three attempts on every branch and the
+        supervisor before :provider-error-limit abandoned them. False starts
+        the run anyway and leaves the outage to that limit."}
+
  :run-start-deadline-ms
  {:value 120000 :kind :threshold :capability-tunable? true
   :provenance ["karamazov-iev2" "karamazov-5fyo"]

resources/prompts/endpoint-unreachable.md

@@ -0,0 +1 @@
+the model endpoint {{url}} is not answering ({{reason}}) — start it, then start the run again

resources/userspace.edn

@@ -88,6 +88,7 @@
    :dispatch-refused "prompts/dispatch-refused.md"
    :drift "prompts/drift.md"
    :emergency-review "prompts/emergency-review.md"
+   :endpoint-unreachable "prompts/endpoint-unreachable.md"
    :eval-calls-main "prompts/eval-calls-main.md"
    :eval-error "prompts/eval-error.md"
    :eval-image "prompts/eval-image.md"

src/samizdat/agent/oversight.clj

@@ -39,7 +39,8 @@
   trigger: it used to fire the pass every two minutes whether or not the
   implementer had done anything, which evaluated no turn in particular and
   spent looks on a run that was idle."
-  (:require [clojure.tools.logging :as log]
+  (:require [clojure.string :as str]
+            [clojure.tools.logging :as log]
             [ebb.core :as ebb]
             [samizdat.cancel :as cancel]
             [samizdat.events :as events]
@@ -142,6 +143,14 @@
                  (not (contains? wanted (keyword (name (:severity %))))))
             findings)))
 
+(defn evidence-text
+  "A finding's measurements as words — `turns 1, retried 10` — for a notice
+  a person reads in the agent log as well as the branch. Printing the map
+  put `{:turns 1, :retried 10}` in front of both (karamazov-8cih)."
+  [evidence]
+  (str/join ", " (for [[k v] evidence]
+                   (str (str/replace (name k) "-" " ") " " v))))
+
 (defn- react!
   "Say something about one finding, through the one write path.
 
@@ -159,7 +168,7 @@
     :payload {:text (prompt/render "watch-intervention"
                                    {:kind (name kind)
                                     :detail detail
-                                    :evidence (pr-str evidence)})}})
+                                    :evidence (evidence-text evidence)})}})
   (journal/note! conn run-id :watch-intervention
                  {:data {:finding kind :evidence evidence}})
   (log/info "watch: raised" kind "-" detail))

src/samizdat/api/client.clj

@@ -165,24 +165,22 @@
 (def start-timeout-ms
   "How long to wait for POST /v1/runs.
 
-  Deliberately longer than the 30s api.control/start-run! itself waits on
-  (deref promised 30000). That endpoint does not answer when the run row is
-  written — beam/run! opens every branch in the beam first and only then
-  calls on-start — so a start can legitimately take tens of seconds, and how
-  long depends on the beam width and on what else the machine is doing.
-
-  Under the shared 10s default this reported a failure for a run that was
-  starting normally: the row was already committed, so the user was told the
-  run had failed AND left with a live run consuming provider spend, with the
-  same request succeeding under curl. Any client bound tighter than the
-  server's own budget has that bug; this one has to outlast it."
-  40000)
+  Longer than the server's own wait, gates.edn :run-start-deadline-ms (120 s
+  shipped), because a start can legitimately take that long and the server
+  answers when the run row exists, not before. A client bound tighter than
+  the server reports a failure for a run that started normally — the row
+  already committed, the run already spending — and a re-post starts a
+  second one (karamazov-5fyo). This stayed at 40 s after the server's wait
+  went to 120 s; control-test pins the ordering against the shipped policy.
+  A project that raises its deadline past this has to raise it here too."
+  150000)
 
 (defn start-run!
   "Start a fresh run and return its id in the body.
 
-  The server answers 503 when the beam does not come up inside its 30s window,
-  which `result` already turns into {:ok false}. It used to answer 200 with an
+  The server answers 503 when the run row does not exist inside its deadline
+  or the model endpoint does not answer, which `result` already turns into
+  {:ok false}. It used to answer 200 with an
   `{:error ...}` body instead, so a plain 2xx was not proof a run existed; the
   unwrap below is kept as a belt-and-braces check on that older shape, since a
   caller that mistakes a refusal for a started run goes on to poll a run id

src/samizdat/api/control.clj

@@ -29,11 +29,13 @@
             [clojure.string :as str]
             [clojure.tools.logging :as log]
             [samizdat.agent.beam :as beam]
+            [samizdat.agent.gates :as gates]
             [samizdat.agent.live :as live]
             [samizdat.agent.resume :as resume]
             [samizdat.approval :as approval]
             [samizdat.cancel :as cancel]
             [samizdat.config :as config]
+            [samizdat.llm.client :as llm]
             [samizdat.llm.registry :as registry]
             [samizdat.prompt :as prompt]
             [samizdat.store.grants :as grants]
@@ -95,6 +97,38 @@
        model (assoc :model model)
        effort (assoc :reasoning-effort effort)))))
 
+;; A run's endpoint is asked what it is before the run starts (llm/with-
+;; discovery — remembered per endpoint once it answers), and one that does not
+;; answer at all refuses the run here rather than letting every branch retry
+;; into :provider-error-limit (karamazov-rhrf). Whether to refuse is policy.
+;;
+;; A project's gates.edn and role map are its OWN copies, and what a later
+;; release adds reaches one only when it adopts the new file — endless-flight
+;; had neither the key nor the prompt, and a throwing read made every run
+;; start a 500. So an absent key is gates/threshold's nil, the behaviour from
+;; before the key existed, and absent prose falls back to what the endpoint
+;; itself said.
+(defn- unreachable-message [url why]
+  (try (str/trim (prompt/render "endpoint-unreachable" {:url url :reason why}))
+       (catch Throwable e
+         (log/warn "endpoint-unreachable prompt:" (ex-message e))
+         (str url " - " why))))
+
+(defn discovered
+  "`llm-config` with what its endpoint said about itself, and the refusal to
+  answer with when it said nothing and gates.edn :endpoint-preflight is on:
+  `[llm-config refusal-or-nil]`. The llm config never carries :unreachable
+  onward — that is this check's, not the run's."
+  [llm-config]
+  (let [probed (llm/with-discovery llm-config)
+        why (:unreachable probed)]
+    [(dissoc probed :unreachable)
+     (when (and why (gates/threshold :endpoint-preflight))
+       (log/warn "refusing a run: model endpoint" (:base-url probed) "-" why)
+       {:status 503
+        :body {:error {:message (unreachable-message (:base-url probed) why)
+                       :type "endpoint_unreachable"}}})]))
+
 (defn start-run!
   "Kick off a run in the background and return its id immediately.
 
@@ -117,8 +151,9 @@
     {:status 400
      :body {:error {:message "a run needs a non-blank `problem`"
                     :type "invalid_request_error"}}}
-  (let [llm-config (run-llm-config config (:llm config) body)
-        adapter (registry/adapter-for (:provider llm-config))
+  (let [[llm-config refusal] (discovered (run-llm-config config (:llm config) body))]
+  (or refusal
+  (let [adapter (registry/adapter-for (:provider llm-config))
         abort (atom false)
         promised (promise)
         cancel* (atom nil)
@@ -182,7 +217,7 @@
       {:status 503
        :body {:error {:message (str/trim
                                 (prompt/render "run-start-timeout"
-                                               {:seconds (quot start-deadline 1000)}))}}})))))
+                                               {:seconds (quot start-deadline 1000)}))}}})))))))
 
 (defn abort!
   "Stop a run without asking it to cooperate. Cancels the run task, which is
@@ -241,8 +276,10 @@
       :else
     ;; A resume may name an arm too — a run that crashed on one model can be
     ;; picked up on another, and saying nothing keeps the original.
-    (let [llm-config (run-llm-config config (or recorded (:llm config)) body)
-          adapter (registry/adapter-for (:provider llm-config))
+    (let [[llm-config refusal] (discovered (run-llm-config config (or recorded (:llm config)) body))]
+    (if refusal
+      (assoc-in refusal [:body :run_id] run-id)
+    (let [adapter (registry/adapter-for (:provider llm-config))
           abort (atom false)
           max-turns (or (:max_turns body) (:max-turns body))
           ;; A run already driven by this process is not resumed: a second
@@ -283,7 +320,7 @@
       ;; max_turns extension was reported as the old budget more often than
       ;; not.
       {:body {:run_id run-id :status "resuming"
-              :max_turns (or max-turns (:max_turns (runs/get-run conn run-id)))}}))))))
+              :max_turns (or max-turns (:max_turns (runs/get-run conn run-id)))}}))))))))
 (defn- grant-pattern
   "The pattern from a grant payload. Accepts a map (what body-json yields), a
   bare string, or nil. Blank is not a pattern — an unset form posts empty

src/samizdat/api/openai.clj

@@ -74,7 +74,7 @@
 (defn chat-completion
   "Run the harness on the last user message and answer in OpenAI's shape."
   [{:keys [conn config]} body]
-  (let [llm-config (:llm config)
+  (let [[llm-config refusal] (api-control/discovered (:llm config))
         adapter (registry/adapter-for (:provider llm-config))
         model (:model llm-config)
         messages (or (:messages body) (get body "messages"))
@@ -84,6 +84,9 @@
       {:status 400
        :body {:error {:message "no user message in `messages`" :type "invalid_request_error"}}}
 
+      ;; The endpoint did not answer at all (gates.edn :endpoint-preflight).
+      refusal refusal
+
       ;; The bypass. Same model, same question, no verification.
       (or (:raw body) (get body "raw"))
       (let [r (llm/chat adapter llm-config messages)]

src/samizdat/core.clj

@@ -97,9 +97,13 @@
                         "— available:" (pr-str (take 10 models)))
               :model-missing)))
       (catch Throwable e
-        (log/warn "provider warm-up failed:" (ex-message e)
-                  "— the harness will start, but https is expected to stay broken"
-                  "in this process once nREPL loads")
+        ;; This used to say https would stay broken once nREPL loaded — true
+        ;; of an ordering bug fixed upstream long ago (see the ns docstring),
+        ;; and wrong about the usual cause, a model server not started yet
+        ;; (karamazov-mxf9).
+        (log/warn "provider" provider "is not reachable:" (ex-message e)
+                  "— the harness will start; with gates.edn :endpoint-preflight on,"
+                  "a run is refused until the endpoint answers")
         :failed))))
 
 (defn start-nrepl!

src/samizdat/llm/client.clj

@@ -45,6 +45,7 @@
             [ebb.core :as ebb]
             [jolt.http-client :as http]
             [samizdat.cancel :as cancel]
+            [samizdat.config :as config]
             [samizdat.lexicon :as lexicon]
             [samizdat.llm.adapter :as adapter]
             [samizdat.llm.message :as message]
@@ -441,10 +442,23 @@
                             ;; transient blip.
                             {:outcome :fatal :reason :timeout
                              :error (str "read timeout: " m)}
-                            ;; A real transport failure — connection reset, TLS
-                            ;; error, connect timeout — is what retrying exists
-                            ;; for.
-                            {:outcome :retry :error (str "transport: " m)}))))
+                            (cond
+                              ;; A cancel is the run being told to stop, and
+                              ;; the ladder is not the place to answer it.
+                              (cancel/control-signal? e) (throw e)
+                              ;; A real transport failure — connection reset,
+                              ;; refused, TLS error, connect timeout — is an
+                              ;; IOException, and is what retrying exists for.
+                              (instance? java.io.IOException e)
+                              {:outcome :retry :error (str "transport: " m)}
+                              ;; Anything else was raised by the harness's own
+                              ;; code inside the call. Run 74ddebb8 retried an
+                              ;; ebb fork-in-finally error as `transport:` 2004
+                              ;; times and read it as an unreliable provider
+                              ;; (karamazov-q8dy); a retry only reproduces a bug.
+                              :else
+                              {:outcome :fatal :reason :harness-error
+                               :error (str "harness: " m)})))))
              errors (conj errors (:error result))]
          (cond
            (= :ok (:outcome result))
@@ -538,9 +552,12 @@
   cannot make a single request. A probe asks; it does not hope.
 
   `/props` is llama.cpp's own endpoint and `total_slots` is the field only it
-  serves. Anything else — a 404, a hosted provider's error page, a connection
-  refused — is `nil`, meaning `not llama.cpp`, which is the safe answer in
-  every direction.
+  serves. Any other ANSWER — a 404, a hosted provider's error page — is
+  `nil`, meaning `not llama.cpp`. No answer at all (refused, unresolvable,
+  timed out) is `{:unreachable message}`: it says nothing about what the
+  endpoint is, and reading it as `not llama.cpp` cost a harness started
+  before its model server its prefix caching and fold role for the life of
+  the process (karamazov-k9q3).
 
   `total_slots` is worth having on its own: RFC-005 said an explicit `:slots`
   table was the only option because `a slot count is a property of how the
@@ -556,10 +573,12 @@
                           :throw-exceptions false})]
       (when (<= 200 (:status resp) 299)
         (llama-props->probe (decode (:body resp)))))
+    (catch java.io.IOException e
+      {:unreachable (str (ex-message e))})
     (catch Throwable _
-      ;; Unreachable, not-llama.cpp and malformed are the same answer here, and
-      ;; none of them is a reason not to start: the harness must come up
-      ;; against an endpoint that is merely slow to boot.
+      ;; A malformed answer is still an answer, and not llama.cpp's. None of
+      ;; these is a reason not to start: the harness must come up against an
+      ;; endpoint that is merely slow to boot.
       nil)))
 
 (def ^:private fold-role-refusal-re
@@ -615,6 +634,51 @@
       (fold-role-from-status (:status resp) (:body resp)))
     (catch Throwable _ nil)))
 
+;; What each endpoint said about itself, by base URL: the probe's facts plus
+;; the fold role, or {} for an endpoint that answered and is not llama.cpp.
+;; Only ANSWERS are kept. An endpoint that was down when asked is asked again
+;; the next time, so a model server started after the harness is found by the
+;; next run rather than never (karamazov-k9q3).
+(defonce ^:private discoveries (atom {}))
+
+(defn forget-discoveries!
+  "Drop every remembered endpoint answer — a test's reset, or a REPL's after
+  swapping the model behind a server."
+  []
+  (reset! discoveries {}))
+
+(defn- discover
+  "Ask the endpoint what it is. `{:unreachable message}` when nothing answered,
+  else the facts to merge into its llm config."
+  [llm]
+  (let [probed (probe-llama-cpp llm)]
+    (if (:unreachable probed)
+      probed
+      (let [fold-role (when (:llama-cpp? probed)
+                        (probe-fold-role (config/apply-discovery llm probed)))]
+        (cond-> (or probed {})
+          fold-role (assoc :fold-role fold-role))))))
+
+(defn with-discovery
+  "`llm` with what its endpoint said about itself merged in (see
+  `config/apply-discovery`), asking it once per endpoint per process. When it
+  did not answer, `llm` comes back carrying `:unreachable` — the reason — and
+  nothing is remembered. An llm with no :base-url is not asked."
+  [llm]
+  (let [url (:base-url llm)]
+    (if (str/blank? (str url))
+      llm
+      (let [facts (or (get @discoveries url)
+                      (let [found (discover llm)]
+                        (when-not (:unreachable found)
+                          (swap! discoveries assoc url found))
+                        found))]
+        (if (:unreachable facts)
+          (assoc llm :unreachable (:unreachable facts))
+          (let [fold-role (:fold-role facts)]
+            (cond-> (config/apply-discovery llm (not-empty (dissoc facts :fold-role)))
+              fold-role (assoc :fold-role fold-role))))))))
+
 (defn list-models
   "Model ids the endpoint advertises, or [] when it has no such endpoint."
   [adapter config]