feat(health): --health-cmd auto — the proposer writes and owns the check

README.md

@@ -578,6 +578,22 @@ names an executable, run with no arguments in the workdir; exit 0 means healthy.
 - The path is resolved and checked at start-up (a missing check aborts) and is
   saved for `specstride resume`.
 
+**`--health-cmd auto`: the proposer owns the check.** When no operator script exists
+(a new lab, a new feature), the proposer writes it. Every pass is told the path
+(`<feature state>/health-check`) and the contract: no arguments, exit 0 only when
+everything the phase uses is healthy, one line per problem, under a minute, read-only,
+and aware of INTENDED degraded states (a factory baseline is not a fault). Specstride
+adopts it as soon as it is executable and runs it exactly like an operator's check.
+
+- Until it exists, a yield whose job needs healthy infrastructure is **refused**
+  ("write the health check first"); `"requires_healthy_infra": false` jobs still run.
+- A hollow check (empty, or nothing but `exit 0`/`true`) or one with a shell syntax
+  error is refused, and the pass is told why.
+- Every change to it between passes is emitted (`health_check_changed`) and shown to
+  the next pass as a diff; the proposer must cite the file in its gate evidence, so a
+  check weakened to pass is in front of the critic.
+- An explicit `--health-cmd PATH` always wins over `auto`.
+
 Independently of the health check, a yield on `file_exists`/`file_stable`/`grep`
 now also ends early when the job specstride launched or adopted is gone and the
 condition is still unmet (`yield_ended_early reason=job_died`), instead of

lib/test_proposer_yield.py

@@ -684,3 +684,77 @@ def test_a_launch_yield_waiting_on_a_file_nobody_writes_ends_when_the_job_exits(
     assert early and early[0]["reason"] == "job_died", result.stderr
     assert int(early[0]["waited_sec"]) < 60
     assert result.returncode != 9
+
+
+# ── --health-cmd auto: the proposer writes and owns the check ────────────────
+
+
+def _auto_path(tmp_path):
+    return tmp_path / ".specstride" / "features" / "f" / "health-check"
+
+
+def _auto_env():
+    return {"SPECSTRIDE_HEALTH_CMD": "auto"}
+
+
+def test_auto_with_no_check_tells_the_pass_to_write_one(tmp_path):
+    _run(tmp_path, _agent(tmp_path, "exit 0\n"), max_iter=1, env_extra=_auto_env())
+    prompt = (tmp_path / "prompts.log").read_text()
+    assert "NOT WRITTEN YET (missing)" in prompt
+    assert str(_auto_path(tmp_path)) in prompt
+
+
+def test_auto_refuses_a_healthy_infra_yield_until_the_check_exists(tmp_path):
+    document = _yield_json(job={"mode": "launch", "argv": ["/bin/sh", "-c", "sleep 1"]})
+    result, events = _run(tmp_path, _agent(tmp_path, _writes_yield(tmp_path, document)),
+                          max_iter=1, env_extra=_auto_env())
+    invalid = [e for e in events if e["event"] == "yield_invalid"]
+    assert invalid and "health check missing" in invalid[0]["reason"], result.stderr
+    assert not [e for e in events if e["event"] == "yield_job_start"]
+
+
+def test_auto_still_allows_a_yield_that_takes_the_infra_down_on_purpose(tmp_path):
+    document = _yield_json(job={"mode": "launch", "argv": ["/bin/sh", "-c", "sleep 1"]},
+                           requires_healthy_infra=False)
+    result, events = _run(tmp_path, _agent(tmp_path, _writes_yield(tmp_path, document)),
+                          max_iter=1, env_extra=_auto_env())
+    assert [e for e in events if e["event"] == "yield_resume"], result.stderr
+
+
+def test_auto_refuses_a_hollow_check(tmp_path):
+    path = _auto_path(tmp_path)
+    path.parent.mkdir(parents=True)
+    path.write_text("#!/bin/bash\n# always fine\nset -e\nexit 0\n")
+    path.chmod(0o755)
+    _run(tmp_path, _agent(tmp_path, "exit 0\n"), max_iter=1, env_extra=_auto_env())
+    assert "refused: it does nothing but succeed" in (tmp_path / "prompts.log").read_text()
+
+
+def test_auto_runs_the_check_the_proposer_wrote(tmp_path):
+    """Pass 1 writes a failing check; pass 2 starts with its output."""
+    path = _auto_path(tmp_path)
+    body = (f'if [ ! -e "{path}" ]; then\n'
+            f'  mkdir -p "$(dirname "{path}")"\n'
+            f"  printf '#!/bin/bash\\necho \"leaf01 bgp: inactive\"\\nexit 2\\n' > \"{path}\"\n"
+            f'  chmod +x "{path}"\n'
+            'fi\nexit 0\n')
+    result, events = _run(tmp_path, _agent(tmp_path, body), max_iter=2, env_extra=_auto_env())
+    passes = (tmp_path / "prompts.log").read_text().split("===PASS-END===")
+    assert "INFRASTRUCTURE HEALTH CHECK FAILED" not in passes[0]
+    assert "INFRASTRUCTURE HEALTH CHECK FAILED" in passes[1], result.stderr
+    assert "leaf01 bgp: inactive" in passes[1]
+    assert [e for e in events if e["event"] == "health_check_adopted"]
+
+
+def test_auto_shows_a_changed_check_as_a_diff(tmp_path):
+    path = _auto_path(tmp_path)
+    path.parent.mkdir(parents=True)
+    path.write_text("#!/bin/bash\nping -c1 -W1 127.0.0.1 >/dev/null || exit 3\n")
+    path.chmod(0o755)
+    body = (f'if ! grep -q weakened "{path}"; then\n'
+            f"  printf '#!/bin/bash\\n# weakened\\ntrue\\necho ok\\n' > \"{path}\"\nfi\nexit 0\n")
+    result, events = _run(tmp_path, _agent(tmp_path, body), max_iter=2, env_extra=_auto_env())
+    passes = (tmp_path / "prompts.log").read_text().split("===PASS-END===")
+    assert "health check CHANGED" in passes[1], result.stderr
+    assert "+# weakened" in passes[1] and "-ping -c1" in passes[1]
+    assert [e for e in events if e["event"] == "health_check_changed"]

orchestrator.sh

@@ -118,11 +118,15 @@ OPTIONS
                         commands, never the discovered ones; every phase then
                         needs a declared command). Also
                         SPECSTRIDE_VERIFICATION_COMMANDS.
-  --health-cmd PATH     Infrastructure health check: an executable run (no args,
+  --health-cmd PATH|auto
+                        Infrastructure health check: an executable run (no args,
                         in the workdir) before every proposer pass and every few
                         polls of a yield wait. Non-zero exit = unhealthy: the pass
                         is told so first, and a yield wait on a job that needs a
                         healthy environment ends early. Also SPECSTRIDE_HEALTH_CMD.
+                        `auto`: the proposer writes and owns the check at
+                        <feature state>/health-check (told so in its prompt);
+                        healthy-infra yields are refused until it exists.
   --generate-tests DIR  Safely scaffold tests below this absolute directory (default:
                          <workdir>/testautomation/<feature>/generated).
                         Existing changed artifacts are never overwritten. Supplying
@@ -865,7 +869,11 @@ fi
 # ── infrastructure health check ──────────────────────────────────────────────
 # Resolved and refused here, once, like the declared commands: a health check
 # that silently does not exist is a loop that silently runs blind.
-if [[ -n "$HEALTH_CMD" ]]; then
+if [[ "$HEALTH_CMD" == auto ]]; then
+  # The proposer writes and owns the check (proposer.sh resolve_health_check).
+  export SPECSTRIDE_HEALTH_CMD=auto
+  export SPECSTRIDE_HEALTH_AUTO_PATH="${SPECSTRIDE_HEALTH_AUTO_PATH:-$FEATURE_DIR/health-check}"
+elif [[ -n "$HEALTH_CMD" ]]; then
   case "$HEALTH_CMD" in /*) : ;; *) HEALTH_CMD="$LAUNCH_DIR/$HEALTH_CMD" ;; esac
   if [[ ! -f "$HEALTH_CMD" || ! -x "$HEALTH_CMD" ]]; then
     echo "orchestrator.sh: --health-cmd is not an executable file: $HEALTH_CMD" >&2
@@ -976,7 +984,11 @@ if [[ "$OTEL" == "true" ]]; then
   log "  otel     : $(specstride_telemetry_status_line otel "$OTEL_URL" "$SPECSTRIDE_EVENTS")"
 fi
 log "  verify   : $VERIFICATION$( [[ "$VERIFICATION" != "off" ]] && echo "  plan: $TEST_PLAN" )$( [[ -n "$GENERATE_TESTS" ]] && echo "  scaffolds: $GENERATE_TESTS" )"
-log "  health   : ${HEALTH_CMD:-none — the loop cannot see its infrastructure (--health-cmd)}"
+if [[ "$HEALTH_CMD" == auto ]]; then
+  log "  health   : auto — written by the proposer at ${SPECSTRIDE_HEALTH_AUTO_PATH#"$WORKDIR"/}$( [[ -x "$SPECSTRIDE_HEALTH_AUTO_PATH" ]] && echo " (present)" || echo " (not written yet)" )"
+else
+  log "  health   : ${HEALTH_CMD:-none — the loop cannot see its infrastructure (--health-cmd)}"
+fi
 log "  resume   : phase ${CUR_PHASE:-<all approved>}$( [[ -n "$START_PHASE" ]] && echo " (--start-phase)" )"
 log "  run_id   : $SPECSTRIDE_RUN_ID"
 log "  stop with: touch $STOP_FLAG"
@@ -1562,6 +1574,10 @@ emit_yield_contract() {
   echo "  with its resume condition unmet, or when the operator's infrastructure health"
   echo "  check keeps failing while it runs. For a job that takes the infrastructure down"
   echo "  on purpose (a re-provision, a rebuild), add \`\"requires_healthy_infra\": false\`."
+  if [[ "${SPECSTRIDE_HEALTH_CMD:-}" == auto ]]; then
+    echo "- This run's infrastructure health check is YOURS: ${SPECSTRIDE_HEALTH_AUTO_PATH#"$WORKDIR"/}."
+    echo "  Cite it by that path in GATE evidence whenever the phase touched live infrastructure."
+  fi
   echo "A yield is not a failure: it does not count as an error, it is not a stall, and"
   echo "by default it does not even burn an iteration. Yielding on a 90-minute job is"
   echo "the CORRECT move; waiting for it inside the pass is the expensive one."

proposer.sh

@@ -1327,7 +1327,12 @@ EOF2
 # consecutive failures (one failure can be a blip; a fault that persists is not).
 : "${SPECSTRIDE_HEALTH_EVERY:=10}"
 : "${SPECSTRIDE_HEALTH_FAILS:=2}"
+# `auto`: the PROPOSER owns the check. It writes an executable at
+# SPECSTRIDE_HEALTH_AUTO_PATH (default <feature dir>/health-check) the first time a
+# phase touches live infrastructure, and specstride adopts it from then on.
+: "${SPECSTRIDE_HEALTH_AUTO_PATH:=$FEATURE_DIR/health-check}"
 HEALTH_OUT=""; HEALTH_RC=0
+HEALTH_ACTIVE=""; HEALTH_PROBLEM=""
 
 YIELD_REASON=""; YIELD_MODE=""; YIELD_JOB_CWD=""; YIELD_JOB_LOG=""
 YIELD_DEADLINE=""; YIELD_PREDICATE=""; YIELD_ON_RESUME=""; YIELD_ADOPT_PID=""
@@ -1632,10 +1637,75 @@ yield_take_job() {
 
 # Run the operator's health check once. Returns 0 when healthy (or when no check
 # is configured); otherwise sets HEALTH_RC and HEALTH_OUT (bounded tail).
-run_health_check() {
+# A proposer-written check is only worth running if it can fail. Refuse the
+# hollow shapes a model under pressure writes: empty, or nothing but `exit 0`.
+health_lint() {
+  local path="$1" body
+  body="$(grep -vE '^[[:space:]]*(#|$)' "$path" 2>/dev/null \
+          | grep -vE '^[[:space:]]*(set[[:space:]].*|exit[[:space:]]+0|true|:)[[:space:]]*;?[[:space:]]*$')"
+  [[ -n "$body" ]] || { echo "it does nothing but succeed (empty, or only 'exit 0'/'true')"; return 1; }
+  if head -n1 "$path" | grep -qE '^#!.*\b(ba|da|z|k)?sh\b'; then
+    bash -n "$path" 2>/dev/null || { echo "it has a shell syntax error (bash -n)"; return 1; }
+  fi
+  return 0
+}
+
+# Which check runs now. Sets HEALTH_ACTIVE (empty = none) and HEALTH_PROBLEM
+# (why an `auto` check is not usable yet).
+resolve_health_check() {
+  HEALTH_ACTIVE=""; HEALTH_PROBLEM=""
   [[ -n "$SPECSTRIDE_HEALTH_CMD" ]] || return 0
+  if [[ "$SPECSTRIDE_HEALTH_CMD" != auto ]]; then HEALTH_ACTIVE="$SPECSTRIDE_HEALTH_CMD"; return 0; fi
+  local path="$SPECSTRIDE_HEALTH_AUTO_PATH" why
+  if [[ ! -e "$path" ]]; then HEALTH_PROBLEM="missing"; return 0; fi
+  if [[ ! -x "$path" ]]; then HEALTH_PROBLEM="not executable (chmod +x it)"; return 0; fi
+  if ! why="$(health_lint "$path")"; then HEALTH_PROBLEM="refused: $why"; return 0; fi
+  HEALTH_ACTIVE="$path"
+}
+
+# An `auto` check the proposer changed since the last pass: say so, with the diff,
+# so weakening it is visible to the next pass and (cited in evidence) the critic.
+health_change_block() {
+  [[ "$SPECSTRIDE_HEALTH_CMD" == auto && -f "$SPECSTRIDE_HEALTH_AUTO_PATH" ]] || return 0
+  local prev="$FEATURE_DIR/.health-check.accepted" diff
+  if [[ ! -f "$prev" ]]; then
+    cp "$SPECSTRIDE_HEALTH_AUTO_PATH" "$prev" 2>/dev/null
+    specstride_emit health_check_adopted path "$SPECSTRIDE_HEALTH_AUTO_PATH"
+    return 0
+  fi
+  cmp -s "$prev" "$SPECSTRIDE_HEALTH_AUTO_PATH" && return 0
+  diff="$(diff -u "$prev" "$SPECSTRIDE_HEALTH_AUTO_PATH" | head -n 80)"
+  cp "$SPECSTRIDE_HEALTH_AUTO_PATH" "$prev" 2>/dev/null
+  specstride_emit health_check_changed path "$SPECSTRIDE_HEALTH_AUTO_PATH"
+  printf '## Your infrastructure health check CHANGED since the last pass\n'
+  printf 'File: %s. The change is recorded and must be cited in your evidence:\n```diff\n%s\n```\n' \
+    "$SPECSTRIDE_HEALTH_AUTO_PATH" "$diff"
+  printf 'A check made weaker so that it passes is a defect the critic will reject.\n'
+}
+
+# `auto` with no usable check yet: what the pass is told.
+health_missing_block() {
+  cat <<EOF2
+## Infrastructure health check: NOT WRITTEN YET ($HEALTH_PROBLEM)
+This run expects YOU to own the health check of the live infrastructure it works
+on. If this phase touches live infrastructure (a lab, cluster, VM, device), FIRST
+write an executable at:
+  $SPECSTRIDE_HEALTH_AUTO_PATH
+It runs with no arguments in the workdir and must exit 0 ONLY when everything this
+phase uses is healthy (reachability, services/units, control-plane sessions, the
+data path), print one line per problem otherwise, finish in under a minute, change
+nothing, and know which degraded states are INTENDED (e.g. a factory baseline).
+Specstride adopts it as soon as it exists and runs it before every pass and while
+it waits on your yielded jobs. Until it exists, a yield on a job that needs healthy
+infrastructure is refused. If this phase touches no live infrastructure, ignore this.
+EOF2
+}
+
+run_health_check() {
+  resolve_health_check
+  [[ -n "$HEALTH_ACTIVE" ]] || return 0
   local out rc
-  out="$(cd "$WORKDIR" && timeout "$SPECSTRIDE_HEALTH_TIMEOUT" "$SPECSTRIDE_HEALTH_CMD" 2>&1)"
+  out="$(cd "$WORKDIR" && timeout "$SPECSTRIDE_HEALTH_TIMEOUT" "$HEALTH_ACTIVE" 2>&1)"
   rc=$?
   (( rc == 0 )) && return 0
   HEALTH_RC="$rc"
@@ -1647,7 +1717,7 @@ run_health_check() {
 health_block() {
   cat <<EOF2
 ## INFRASTRUCTURE HEALTH CHECK FAILED — read this before anything else
-Specstride ran the operator's health check (\`$SPECSTRIDE_HEALTH_CMD\`) before this
+Specstride ran the health check (\`$HEALTH_ACTIVE\`) before this
 pass and it exited $HEALTH_RC. Last lines of its output:
 \`\`\`
 $HEALTH_OUT
@@ -2226,7 +2296,11 @@ for (( i=1; i<=MAX_ITER; i++ )); do
   if ! run_health_check; then
     specstride_emit health_check_failed iter "$i" where pass_start rc "$HEALTH_RC"
     pass_prompt="${pass_prompt}"$'\n\n'"$(health_block)"
+  elif [[ -n "$HEALTH_PROBLEM" ]]; then
+    pass_prompt="${pass_prompt}"$'\n\n'"$(health_missing_block)"
   fi
+  health_change="$(health_change_block)"
+  [[ -n "$health_change" ]] && pass_prompt="${pass_prompt}"$'\n\n'"${health_change}"
   # Consumed: the sidecar describes the pass that just ended, never an older one.
   rm -f "$KILL_SIDECAR" "$KILL_SIDECAR.path"
   # Likewise the yield artifact: only what THIS pass writes may be acted on.
@@ -2318,7 +2392,16 @@ for (( i=1; i<=MAX_ITER; i++ )); do
       rm -f "$YIELD_ARTIFACT"
       exit 9
     fi
-    if yield_take_job; then
+    resolve_health_check
+    if [[ "$YIELD_NEEDS_HEALTHY" == "true" && "$SPECSTRIDE_HEALTH_CMD" == auto && -z "$HEALTH_ACTIVE" ]]; then
+      # The forcing function of `auto`: the long jobs are exactly where running
+      # blind costs hours, so they wait until the check exists.
+      echo "proposer.sh: refusing the yield — this run's health check is $HEALTH_PROBLEM ($SPECSTRIDE_HEALTH_AUTO_PATH). Write it first, or mark a job that takes the infrastructure down on purpose with requires_healthy_infra: false." >&2
+      specstride_emit yield_invalid iter "$i" reason "health check $HEALTH_PROBLEM" \
+        artifact "$YIELD_ARTIFACT"
+      rm -f "$YIELD_ARTIFACT"
+      YIELD_INDEX=$(( YIELD_INDEX - 1 ))
+    elif yield_take_job; then
       YIELD_STARTED_AT="$(date +%s)"
       specstride_emit pass_yield iter "$i" reason "$YIELD_REASON" predicate_kind "$YIELD_PREDICATE" \
         deadline_sec "$YIELD_DEADLINE" job_mode "$YIELD_MODE" job_log "$YIELD_JOB_LOG" \