diff --git a/README.md b/README.md index 0d71a78..670f60b 100644 --- a/README.md +++ b/README.md @@ -578,6 +578,22 @@ names an executable, run with no arguments in the workdir; exit 0 means healthy. - The path is resolved and checked at start-up (a missing check aborts) and is saved for `specstride resume`. +**`--health-cmd auto`: the proposer owns the check.** When no operator script exists +(a new lab, a new feature), the proposer writes it. Every pass is told the path +(`/health-check`) and the contract: no arguments, exit 0 only when +everything the phase uses is healthy, one line per problem, under a minute, read-only, +and aware of INTENDED degraded states (a factory baseline is not a fault). Specstride +adopts it as soon as it is executable and runs it exactly like an operator's check. + +- Until it exists, a yield whose job needs healthy infrastructure is **refused** + ("write the health check first"); `"requires_healthy_infra": false` jobs still run. +- A hollow check (empty, or nothing but `exit 0`/`true`) or one with a shell syntax + error is refused, and the pass is told why. +- Every change to it between passes is emitted (`health_check_changed`) and shown to + the next pass as a diff; the proposer must cite the file in its gate evidence, so a + check weakened to pass is in front of the critic. +- An explicit `--health-cmd PATH` always wins over `auto`. + Independently of the health check, a yield on `file_exists`/`file_stable`/`grep` now also ends early when the job specstride launched or adopted is gone and the condition is still unmet (`yield_ended_early reason=job_died`), instead of diff --git a/lib/test_proposer_yield.py b/lib/test_proposer_yield.py index 15b0e0e..5217145 100644 --- a/lib/test_proposer_yield.py +++ b/lib/test_proposer_yield.py @@ -684,3 +684,77 @@ def test_a_launch_yield_waiting_on_a_file_nobody_writes_ends_when_the_job_exits( assert early and early[0]["reason"] == "job_died", result.stderr assert int(early[0]["waited_sec"]) < 60 assert result.returncode != 9 + + +# ── --health-cmd auto: the proposer writes and owns the check ──────────────── + + +def _auto_path(tmp_path): + return tmp_path / ".specstride" / "features" / "f" / "health-check" + + +def _auto_env(): + return {"SPECSTRIDE_HEALTH_CMD": "auto"} + + +def test_auto_with_no_check_tells_the_pass_to_write_one(tmp_path): + _run(tmp_path, _agent(tmp_path, "exit 0\n"), max_iter=1, env_extra=_auto_env()) + prompt = (tmp_path / "prompts.log").read_text() + assert "NOT WRITTEN YET (missing)" in prompt + assert str(_auto_path(tmp_path)) in prompt + + +def test_auto_refuses_a_healthy_infra_yield_until_the_check_exists(tmp_path): + document = _yield_json(job={"mode": "launch", "argv": ["/bin/sh", "-c", "sleep 1"]}) + result, events = _run(tmp_path, _agent(tmp_path, _writes_yield(tmp_path, document)), + max_iter=1, env_extra=_auto_env()) + invalid = [e for e in events if e["event"] == "yield_invalid"] + assert invalid and "health check missing" in invalid[0]["reason"], result.stderr + assert not [e for e in events if e["event"] == "yield_job_start"] + + +def test_auto_still_allows_a_yield_that_takes_the_infra_down_on_purpose(tmp_path): + document = _yield_json(job={"mode": "launch", "argv": ["/bin/sh", "-c", "sleep 1"]}, + requires_healthy_infra=False) + result, events = _run(tmp_path, _agent(tmp_path, _writes_yield(tmp_path, document)), + max_iter=1, env_extra=_auto_env()) + assert [e for e in events if e["event"] == "yield_resume"], result.stderr + + +def test_auto_refuses_a_hollow_check(tmp_path): + path = _auto_path(tmp_path) + path.parent.mkdir(parents=True) + path.write_text("#!/bin/bash\n# always fine\nset -e\nexit 0\n") + path.chmod(0o755) + _run(tmp_path, _agent(tmp_path, "exit 0\n"), max_iter=1, env_extra=_auto_env()) + assert "refused: it does nothing but succeed" in (tmp_path / "prompts.log").read_text() + + +def test_auto_runs_the_check_the_proposer_wrote(tmp_path): + """Pass 1 writes a failing check; pass 2 starts with its output.""" + path = _auto_path(tmp_path) + body = (f'if [ ! -e "{path}" ]; then\n' + f' mkdir -p "$(dirname "{path}")"\n' + f" printf '#!/bin/bash\\necho \"leaf01 bgp: inactive\"\\nexit 2\\n' > \"{path}\"\n" + f' chmod +x "{path}"\n' + 'fi\nexit 0\n') + result, events = _run(tmp_path, _agent(tmp_path, body), max_iter=2, env_extra=_auto_env()) + passes = (tmp_path / "prompts.log").read_text().split("===PASS-END===") + assert "INFRASTRUCTURE HEALTH CHECK FAILED" not in passes[0] + assert "INFRASTRUCTURE HEALTH CHECK FAILED" in passes[1], result.stderr + assert "leaf01 bgp: inactive" in passes[1] + assert [e for e in events if e["event"] == "health_check_adopted"] + + +def test_auto_shows_a_changed_check_as_a_diff(tmp_path): + path = _auto_path(tmp_path) + path.parent.mkdir(parents=True) + path.write_text("#!/bin/bash\nping -c1 -W1 127.0.0.1 >/dev/null || exit 3\n") + path.chmod(0o755) + body = (f'if ! grep -q weakened "{path}"; then\n' + f" printf '#!/bin/bash\\n# weakened\\ntrue\\necho ok\\n' > \"{path}\"\nfi\nexit 0\n") + result, events = _run(tmp_path, _agent(tmp_path, body), max_iter=2, env_extra=_auto_env()) + passes = (tmp_path / "prompts.log").read_text().split("===PASS-END===") + assert "health check CHANGED" in passes[1], result.stderr + assert "+# weakened" in passes[1] and "-ping -c1" in passes[1] + assert [e for e in events if e["event"] == "health_check_changed"] diff --git a/orchestrator.sh b/orchestrator.sh index ab91c30..5a600df 100755 --- a/orchestrator.sh +++ b/orchestrator.sh @@ -118,11 +118,15 @@ OPTIONS commands, never the discovered ones; every phase then needs a declared command). Also SPECSTRIDE_VERIFICATION_COMMANDS. - --health-cmd PATH Infrastructure health check: an executable run (no args, + --health-cmd PATH|auto + Infrastructure health check: an executable run (no args, in the workdir) before every proposer pass and every few polls of a yield wait. Non-zero exit = unhealthy: the pass is told so first, and a yield wait on a job that needs a healthy environment ends early. Also SPECSTRIDE_HEALTH_CMD. + `auto`: the proposer writes and owns the check at + /health-check (told so in its prompt); + healthy-infra yields are refused until it exists. --generate-tests DIR Safely scaffold tests below this absolute directory (default: /testautomation//generated). Existing changed artifacts are never overwritten. Supplying @@ -865,7 +869,11 @@ fi # ── infrastructure health check ────────────────────────────────────────────── # Resolved and refused here, once, like the declared commands: a health check # that silently does not exist is a loop that silently runs blind. -if [[ -n "$HEALTH_CMD" ]]; then +if [[ "$HEALTH_CMD" == auto ]]; then + # The proposer writes and owns the check (proposer.sh resolve_health_check). + export SPECSTRIDE_HEALTH_CMD=auto + export SPECSTRIDE_HEALTH_AUTO_PATH="${SPECSTRIDE_HEALTH_AUTO_PATH:-$FEATURE_DIR/health-check}" +elif [[ -n "$HEALTH_CMD" ]]; then case "$HEALTH_CMD" in /*) : ;; *) HEALTH_CMD="$LAUNCH_DIR/$HEALTH_CMD" ;; esac if [[ ! -f "$HEALTH_CMD" || ! -x "$HEALTH_CMD" ]]; then echo "orchestrator.sh: --health-cmd is not an executable file: $HEALTH_CMD" >&2 @@ -976,7 +984,11 @@ if [[ "$OTEL" == "true" ]]; then log " otel : $(specstride_telemetry_status_line otel "$OTEL_URL" "$SPECSTRIDE_EVENTS")" fi log " verify : $VERIFICATION$( [[ "$VERIFICATION" != "off" ]] && echo " plan: $TEST_PLAN" )$( [[ -n "$GENERATE_TESTS" ]] && echo " scaffolds: $GENERATE_TESTS" )" -log " health : ${HEALTH_CMD:-none — the loop cannot see its infrastructure (--health-cmd)}" +if [[ "$HEALTH_CMD" == auto ]]; then + log " health : auto — written by the proposer at ${SPECSTRIDE_HEALTH_AUTO_PATH#"$WORKDIR"/}$( [[ -x "$SPECSTRIDE_HEALTH_AUTO_PATH" ]] && echo " (present)" || echo " (not written yet)" )" +else + log " health : ${HEALTH_CMD:-none — the loop cannot see its infrastructure (--health-cmd)}" +fi log " resume : phase ${CUR_PHASE:-}$( [[ -n "$START_PHASE" ]] && echo " (--start-phase)" )" log " run_id : $SPECSTRIDE_RUN_ID" log " stop with: touch $STOP_FLAG" @@ -1562,6 +1574,10 @@ emit_yield_contract() { echo " with its resume condition unmet, or when the operator's infrastructure health" echo " check keeps failing while it runs. For a job that takes the infrastructure down" echo " on purpose (a re-provision, a rebuild), add \`\"requires_healthy_infra\": false\`." + if [[ "${SPECSTRIDE_HEALTH_CMD:-}" == auto ]]; then + echo "- This run's infrastructure health check is YOURS: ${SPECSTRIDE_HEALTH_AUTO_PATH#"$WORKDIR"/}." + echo " Cite it by that path in GATE evidence whenever the phase touched live infrastructure." + fi echo "A yield is not a failure: it does not count as an error, it is not a stall, and" echo "by default it does not even burn an iteration. Yielding on a 90-minute job is" echo "the CORRECT move; waiting for it inside the pass is the expensive one." diff --git a/proposer.sh b/proposer.sh index 4f367e0..39f34b2 100755 --- a/proposer.sh +++ b/proposer.sh @@ -1327,7 +1327,12 @@ EOF2 # consecutive failures (one failure can be a blip; a fault that persists is not). : "${SPECSTRIDE_HEALTH_EVERY:=10}" : "${SPECSTRIDE_HEALTH_FAILS:=2}" +# `auto`: the PROPOSER owns the check. It writes an executable at +# SPECSTRIDE_HEALTH_AUTO_PATH (default /health-check) the first time a +# phase touches live infrastructure, and specstride adopts it from then on. +: "${SPECSTRIDE_HEALTH_AUTO_PATH:=$FEATURE_DIR/health-check}" HEALTH_OUT=""; HEALTH_RC=0 +HEALTH_ACTIVE=""; HEALTH_PROBLEM="" YIELD_REASON=""; YIELD_MODE=""; YIELD_JOB_CWD=""; YIELD_JOB_LOG="" YIELD_DEADLINE=""; YIELD_PREDICATE=""; YIELD_ON_RESUME=""; YIELD_ADOPT_PID="" @@ -1632,10 +1637,75 @@ yield_take_job() { # Run the operator's health check once. Returns 0 when healthy (or when no check # is configured); otherwise sets HEALTH_RC and HEALTH_OUT (bounded tail). -run_health_check() { +# A proposer-written check is only worth running if it can fail. Refuse the +# hollow shapes a model under pressure writes: empty, or nothing but `exit 0`. +health_lint() { + local path="$1" body + body="$(grep -vE '^[[:space:]]*(#|$)' "$path" 2>/dev/null \ + | grep -vE '^[[:space:]]*(set[[:space:]].*|exit[[:space:]]+0|true|:)[[:space:]]*;?[[:space:]]*$')" + [[ -n "$body" ]] || { echo "it does nothing but succeed (empty, or only 'exit 0'/'true')"; return 1; } + if head -n1 "$path" | grep -qE '^#!.*\b(ba|da|z|k)?sh\b'; then + bash -n "$path" 2>/dev/null || { echo "it has a shell syntax error (bash -n)"; return 1; } + fi + return 0 +} + +# Which check runs now. Sets HEALTH_ACTIVE (empty = none) and HEALTH_PROBLEM +# (why an `auto` check is not usable yet). +resolve_health_check() { + HEALTH_ACTIVE=""; HEALTH_PROBLEM="" [[ -n "$SPECSTRIDE_HEALTH_CMD" ]] || return 0 + if [[ "$SPECSTRIDE_HEALTH_CMD" != auto ]]; then HEALTH_ACTIVE="$SPECSTRIDE_HEALTH_CMD"; return 0; fi + local path="$SPECSTRIDE_HEALTH_AUTO_PATH" why + if [[ ! -e "$path" ]]; then HEALTH_PROBLEM="missing"; return 0; fi + if [[ ! -x "$path" ]]; then HEALTH_PROBLEM="not executable (chmod +x it)"; return 0; fi + if ! why="$(health_lint "$path")"; then HEALTH_PROBLEM="refused: $why"; return 0; fi + HEALTH_ACTIVE="$path" +} + +# An `auto` check the proposer changed since the last pass: say so, with the diff, +# so weakening it is visible to the next pass and (cited in evidence) the critic. +health_change_block() { + [[ "$SPECSTRIDE_HEALTH_CMD" == auto && -f "$SPECSTRIDE_HEALTH_AUTO_PATH" ]] || return 0 + local prev="$FEATURE_DIR/.health-check.accepted" diff + if [[ ! -f "$prev" ]]; then + cp "$SPECSTRIDE_HEALTH_AUTO_PATH" "$prev" 2>/dev/null + specstride_emit health_check_adopted path "$SPECSTRIDE_HEALTH_AUTO_PATH" + return 0 + fi + cmp -s "$prev" "$SPECSTRIDE_HEALTH_AUTO_PATH" && return 0 + diff="$(diff -u "$prev" "$SPECSTRIDE_HEALTH_AUTO_PATH" | head -n 80)" + cp "$SPECSTRIDE_HEALTH_AUTO_PATH" "$prev" 2>/dev/null + specstride_emit health_check_changed path "$SPECSTRIDE_HEALTH_AUTO_PATH" + printf '## Your infrastructure health check CHANGED since the last pass\n' + printf 'File: %s. The change is recorded and must be cited in your evidence:\n```diff\n%s\n```\n' \ + "$SPECSTRIDE_HEALTH_AUTO_PATH" "$diff" + printf 'A check made weaker so that it passes is a defect the critic will reject.\n' +} + +# `auto` with no usable check yet: what the pass is told. +health_missing_block() { + cat <&1)" + out="$(cd "$WORKDIR" && timeout "$SPECSTRIDE_HEALTH_TIMEOUT" "$HEALTH_ACTIVE" 2>&1)" rc=$? (( rc == 0 )) && return 0 HEALTH_RC="$rc" @@ -1647,7 +1717,7 @@ run_health_check() { health_block() { cat <&2 + specstride_emit yield_invalid iter "$i" reason "health check $HEALTH_PROBLEM" \ + artifact "$YIELD_ARTIFACT" + rm -f "$YIELD_ARTIFACT" + YIELD_INDEX=$(( YIELD_INDEX - 1 )) + elif yield_take_job; then YIELD_STARTED_AT="$(date +%s)" specstride_emit pass_yield iter "$i" reason "$YIELD_REASON" predicate_kind "$YIELD_PREDICATE" \ deadline_sec "$YIELD_DEADLINE" job_mode "$YIELD_MODE" job_log "$YIELD_JOB_LOG" \