Retry AutoResearch worker restarts safely

autoresearch/prefill/supervisor.py

@@ -219,10 +219,50 @@ def deploy_candidate(worker_ssh: str, chunk_tokens: int) -> None:
 os.chmod(t,0o644)
 t.replace(p)
 PY
-launchctl bootout gui/$(id -u)/ai.kakeya.prefill-worker 2>/dev/null || true
-sleep 3
-launchctl bootstrap gui/$(id -u) \"$HOME/Library/LaunchAgents/ai.kakeya.prefill-worker.plist\"
-launchctl kickstart -k gui/$(id -u)/ai.kakeya.prefill-worker
+domain=gui/$(id -u)
+label=ai.kakeya.prefill-worker
+service=\"$domain/$label\"
+plist=\"$HOME/Library/LaunchAgents/$label.plist\"
+launchctl bootout \"$service\" 2>/dev/null || true
+for delay in 1 1 2 3 5; do
+    if ! launchctl print \"$service\" >/dev/null 2>&1; then
+        break
+    fi
+    sleep \"$delay\"
+done
+if launchctl print \"$service\" >/dev/null 2>&1; then
+    echo \"worker service did not unload\" >&2
+    exit 70
+fi
+loaded=0
+for delay in 1 2 3 5; do
+    if launchctl bootstrap \"$domain\" \"$plist\"; then
+        loaded=1
+        break
+    fi
+    if launchctl print \"$service\" >/dev/null 2>&1; then
+        loaded=1
+        break
+    fi
+    sleep \"$delay\"
+done
+if [ \"$loaded\" -ne 1 ]; then
+    echo \"worker service did not bootstrap\" >&2
+    exit 71
+fi
+launchctl kickstart -k \"$service\"
+for attempt in $(seq 1 120); do
+    if nc -G 2 -z 127.0.0.1 53051 >/dev/null 2>&1; then
+        exit 0
+    fi
+    if ! launchctl print \"$service\" >/dev/null 2>&1; then
+        echo \"worker service disappeared during startup\" >&2
+        exit 72
+    fi
+    sleep 1
+done
+echo \"worker did not become ready on port 53051\" >&2
+exit 73
 """
     subprocess.run(
         ["ssh", "-o", "BatchMode=yes", worker_ssh, remote],
@@ -563,7 +603,14 @@ def run_iteration(args, iteration: int) -> dict:
         candidate_path.write_bytes(previous_candidate)
         _restore(state_path, previous_state)
         _restore(ledger_path, previous_ledger)
-        deploy_candidate(args.worker_ssh, previous_chunk)
+        try:
+            deploy_candidate(args.worker_ssh, previous_chunk)
+        except Exception as rollback_exc:
+            print(
+                "[autoresearch] phase=rollback-worker-failed "
+                f"error={type(rollback_exc).__name__}: {rollback_exc}",
+                flush=True,
+            )
         raise
 
 

tests/inference_engine/bench/test_autoresearch_supervisor.py

@@ -1,6 +1,7 @@
 from autoresearch.prefill.supervisor import (
     append_result,
     best_kept,
+    deploy_candidate,
     parse_research_verdict,
     read_results,
     render_candidate,
@@ -139,6 +140,30 @@ def test_append_result_migrates_legacy_results_header(tmp_path):
     assert rows[1]["hypothesis_sha256"] == "sha"
 
 
+def test_worker_deploy_waits_for_unload_and_readiness(monkeypatch):
+    captured = []
+
+    class Result:
+        stdout = "--prefill-compute-chunk-tokens 128"
+
+    def fake_run(command, **kwargs):
+        captured.append((command, kwargs))
+        return Result()
+
+    monkeypatch.setattr(
+        "autoresearch.prefill.supervisor._wait_port",
+        lambda *_args: None,
+    )
+    monkeypatch.setattr("subprocess.run", fake_run)
+    deploy_candidate("allens", 128)
+    remote = captured[0][0][-1]
+    assert captured[0][1]["check"] is True
+    assert "worker service did not unload" in remote
+    assert "launchctl bootstrap" in remote
+    assert "nc -G 2 -z 127.0.0.1 53051" in remote
+    assert "a[i+1]='128'" in remote
+
+
 def test_supervisor_predeploys_before_real_strategy_proposal():
     source = (
         Path(__file__).resolve().parents[3]