diff --git a/autoresearch/prefill/supervisor.py b/autoresearch/prefill/supervisor.py index ad48046..d42b83c 100644 --- a/autoresearch/prefill/supervisor.py +++ b/autoresearch/prefill/supervisor.py @@ -219,10 +219,50 @@ def deploy_candidate(worker_ssh: str, chunk_tokens: int) -> None: os.chmod(t,0o644) t.replace(p) PY -launchctl bootout gui/$(id -u)/ai.kakeya.prefill-worker 2>/dev/null || true -sleep 3 -launchctl bootstrap gui/$(id -u) \"$HOME/Library/LaunchAgents/ai.kakeya.prefill-worker.plist\" -launchctl kickstart -k gui/$(id -u)/ai.kakeya.prefill-worker +domain=gui/$(id -u) +label=ai.kakeya.prefill-worker +service=\"$domain/$label\" +plist=\"$HOME/Library/LaunchAgents/$label.plist\" +launchctl bootout \"$service\" 2>/dev/null || true +for delay in 1 1 2 3 5; do + if ! launchctl print \"$service\" >/dev/null 2>&1; then + break + fi + sleep \"$delay\" +done +if launchctl print \"$service\" >/dev/null 2>&1; then + echo \"worker service did not unload\" >&2 + exit 70 +fi +loaded=0 +for delay in 1 2 3 5; do + if launchctl bootstrap \"$domain\" \"$plist\"; then + loaded=1 + break + fi + if launchctl print \"$service\" >/dev/null 2>&1; then + loaded=1 + break + fi + sleep \"$delay\" +done +if [ \"$loaded\" -ne 1 ]; then + echo \"worker service did not bootstrap\" >&2 + exit 71 +fi +launchctl kickstart -k \"$service\" +for attempt in $(seq 1 120); do + if nc -G 2 -z 127.0.0.1 53051 >/dev/null 2>&1; then + exit 0 + fi + if ! launchctl print \"$service\" >/dev/null 2>&1; then + echo \"worker service disappeared during startup\" >&2 + exit 72 + fi + sleep 1 +done +echo \"worker did not become ready on port 53051\" >&2 +exit 73 """ subprocess.run( ["ssh", "-o", "BatchMode=yes", worker_ssh, remote], @@ -563,7 +603,14 @@ def run_iteration(args, iteration: int) -> dict: candidate_path.write_bytes(previous_candidate) _restore(state_path, previous_state) _restore(ledger_path, previous_ledger) - deploy_candidate(args.worker_ssh, previous_chunk) + try: + deploy_candidate(args.worker_ssh, previous_chunk) + except Exception as rollback_exc: + print( + "[autoresearch] phase=rollback-worker-failed " + f"error={type(rollback_exc).__name__}: {rollback_exc}", + flush=True, + ) raise diff --git a/tests/inference_engine/bench/test_autoresearch_supervisor.py b/tests/inference_engine/bench/test_autoresearch_supervisor.py index c3ef377..d9e71d8 100644 --- a/tests/inference_engine/bench/test_autoresearch_supervisor.py +++ b/tests/inference_engine/bench/test_autoresearch_supervisor.py @@ -1,6 +1,7 @@ from autoresearch.prefill.supervisor import ( append_result, best_kept, + deploy_candidate, parse_research_verdict, read_results, render_candidate, @@ -139,6 +140,30 @@ def test_append_result_migrates_legacy_results_header(tmp_path): assert rows[1]["hypothesis_sha256"] == "sha" +def test_worker_deploy_waits_for_unload_and_readiness(monkeypatch): + captured = [] + + class Result: + stdout = "--prefill-compute-chunk-tokens 128" + + def fake_run(command, **kwargs): + captured.append((command, kwargs)) + return Result() + + monkeypatch.setattr( + "autoresearch.prefill.supervisor._wait_port", + lambda *_args: None, + ) + monkeypatch.setattr("subprocess.run", fake_run) + deploy_candidate("allens", 128) + remote = captured[0][0][-1] + assert captured[0][1]["check"] is True + assert "worker service did not unload" in remote + assert "launchctl bootstrap" in remote + assert "nc -G 2 -z 127.0.0.1 53051" in remote + assert "a[i+1]='128'" in remote + + def test_supervisor_predeploys_before_real_strategy_proposal(): source = ( Path(__file__).resolve().parents[3]