From 0a47a5c7f83681da27cedcf46b67f49f46dfbcf4 Mon Sep 17 00:00:00 2001 From: Luke Inglis Date: Thu, 20 Aug 2026 13:56:16 -0400 Subject: [PATCH 1/3] feat: show in-progress candidates in outer-loop status Add worktree scanning and agent phase detection to `factory outer-loop status` so users can see which candidates are actively being evaluated, what phase each is in, and how long they've been running. Closes #1360 Co-Authored-By: Claude Opus 4.6 (1M context) --- factory/cli/outer_loop.py | 125 ++++++++++++++++++- tests/test_outer_loop_status.py | 215 ++++++++++++++++++++++++++++++++ 2 files changed, 338 insertions(+), 2 deletions(-) create mode 100644 tests/test_outer_loop_status.py diff --git a/factory/cli/outer_loop.py b/factory/cli/outer_loop.py index f8ddb9a52..359651087 100644 --- a/factory/cli/outer_loop.py +++ b/factory/cli/outer_loop.py @@ -4,8 +4,10 @@ import argparse import json +import re import shutil import sys +import time from collections.abc import Callable from pathlib import Path from typing import TYPE_CHECKING @@ -433,6 +435,65 @@ def _cmd_evolve(args: argparse.Namespace) -> int: return 0 +def _scan_eval_worktrees(project_path: Path) -> dict[str, Path]: + """Scan .eval-worktrees/ for active worktree directories. + + Returns {label: worktree_path} where label is extracted from `wt-{label}-{uuid8hex}`. + """ + wt_base = project_path.parent / ".eval-worktrees" + result: dict[str, Path] = {} + try: + entries = list(wt_base.iterdir()) + except (FileNotFoundError, PermissionError): + return result + for entry in entries: + try: + if not entry.is_dir() or not entry.name.startswith("wt-"): + continue + except (PermissionError, OSError): + continue + name = entry.name + if re.fullmatch(r"wt-.+-[0-9a-f]{8}", name): + label = name[3:-(8 + 1)] # strip "wt-" prefix and "-{8hex}" suffix + result[label] = entry + return result + + +def _get_last_agent_phase(wt_path: Path) -> str | None: + """Read the last agent.started event from a worktree's events.jsonl.""" + events_path = wt_path / ".factory" / "events.jsonl" + try: + text = events_path.read_text() + except (FileNotFoundError, PermissionError, OSError): + return None + last_role: str | None = None + for line in text.splitlines(): + line = line.strip() + if not line: + continue + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + if event.get("type") == "agent.started": + last_role = event.get("role") + return last_role + + +def _format_elapsed(seconds: float) -> str: + """Format elapsed seconds as a human-readable string.""" + seconds = max(0.0, seconds) + total = int(seconds) + if total < 60: + return f"{total}s" + if total < 3600: + m, s = divmod(total, 60) + return f"{m}m{s:02d}s" + h, remainder = divmod(total, 3600) + m = remainder // 60 + return f"{h}h{m:02d}m" + + def _cmd_status(args: argparse.Namespace) -> int: """Show outer loop progress and metrics.""" project_path = Path(getattr(args, "project_path", ".")).resolve() @@ -455,7 +516,53 @@ def _cmd_status(args: argparse.Namespace) -> int: if state: print(f"Generation: {state.generation}") - print(f"Total evaluations: {state.total_evaluations}") + gen = state.generation + else: + gen = 0 + + modes = registry.list_modes() + gen_prefix = f"evolve-gen{gen}-" + eval_prefix = f"evolve-gen{gen}-eval-" + gen_modes = [m for m in modes if m.startswith(gen_prefix) and not m.startswith(eval_prefix)] + + runs_dir = project_path / ".factory" / "outer_loop" / "runs" + worktrees = _scan_eval_worktrees(project_path) + + completed: list[str] = [] + in_progress: list[tuple[str, str | None, float, str]] = [] + pending: list[str] = [] + now = time.time() + + for mode_name in gen_modes: + summary_path = runs_dir / mode_name / "cycle_summary.json" + if summary_path.exists(): + completed.append(mode_name) + continue + matched_wt: str | None = None + for label, wt_path in worktrees.items(): + if mode_name.startswith(label) or label.startswith(mode_name[:12]): + matched_wt = label + break + if matched_wt is not None: + wt_path = worktrees[matched_wt] + phase = _get_last_agent_phase(wt_path) + try: + elapsed = now - wt_path.stat().st_mtime + except OSError: + elapsed = 0.0 + wt_dir_name = wt_path.name + in_progress.append((mode_name, phase, elapsed, wt_dir_name)) + else: + pending.append(mode_name) + + n_in_progress = len(in_progress) + total_evals = state.total_evaluations if state else 0 + if n_in_progress: + print(f"Total evaluations: {total_evals} ({n_in_progress} in progress)") + else: + print(f"Total evaluations: {total_evals}") + + if state: print(f"Best score: {state.best_score:.4f}") print(f"Budget remaining: {state.budget_remaining}") if state.convergence_reason: @@ -465,9 +572,23 @@ def _cmd_status(args: argparse.Namespace) -> int: else: print("No checkpoint found — outer loop not started.") - modes = registry.list_modes() print(f"Ephemeral modes: {len(modes)}") + if in_progress: + print() + print("In progress:") + for mode_name, phase, elapsed, wt_dir in in_progress: + phase_str = f"[{phase}]" if phase else "[starting]" + print(f" {mode_name} {phase_str} {_format_elapsed(elapsed)} {wt_dir}") + + if completed: + print() + print(f"Completed: {len(completed)}") + + if pending: + print() + print(f"Pending: {len(pending)}") + traj_path = project_path / ".factory" / "outer_loop" / "trajectory.jsonl" if traj_path.exists(): lines = traj_path.read_text().strip().splitlines() diff --git a/tests/test_outer_loop_status.py b/tests/test_outer_loop_status.py new file mode 100644 index 000000000..7203dce68 --- /dev/null +++ b/tests/test_outer_loop_status.py @@ -0,0 +1,215 @@ +"""Tests for outer-loop status in-progress candidate detection.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from factory.cli.outer_loop import ( + _format_elapsed, + _get_last_agent_phase, + _scan_eval_worktrees, +) + + +class TestScanEvalWorktrees: + def test_no_directory(self, tmp_path: Path) -> None: + project = tmp_path / "project" + project.mkdir() + result = _scan_eval_worktrees(project) + assert result == {} + + def test_empty_directory(self, tmp_path: Path) -> None: + project = tmp_path / "project" + project.mkdir() + (tmp_path / ".eval-worktrees").mkdir() + result = _scan_eval_worktrees(project) + assert result == {} + + def test_valid_worktrees(self, tmp_path: Path) -> None: + project = tmp_path / "project" + project.mkdir() + wt_base = tmp_path / ".eval-worktrees" + wt_base.mkdir() + (wt_base / "wt-evolve-gen0-abcd1234").mkdir() + (wt_base / "wt-mymode-deadbeef").mkdir() + (wt_base / "not-a-worktree").mkdir() + (wt_base / "wt-bad").mkdir() # no uuid suffix + + result = _scan_eval_worktrees(project) + assert "evolve-gen0" in result + assert "mymode" in result + assert len(result) == 2 + + def test_label_with_hyphens(self, tmp_path: Path) -> None: + """Labels like 'evolve-gen0-0d4cc86a' where the label itself has hyphens.""" + project = tmp_path / "project" + project.mkdir() + wt_base = tmp_path / ".eval-worktrees" + wt_base.mkdir() + (wt_base / "wt-evolve-gen0-0d4cc86a-7a8111fd").mkdir() + + result = _scan_eval_worktrees(project) + assert "evolve-gen0-0d4cc86a" in result + + def test_race_condition_dir_disappears(self, tmp_path: Path) -> None: + project = tmp_path / "project" + project.mkdir() + wt_base = tmp_path / ".eval-worktrees" + wt_base.mkdir() + wt = wt_base / "wt-test-12345678" + wt.mkdir() + result = _scan_eval_worktrees(project) + assert "test" in result + + def test_permission_error(self, tmp_path: Path) -> None: + project = tmp_path / "project" + project.mkdir() + result = _scan_eval_worktrees(project) + assert result == {} + + def test_file_not_dir_skipped(self, tmp_path: Path) -> None: + project = tmp_path / "project" + project.mkdir() + wt_base = tmp_path / ".eval-worktrees" + wt_base.mkdir() + (wt_base / "wt-file-12345678").write_text("not a dir") + result = _scan_eval_worktrees(project) + assert result == {} + + +class TestGetLastAgentPhase: + def test_missing_file(self, tmp_path: Path) -> None: + assert _get_last_agent_phase(tmp_path) is None + + def test_empty_file(self, tmp_path: Path) -> None: + events_dir = tmp_path / ".factory" + events_dir.mkdir(parents=True) + (events_dir / "events.jsonl").write_text("") + assert _get_last_agent_phase(tmp_path) is None + + def test_valid_events(self, tmp_path: Path) -> None: + events_dir = tmp_path / ".factory" + events_dir.mkdir(parents=True) + lines = [ + json.dumps({"type": "agent.started", "role": "researcher"}), + json.dumps({"type": "agent.completed", "role": "researcher"}), + json.dumps({"type": "agent.started", "role": "builder"}), + ] + (events_dir / "events.jsonl").write_text("\n".join(lines) + "\n") + assert _get_last_agent_phase(tmp_path) == "builder" + + def test_no_agent_started_events(self, tmp_path: Path) -> None: + events_dir = tmp_path / ".factory" + events_dir.mkdir(parents=True) + lines = [ + json.dumps({"type": "cycle.started"}), + json.dumps({"type": "agent.completed", "role": "builder"}), + ] + (events_dir / "events.jsonl").write_text("\n".join(lines) + "\n") + assert _get_last_agent_phase(tmp_path) is None + + def test_malformed_json(self, tmp_path: Path) -> None: + events_dir = tmp_path / ".factory" + events_dir.mkdir(parents=True) + lines = [ + json.dumps({"type": "agent.started", "role": "researcher"}), + "this is not json{{{", + json.dumps({"type": "agent.started", "role": "builder"}), + ] + (events_dir / "events.jsonl").write_text("\n".join(lines) + "\n") + assert _get_last_agent_phase(tmp_path) == "builder" + + def test_blank_lines(self, tmp_path: Path) -> None: + events_dir = tmp_path / ".factory" + events_dir.mkdir(parents=True) + content = ( + json.dumps({"type": "agent.started", "role": "strategist"}) + + "\n\n\n" + ) + (events_dir / "events.jsonl").write_text(content) + assert _get_last_agent_phase(tmp_path) == "strategist" + + +class TestFormatElapsed: + @pytest.mark.parametrize( + ("seconds", "expected"), + [ + (0, "0s"), + (5, "5s"), + (59, "59s"), + (60, "1m00s"), + (61, "1m01s"), + (192, "3m12s"), + (3599, "59m59s"), + (3600, "1h00m"), + (3900, "1h05m"), + (7261, "2h01m"), + (-5, "0s"), + ], + ) + def test_format(self, seconds: float, expected: str) -> None: + assert _format_elapsed(seconds) == expected + + +class TestStatusClassification: + """Test the classification logic: completed vs in-progress vs pending.""" + + def _setup_project(self, tmp_path: Path, gen: int = 0) -> Path: + project = tmp_path / "project" + project.mkdir() + ol_dir = project / ".factory" / "outer_loop" + ol_dir.mkdir(parents=True) + return project + + def test_completed_mode(self, tmp_path: Path) -> None: + project = self._setup_project(tmp_path) + mode = "evolve-gen0-abc12345" + runs_dir = project / ".factory" / "outer_loop" / "runs" / mode + runs_dir.mkdir(parents=True) + (runs_dir / "cycle_summary.json").write_text(json.dumps({"score": 0.5})) + + summary_path = runs_dir / "cycle_summary.json" + assert summary_path.exists() + + def test_in_progress_mode(self, tmp_path: Path) -> None: + project = self._setup_project(tmp_path) + mode = "evolve-gen0-abc12345" + + wt_base = tmp_path / ".eval-worktrees" + wt_base.mkdir() + wt = wt_base / "wt-evolve-gen0-deadbeef" + wt.mkdir() + events_dir = wt / ".factory" + events_dir.mkdir(parents=True) + (events_dir / "events.jsonl").write_text( + json.dumps({"type": "agent.started", "role": "builder"}) + "\n" + ) + + worktrees = _scan_eval_worktrees(project) + assert len(worktrees) == 1 + label = list(worktrees.keys())[0] + assert label == "evolve-gen0" + + runs_dir = project / ".factory" / "outer_loop" / "runs" / mode + runs_dir.mkdir(parents=True) + summary_path = runs_dir / "cycle_summary.json" + assert not summary_path.exists() + + wt_path = worktrees[label] + phase = _get_last_agent_phase(wt_path) + assert phase == "builder" + + def test_pending_mode(self, tmp_path: Path) -> None: + project = self._setup_project(tmp_path) + mode = "evolve-gen0-abc12345" + + runs_dir = project / ".factory" / "outer_loop" / "runs" / mode + runs_dir.mkdir(parents=True) + summary_path = runs_dir / "cycle_summary.json" + assert not summary_path.exists() + + worktrees = _scan_eval_worktrees(project) + assert len(worktrees) == 0 From bd38135a1aa4d89e74b2ce0666f6782a0fb0cfc9 Mon Sep 17 00:00:00 2001 From: Luke Inglis Date: Thu, 20 Aug 2026 13:58:41 -0400 Subject: [PATCH 2/3] fix: use correct event field name 'agent' instead of 'role' in _get_last_agent_phase The emit_event() call uses agent= kwarg, producing {"agent": "builder"} in events.jsonl. _get_last_agent_phase() was reading event.get("role") which always returned None. Updated source and all test fixtures to match the actual event schema. Closes #1361 --- factory/cli/outer_loop.py | 2 +- tests/test_outer_loop_status.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/factory/cli/outer_loop.py b/factory/cli/outer_loop.py index 359651087..e0ede5607 100644 --- a/factory/cli/outer_loop.py +++ b/factory/cli/outer_loop.py @@ -476,7 +476,7 @@ def _get_last_agent_phase(wt_path: Path) -> str | None: except json.JSONDecodeError: continue if event.get("type") == "agent.started": - last_role = event.get("role") + last_role = event.get("agent") return last_role diff --git a/tests/test_outer_loop_status.py b/tests/test_outer_loop_status.py index 7203dce68..1adf6a7e6 100644 --- a/tests/test_outer_loop_status.py +++ b/tests/test_outer_loop_status.py @@ -94,9 +94,9 @@ def test_valid_events(self, tmp_path: Path) -> None: events_dir = tmp_path / ".factory" events_dir.mkdir(parents=True) lines = [ - json.dumps({"type": "agent.started", "role": "researcher"}), - json.dumps({"type": "agent.completed", "role": "researcher"}), - json.dumps({"type": "agent.started", "role": "builder"}), + json.dumps({"type": "agent.started", "agent": "researcher"}), + json.dumps({"type": "agent.completed", "agent": "researcher"}), + json.dumps({"type": "agent.started", "agent": "builder"}), ] (events_dir / "events.jsonl").write_text("\n".join(lines) + "\n") assert _get_last_agent_phase(tmp_path) == "builder" @@ -106,7 +106,7 @@ def test_no_agent_started_events(self, tmp_path: Path) -> None: events_dir.mkdir(parents=True) lines = [ json.dumps({"type": "cycle.started"}), - json.dumps({"type": "agent.completed", "role": "builder"}), + json.dumps({"type": "agent.completed", "agent": "builder"}), ] (events_dir / "events.jsonl").write_text("\n".join(lines) + "\n") assert _get_last_agent_phase(tmp_path) is None @@ -115,9 +115,9 @@ def test_malformed_json(self, tmp_path: Path) -> None: events_dir = tmp_path / ".factory" events_dir.mkdir(parents=True) lines = [ - json.dumps({"type": "agent.started", "role": "researcher"}), + json.dumps({"type": "agent.started", "agent": "researcher"}), "this is not json{{{", - json.dumps({"type": "agent.started", "role": "builder"}), + json.dumps({"type": "agent.started", "agent": "builder"}), ] (events_dir / "events.jsonl").write_text("\n".join(lines) + "\n") assert _get_last_agent_phase(tmp_path) == "builder" @@ -126,7 +126,7 @@ def test_blank_lines(self, tmp_path: Path) -> None: events_dir = tmp_path / ".factory" events_dir.mkdir(parents=True) content = ( - json.dumps({"type": "agent.started", "role": "strategist"}) + json.dumps({"type": "agent.started", "agent": "strategist"}) + "\n\n\n" ) (events_dir / "events.jsonl").write_text(content) @@ -185,7 +185,7 @@ def test_in_progress_mode(self, tmp_path: Path) -> None: events_dir = wt / ".factory" events_dir.mkdir(parents=True) (events_dir / "events.jsonl").write_text( - json.dumps({"type": "agent.started", "role": "builder"}) + "\n" + json.dumps({"type": "agent.started", "agent": "builder"}) + "\n" ) worktrees = _scan_eval_worktrees(project) From f7ba10e3a47a034a684c1aaa5cf0c059a697ec86 Mon Sep 17 00:00:00 2001 From: Luke Inglis Date: Thu, 20 Aug 2026 14:11:43 -0400 Subject: [PATCH 3/3] fix: consume matched worktrees to prevent false-positive in-progress status When one evaluation worktree existed, all modes in the same generation matched via the shared prefix (e.g. "evolve-gen0-"), causing them all to show as "in progress" instead of just the one actually evaluating. Delete the worktree key from the dict after matching so each worktree can only match one mode. Adds a regression test with 3 modes and 1 worktree. Co-Authored-By: Claude Opus 4.6 (1M context) --- factory/cli/outer_loop.py | 1 + tests/test_outer_loop_status.py | 73 +++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/factory/cli/outer_loop.py b/factory/cli/outer_loop.py index e0ede5607..722deba2e 100644 --- a/factory/cli/outer_loop.py +++ b/factory/cli/outer_loop.py @@ -545,6 +545,7 @@ def _cmd_status(args: argparse.Namespace) -> int: break if matched_wt is not None: wt_path = worktrees[matched_wt] + del worktrees[matched_wt] phase = _get_last_agent_phase(wt_path) try: elapsed = now - wt_path.stat().st_mtime diff --git a/tests/test_outer_loop_status.py b/tests/test_outer_loop_status.py index 1adf6a7e6..120dd28bc 100644 --- a/tests/test_outer_loop_status.py +++ b/tests/test_outer_loop_status.py @@ -213,3 +213,76 @@ def test_pending_mode(self, tmp_path: Path) -> None: worktrees = _scan_eval_worktrees(project) assert len(worktrees) == 0 + + def test_multi_mode_single_worktree_consume_match(self, tmp_path: Path) -> None: + """One worktree must match at most one mode; unmatched modes stay pending.""" + import time + + project = self._setup_project(tmp_path) + runs_dir = project / ".factory" / "outer_loop" / "runs" + + modes = [ + "evolve-gen0-aaa11111", + "evolve-gen0-bbb22222", + "evolve-gen0-ccc33333", + ] + + # ccc33333 is completed (has cycle_summary) + (runs_dir / modes[2]).mkdir(parents=True) + (runs_dir / modes[2] / "cycle_summary.json").write_text( + json.dumps({"score": 0.75}) + ) + # aaa11111 and bbb22222 have no cycle_summary + (runs_dir / modes[0]).mkdir(parents=True) + (runs_dir / modes[1]).mkdir(parents=True) + + # One worktree whose label is "evolve-gen0" — shared prefix for all 3 modes + wt_base = tmp_path / ".eval-worktrees" + wt_base.mkdir() + wt = wt_base / "wt-evolve-gen0-deadbeef" + wt.mkdir() + events_dir = wt / ".factory" + events_dir.mkdir(parents=True) + (events_dir / "events.jsonl").write_text( + json.dumps({"type": "agent.started", "agent": "builder"}) + "\n" + ) + + # Reproduce the classification logic from _cmd_status + worktrees = _scan_eval_worktrees(project) + assert len(worktrees) == 1 + + completed: list[str] = [] + in_progress: list[tuple[str, str | None, float, str]] = [] + pending: list[str] = [] + now = time.time() + + for mode_name in modes: + summary_path = runs_dir / mode_name / "cycle_summary.json" + if summary_path.exists(): + completed.append(mode_name) + continue + matched_wt: str | None = None + for label, wt_path in worktrees.items(): + if mode_name.startswith(label) or label.startswith(mode_name[:12]): + matched_wt = label + break + if matched_wt is not None: + wt_path = worktrees[matched_wt] + del worktrees[matched_wt] + phase = _get_last_agent_phase(wt_path) + try: + elapsed = now - wt_path.stat().st_mtime + except OSError: + elapsed = 0.0 + wt_dir_name = wt_path.name + in_progress.append((mode_name, phase, elapsed, wt_dir_name)) + else: + pending.append(mode_name) + + assert len(completed) == 1, f"Expected 1 completed, got {completed}" + assert completed[0] == "evolve-gen0-ccc33333" + assert len(in_progress) == 1, f"Expected 1 in-progress, got {in_progress}" + assert in_progress[0][0] == "evolve-gen0-aaa11111" + assert in_progress[0][1] == "builder" + assert len(pending) == 1, f"Expected 1 pending, got {pending}" + assert pending[0] == "evolve-gen0-bbb22222"