Skip to content

Commit 9088e0e

Browse files
recuu-pfegclaude
andcommitted
feat: stall detection, worktree cleanup, plan-limits concurrency, reviewer parallel confirmed
- runner.ts: agent stall detection (3min warn, 5min kill via SIGTERM) - sprint-complete.ts: auto-cleanup orphaned worktrees on sprint complete - slot-scheduler.ts: reconfigure() for dynamic concurrency from plan limits - api-client.ts: fetchPlanLimits() for builder/cloud-engineer concurrency - run-loop.ts: fetch plan limits at startup, start stall detection - Reviewer already parallel via onExit callbacks (no changes needed) - 3 new test files (20 tests), 97 total pass Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 842fb35 commit 9088e0e

10 files changed

Lines changed: 443 additions & 4 deletions
Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
import { describe, it, expect } from "vitest";
2+
import { SlotScheduler } from "../slot-scheduler.js";
3+
4+
describe("SlotScheduler.reconfigure", () => {
5+
it("increases concurrency by adding new slots", () => {
6+
const scheduler = new SlotScheduler([
7+
{ role: "builder", maxConcurrency: 1 },
8+
]);
9+
expect(scheduler.getSlots()).toHaveLength(1);
10+
11+
scheduler.reconfigure("builder", 3);
12+
13+
const slots = scheduler.getSlots().filter((s) => s.role === "builder");
14+
expect(slots).toHaveLength(3);
15+
expect(scheduler.getMaxConcurrency("builder")).toBe(3);
16+
});
17+
18+
it("decreases concurrency by removing idle slots", () => {
19+
const scheduler = new SlotScheduler([
20+
{ role: "builder", maxConcurrency: 3 },
21+
]);
22+
expect(scheduler.getSlots().filter((s) => s.role === "builder")).toHaveLength(3);
23+
24+
scheduler.reconfigure("builder", 1);
25+
26+
const slots = scheduler.getSlots().filter((s) => s.role === "builder");
27+
expect(slots).toHaveLength(1);
28+
});
29+
30+
it("does not remove running slots", () => {
31+
const scheduler = new SlotScheduler([
32+
{ role: "builder", maxConcurrency: 3 },
33+
]);
34+
35+
// Assign tasks to 2 slots
36+
const slot1 = scheduler.acquireSlot("builder")!;
37+
scheduler.assignTask(slot1, "task-1");
38+
const slot2 = scheduler.acquireSlot("builder")!;
39+
scheduler.assignTask(slot2, "task-2");
40+
41+
// Try to reduce to 1 — only the idle slot should be removed
42+
scheduler.reconfigure("builder", 1);
43+
44+
const slots = scheduler.getSlots().filter((s) => s.role === "builder");
45+
expect(slots.length).toBe(2); // 2 running can't be removed
46+
});
47+
48+
it("no-ops when concurrency is unchanged", () => {
49+
const scheduler = new SlotScheduler([
50+
{ role: "builder", maxConcurrency: 2 },
51+
]);
52+
53+
scheduler.reconfigure("builder", 2);
54+
55+
expect(scheduler.getSlots().filter((s) => s.role === "builder")).toHaveLength(2);
56+
});
57+
58+
it("getMaxConcurrency returns updated value", () => {
59+
const scheduler = new SlotScheduler([
60+
{ role: "builder", maxConcurrency: 1 },
61+
]);
62+
expect(scheduler.getMaxConcurrency("builder")).toBe(1);
63+
64+
scheduler.reconfigure("builder", 5);
65+
expect(scheduler.getMaxConcurrency("builder")).toBe(5);
66+
});
67+
68+
it("getMaxConcurrency returns 0 for unknown role", () => {
69+
const scheduler = new SlotScheduler([]);
70+
expect(scheduler.getMaxConcurrency("unknown")).toBe(0);
71+
});
72+
73+
it("reconfigures multiple roles independently", () => {
74+
const scheduler = new SlotScheduler([
75+
{ role: "builder", maxConcurrency: 2 },
76+
{ role: "cloud-engineer", maxConcurrency: 1 },
77+
]);
78+
79+
scheduler.reconfigure("builder", 3);
80+
scheduler.reconfigure("cloud-engineer", 2);
81+
82+
expect(scheduler.getSlots().filter((s) => s.role === "builder")).toHaveLength(3);
83+
expect(scheduler.getSlots().filter((s) => s.role === "cloud-engineer")).toHaveLength(2);
84+
});
85+
});
86+
87+
describe("SlotScheduler plan-limits integration", () => {
88+
it("free tier: 1 builder, 1 cloud-engineer", () => {
89+
const scheduler = new SlotScheduler([
90+
{ role: "builder", maxConcurrency: 2 },
91+
{ role: "cloud-engineer", maxConcurrency: 1 },
92+
]);
93+
94+
// Simulate free plan limits
95+
const limits = { max_builders: 1, max_cloud_engineers: 1 };
96+
scheduler.reconfigure("builder", limits.max_builders);
97+
scheduler.reconfigure("cloud-engineer", limits.max_cloud_engineers);
98+
99+
expect(scheduler.getSlots().filter((s) => s.role === "builder")).toHaveLength(1);
100+
expect(scheduler.getSlots().filter((s) => s.role === "cloud-engineer")).toHaveLength(1);
101+
});
102+
103+
it("pro tier: 3 builders, 1 cloud-engineer", () => {
104+
const scheduler = new SlotScheduler([
105+
{ role: "builder", maxConcurrency: 2 },
106+
{ role: "cloud-engineer", maxConcurrency: 1 },
107+
]);
108+
109+
// Simulate pro plan limits
110+
const limits = { max_builders: 3, max_cloud_engineers: 1 };
111+
scheduler.reconfigure("builder", limits.max_builders);
112+
scheduler.reconfigure("cloud-engineer", limits.max_cloud_engineers);
113+
114+
expect(scheduler.getSlots().filter((s) => s.role === "builder")).toHaveLength(3);
115+
expect(scheduler.acquireSlot("builder")).not.toBeNull();
116+
});
117+
});
Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
2+
import { TIMEOUTS } from "../constants.js";
3+
4+
describe("stall detection constants", () => {
5+
it("AGENT_STALL_WARN is 3 minutes", () => {
6+
expect(TIMEOUTS.AGENT_STALL_WARN).toBe(180_000);
7+
});
8+
9+
it("AGENT_STALL_KILL is 5 minutes", () => {
10+
expect(TIMEOUTS.AGENT_STALL_KILL).toBe(300_000);
11+
});
12+
13+
it("AGENT_STALL_KILL > AGENT_STALL_WARN", () => {
14+
expect(TIMEOUTS.AGENT_STALL_KILL).toBeGreaterThan(TIMEOUTS.AGENT_STALL_WARN);
15+
});
16+
});
17+
18+
describe("stall detection logic", () => {
19+
it("detects stall when idle duration exceeds kill threshold", () => {
20+
const now = Date.now();
21+
const lastActivityAt = now - TIMEOUTS.AGENT_STALL_KILL - 1000;
22+
const idleDuration = now - lastActivityAt;
23+
24+
expect(idleDuration).toBeGreaterThanOrEqual(TIMEOUTS.AGENT_STALL_KILL);
25+
});
26+
27+
it("emits warning when idle duration exceeds warn threshold but not kill", () => {
28+
const now = Date.now();
29+
const lastActivityAt = now - TIMEOUTS.AGENT_STALL_WARN - 1000;
30+
const idleDuration = now - lastActivityAt;
31+
32+
expect(idleDuration).toBeGreaterThanOrEqual(TIMEOUTS.AGENT_STALL_WARN);
33+
expect(idleDuration).toBeLessThan(TIMEOUTS.AGENT_STALL_KILL);
34+
});
35+
36+
it("does not trigger when activity is recent", () => {
37+
const now = Date.now();
38+
const lastActivityAt = now - 10_000; // 10 seconds ago
39+
const idleDuration = now - lastActivityAt;
40+
41+
expect(idleDuration).toBeLessThan(TIMEOUTS.AGENT_STALL_WARN);
42+
});
43+
44+
it("resets stall warning on new activity", () => {
45+
let stallWarned = true;
46+
// Simulate stdout data arriving
47+
const lastActivityAt = Date.now();
48+
stallWarned = false;
49+
50+
expect(stallWarned).toBe(false);
51+
expect(lastActivityAt).toBeGreaterThan(0);
52+
});
53+
});
Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
import { describe, it, expect } from "vitest";
2+
import { parseWorktreeList } from "../commands/sprint-complete.js";
3+
4+
describe("parseWorktreeList", () => {
5+
it("parses porcelain output with branches", () => {
6+
const output = `worktree /Users/test/repo
7+
branch refs/heads/main
8+
9+
worktree /Users/test/repo/.worktrees/agent-builder-1-abc12345
10+
branch refs/heads/agent/builder-1-abc12345
11+
12+
worktree /Users/test/repo/.worktrees/agent-builder-2-def67890
13+
branch refs/heads/agent/builder-2-def67890
14+
15+
`;
16+
const result = parseWorktreeList(output);
17+
18+
expect(result).toHaveLength(3);
19+
expect(result[0]).toEqual({ path: "/Users/test/repo", branch: "refs/heads/main" });
20+
expect(result[1]).toEqual({
21+
path: "/Users/test/repo/.worktrees/agent-builder-1-abc12345",
22+
branch: "refs/heads/agent/builder-1-abc12345",
23+
});
24+
expect(result[2]).toEqual({
25+
path: "/Users/test/repo/.worktrees/agent-builder-2-def67890",
26+
branch: "refs/heads/agent/builder-2-def67890",
27+
});
28+
});
29+
30+
it("handles entries without branches (detached HEAD)", () => {
31+
const output = `worktree /Users/test/repo
32+
branch refs/heads/main
33+
34+
worktree /Users/test/repo/.worktrees/detached
35+
HEAD abc1234567890
36+
detached
37+
38+
`;
39+
const result = parseWorktreeList(output);
40+
41+
expect(result).toHaveLength(2);
42+
expect(result[0].branch).toBe("refs/heads/main");
43+
expect(result[1].branch).toBeNull();
44+
});
45+
46+
it("returns empty for empty input", () => {
47+
expect(parseWorktreeList("")).toEqual([]);
48+
});
49+
50+
it("identifies agent worktree branches", () => {
51+
const output = `worktree /repo
52+
branch refs/heads/main
53+
54+
worktree /repo/.worktrees/agent-builder-abc
55+
branch refs/heads/agent/builder-abc
56+
57+
worktree /repo/.worktrees/feature-branch
58+
branch refs/heads/feature/new-feature
59+
60+
`;
61+
const result = parseWorktreeList(output);
62+
const agentWorktrees = result.filter((w) => w.branch?.startsWith("refs/heads/agent/"));
63+
64+
expect(agentWorktrees).toHaveLength(1);
65+
expect(agentWorktrees[0].branch).toBe("refs/heads/agent/builder-abc");
66+
});
67+
});

src/api-client.ts

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,13 @@ export interface ApiClient {
115115
putAgentMemory(agentName: string, key: string, data: { type: string; content: string; shared?: boolean; tags?: string }): Promise<void>;
116116
fetchRelevantFailures(): Promise<Array<{ summary: string; failure_type: string; agent_name: string | null; created_at: string }>>;
117117
recordFailure(data: { task_id: string; failure_type: string; summary: string; agent_name?: string; sprint?: number; review_comment?: string; files_involved?: string }): Promise<void>;
118+
/** Fetch plan limits (max concurrent builders, etc.) */
119+
fetchPlanLimits(): Promise<PlanLimits>;
120+
}
121+
122+
export interface PlanLimits {
123+
max_builders: number;
124+
max_cloud_engineers: number;
118125
}
119126

120127
/** Create standard auth headers for API requests. */
@@ -368,5 +375,16 @@ export function createApiClient(apiUrl: string, apiKey: string): ApiClient {
368375
});
369376
} catch { /* best-effort */ }
370377
},
378+
379+
async fetchPlanLimits(): Promise<PlanLimits> {
380+
try {
381+
const res = await fetch(`${apiUrl}/api/v1/workspace/plan-limits`, { headers });
382+
if (res.ok) {
383+
return (await res.json()) as PlanLimits;
384+
}
385+
} catch { /* non-fatal */ }
386+
// Default: free tier limits
387+
return { max_builders: 1, max_cloud_engineers: 1 };
388+
},
371389
};
372390
}

src/commands/run-loop.ts

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,13 +95,24 @@ export async function runLoop(cliArgs: CliArgs, runner: AgentRunner, shutdownSta
9595

9696
const POLL_INTERVAL_MS = INTERVALS.POLL;
9797

98-
// Parallel agent slots
98+
// Parallel agent slots — start with defaults, then reconfigure from plan limits
9999
const { SlotScheduler } = await import("../slot-scheduler.js");
100100
const scheduler = new SlotScheduler([
101101
{ role: "builder", maxConcurrency: 2 },
102102
{ role: "cloud-engineer", maxConcurrency: 1 },
103103
]);
104104

105+
// Fetch plan limits and reconfigure scheduler
106+
try {
107+
const limits = await api.fetchPlanLimits();
108+
scheduler.reconfigure("builder", limits.max_builders);
109+
scheduler.reconfigure("cloud-engineer", limits.max_cloud_engineers);
110+
ui.info(`[plan] Builder concurrency: ${limits.max_builders}, Cloud-engineer: ${limits.max_cloud_engineers}`);
111+
} catch { /* non-fatal — use defaults */ }
112+
113+
// Start stall detection for agent processes
114+
runner.startStallDetection();
115+
105116
while (!shutdownState.shuttingDown) {
106117
try {
107118
sprintData = await api.fetchSprintData();

src/commands/shutdown.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ export function setupShutdownHandlers(runner: AgentRunner, state: ShutdownState)
2727
ui.warn("Shutting down...");
2828
state.activeWsServer?.stop().catch(() => {});
2929
state.activeManager?.stop();
30+
runner.stopStallDetection();
3031
for (const agent of runner.status()) { ui.info(`Stopping agent: ${agent.name}`); runner.stop(agent.name); }
3132
setTimeout(() => { ui.shutdown(); process.exit(0); }, 3000);
3233
};

src/commands/sprint-complete.ts

Lines changed: 65 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,34 @@
22
* Sprint complete command
33
*/
44

5-
import { createApiClient } from "../api-client.js";
5+
import { createApiClient, createAuthHeaders } from "../api-client.js";
66
import { logError, CLI_ERR } from "../error-logger.js";
77
import * as ui from "../ui.js";
88
import { execSync } from "node:child_process";
99

10+
interface WorktreeEntry {
11+
path: string;
12+
branch: string | null;
13+
}
14+
15+
/**
16+
* Parse `git worktree list --porcelain` output into structured entries.
17+
*/
18+
export function parseWorktreeList(output: string): WorktreeEntry[] {
19+
const entries: WorktreeEntry[] = [];
20+
let current: Partial<WorktreeEntry> = {};
21+
for (const line of output.split("\n")) {
22+
if (line.startsWith("worktree ")) {
23+
if (current.path) entries.push({ path: current.path, branch: current.branch ?? null });
24+
current = { path: line.slice("worktree ".length) };
25+
} else if (line.startsWith("branch ")) {
26+
current.branch = line.slice("branch ".length);
27+
}
28+
}
29+
if (current.path) entries.push({ path: current.path, branch: current.branch ?? null });
30+
return entries;
31+
}
32+
1033
export async function handleSprintComplete(apiUrl: string, apiKey: string, push: boolean): Promise<void> {
1134
const api = createApiClient(apiUrl, apiKey);
1235
ui.intro();
@@ -47,7 +70,47 @@ export async function handleSprintComplete(apiUrl: string, apiKey: string, push:
4770
process.exit(1);
4871
}
4972
}
50-
// Proposals are now triggered at retrospective phase, not on complete
73+
// Clean up orphaned worktrees
74+
try {
75+
const worktreeOutput = execSync("git worktree list --porcelain", { stdio: "pipe" }).toString();
76+
const worktrees = parseWorktreeList(worktreeOutput);
77+
let cleaned = 0;
78+
for (const wt of worktrees) {
79+
if (wt.branch?.startsWith("refs/heads/agent/")) {
80+
try {
81+
execSync(`git worktree remove "${wt.path}" --force`, { stdio: "pipe" });
82+
cleaned++;
83+
} catch { /* non-fatal — worktree may be in use */ }
84+
try {
85+
const branchName = wt.branch.replace("refs/heads/", "");
86+
execSync(`git branch -D "${branchName}"`, { stdio: "pipe" });
87+
} catch { /* non-fatal */ }
88+
}
89+
}
90+
execSync("git worktree prune", { stdio: "pipe" });
91+
if (cleaned > 0) ui.step(`Cleaned up ${cleaned} orphaned worktree(s)`);
92+
} catch (err) {
93+
ui.warn(`Failed to clean up worktrees: ${err instanceof Error ? err.message : err}`);
94+
}
95+
96+
// Rule scoring: apply decay and report stale/remove rules
97+
try {
98+
const decayRes = await fetch(`${apiUrl}/api/v1/rule-evaluations/decay`, {
99+
method: "POST",
100+
headers: createAuthHeaders(apiKey),
101+
body: JSON.stringify({ sprint_number: sprint.number }),
102+
});
103+
if (decayRes.ok) {
104+
const decay = await decayRes.json() as { rules_decayed: number; decayed: Array<{ rule_id: string; score: number; status: string }> };
105+
if (decay.rules_decayed > 0) {
106+
ui.info(`[rule-eval] Decayed ${decay.rules_decayed} rule(s)`);
107+
const stale = decay.decayed.filter((r) => r.status === "stale");
108+
const remove = decay.decayed.filter((r) => r.status === "remove");
109+
if (stale.length > 0) ui.warn(`[rule-eval] ${stale.length} rule(s) stale — consider reviewing`);
110+
if (remove.length > 0) ui.warn(`[rule-eval] ${remove.length} rule(s) below threshold — candidates for removal`);
111+
}
112+
}
113+
} catch { /* non-fatal */ }
51114

52115
ui.outro(`Sprint #${sprint.number} complete`);
53116
}

0 commit comments

Comments
 (0)