From d34ab444056260b109fb324ea3ecef64f302e6c7 Mon Sep 17 00:00:00 2001 From: Bailey Dixon Date: Mon, 13 Jul 2026 17:17:03 -0400 Subject: [PATCH] fix: recover expired runtime approvals --- CHANGELOG.md | 10 +++ DEVLOG.md | 21 +++++ package.json | 2 +- src/components/agents/run-approval-card.tsx | 10 ++- src/server/routers/agent-run.ts | 71 +++++++++++++++ .../services/__tests__/agent-run.test.ts | 87 ++++++++++++++++++- .../services/__tests__/run-dispatcher.test.ts | 69 ++++++++++++++- src/server/services/dispatch/hermes-runs.ts | 8 +- .../services/dispatch/run-dispatcher.ts | 30 +++++-- src/server/services/dispatch/types.ts | 21 +++++ tests/unit/hermes-runs.test.ts | 26 ++++++ 11 files changed, 342 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cbe8e9ae..70ad3a34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,16 @@ date, grouped by `Added` / `Changed` / `Fixed` / `Removed`. ## [Unreleased] +## [2026-07-13] — v0.10.2 · Expired approval recovery + +### Changed + +- **Approving an expired runtime session now recovers the work.** If a provider has swept or lost the run behind a permission card, Forge truthfully marks the orphan stalled and immediately starts a fresh run with the same agent and engagement mode. The operator is told that the old approval was not applied and may be requested again. + +### Fixed + +- **Runtime approval cards can no longer become immortal.** The run worker now continues polling provider-backed WAITING runs, distinguishes a missing provider run from successful completion, clears expired approval state, and surfaces the failed attempt on the issue and Command Center recovery queue. + ## [2026-07-13] — v0.10.1 · Human-action visibility ### Changed diff --git a/DEVLOG.md b/DEVLOG.md index 00cdb6eb..e84da6d9 100644 --- a/DEVLOG.md +++ b/DEVLOG.md @@ -2,6 +2,27 @@ > Append-only session log. Read at session start. Update at session end. +## 2026-07-13 — AXI-102 expired runtime approval recovery + +Diagnosed AXI-102's failed approval against Hermes: Forge retained WAITING run +`cmrjhkjbf037mo1089vzolecc`, but the gateway had already swept provider run +`run_10446861a003426f88b2c692d30a2542` and returned `run_not_found`. The +approval visibility release had intentionally excluded live approvals from +generic stale recovery, while the worker still polled only ACTIVE runs, so the +orphan could not self-reconcile. + +Hermes now reports provider 404s as explicit missing-run state rather than a +successful completion. The worker polls ACTIVE and WAITING provider runs, +retires swept approvals as STALLED, clears their pending decision data, and +posts the existing failure evidence back to issue surfaces. If an operator's +approval discovers the expiry first, Forge retires the orphan and emits a +fresh same-agent/same-mode assignment atomically; the UI explicitly says the +old approval was not applied and may be requested again. + +Verification for v0.10.2: `pnpm ci:local` passed — lint (existing repository +warnings only), typecheck, the full Vitest suite (**1,167 passed; 1 skipped**), +a fresh Next.js production build, and the full Playwright suite (**37 passed**). + ## 2026-07-13 — AXI-102 human-action visibility Traced AXI-102 from production: its intentionally unassigned Backlog issue had diff --git a/package.json b/package.json index 5c80ec06..d608a5ea 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "forge", - "version": "0.10.1", + "version": "0.10.2", "private": true, "description": "Forge — a fast, minimalist, keyboard-driven project management platform with pluggable agents.", "license": "MIT", diff --git a/src/components/agents/run-approval-card.tsx b/src/components/agents/run-approval-card.tsx index 9226147b..8d193741 100644 --- a/src/components/agents/run-approval-card.tsx +++ b/src/components/agents/run-approval-card.tsx @@ -43,8 +43,14 @@ export function RunApprovalCard({ const utils = trpc.useUtils(); const approval = readPendingApproval(pendingApproval); const respond = trpc.agentRun.respondApproval.useMutation({ - onSuccess: (_result, variables) => { - toast.success(variables.decision === "approve" ? "Permission granted" : "Run stopped"); + onSuccess: (result, variables) => { + if (result.decision === "expired") { + toast.warning("Runtime session expired; started a fresh run", { + description: "The old approval was not applied. The agent may request permission again.", + }); + } else { + toast.success(variables.decision === "approve" ? "Permission granted" : "Run stopped"); + } onResolved?.(); }, onError: (error) => toast.error(error.message), diff --git a/src/server/routers/agent-run.ts b/src/server/routers/agent-run.ts index c615866f..667ec7df 100644 --- a/src/server/routers/agent-run.ts +++ b/src/server/routers/agent-run.ts @@ -10,6 +10,7 @@ import { STALE_RUN_MS } from "@/server/services/agent-presence"; import { appendRunEvent, finishRun } from "@/server/services/agent-run"; import { resolveRunApproval } from "@/server/services/run-approval-lifecycle"; import { getRunsConnectorForAgent, resolveRunEngine } from "@/server/services/dispatch/registry"; +import { isRunNotFoundError } from "@/server/services/dispatch/types"; import { FORGE_RUN_CONTRACT_VERSION } from "@/server/services/engagement-mode"; import { buildRuntimePolicySnapshot } from "@/lib/runtime-enforcement"; import { runProtocolDiagnostics } from "@/server/services/run-protocol-diagnostics"; @@ -1224,11 +1225,13 @@ export const agentRunRouter = router({ issueId: true, agentId: true, status: true, + engagementMode: true, externalRunId: true, awaitingApprovalAt: true, agent: { select: { provider: true, + profileKey: true, runtime: { select: { adapterKey: true, endpoint: true, secret: true, config: true, disabledAt: true, name: true } }, }, }, @@ -1259,6 +1262,74 @@ export const agentRunRouter = router({ try { await connector.approve?.(run.externalRunId, input.scope); } catch (err) { + if (isRunNotFoundError(err)) { + const dispatchReason = { + mode: "APPROVAL_EXPIRED_RESTART", + picked: run.agent.profileKey, + engagementMode: run.engagementMode, + restartedFromRunId: run.id, + decidedAt: new Date().toISOString(), + }; + await ctx.db.$transaction(async (tx) => { + await resolveRunApproval(tx, { + runId: run.id, + workspaceId: ctx.workspaceId, + issueId: run.issueId, + agentId: run.agentId, + source: "operator", + decision: "expired", + currentStep: "approval target expired · restarting", + }); + await finishRun(tx, { + runId: run.id, + workspaceId: ctx.workspaceId, + issueId: run.issueId, + agentId: run.agentId, + status: "STALLED", + summary: + "The provider no longer recognizes this run. Its runtime session expired " + + "before the approval could be delivered; Forge started a fresh run.", + actorId: ctx.session.user.id, + }); + const issue = await tx.issue.findUniqueOrThrow({ + where: { id: run.issueId }, + select: { assignedAgentId: true }, + }); + await tx.issue.update({ + where: { id: run.issueId }, + data: { + assignedAgentId: run.agentId, + queued: false, + dispatchReason, + }, + }); + await recordChange(tx, { + workspaceId: ctx.workspaceId, + actorId: ctx.session.user.id, + actorAgentId: ctx.apiKey?.linkedAgentId ?? null, + entity: "Issue", + entityId: run.issueId, + action: "restart-expired-approval-run", + before: { assignedAgentId: issue.assignedAgentId }, + after: { assignedAgentId: run.agentId, engagementMode: run.engagementMode }, + eventKind: EventKind.AGENT_ASSIGNED, + subjectType: "issue", + subjectId: run.issueId, + payload: { + agentId: run.agentId, + previousAgentId: issue.assignedAgentId, + engagementMode: run.engagementMode, + restartedFromRunId: run.id, + dispatchReason, + }, + }); + }); + return { + ok: true as const, + decision: "expired" as const, + restarted: true as const, + }; + } throw new TRPCError({ code: "INTERNAL_SERVER_ERROR", message: `Approval didn't reach the runtime — the run is still blocked. ${ diff --git a/src/server/services/__tests__/agent-run.test.ts b/src/server/services/__tests__/agent-run.test.ts index cdd76113..4c6948b6 100644 --- a/src/server/services/__tests__/agent-run.test.ts +++ b/src/server/services/__tests__/agent-run.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, afterAll, afterEach } from "vitest"; +import { describe, it, expect, afterAll, afterEach, vi } from "vitest"; import { AgentRunStatus, EngagementMode, EventKind, RuntimeKind } from "@prisma/client"; import { abandonRunsForAgentReassignment, @@ -21,6 +21,7 @@ import { captureRunApproval, resolveRunApproval } from "@/server/services/run-ap const fixtures: TestFixture[] = []; afterEach(async () => { + vi.restoreAllMocks(); while (fixtures.length) { const f = fixtures.pop()!; await f.cleanup(); @@ -428,6 +429,90 @@ describe("agent-run lifecycle", () => { expect(runs[1]?.assignmentEventId).not.toBeNull(); }); + it("retires and restarts an approval whose provider run expired", async () => { + const fixture = await createWorkspaceFixture({ keyPrefix: "ARE" }); + fixtures.push(fixture); + const prisma = getPrisma(); + const runtime = await prisma.runtime.create({ + data: { + workspaceId: fixture.workspace.id, + name: "Hermes expired run", + kind: RuntimeKind.REMOTE_HTTP, + adapterKey: "hermes", + endpoint: "http://hermes.invalid/v1", + }, + }); + const agent = await createAgent(fixture.workspace.id, "are-a1"); + await prisma.agent.update({ + where: { id: agent.id }, + data: { provider: "HERMES", runtimeId: runtime.id }, + }); + const issue = await createIssue(fixture); + const run = await prisma.agentRun.create({ + data: { + workspaceId: fixture.workspace.id, + issueId: issue.id, + agentId: agent.id, + status: AgentRunStatus.WAITING, + engagementMode: EngagementMode.RESEARCH, + externalRunId: "run_expired", + awaitingApprovalAt: new Date(), + pendingApproval: { command: "git clone example", choices: ["session", "deny"] }, + }, + }); + vi.stubGlobal( + "fetch", + vi.fn(async () => + Response.json( + { error: { message: "Run not found: run_expired", code: "run_not_found" } }, + { status: 404 }, + ), + ), + ); + + const caller = agentRunRouter.createCaller(await buildContext(fixture)); + const result = await caller.respondApproval({ + runId: run.id, + decision: "approve", + scope: "session", + }); + expect(result).toMatchObject({ decision: "expired", restarted: true }); + + const expired = await prisma.agentRun.findUniqueOrThrow({ where: { id: run.id } }); + expect(expired.status).toBe(AgentRunStatus.STALLED); + expect(expired.awaitingApprovalAt).toBeNull(); + expect(expired.pendingApproval).toBeNull(); + expect(expired.summary).toMatch(/started a fresh run/i); + + const updatedIssue = await prisma.issue.findUniqueOrThrow({ where: { id: issue.id } }); + expect(updatedIssue.assignedAgentId).toBe(agent.id); + expect(updatedIssue.dispatchReason).toMatchObject({ + mode: "APPROVAL_EXPIRED_RESTART", + restartedFromRunId: run.id, + }); + const restart = await prisma.activityEvent.findFirstOrThrow({ + where: { + workspaceId: fixture.workspace.id, + kind: EventKind.AGENT_ASSIGNED, + subjectId: issue.id, + }, + orderBy: { createdAt: "desc" }, + }); + expect(restart.payload).toMatchObject({ + agentId: agent.id, + restartedFromRunId: run.id, + engagementMode: EngagementMode.RESEARCH, + }); + const fresh = await prisma.agentRun.findMany({ + where: { issueId: issue.id, agentId: agent.id }, + orderBy: { startedAt: "asc" }, + }); + expect(fresh).toHaveLength(2); + expect(fresh[1]?.status).toBe(AgentRunStatus.ACTIVE); + expect(fresh[1]?.engagementMode).toBe(EngagementMode.RESEARCH); + expect(fresh[1]?.assignmentEventId).toBe(restart.id); + }); + it("marks output started on the first substantive run event", async () => { const fixture = await createWorkspaceFixture({ keyPrefix: "ARO" }); fixtures.push(fixture); diff --git a/src/server/services/__tests__/run-dispatcher.test.ts b/src/server/services/__tests__/run-dispatcher.test.ts index 2561bdea..2e873265 100644 --- a/src/server/services/__tests__/run-dispatcher.test.ts +++ b/src/server/services/__tests__/run-dispatcher.test.ts @@ -1,5 +1,12 @@ -import { describe, it, expect, afterAll, afterEach } from "vitest"; -import { AgentProvider, EngagementMode, EventKind, RuntimeKind, RunEngine } from "@prisma/client"; +import { describe, it, expect, afterAll, afterEach, vi } from "vitest"; +import { + AgentProvider, + AgentRunStatus, + EngagementMode, + EventKind, + RuntimeKind, + RunEngine, +} from "@prisma/client"; import { recordChange } from "@/server/audit"; import { dispatchTriggerContext, @@ -16,6 +23,7 @@ import { const fixtures: TestFixture[] = []; afterEach(async () => { + vi.restoreAllMocks(); while (fixtures.length) { const f = fixtures.pop()!; await f.cleanup(); @@ -36,6 +44,63 @@ describe("runs dispatcher", () => { expect(context).toContain("plans.addSteps"); }); + it("retires a WAITING approval after the provider swept its run", async () => { + const fixture = await createWorkspaceFixture({ keyPrefix: "RDE" }); + fixtures.push(fixture); + const prisma = getPrisma(); + const runtime = await prisma.runtime.create({ + data: { + workspaceId: fixture.workspace.id, + ownerId: fixture.user.id, + name: "Hermes TTL test", + kind: RuntimeKind.REMOTE_HTTP, + adapterKey: "hermes", + endpoint: "http://hermes.invalid/v1", + providersAvailable: [AgentProvider.HERMES], + }, + }); + const agent = await prisma.agent.create({ + data: { + workspaceId: fixture.workspace.id, + name: "expired runner", + profileKey: "expired-runner", + provider: AgentProvider.HERMES, + runEngine: RunEngine.RUNS, + runtimeId: runtime.id, + status: "ONLINE", + }, + }); + const issue = await createIssue(fixture, { title: "expired approval" }); + const run = await prisma.agentRun.create({ + data: { + workspaceId: fixture.workspace.id, + issueId: issue.id, + agentId: agent.id, + status: AgentRunStatus.WAITING, + externalRunId: "run_expired_poll", + awaitingApprovalAt: new Date(), + pendingApproval: { command: "git clone example", choices: ["session", "deny"] }, + }, + }); + vi.stubGlobal( + "fetch", + vi.fn(async () => + Response.json( + { error: { message: "Run not found: run_expired_poll", code: "run_not_found" } }, + { status: 404 }, + ), + ), + ); + + const tick = await ingestRunsDispatch(); + expect(tick.polled).toBeGreaterThanOrEqual(1); + const after = await prisma.agentRun.findUniqueOrThrow({ where: { id: run.id } }); + expect(after.status).toBe(AgentRunStatus.STALLED); + expect(after.awaitingApprovalAt).toBeNull(); + expect(after.pendingApproval).toBeNull(); + expect(after.summary).toMatch(/provider no longer recognizes this run/i); + }); + it("starts the provider run when the durable inbox already precreated AgentRun", async () => { const previousE2E = process.env.FORGE_E2E; process.env.FORGE_E2E = "1"; diff --git a/src/server/services/dispatch/hermes-runs.ts b/src/server/services/dispatch/hermes-runs.ts index 99cbf747..b1338c9f 100644 --- a/src/server/services/dispatch/hermes-runs.ts +++ b/src/server/services/dispatch/hermes-runs.ts @@ -7,6 +7,7 @@ import type { RunStatus, StartedRun, } from "./types"; +import { RunNotFoundError } from "./types"; function mapStatus(raw: string): RunStatus["state"] { const s = raw.toLowerCase(); @@ -122,6 +123,7 @@ export function makeHermesRunsConnector(opts?: { if (!res.ok) { const text = await res.text().catch(() => ""); logger.warn({ externalRunId, status: res.status }, "hermes-runs: approval rejected"); + if (res.status === 404) throw new RunNotFoundError(externalRunId, text || undefined); throw new Error(text || `Hermes approval failed (${res.status})`); } }; @@ -340,8 +342,9 @@ export function makeHermesRunsConnector(opts?: { headers: { accept: "application/json", ...authHeaders() }, }); if (!res.ok) { - // 404 = run swept (TTL) — treat as completed so the poller stops. - if (res.status === 404) return { state: "completed" }; + // A swept provider run is not a successful completion. Preserve the + // distinction so Forge can retire an orphaned approval truthfully. + if (res.status === 404) return { state: "not_found" }; throw new Error(`Run status failed (${res.status})`); } const json = (await res.json()) as Record; @@ -383,6 +386,7 @@ export function makeHermesRunsConnector(opts?: { if (!res.ok) { const text = await res.text().catch(() => ""); logger.warn({ externalRunId, status: res.status }, "hermes-runs: stop rejected"); + if (res.status === 404) throw new RunNotFoundError(externalRunId, text || undefined); throw new Error(text || `Hermes stop failed (${res.status})`); } }, diff --git a/src/server/services/dispatch/run-dispatcher.ts b/src/server/services/dispatch/run-dispatcher.ts index 183ba144..52b9155e 100644 --- a/src/server/services/dispatch/run-dispatcher.ts +++ b/src/server/services/dispatch/run-dispatcher.ts @@ -604,8 +604,8 @@ async function startUnbackedAgentRuns(limit: number): Promise { * When an agent calls `runs.setWaiting({ blocking: true })` its provider turn * ends and the AgentRun goes WAITING with a stashed `externalRunId`. Nothing * else re-invokes it: `startUnbackedAgentRuns` only catches ACTIVE/unbacked - * runs and `pollActiveRuns` only polls ACTIVE — so an operator reply (the - * Nudge button or an `@agent` comment) had no effect for RUNS agents, even + * runs, while polling only mirrors the existing provider turn — so an + * operator reply (the Nudge button or an `@agent` comment) had no effect for RUNS agents, even * though the issue panel invited one. Here we pick up a WAITING run once a * comment lands *after* it paused, fold the waiting reason + the reply(s) into * a fresh turn, and flip it back to ACTIVE (the provider connector opens a new @@ -836,11 +836,17 @@ async function applyPolledCostDelta( async function pollActiveRuns(): Promise { const runs = await db.agentRun.findMany({ - where: { status: AgentRunStatus.ACTIVE, externalRunId: { not: null } }, + // A WAITING approval remains provider-owned work. Keep polling it so a + // gateway restart/TTL sweep cannot leave an immortal approval in Forge. + where: { + status: { in: [AgentRunStatus.ACTIVE, AgentRunStatus.WAITING] }, + externalRunId: { not: null }, + }, orderBy: { lastEventAt: "asc" }, take: POLL_BATCH, select: { id: true, + status: true, workspaceId: true, issueId: true, agentId: true, @@ -1002,7 +1008,11 @@ async function pollActiveRuns(): Promise { if (clearApproval) { await tx.agentRun.update({ where: { id: run.id }, - data: { awaitingApprovalAt: null }, + data: { + status: AgentRunStatus.ACTIVE, + awaitingApprovalAt: null, + pendingApproval: Prisma.DbNull, + }, }); } return; @@ -1013,9 +1023,14 @@ async function pollActiveRuns(): Promise { await tx.agentRun.update({ where: { id: run.id }, data: { + ...(run.status === AgentRunStatus.WAITING + ? { status: AgentRunStatus.ACTIVE } + : {}), lastEventAt: new Date(), ...(stepChanged ? { currentStep: step } : {}), - ...(clearApproval ? { awaitingApprovalAt: null } : {}), + ...(clearApproval + ? { awaitingApprovalAt: null, pendingApproval: Prisma.DbNull } + : {}), }, }); }) @@ -1042,6 +1057,11 @@ async function pollActiveRuns(): Promise { ? "ABANDONED" : "STALLED"; let summary = status.output ?? run.summary ?? null; + if (status.state === "not_found") { + summary = + "The provider no longer recognizes this run. Its runtime session expired or was " + + "discarded before the pending approval could be resolved; start a fresh run."; + } if (terminal === "COMPLETED" && !hasForgeCompletionMeta(run.completionMeta)) { terminal = "STALLED"; summary = diff --git a/src/server/services/dispatch/types.ts b/src/server/services/dispatch/types.ts index d10265d1..50ab3f6a 100644 --- a/src/server/services/dispatch/types.ts +++ b/src/server/services/dispatch/types.ts @@ -108,6 +108,7 @@ export interface RunStatus { | "completed" | "failed" | "cancelled" + | "not_found" | "unknown"; /** Last lifecycle event name, e.g. "tool.completed" — drives currentStep. */ lastEvent?: string; @@ -116,6 +117,26 @@ export interface RunStatus { usage?: { tokensIn?: number; tokensOut?: number; tokensCached?: number; costUsd?: number }; } +/** A provider-side run id has expired or was discarded by the runtime. */ +export class RunNotFoundError extends Error { + readonly code = "RUN_NOT_FOUND"; + + constructor( + readonly externalRunId: string, + message?: string, + ) { + super(message || `Run not found: ${externalRunId}`); + this.name = "RunNotFoundError"; + } +} + +export function isRunNotFoundError(error: unknown): error is RunNotFoundError { + return ( + error instanceof RunNotFoundError || + (error instanceof Error && "code" in error && error.code === "RUN_NOT_FOUND") + ); +} + /** * A dispatch connector for one provider family. `subscribe` resolves when * the run reaches a terminal state (or `signal` aborts); it must always diff --git a/tests/unit/hermes-runs.test.ts b/tests/unit/hermes-runs.test.ts index 261e686f..e0b05c66 100644 --- a/tests/unit/hermes-runs.test.ts +++ b/tests/unit/hermes-runs.test.ts @@ -120,4 +120,30 @@ describe("makeHermesRunsConnector", () => { body: { choice: "once" }, }); }); + + it("distinguishes an expired provider run from successful completion", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => + Response.json( + { + error: { + message: "Run not found: run_expired", + code: "run_not_found", + }, + }, + { status: 404 }, + ), + ), + ); + + const connector = makeHermesRunsConnector({ baseUrl: "http://hermes.local/v1" }); + await expect(connector.getStatus?.("run_expired")).resolves.toMatchObject({ + state: "not_found", + }); + await expect(connector.approve?.("run_expired", "session")).rejects.toMatchObject({ + code: "RUN_NOT_FOUND", + externalRunId: "run_expired", + }); + }); });