From ff5845cd3fa91d67fe5426b8625424c6f83d6b16 Mon Sep 17 00:00:00 2001 From: AlekShu <108252944+AlekShu@users.noreply.github.com> Date: Wed, 14 Oct 2026 12:24:28 +0000 Subject: [PATCH 01/23] fix(session): clear transient probe errors and stop dead probe-error records --- v2/src/session-service.ts | 65 ++++- v2/test/fast/session-service.test.ts | 402 +++++++++++++++++++++++++++ 2 files changed, 453 insertions(+), 14 deletions(-) diff --git a/v2/src/session-service.ts b/v2/src/session-service.ts index 7550f2143..24dbbb784 100644 --- a/v2/src/session-service.ts +++ b/v2/src/session-service.ts @@ -604,6 +604,12 @@ const LEGACY_RUNTIME_DIAGNOSTICS = [ "tmux process failed:", ]; +// Status whose record claims a live agent runtime. Only these carry a runtime +// probe failure as a session error; any other status ignores it. +function expectsLiveRuntime(status: SessionStatus): boolean { + return status === "running" || status === "spawning"; +} + function recoverableRuntimeError(session: SessionRecord): boolean { if (session.error === INFERRED_RUNTIME_EXIT_ERROR) { return ( @@ -611,6 +617,12 @@ function recoverableRuntimeError(session: SessionRecord): boolean { (session.errorSource === undefined && session.launchCommand !== "") ); } + return isRuntimeProbeError(session); +} + +// Retained error that only records a failed runtime probe (runtime_probe +// source, or a legacy unsourced tmux/ps diagnostic): void once a probe answers. +function isRuntimeProbeError(session: SessionRecord): boolean { if ( !session.error?.startsWith(STATUS_DETECTION_ERROR_PREFIX) || session.error.startsWith(REPORTING_DETECTION_ERROR_PREFIX) @@ -11105,6 +11117,10 @@ export class SessionService { const driftedSessions: { id: string; project: string }[] = []; for (const session of candidates) { + // A record that carried a probe error is stopped at boot but never + // reboot-restored. Read from the listed snapshot: a concurrent heal + // between list and read must not turn it into a plain drifted record. + const healedProbeError = this.probeErrorHealable(session); const runtime = await this.readRuntimeSnapshot(session); const current = readSession(this.config.dataDir, session.id); if (!current) continue; @@ -11138,7 +11154,7 @@ export class SessionService { if (persisted.status === "missing") continue; } drifted += 1; - if (reconciled.session.status === "stopped") { + if (reconciled.session.status === "stopped" && !healedProbeError) { driftedSessions.push({ id: session.id, project: session.project }); } } else { @@ -21643,7 +21659,8 @@ export class SessionService { }; } if ( - hasRetainedSessionError(session, this.lastClassifiedLogStates.get(session.id)) || + (hasRetainedSessionError(session, this.lastClassifiedLogStates.get(session.id)) && + !this.probeErrorHealable(session)) || session.serverErrorAt ) return { session, runtime }; @@ -21734,7 +21751,8 @@ export class SessionService { if (latest.status !== session.status) { return { session: latest, runtime }; } - if (hasRetainedSessionError(latest)) return { session: latest, runtime: confirmedRuntime }; + if (hasRetainedSessionError(latest) && !this.probeErrorHealable(latest)) + return { session: latest, runtime: confirmedRuntime }; const terminalUnavailable = !workspaceGone && (!confirmedRuntime.runtimeAlive || !confirmedRuntime.paneUsable); @@ -21936,6 +21954,12 @@ export class SessionService { ); } + // A retained runtime-probe error does not block reconciling a runtime that + // answers dead: the stop/errored write replaces it in one write. + private probeErrorHealable(record: SessionRecord): boolean { + return isRuntimeProbeError(record) && !this.restoreOwnsRuntimeRecovery(record.id); + } + private async readFinalTokenUsage( session: SessionRecord, runtime: SessionRuntimeSnapshot, @@ -22008,7 +22032,7 @@ export class SessionService { probeUnresponsive: true, diagnostic, }, - state: "error", + state: expectsLiveRuntime(retained.status) ? "error" : statusFallbackState(retained), source: "status", workspacePresent: false, serverError: false, @@ -22023,8 +22047,14 @@ export class SessionService { ): SessionRecord { const latest = readSession(this.config.dataDir, session.id); if (!latest) throw new SessionResourceNotFoundError(`Session not found: ${session.id}`); - if (isTerminalSessionStatus(latest.status)) return latest; const message = STATUS_DETECTION_ERROR_PREFIX + diagnostic; + // Resource readout keeps stamping any non-terminal owner: it has its own + // clear path and its retained error protects sidecars from the reaper. + if ( + isTerminalSessionStatus(latest.status) || + (!expectsLiveRuntime(latest.status) && !message.startsWith(REPORTING_DETECTION_ERROR_PREFIX)) + ) + return latest; if (latest.error?.startsWith(REPORTING_DETECTION_ERROR_PREFIX)) return latest; if (latest.error?.startsWith(STATUS_DETECTION_ERROR_PREFIX) && !recoverableRuntimeError(latest)) return latest; @@ -22145,7 +22175,17 @@ export class SessionService { const workspace = probeWorkspace(session.worktreePath); let effectiveSession = session; let runtimeOwnerChanged = false; - if (runtime.probeUnresponsive || workspace.diagnostic) { + const probeFailed = runtime.probeUnresponsive || workspace.diagnostic !== undefined; + const runtimeLive = + workspace.exists && runtime.runtimeAlive && runtime.paneUsable && runtime.processAlive; + // A live runtime voids any recoverable error. A dead one voids a probe + // error only off running/spawning: a dead running record heals inside + // reconcileUnexpectedStop, in the same write that stops it. + const clearableError = (record: SessionRecord): boolean => + runtimeLive + ? recoverableRuntimeError(record) + : !expectsLiveRuntime(record.status) && isRuntimeProbeError(record); + if (probeFailed && expectsLiveRuntime(session.status)) { effectiveSession = this.retainDetectionError( session, runtime.diagnostic ?? @@ -22156,13 +22196,10 @@ export class SessionService { : "other", ); } else if ( + !probeFailed && session.status !== "errored" && - workspace.exists && - runtime.runtimeAlive && - runtime.paneUsable && - runtime.processAlive && !session.serverErrorAt && - recoverableRuntimeError(session) + clearableError(session) ) { const latest = readSession(this.config.dataDir, session.id); if (!latest) throw new SessionResourceNotFoundError(`Session not found: ${session.id}`); @@ -22171,11 +22208,12 @@ export class SessionService { !sameSessionRuntimeOwner(latest, session) || latest.agentLaunchId !== session.agentLaunchId; if ( !isTerminalSessionStatus(latest.status) && + latest.status !== "errored" && !runtimeOwnerChanged && latest.error === session.error && latest.errorSource === session.errorSource && !latest.serverErrorAt && - recoverableRuntimeError(latest) && + clearableError(latest) && !this.restoreOwnsRuntimeRecovery(session.id) ) { const { error: _ignoredError, errorSource: _ignoredErrorSource, ...recovered } = latest; @@ -22240,8 +22278,7 @@ export class SessionService { // Set from whichever structured artifact the branches below already read. let agentActivityAt: Date | null = null; if ( - runtime.probeUnresponsive || - workspace.diagnostic || + (probeFailed && expectsLiveRuntime(effectiveSession.status)) || hasRetainedSessionError(effectiveSession) ) { state = "error"; diff --git a/v2/test/fast/session-service.test.ts b/v2/test/fast/session-service.test.ts index 45a7f53d6..83b2d63c4 100644 --- a/v2/test/fast/session-service.test.ts +++ b/v2/test/fast/session-service.test.ts @@ -19596,6 +19596,78 @@ describe("SessionService", () => { expect(result.driftedSessions).toEqual([]); }); + describe("boot heal of a runtime-probe error", () => { + const PROBE_ERROR = "Session status detection failed: tmux list-windows failed: "; + + function seedRestoreAfterRebootFleet() { + loadConfigMock.mockReturnValue({ + ...baseConfig(), + projects: { + api: { + path: "/repo/api", + defaultBranch: "main", + sessionPrefix: "api", + worktree: true, + restoreAfterReboot: true, + symlinks: [], + sidecars: {}, + sources: {}, + triggers: {}, + }, + }, + }); + const sessions = createSessionStore(); + sessions.set( + "api-1", + runningSession({ id: "api-1", error: PROBE_ERROR, errorSource: "runtime_probe" }), + ); + sessions.set("api-2", runningSession({ id: "api-2" })); + return sessions; + } + + it("stops a probe-error record at boot without handing it to reboot restore", async () => { + const sessions = seedRestoreAfterRebootFleet(); + getTmuxSessionPresenceMock.mockResolvedValue({ present: false, unresponsive: false }); + const service = await createDisposedSessionService(); + + const pending = service.reconcileStoppedSessions(); + await vi.advanceTimersByTimeAsync(1_000); + const result = await pending; + + expect(sessions.get("api-1")?.status).toBe("stopped"); + expect(sessions.get("api-1")).not.toHaveProperty("error"); + expect(sessions.get("api-1")).not.toHaveProperty("errorSource"); + expect(sessions.get("api-2")?.status).toBe("stopped"); + expect(result.drifted).toBe(2); + expect(result.driftedSessions).toEqual([{ id: "api-2", project: "api" }]); + + const restoreSpy = vi.spyOn(service, "restore").mockResolvedValue({} as never); + await service.restoreRebootedSessions(result.driftedSessions); + expect(restoreSpy).toHaveBeenCalledTimes(1); + expect(restoreSpy).toHaveBeenCalledWith("api-2"); + expect(restoreSpy).not.toHaveBeenCalledWith("api-1"); + }); + + it("keeps a record healed between list and read out of reboot restore", async () => { + const sessions = seedRestoreAfterRebootFleet(); + getTmuxSessionPresenceMock.mockImplementation(async (name: string) => { + const listed = sessions.get("api-1"); + if (name === "api-1" && listed?.error) { + const { error: _error, errorSource: _errorSource, ...healed } = listed; + sessions.set("api-1", { ...healed, status: "stopped" }); + } + return { present: false, unresponsive: false }; + }); + const service = await createDisposedSessionService(); + + const pending = service.reconcileStoppedSessions(); + await vi.advanceTimersByTimeAsync(1_000); + const result = await pending; + + expect(result.driftedSessions).toEqual([{ id: "api-2", project: "api" }]); + }); + }); + it.each(["claude", "codex", "opencode", "cursor"] as const)( "retains final %s usage when boot reconciliation runs before the first get", async (agent) => { @@ -26607,6 +26679,336 @@ describe("SessionService", () => { expect(sessions.get(record.id)?.errorSource).toBe(patch.errorSource); }); + describe("runtime probe errors on non-running and dead records", () => { + const PROBE_ERROR = "Session status detection failed: tmux list-windows failed: "; + + function failProbe(): void { + getTmuxSessionPresenceMock.mockResolvedValue({ + present: false, + unresponsive: true, + diagnostic: "tmux list-windows failed: ", + }); + } + + function answerDead(): void { + getTmuxSessionPresenceMock.mockResolvedValue({ present: false, unresponsive: false }); + } + + async function getSettled(service: { get(id: string): Promise }, id: string) { + const pending = service.get(id); + await vi.advanceTimersByTimeAsync(1_000); + return pending; + } + + function spyNotify(service: unknown) { + return vi + .spyOn( + service as { notifyAttention(view: SessionView, state: "error"): Promise }, + "notifyAttention", + ) + .mockResolvedValue(); + } + + it.each([ + { status: "stopped" as const, state: "stopped" }, + { status: "paused" as const, state: "stopped" }, + { status: "budget_limited" as const, state: "stopped" }, + { + status: "errored" as const, + state: "error", + error: "Agent runtime exited unexpectedly.", + errorSource: "inferred_runtime_exit" as const, + }, + ])("leaves a $status record untouched on a failed probe", async ({ state, ...patch }) => { + const sessions = createSessionStore(runningSession(patch)); + const before = clone(sessions.get("api-1")); + failProbe(); + const service = await createDisposedSessionService(); + + const result = await service.get("api-1"); + + expect(result.state).toBe(state); + expect(sessions.get("api-1")).toEqual(before); + expect(writeSessionMock).not.toHaveBeenCalled(); + }); + + it("leaves a stopped record untouched when the runtime probe throws", async () => { + const sessions = createSessionStore(runningSession({ status: "stopped" })); + const before = clone(sessions.get("api-1")); + tmuxSessionExistsMock.mockRejectedValue(new Error("boom")); + const service = await createDisposedSessionService(); + + expect((await service.get("api-1")).state).toBe("stopped"); + expect(sessions.get("api-1")).toEqual(before); + expect(writeSessionMock).not.toHaveBeenCalled(); + }); + + it("fires no attention notification for a stopped record under a failed probe", async () => { + createSessionStore(runningSession({ status: "stopped" })); + failProbe(); + const service = await createDisposedSessionService(); + const notify = spyNotify(service); + const internals = sessionServiceInternals(service); + + await internals.pollAttentionStates(false); + await internals.pollAttentionStates(false); + + expect(notify).not.toHaveBeenCalled(); + expect(internals.attentionStates.has("api-1")).toBe(false); + }); + + it("still stamps a running record on a failed probe", async () => { + const sessions = createSessionStore(runningSession()); + failProbe(); + const service = await createDisposedSessionService(); + + expect((await service.get("api-1")).state).toBe("error"); + expect(sessions.get("api-1")?.error).toBe(PROBE_ERROR); + expect(sessions.get("api-1")?.errorSource).toBe("runtime_probe"); + }); + + it("stops a dead running record and drops its probe error in one write", async () => { + const sessions = createSessionStore(runningSession()); + failProbe(); + const service = await createDisposedSessionService(); + const notify = spyNotify(service); + expect((await service.get("api-1")).state).toBe("error"); + expect(sessions.get("api-1")?.errorSource).toBe("runtime_probe"); + writeSessionMock.mockClear(); + + answerDead(); + const result = await getSettled(service, "api-1"); + + expect(result.state).toBe("stopped"); + expect(result.status).toBe("stopped"); + expect(sessions.get("api-1")?.status).toBe("stopped"); + expect(sessions.get("api-1")?.error).toBeUndefined(); + expect(sessions.get("api-1")?.errorSource).toBeUndefined(); + const intermediate = writeSessionMock.mock.calls + .map((call) => call[1] as SessionRecord) + .filter( + (record) => + record.id === "api-1" && record.status === "running" && record.error === undefined, + ); + expect(intermediate).toEqual([]); + expect(createTmuxSessionMock).not.toHaveBeenCalled(); + const internals = sessionServiceInternals(service); + await internals.pollAttentionStates(false); + expect(notify).not.toHaveBeenCalled(); + expect(sendDesktopNotificationMock).not.toHaveBeenCalled(); + }); + + it("errors a running probe-error record whose pane lives but agent process died", async () => { + const sessions = createSessionStore( + runningSession({ error: PROBE_ERROR, errorSource: "runtime_probe" }), + ); + tmuxSessionExistsMock.mockResolvedValue(true); + tmuxPaneDeadMock.mockResolvedValue(false); + isProcessRunningInTmuxMock.mockResolvedValue(false); + const service = await createDisposedSessionService(); + const notify = spyNotify(service); + + await getSettled(service, "api-1"); + + expect(sessions.get("api-1")).toMatchObject({ + status: "errored", + error: "Agent runtime exited unexpectedly.", + errorSource: "inferred_runtime_exit", + }); + const internals = sessionServiceInternals(service); + await internals.pollAttentionStates(false); + await internals.pollAttentionStates(false); + expect(notify).toHaveBeenCalledTimes(1); + expect(notify).toHaveBeenCalledWith(expect.objectContaining({ state: "error" }), "error"); + expect(createTmuxSessionMock).not.toHaveBeenCalled(); + }); + + it("errors a running probe-error record whose worktree is missing", async () => { + const sessions = createSessionStore( + runningSession({ error: PROBE_ERROR, errorSource: "runtime_probe" }), + ); + answerDead(); + probeWorkspaceMock.mockReturnValue({ exists: false, missing: true }); + const service = await createDisposedSessionService(); + const notify = spyNotify(service); + + await service.get("api-1"); + + expect(sessions.get("api-1")).toMatchObject({ + status: "errored", + error: "Agent worktree is missing.", + errorSource: "other", + }); + expect(createTmuxSessionMock).not.toHaveBeenCalled(); + const internals = sessionServiceInternals(service); + await internals.pollAttentionStates(false); + await internals.pollAttentionStates(false); + expect(notify).toHaveBeenCalledTimes(1); + const settled = clone(sessions.get("api-1")); + await service.get("api-1"); + expect(sessions.get("api-1")).toEqual(settled); + }); + + it("leaves a dead probe-error record untouched without runtime reconcile", async () => { + const sessions = createSessionStore( + runningSession({ error: PROBE_ERROR, errorSource: "runtime_probe" }), + ); + const before = clone(sessions.get("api-1")); + answerDead(); + const service = await createDisposedSessionService(); + const classify = sessionServiceInternals(service).classifySessionRecord as ( + session: SessionRecord, + options: { scanPane: boolean; reconcileRuntime: boolean }, + ) => Promise; + + await classify.call(service, clone(before) as SessionRecord, { + scanPane: false, + reconcileRuntime: false, + }); + + expect(sessions.get("api-1")).toEqual(before); + expect(createTmuxSessionMock).not.toHaveBeenCalled(); + }); + + it.each([ + { status: "stopped" as const, expected: "stopped" as const }, + { status: "running" as const, expected: "stopped" as const }, + ])("heals a legacy unsourced probe error on a $status record", async ({ status, expected }) => { + const sessions = createSessionStore(runningSession({ status, error: PROBE_ERROR })); + answerDead(); + const service = await createDisposedSessionService(); + + const result = await getSettled(service, "api-1"); + + expect(result.state).toBe("stopped"); + expect(sessions.get("api-1")?.status).toBe(expected); + expect(sessions.get("api-1")?.error).toBeUndefined(); + }); + + it.each([ + { + error: "Agent runtime exited unexpectedly.", + errorSource: "inferred_runtime_exit" as const, + }, + { + error: "Session status detection failed: workspace stat failed: permission denied", + errorSource: "other" as const, + }, + { error: "Session status detection failed: classifier unavailable" }, + { error: "Provider request failed" }, + { + error: PROBE_ERROR, + errorSource: "runtime_probe" as const, + serverErrorAt: "2026-03-18T10:01:00.000Z", + }, + { error: "Session status detection failed: Resource readout api-1--dev: x" }, + { status: "errored" as const, error: PROBE_ERROR, errorSource: "runtime_probe" as const }, + ])("keeps a real error on a dead record: $error ($status)", async (patch) => { + const sessions = createSessionStore(runningSession(patch)); + const before = sessions.get("api-1"); + answerDead(); + const service = await createDisposedSessionService(); + + await getSettled(service, "api-1"); + + expect(sessions.get("api-1")?.status).toBe(before?.status); + expect(sessions.get("api-1")?.error).toBe(before?.error); + expect(sessions.get("api-1")?.errorSource).toBe(before?.errorSource); + }); + + it("never relaunches or delivers queued work while a dead probe-error record heals", async () => { + const sessions = createSessionStore( + runningSession({ + error: PROBE_ERROR, + errorSource: "runtime_probe", + queuedMessages: { messages: ["held message"], awaitingPrompt: false }, + }), + ); + answerDead(); + const service = await createDisposedSessionService(); + const internals = sessionServiceInternals(service) as SessionServiceInternals & { + ensureDeliveryRunner(sessionId: string): void; + }; + + internals.ensureDeliveryRunner("api-1"); + await vi.advanceTimersByTimeAsync(1_000); + await getSettled(service, "api-1"); + await waitForRealTime(() => expect(internals.deliveryRuns.has("api-1")).toBe(false)); + + expect(sessions.get("api-1")?.status).toBe("stopped"); + expect(sessions.get("api-1")?.queuedMessages?.messages).toEqual(["held message"]); + expect(createTmuxSessionMock).not.toHaveBeenCalled(); + expect(sendMessageToTmuxMock).not.toHaveBeenCalled(); + const events = logSpurEventMock.mock.calls.map(([, entry]) => entry.event); + expect(events).not.toContain("session.recover.completed"); + expect(events).not.toContain("session.message.sent"); + }); + + it("keeps a dead probe-error record while restore owns its recovery", async () => { + const sessions = createSessionStore( + runningSession({ error: PROBE_ERROR, errorSource: "runtime_probe" }), + ); + answerDead(); + const service = await createDisposedSessionService(); + const lifecycle = (service as unknown as { lifecycle: SessionLifecycleRegister }).lifecycle; + lifecycle.begin("restore", ["api-1"]); + + await getSettled(service, "api-1"); + + expect(sessions.get("api-1")).toMatchObject({ + status: "running", + error: PROBE_ERROR, + errorSource: "runtime_probe", + }); + }); + + it("keeps stamping a stopped owner on a Resource readout failure", async () => { + const sessions = createSessionStore( + runningSession({ status: "stopped", sidecarNames: ["dev"] }), + ); + answerDead(); + const { TmuxProbeUnknownError } = await import("../../src/runtime-tmux.js"); + sidecarTmuxAliveMock.mockRejectedValue(new TmuxProbeUnknownError("resource unavailable")); + const service = await createDisposedSessionService(); + + await service.get("api-1"); + + expect(sessions.get("api-1")?.error).toMatch( + /^Session status detection failed: Resource readout /, + ); + expect(sessions.get("api-1")?.errorSource).toBe("other"); + }); + + it("refuses a send to a stopped record under a failed probe without stamping it", async () => { + const sessions = createSessionStore(runningSession({ status: "stopped" })); + failProbe(); + const service = await createDisposedSessionService(); + const { TmuxProbeUnknownError } = await import("../../src/runtime-tmux.js"); + + await expect(service.send("api-1", { message: "x" })).rejects.toBeInstanceOf( + TmuxProbeUnknownError, + ); + + expect(sessions.get("api-1")?.error).toBeUndefined(); + expect(createTmuxSessionMock).not.toHaveBeenCalled(); + }); + + it("heals a stopped probe-error record into the plain stopped record", async () => { + const plain = runningSession({ status: "stopped" }); + const sessions = createSessionStore({ + ...plain, + error: PROBE_ERROR, + errorSource: "runtime_probe", + }); + answerDead(); + const service = await createDisposedSessionService(); + + expect((await service.get("api-1")).state).toBe("stopped"); + + expect({ ...sessions.get("api-1"), updatedAt: plain.updatedAt }).toEqual(plain); + }); + }); + it("does not promote a stale errored record when the pane is dead", async () => { const sessions = createSessionStore(); sessions.set("api-1", { From 6a60147cea11b468f041d4202b1ad3dc750fe397 Mon Sep 17 00:00:00 2001 From: AlekShu <108252944+AlekShu@users.noreply.github.com> Date: Wed, 14 Oct 2026 12:51:05 +0000 Subject: [PATCH 02/23] fix(session): gate stopped wake relaunch by admission with bounded daily retry --- docs/configuration.md | 4 +- v2/src/session-service.ts | 121 +++++++++- v2/src/types.ts | 3 + v2/test/fast/metadata.test.ts | 1 + v2/test/fast/session-service.test.ts | 346 +++++++++++++++++++++++++++ 5 files changed, 469 insertions(+), 6 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index d3fd032f7..2abef5470 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -114,7 +114,7 @@ Type/constraint/default per key; full validation source `v2/src/config.ts`. ## Admission control -Cap limits concurrent live sessions at spawn, restore, and the wake of a stale-parked or memory-shed session. Over cap: `429` naming stop candidates. Reporting route: [`GET /headroom`](daemon-api.md). CLI: [`spur doctor`](commands.md#doctor). Schema `v2/src/config.ts`, enforcement `v2/src/session-service.ts`. +Cap limits concurrent live sessions at spawn, restore, the wake of a stale-parked or memory-shed session, and the interval or daily wake relaunch of a stopped or paused session. A denied interval wake waits for its next interval; a denied daily wake retries every 15 minutes until its next scheduled time, then that occurrence is skipped. Over cap: `429` naming stop candidates. Reporting route: [`GET /headroom`](daemon-api.md). CLI: [`spur doctor`](commands.md#doctor). Schema `v2/src/config.ts`, enforcement `v2/src/session-service.ts`. The shed's session stop writes `status: "stopped"`, `stopReason: "memory_shed"`. Queued trigger batches for such a session are held, not dropped, and its wake passes the admission gate. @@ -163,7 +163,7 @@ Event names by source: - Webhook request: `POST ` with `Authorization: Bearer ` and `Content-Type: application/json`; runtime behavior and limits: `v2/src/event-sources/webhook.ts`. -Other event names (see source file for trigger conditions): `trigger.spawn.suppressed`, `source.poll.disabled`, `source.poll.enabled`, `gh.poll_cycle`, `gh.usage`, `gh.poll_budget_paused`, `session.message.{sent,delivery_recovered,delivery_failed,requeued,submit_unconfirmed,queued_ahead,queue_removed,stalled}`, `session.spawn.{orphan_stopped,launch_unconfirmed,launch_submitted}`, `session.controls.delivery_recovered`, `session.todo.{nudge_failed,nudge_disabled,nudge_exhausted}`, `session.{complete,pause,self_destruct,desk_complete,handoff}.{completed,failed}`, `session.wake.{failed,daily_failed,interval_failed,sent,daily_sent,interval_sent,interval_cancelled,daily_cancelled,suppressed}`, `session.wake.token_budget_blocked`, `session.token_budget.exhausted`, `session.token_budget.unenforced`, `session.token_budget.teardown_failed`, `session.attention_monitor.{failed,session_failed,slow}`, `session.runtime.{probe_unresponsive,pane_child_fallback}`, `session.agent_process.{pane_pid_unreadable,capture_unavailable,capture_blind,survivors}`, `session.{handoff,respawn}.startup_attachment_missing`, `session.server_error.reactivation_exhausted`, `session.sidecar.{start_noop,start_rejected,launch_reap_skipped}`, `session.submit.{timeout,confirmed,released,failure_dismissed}`, `session.subscription.{delivery_failed,spawn_failed}`, `session.recover.context_unconfirmed`, `trigger.send.{failed,suppressed_admission,suppressed_launch_pending}`, `host.disk.low`, `daemon.admission.startup`, `daemon.registry.{count,warning}`, `daemon.auto_update.{started,failed,retry,skipped,suppressed,paused,config_invalid,disarm_failed}`, `daemon.deploy_switch.{started,rejected}`, `cli.update.{started,rolled_back,abandoned}`. +Other event names (see source file for trigger conditions): `trigger.spawn.suppressed`, `source.poll.disabled`, `source.poll.enabled`, `gh.poll_cycle`, `gh.usage`, `gh.poll_budget_paused`, `session.message.{sent,delivery_recovered,delivery_failed,requeued,submit_unconfirmed,queued_ahead,queue_removed,stalled}`, `session.spawn.{orphan_stopped,launch_unconfirmed,launch_submitted}`, `session.controls.delivery_recovered`, `session.todo.{nudge_failed,nudge_disabled,nudge_exhausted}`, `session.{complete,pause,self_destruct,desk_complete,handoff}.{completed,failed}`, `session.wake.{failed,daily_failed,daily_exhausted,interval_failed,sent,daily_sent,interval_sent,interval_cancelled,daily_cancelled,suppressed}`, `session.wake.token_budget_blocked`, `session.token_budget.exhausted`, `session.token_budget.unenforced`, `session.token_budget.teardown_failed`, `session.attention_monitor.{failed,session_failed,slow}`, `session.runtime.{probe_unresponsive,pane_child_fallback}`, `session.agent_process.{pane_pid_unreadable,capture_unavailable,capture_blind,survivors}`, `session.{handoff,respawn}.startup_attachment_missing`, `session.server_error.reactivation_exhausted`, `session.sidecar.{start_noop,start_rejected,launch_reap_skipped}`, `session.submit.{timeout,confirmed,released,failure_dismissed}`, `session.subscription.{delivery_failed,spawn_failed}`, `session.recover.context_unconfirmed`, `trigger.send.{failed,suppressed_admission,suppressed_launch_pending}`, `host.disk.low`, `daemon.admission.startup`, `daemon.registry.{count,warning}`, `daemon.auto_update.{started,failed,retry,skipped,suppressed,paused,config_invalid,disarm_failed}`, `daemon.deploy_switch.{started,rejected}`, `cli.update.{started,rolled_back,abandoned}`. ## Auto update diff --git a/v2/src/session-service.ts b/v2/src/session-service.ts index 24dbbb784..e71d2e715 100644 --- a/v2/src/session-service.ts +++ b/v2/src/session-service.ts @@ -508,6 +508,8 @@ import { type SharedMemoryScope, type StartSidecarRequest, type SessionRecord, + type SessionDailyWakeState, + type SessionIntervalWakeState, type CursorRestoreBoundary, type SessionSlots, type SessionStatus, @@ -647,6 +649,7 @@ const CLAUDE_SERVER_ERROR_REACTIVATION_PROMPT = const CLAUDE_SERVER_ERROR_REACTIVATION_MS = 30 * 60 * 1000; const PIPELINE_POLL_INTERVAL_MS = 1_000; const SCHEDULED_WAKE_POLL_INTERVAL_MS = 1_000; +const DAILY_WAKE_ADMISSION_RETRY_MS = 15 * 60_000; const SIDECAR_REAPER_INTERVAL_MS = 60_000; const MEMORY_SHED_INTERVAL_MS = 1_000; // Consecutive 1s ticks above the restore floor plus margin required before @@ -1402,6 +1405,17 @@ function isRestorableStatus(status: SessionStatus): boolean { return status === "running" || status === "stopped" || status === "paused"; } +// Interval or daily wake relaunch of a stopped or paused session. Stale-parked +// and memory-shed sessions keep their own gate inside ensureSessionReadyForSend. +function wakeRelaunchNeedsAdmission( + session: Pick, +): boolean { + if (session.status === "paused") return true; + return ( + session.status === "stopped" && session.stopReason !== "memory_shed" && !isStaleParked(session) + ); +} + // A launch or Send now whose submit never confirmed: the prompt may still sit // unsubmitted in the composer, and any typed send (its line clear) would erase // it. Every path that types into the pane checks this one predicate. @@ -5108,7 +5122,10 @@ export class SessionService { return null; }, ); - if (claimedNextDueAt !== null) { + if ( + claimedNextDueAt !== null && + !this.deferIntervalWakeForAdmission(session, intervalWake, claimedNextDueAt) + ) { const nextDueAt = claimedNextDueAt; try { await this.send(session.id, { @@ -5412,6 +5429,9 @@ export class SessionService { ) { continue; } + // Every claim write drops the admission retry marker: the claimed + // occurrence is the one now in flight. + const { admissionDeferredOccurrence: _deferredOccurrence, ...dailySchedule } = dailyWake; const claimedDailyNextDueAt = await this.withWorkspaceLifecycleLocks( session.id, async (): Promise => { @@ -5460,7 +5480,7 @@ export class SessionService { const skipped: SessionRecord = { ...currentDailyWakeSession, dailyWake: { - ...dailyWake, + ...dailySchedule, nextDueAt: new Date(now + 24 * 60 * 60 * 1000).toISOString(), }, updatedAt: nowIso(), @@ -5481,7 +5501,7 @@ export class SessionService { const updated: SessionRecord = { ...currentDailyWakeSession, dailyWake: { - ...dailyWake, + ...dailySchedule, nextDueAt: nextDueAt.toISOString(), }, updatedAt: nowIso(), @@ -5493,6 +5513,17 @@ export class SessionService { }, ); if (claimedDailyNextDueAt !== null) { + const admissionDenial = this.wakeAdmissionDenial(session.id); + if (admissionDenial) { + await this.deferDailyWakeForAdmission( + session, + dailyWake, + claimedDailyNextDueAt, + admissionDenial, + now, + ); + continue; + } try { await this.send(session.id, { message: this.formatDailyWakeMessage( @@ -5560,6 +5591,87 @@ export class SessionService { } } + // Runs after the occurrence claim: a denial before it would leave the wake + // due and retry every tick. + private wakeAdmissionDenial(sessionId: string): SessionAdmissionDeniedError | null { + const current = readSession(this.config.dataDir, sessionId); + if (!current || !wakeRelaunchNeedsAdmission(current)) return null; + try { + this.assertAdmissible(current.project, "wake"); + return null; + } catch (error) { + if (error instanceof SessionAdmissionDeniedError) return error; + throw error; + } + } + + // Denied interval occurrence stays consumed: the claim already advanced + // nextDueAt to the next interval. + private deferIntervalWakeForAdmission( + session: SessionRecord, + intervalWake: SessionIntervalWakeState, + nextDueAt: string, + ): boolean { + const denial = this.wakeAdmissionDenial(session.id); + if (!denial) return false; + this.logEvent("session.wake.interval_deferred", { + level: "warn", + sessionId: session.id, + projectId: session.project, + message: `Deferred interval wake for ${session.id}: ${denial.message}`, + details: { nextDueAt, intervalMs: intervalWake.intervalMs, reason: "admission" }, + }); + return true; + } + + // Denied daily occurrence retries every DAILY_WAKE_ADMISSION_RETRY_MS until + // its next scheduled slot, where the claim already points; past that it is + // skipped. The occurrence is persisted so a restart keeps the window. + private async deferDailyWakeForAdmission( + session: SessionRecord, + dailyWake: SessionDailyWakeState, + claimedNextDueAt: string, + denial: SessionAdmissionDeniedError, + now: number, + ): Promise { + const occurrence = dailyWake.admissionDeferredOccurrence ?? dailyWake.nextDueAt; + const windowEndMs = resolveNextDailyWakeAt(dailyWake.dailyAt, new Date(occurrence)).getTime(); + const retryAtMs = now + DAILY_WAKE_ADMISSION_RETRY_MS; + if (retryAtMs >= windowEndMs) { + this.logEvent("session.wake.daily_exhausted", { + level: "warn", + sessionId: session.id, + projectId: session.project, + message: `Skipped daily wake occurrence ${occurrence} for ${session.id}: ${denial.message}`, + details: { occurrence, nextDueAt: claimedNextDueAt, dailyAt: dailyWake.dailyAt }, + }); + return; + } + const retryAt = new Date(retryAtMs).toISOString(); + const deferred = await this.withWorkspaceLifecycleLocks(session.id, async () => { + const afterDenial = readSession(this.config.dataDir, session.id); + if (afterDenial?.dailyWake?.nextDueAt !== claimedNextDueAt) return false; + writeSession(this.config.dataDir, { + ...afterDenial, + dailyWake: { + ...afterDenial.dailyWake, + nextDueAt: retryAt, + admissionDeferredOccurrence: occurrence, + }, + updatedAt: nowIso(), + }); + return true; + }); + if (!deferred || dailyWake.admissionDeferredOccurrence !== undefined) return; + this.logEvent("session.wake.daily_deferred", { + level: "warn", + sessionId: session.id, + projectId: session.project, + message: `Deferred daily wake for ${session.id}: ${denial.message}`, + details: { reason: "admission", occurrence, retryAt, dailyAt: dailyWake.dailyAt }, + }); + } + previewConfigConnect(configPath: string): { config: AppConfig; registryPaths: string[]; @@ -13955,9 +14067,10 @@ export class SessionService { cause: error, }); } + const { admissionDeferredOccurrence: _deferredOccurrence, ...dailySchedule } = dailyWake; writeSession(this.config.dataDir, { ...current, - dailyWake: { ...dailyWake, nextDueAt: nextDueAt.toISOString() }, + dailyWake: { ...dailySchedule, nextDueAt: nextDueAt.toISOString() }, updatedAt: nowIso(), }); claimedNextDueAt = nextDueAt.toISOString(); diff --git a/v2/src/types.ts b/v2/src/types.ts index c07741d5f..0a6a2e13c 100644 --- a/v2/src/types.ts +++ b/v2/src/types.ts @@ -1064,6 +1064,9 @@ export interface SessionDailyWakeState { nextDueAt: string; message: string; stopCondition: string; + // Scheduled occurrence retried after an admission denial; nextDueAt is then + // the next retry. Cleared by the next claim. + admissionDeferredOccurrence?: string; } export interface SessionStateSubscription { diff --git a/v2/test/fast/metadata.test.ts b/v2/test/fast/metadata.test.ts index 0270f62d9..caf2d5b03 100644 --- a/v2/test/fast/metadata.test.ts +++ b/v2/test/fast/metadata.test.ts @@ -1774,6 +1774,7 @@ describe("session metadata PR migration", () => { nextDueAt: "2026-03-19T09:30:00.000Z", message: "Check daily state", stopCondition: "Daily checks done", + admissionDeferredOccurrence: "2026-03-19T09:30:00.000Z", }, }; diff --git a/v2/test/fast/session-service.test.ts b/v2/test/fast/session-service.test.ts index 83b2d63c4..00c5e22b2 100644 --- a/v2/test/fast/session-service.test.ts +++ b/v2/test/fast/session-service.test.ts @@ -49481,6 +49481,352 @@ describe("SessionService", () => { service.dispose(); }); + describe("wake relaunch admission", () => { + type WakeInternals = { processScheduledWakes(): Promise }; + + function capAt(maxLiveSessions: number): void { + loadConfigMock.mockReturnValue({ + ...baseConfig(), + admission: { ...baseConfig().admission, maxLiveSessions }, + }); + } + + // Per-pane runtime: a pane exists and runs its agent once it is live or + // relaunched; everything else answers dead. + function trackRuntimes(live: string[]): Set { + const panes = new Set(live); + createTmuxSessionMock.mockImplementation( + async ({ sessionName }: { sessionName: string }) => { + panes.add(sessionName); + }, + ); + tmuxSessionExistsMock.mockImplementation(async (name: string) => panes.has(name)); + isProcessRunningInTmuxMock.mockImplementation(async (name: string) => panes.has(name)); + return panes; + } + + function stoppedRecord(id: string, overrides: Partial = {}): SessionRecord { + return runningSession({ + id, + status: "stopped", + agentSessionId: `native-${id}`, + ...overrides, + }); + } + + function events(event: string, sessionId?: string) { + return logSpurEventMock.mock.calls + .map(([, entry]) => entry) + .filter( + (entry) => + entry.event === event && (sessionId === undefined || entry.sessionId === sessionId), + ); + } + + function wakeSends(text: string): number { + return sendMessageToTmuxMock.mock.calls.filter( + ([, message]) => typeof message === "string" && message.includes(text), + ).length; + } + + async function tick(service: unknown, atMs?: number): Promise { + if (atMs !== undefined) vi.setSystemTime(atMs); + await (service as WakeInternals).processScheduledWakes(); + } + + async function idleTicks(service: unknown, count: number): Promise { + for (let i = 0; i < count; i += 1) { + vi.setSystemTime(Date.now() + 1_000); + await tick(service); + } + } + + const WATCHED = [ + "session.wake.interval_deferred", + "session.wake.interval_sent", + "session.wake.daily_deferred", + "session.wake.daily_exhausted", + "session.wake.daily_sent", + "session.recover.completed", + "session.admission.denied", + ]; + function watchedCount(): number { + return WATCHED.reduce((total, event) => total + events(event).length, 0); + } + + describe("interval", () => { + const DUE = "2026-03-18T10:04:59.000Z"; + const ids = ["api-1", "api-2", "api-3", "api-4"]; + + function intervalWake(id: string) { + return { + nextDueAt: DUE, + intervalMs: 60_000, + message: `interval-${id}`, + stopCondition: "Stop condition", + }; + } + + it("relaunches only up to the cap and consumes denied occurrences", async () => { + capAt(2); + trackRuntimes([]); + mockClaudeJsonlState("waiting"); + const sessions = createSessionStore( + ...ids.map((id) => stoppedRecord(id, { intervalWake: intervalWake(id) })), + ); + const service = await createDisposedSessionService(); + + await tick(service); + + expect(events("session.recover.completed")).toHaveLength(2); + await waitForRealTime(() => { + expect(wakeSends("interval-api-1")).toBe(1); + expect(wakeSends("interval-api-2")).toBe(1); + }); + expect(wakeSends("interval-api-3")).toBe(0); + expect(wakeSends("interval-api-4")).toBe(0); + const deferred = events("session.wake.interval_deferred"); + expect(deferred.map((entry) => entry.sessionId)).toEqual(["api-3", "api-4"]); + expect(deferred.map((entry) => entry.details?.["reason"])).toEqual([ + "admission", + "admission", + ]); + expect(events("session.wake.interval_failed")).toHaveLength(0); + expect(sessions.get("api-3")?.intervalWake?.nextDueAt).toBe("2026-03-18T10:05:59.000Z"); + expect(sessions.get("api-4")?.intervalWake?.nextDueAt).toBe("2026-03-18T10:05:59.000Z"); + + const settled = watchedCount(); + await idleTicks(service, 9); + expect(watchedCount()).toBe(settled); + + service.dispose(); + const restarted = await createDisposedSessionService(); + await idleTicks(restarted, 5); + expect(watchedCount()).toBe(settled); + + await tick(restarted, Date.parse("2026-03-18T10:06:00.000Z")); + expect(events("session.wake.interval_deferred", "api-3")).toHaveLength(2); + expect(events("session.wake.interval_deferred", "api-4")).toHaveLength(2); + await waitForRealTime(() => { + expect(wakeSends("interval-api-1")).toBe(2); + expect(wakeSends("interval-api-2")).toBe(2); + }); + expect(events("session.recover.completed")).toHaveLength(2); + restarted.dispose(); + }); + + it("keeps the parked rollback for a stale-parked record", async () => { + capAt(1); + trackRuntimes(["occupier"]); + mockClaudeJsonlState("waiting"); + const sessions = createSessionStore( + runningSession({ id: "occupier" }), + stoppedRecord("api-1", { + stopReason: "stale_timeout", + intervalWake: intervalWake("api-1"), + }), + ); + const service = await createDisposedSessionService(); + + await tick(service); + + expect(sessions.get("api-1")?.intervalWake?.nextDueAt).toBe(DUE); + const deferred = events("session.wake.interval_deferred", "api-1"); + expect(deferred).toHaveLength(1); + expect(deferred[0]?.details).not.toHaveProperty("reason"); + service.dispose(); + }); + + it("gates every stopped and paused wake, healed or not", async () => { + capAt(1); + trackRuntimes(["occupier"]); + mockClaudeJsonlState("waiting"); + const sessions = createSessionStore( + runningSession({ id: "occupier" }), + stoppedRecord("api-1", { intervalWake: intervalWake("api-1") }), + stoppedRecord("api-2", { status: "paused", intervalWake: intervalWake("api-2") }), + stoppedRecord("api-3", { + status: "budget_limited", + intervalWake: intervalWake("api-3"), + }), + ); + const service = await createDisposedSessionService(); + + await tick(service); + + for (const id of ["api-1", "api-2"]) { + expect(sessions.get(id)?.intervalWake?.nextDueAt).toBe("2026-03-18T10:05:59.000Z"); + expect( + events("session.wake.interval_deferred", id).map((e) => e.details?.["reason"]), + ).toEqual(["admission"]); + } + expect(events("session.admission.denied")).toHaveLength(2); + expect(createTmuxSessionMock).not.toHaveBeenCalled(); + expect(sendMessageToTmuxMock).not.toHaveBeenCalled(); + service.dispose(); + }); + }); + + describe("daily", () => { + // Local slots on the day after the fixture records, in any time zone. + const at = (hours: number, minutes: number, seconds = 0) => + new Date(2026, 2, 19, hours, minutes, seconds).getTime(); + const iso = (ms: number) => new Date(ms).toISOString(); + + function seedDaily(overrides: Partial = {}) { + capAt(1); + const panes = trackRuntimes(["occupier"]); + mockClaudeJsonlState("waiting"); + const sessions = createSessionStore( + runningSession({ id: "occupier" }), + stoppedRecord("api-1", { + dailyWake: { + dailyAt: ["09:00", "09:40"], + nextDueAt: iso(at(9, 0)), + message: "daily-api-1", + stopCondition: "Stop condition", + ...overrides, + }, + }), + ); + const freeSlot = () => { + const occupier = sessions.get("occupier"); + if (occupier) sessions.set("occupier", { ...occupier, status: "stopped" }); + panes.delete("occupier"); + }; + const takeSlot = () => { + const occupier = sessions.get("occupier"); + if (occupier) sessions.set("occupier", { ...occupier, status: "running" }); + panes.add("occupier"); + }; + return { sessions, panes, freeSlot, takeSlot }; + } + + it("retries a denied occurrence until admitted, then sends it once", async () => { + const { sessions, freeSlot } = seedDaily(); + vi.setSystemTime(at(9, 0, 1)); + const service = await createDisposedSessionService(); + + await tick(service); + + expect(events("session.admission.denied")).toHaveLength(1); + const deferred = events("session.wake.daily_deferred"); + expect(deferred).toHaveLength(1); + expect(deferred[0]?.details?.["reason"]).toBe("admission"); + expect(wakeSends("daily-api-1")).toBe(0); + expect(sessions.get("api-1")?.dailyWake).toMatchObject({ + nextDueAt: iso(at(9, 15, 1)), + admissionDeferredOccurrence: iso(at(9, 0)), + }); + const settled = watchedCount(); + await idleTicks(service, 10); + expect(watchedCount()).toBe(settled); + + freeSlot(); + await tick(service, at(9, 15, 2)); + await waitForRealTime(() => expect(wakeSends("daily-api-1")).toBe(1)); + expect(events("session.wake.daily_sent")).toHaveLength(1); + expect(sessions.get("api-1")?.dailyWake?.nextDueAt).toBe(iso(at(9, 40))); + expect(sessions.get("api-1")?.dailyWake).not.toHaveProperty( + "admissionDeferredOccurrence", + ); + await idleTicks(service, 10); + expect(wakeSends("daily-api-1")).toBe(1); + service.dispose(); + }); + + it("skips the occurrence once its retry window reaches the next slot", async () => { + const { sessions } = seedDaily(); + vi.setSystemTime(at(9, 0, 1)); + const service = await createDisposedSessionService(); + + await tick(service); + await idleTicks(service, 10); + await tick(service, at(9, 15, 2)); + await idleTicks(service, 10); + await tick(service, at(9, 30, 3)); + const settled = watchedCount(); + await idleTicks(service, 10); + expect(watchedCount()).toBe(settled); + + expect(wakeSends("daily-api-1")).toBe(0); + expect(events("session.admission.denied")).toHaveLength(3); + expect(events("session.wake.daily_deferred")).toHaveLength(1); + const exhausted = events("session.wake.daily_exhausted"); + expect(exhausted).toHaveLength(1); + expect(exhausted[0]?.details?.["occurrence"]).toBe(iso(at(9, 0))); + expect(sessions.get("api-1")?.dailyWake?.nextDueAt).toBe(iso(at(9, 40))); + expect(sessions.get("api-1")?.dailyWake).not.toHaveProperty( + "admissionDeferredOccurrence", + ); + service.dispose(); + }); + + it("keeps the retry occurrence and budget across a restart", async () => { + const { sessions, freeSlot } = seedDaily(); + vi.setSystemTime(at(9, 0, 1)); + const first = await createDisposedSessionService(); + await tick(first); + first.dispose(); + + const restarted = await createDisposedSessionService(); + const settled = watchedCount(); + await idleTicks(restarted, 5); + expect(watchedCount()).toBe(settled); + + await tick(restarted, at(9, 15, 2)); + expect(events("session.admission.denied")).toHaveLength(2); + expect(events("session.wake.daily_deferred")).toHaveLength(1); + expect(sessions.get("api-1")?.dailyWake?.admissionDeferredOccurrence).toBe(iso(at(9, 0))); + + freeSlot(); + await tick(restarted, at(9, 30, 3)); + await waitForRealTime(() => expect(wakeSends("daily-api-1")).toBe(1)); + await idleTicks(restarted, 5); + expect(wakeSends("daily-api-1")).toBe(1); + restarted.dispose(); + }); + + it("starts a fresh retry budget for the next occurrence", async () => { + const { sessions, panes, freeSlot, takeSlot } = seedDaily(); + vi.setSystemTime(at(9, 0, 1)); + const service = await createDisposedSessionService(); + await tick(service); + freeSlot(); + await tick(service, at(9, 15, 2)); + await waitForRealTime(() => expect(wakeSends("daily-api-1")).toBe(1)); + + const woken = sessions.get("api-1"); + if (woken) sessions.set("api-1", { ...woken, status: "stopped" }); + panes.delete("api-1"); + takeSlot(); + await tick(service, at(9, 40, 1)); + + const deferred = events("session.wake.daily_deferred"); + expect(deferred).toHaveLength(2); + expect(deferred[1]?.details?.["occurrence"]).toBe(iso(at(9, 40))); + service.dispose(); + }); + + it("drops the retry marker on a manual daily dispatch", async () => { + const { sessions, freeSlot } = seedDaily({ + nextDueAt: iso(at(9, 15, 1)), + admissionDeferredOccurrence: iso(at(9, 0)), + }); + freeSlot(); + vi.setSystemTime(at(9, 5)); + const service = await createDisposedSessionService(); + + await service.dispatchWake("api-1", { target: "daily", dispatch: true }); + + expect(sessions.get("api-1")?.dailyWake).not.toHaveProperty( + "admissionDeferredOccurrence", + ); + service.dispose(); + }); + }); + }); + it("fires a far-past-due daily wake once and jumps to the next occurrence", async () => { const sessions = seedShepherdSession({ dailyWake: { From d17d197ad7c0ca6f116f161a51a0a485c2e33130 Mon Sep 17 00:00:00 2001 From: AlekShu <108252944+AlekShu@users.noreply.github.com> Date: Wed, 14 Oct 2026 13:05:01 +0000 Subject: [PATCH 03/23] fix(session): report daily wake occurrence on admitted retry --- v2/src/session-service.ts | 16 ++++++++++------ v2/test/fast/session-service.test.ts | 3 +++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/v2/src/session-service.ts b/v2/src/session-service.ts index e71d2e715..460bdc6f6 100644 --- a/v2/src/session-service.ts +++ b/v2/src/session-service.ts @@ -606,6 +606,13 @@ const LEGACY_RUNTIME_DIAGNOSTICS = [ "tmux process failed:", ]; +// Daily wake schedule without the admission retry marker: every claim write +// drops it, the claimed occurrence is the one now in flight. +function withoutAdmissionDeferral(dailyWake: SessionDailyWakeState): SessionDailyWakeState { + const { admissionDeferredOccurrence: _deferredOccurrence, ...schedule } = dailyWake; + return schedule; +} + // Status whose record claims a live agent runtime. Only these carry a runtime // probe failure as a session error; any other status ignores it. function expectsLiveRuntime(status: SessionStatus): boolean { @@ -5429,9 +5436,7 @@ export class SessionService { ) { continue; } - // Every claim write drops the admission retry marker: the claimed - // occurrence is the one now in flight. - const { admissionDeferredOccurrence: _deferredOccurrence, ...dailySchedule } = dailyWake; + const dailySchedule = withoutAdmissionDeferral(dailyWake); const claimedDailyNextDueAt = await this.withWorkspaceLifecycleLocks( session.id, async (): Promise => { @@ -5538,7 +5543,7 @@ export class SessionService { projectId: session.project, message: `Sent daily wake to ${session.id}`, details: { - nextDueAt: dailyWake.nextDueAt, + nextDueAt: dailyWake.admissionDeferredOccurrence ?? dailyWake.nextDueAt, dailyAt: dailyWake.dailyAt, }, }); @@ -14067,10 +14072,9 @@ export class SessionService { cause: error, }); } - const { admissionDeferredOccurrence: _deferredOccurrence, ...dailySchedule } = dailyWake; writeSession(this.config.dataDir, { ...current, - dailyWake: { ...dailySchedule, nextDueAt: nextDueAt.toISOString() }, + dailyWake: { ...withoutAdmissionDeferral(dailyWake), nextDueAt: nextDueAt.toISOString() }, updatedAt: nowIso(), }); claimedNextDueAt = nextDueAt.toISOString(); diff --git a/v2/test/fast/session-service.test.ts b/v2/test/fast/session-service.test.ts index 00c5e22b2..c4e913f4f 100644 --- a/v2/test/fast/session-service.test.ts +++ b/v2/test/fast/session-service.test.ts @@ -26781,6 +26781,8 @@ describe("SessionService", () => { expect(result.state).toBe("stopped"); expect(result.status).toBe("stopped"); + const { isRestorableSession } = await loadSessionServiceModule(); + expect(isRestorableSession({ ...result, workspaceExists: true })).toBe(true); expect(sessions.get("api-1")?.status).toBe("stopped"); expect(sessions.get("api-1")?.error).toBeUndefined(); expect(sessions.get("api-1")?.errorSource).toBeUndefined(); @@ -49726,6 +49728,7 @@ describe("SessionService", () => { await tick(service, at(9, 15, 2)); await waitForRealTime(() => expect(wakeSends("daily-api-1")).toBe(1)); expect(events("session.wake.daily_sent")).toHaveLength(1); + expect(events("session.wake.daily_sent")[0]?.details?.["nextDueAt"]).toBe(iso(at(9, 0))); expect(sessions.get("api-1")?.dailyWake?.nextDueAt).toBe(iso(at(9, 40))); expect(sessions.get("api-1")?.dailyWake).not.toHaveProperty( "admissionDeferredOccurrence", From 84570326f80b22fc96ce643f5b10e119981a1257 Mon Sep 17 00:00:00 2001 From: AlekShu <108252944+AlekShu@users.noreply.github.com> Date: Wed, 14 Oct 2026 13:14:30 +0000 Subject: [PATCH 04/23] test(session): pin probe heal and wake admission boundaries --- v2/test/fast/session-service.test.ts | 158 +++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) diff --git a/v2/test/fast/session-service.test.ts b/v2/test/fast/session-service.test.ts index c4e913f4f..170aaf291 100644 --- a/v2/test/fast/session-service.test.ts +++ b/v2/test/fast/session-service.test.ts @@ -26964,6 +26964,79 @@ describe("SessionService", () => { }); }); + it("keeps a stopped probe error that carries serverErrorAt", async () => { + const sessions = createSessionStore( + runningSession({ + status: "stopped", + error: PROBE_ERROR, + errorSource: "runtime_probe", + serverErrorAt: "2026-03-18T10:01:00.000Z", + }), + ); + const before = clone(sessions.get("api-1")); + answerDead(); + const service = await createDisposedSessionService(); + + await service.get("api-1"); + + expect(sessions.get("api-1")).toEqual(before); + }); + + it("keeps a stopped probe error while restore owns its recovery", async () => { + const sessions = createSessionStore( + runningSession({ status: "stopped", error: PROBE_ERROR, errorSource: "runtime_probe" }), + ); + const before = clone(sessions.get("api-1")); + answerDead(); + const service = await createDisposedSessionService(); + const lifecycle = (service as unknown as { lifecycle: SessionLifecycleRegister }).lifecycle; + lifecycle.begin("restore", ["api-1"]); + + await getSettled(service, "api-1"); + + expect(sessions.get("api-1")).toEqual(before); + }); + + it.each(["paused", "budget_limited"] as const)( + "drops a probe error from a %s record once the probe answers", + async (status) => { + const sessions = createSessionStore( + runningSession({ status, error: PROBE_ERROR, errorSource: "runtime_probe" }), + ); + answerDead(); + const service = await createDisposedSessionService(); + + await getSettled(service, "api-1"); + + expect(sessions.get("api-1")?.status).toBe(status); + expect(sessions.get("api-1")?.error).toBeUndefined(); + expect(sessions.get("api-1")?.errorSource).toBeUndefined(); + }, + ); + + it.each([ + "runtime state unknown", + "runtime probe (tmux or ps) timed out; runtime state unknown", + "tmux list-windows failed: x", + "tmux list-panes failed: x", + "ps failed: x", + "tmux process failed: x", + ])("heals a legacy unsourced stopped record carrying: %s", async (diagnostic) => { + const sessions = createSessionStore( + runningSession({ + status: "stopped", + error: `Session status detection failed: ${diagnostic}`, + }), + ); + answerDead(); + const service = await createDisposedSessionService(); + + await getSettled(service, "api-1"); + + expect(sessions.get("api-1")?.status).toBe("stopped"); + expect(sessions.get("api-1")?.error).toBeUndefined(); + }); + it("keeps stamping a stopped owner on a Resource readout failure", async () => { const sessions = createSessionStore( runningSession({ status: "stopped", sidecarNames: ["dev"] }), @@ -49639,6 +49712,28 @@ describe("SessionService", () => { service.dispose(); }); + it("keeps the parked rollback for a memory-shed record", async () => { + capAt(1); + trackRuntimes(["occupier"]); + mockClaudeJsonlState("waiting"); + const sessions = createSessionStore( + runningSession({ id: "occupier" }), + stoppedRecord("api-1", { + stopReason: "memory_shed", + intervalWake: intervalWake("api-1"), + }), + ); + const service = await createDisposedSessionService(); + + await tick(service); + + expect(sessions.get("api-1")?.intervalWake?.nextDueAt).toBe(DUE); + const deferred = events("session.wake.interval_deferred", "api-1"); + expect(deferred).toHaveLength(1); + expect(deferred[0]?.details).not.toHaveProperty("reason"); + service.dispose(); + }); + it("gates every stopped and paused wake, healed or not", async () => { capAt(1); trackRuntimes(["occupier"]); @@ -49811,6 +49906,69 @@ describe("SessionService", () => { service.dispose(); }); + it.each([ + { + name: "re-armed", + rearm: (record: SessionRecord): SessionRecord => ({ + ...record, + dailyWake: { + dailyAt: ["09:50"], + nextDueAt: iso(at(9, 50)), + message: "rearmed", + stopCondition: "Stop condition", + }, + }), + }, + { + name: "cancelled", + rearm: (record: SessionRecord): SessionRecord => { + const { dailyWake: _cancelled, ...rest } = record; + return rest; + }, + }, + ])("keeps a $name daily wake when the denial write loses the claim", async ({ rearm }) => { + const { sessions } = seedDaily(); + vi.setSystemTime(at(9, 0, 1)); + const service = await createDisposedSessionService(); + const internals = service as unknown as { + assertAdmissible(...args: unknown[]): unknown; + }; + const original = internals.assertAdmissible.bind(service); + vi.spyOn(internals, "assertAdmissible").mockImplementation((...args: unknown[]) => { + const current = sessions.get("api-1"); + if (current) sessions.set("api-1", rearm(current)); + return original(...args); + }); + + await tick(service); + + expect(events("session.admission.denied")).toHaveLength(1); + expect(events("session.wake.daily_deferred")).toHaveLength(0); + expect(sessions.get("api-1")?.dailyWake).toEqual(rearm(stoppedRecord("api-1")).dailyWake); + service.dispose(); + }); + + it("drops the retry marker when a malformed dailyAt skips the occurrence", async () => { + const { sessions } = seedDaily({ + dailyAt: ["99:99"], + nextDueAt: iso(at(9, 15, 1)), + admissionDeferredOccurrence: iso(at(9, 0)), + }); + vi.setSystemTime(at(9, 15, 2)); + const service = await createDisposedSessionService(); + + await tick(service); + + expect(events("session.wake.daily_failed")).toHaveLength(1); + expect(sessions.get("api-1")?.dailyWake).toMatchObject({ + nextDueAt: iso(at(9, 15, 2) + 24 * 60 * 60 * 1000), + }); + expect(sessions.get("api-1")?.dailyWake).not.toHaveProperty( + "admissionDeferredOccurrence", + ); + service.dispose(); + }); + it("drops the retry marker on a manual daily dispatch", async () => { const { sessions, freeSlot } = seedDaily({ nextDueAt: iso(at(9, 15, 1)), From 471182a9e618275d3920f7316cbd54862de63d83 Mon Sep 17 00:00:00 2001 From: AlekShu <108252944+AlekShu@users.noreply.github.com> Date: Wed, 14 Oct 2026 13:20:31 +0000 Subject: [PATCH 05/23] fix(session): defer one wake record when its admission check throws --- v2/src/session-service.ts | 10 ++-- v2/test/fast/session-service.test.ts | 87 ++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 4 deletions(-) diff --git a/v2/src/session-service.ts b/v2/src/session-service.ts index 460bdc6f6..3ec43a6b3 100644 --- a/v2/src/session-service.ts +++ b/v2/src/session-service.ts @@ -5597,8 +5597,9 @@ export class SessionService { } // Runs after the occurrence claim: a denial before it would leave the wake - // due and retry every tick. - private wakeAdmissionDenial(sessionId: string): SessionAdmissionDeniedError | null { + // due and retry every tick. An unexpected check failure denies only this + // record: rethrowing would abort the wake loop for every later session. + private wakeAdmissionDenial(sessionId: string): Error | null { const current = readSession(this.config.dataDir, sessionId); if (!current || !wakeRelaunchNeedsAdmission(current)) return null; try { @@ -5606,7 +5607,8 @@ export class SessionService { return null; } catch (error) { if (error instanceof SessionAdmissionDeniedError) return error; - throw error; + const message = error instanceof Error ? error.message : String(error); + return new Error(`admission check failed: ${message}`); } } @@ -5636,7 +5638,7 @@ export class SessionService { session: SessionRecord, dailyWake: SessionDailyWakeState, claimedNextDueAt: string, - denial: SessionAdmissionDeniedError, + denial: Error, now: number, ): Promise { const occurrence = dailyWake.admissionDeferredOccurrence ?? dailyWake.nextDueAt; diff --git a/v2/test/fast/session-service.test.ts b/v2/test/fast/session-service.test.ts index 170aaf291..108ad5e72 100644 --- a/v2/test/fast/session-service.test.ts +++ b/v2/test/fast/session-service.test.ts @@ -49629,6 +49629,93 @@ describe("SessionService", () => { return WATCHED.reduce((total, event) => total + events(event).length, 0); } + describe("unexpected admission check failure", () => { + function failFirstCheck(service: unknown): void { + vi.spyOn( + service as { assertAdmissible(...args: unknown[]): unknown }, + "assertAdmissible", + ).mockImplementationOnce(() => { + throw new Error("host memory read blew up"); + }); + } + + function failureEvents(kind: "interval" | "daily") { + return [`session.wake.${kind}_deferred`, `session.wake.${kind}_failed`].flatMap((name) => + events(name), + ); + } + + it("defers only the failing interval record and still wakes the next one", async () => { + capAt(10); + trackRuntimes([]); + mockClaudeJsonlState("waiting"); + const due = "2026-03-18T10:04:59.000Z"; + const wake = (id: string) => ({ + nextDueAt: due, + intervalMs: 60_000, + message: `interval-${id}`, + stopCondition: "Stop condition", + }); + const sessions = createSessionStore( + stoppedRecord("api-1", { intervalWake: wake("api-1") }), + stoppedRecord("api-2", { intervalWake: wake("api-2") }), + ); + const service = await createDisposedSessionService(); + failFirstCheck(service); + + await tick(service); + + await waitForRealTime(() => expect(wakeSends("interval-api-2")).toBe(1)); + expect(wakeSends("interval-api-1")).toBe(0); + const logged = failureEvents("interval"); + expect(logged).toHaveLength(1); + expect(logged[0]).toMatchObject({ + event: "session.wake.interval_deferred", + sessionId: "api-1", + }); + expect(logged[0]?.message).toContain("host memory read blew up"); + expect(sessions.get("api-1")?.intervalWake?.nextDueAt).toBe("2026-03-18T10:05:59.000Z"); + service.dispose(); + }); + + it("defers only the failing daily record and still wakes the next one", async () => { + capAt(10); + trackRuntimes([]); + mockClaudeJsonlState("waiting"); + const at = (hours: number, minutes: number, seconds = 0) => + new Date(2026, 2, 19, hours, minutes, seconds).getTime(); + const wake = (id: string) => ({ + dailyAt: ["09:00", "09:40"], + nextDueAt: new Date(at(9, 0)).toISOString(), + message: `daily-${id}`, + stopCondition: "Stop condition", + }); + const sessions = createSessionStore( + stoppedRecord("api-1", { dailyWake: wake("api-1") }), + stoppedRecord("api-2", { dailyWake: wake("api-2") }), + ); + vi.setSystemTime(at(9, 0, 1)); + const service = await createDisposedSessionService(); + failFirstCheck(service); + + await tick(service); + + await waitForRealTime(() => expect(wakeSends("daily-api-2")).toBe(1)); + expect(wakeSends("daily-api-1")).toBe(0); + const logged = failureEvents("daily"); + expect(logged).toHaveLength(1); + expect(logged[0]).toMatchObject({ + event: "session.wake.daily_deferred", + sessionId: "api-1", + }); + expect(logged[0]?.message).toContain("host memory read blew up"); + expect(sessions.get("api-1")?.dailyWake?.admissionDeferredOccurrence).toBe( + new Date(at(9, 0)).toISOString(), + ); + service.dispose(); + }); + }); + describe("interval", () => { const DUE = "2026-03-18T10:04:59.000Z"; const ids = ["api-1", "api-2", "api-3", "api-4"]; From c27db4db3a4df5271bb8b9dda6d41a7234fbabcc Mon Sep 17 00:00:00 2001 From: AlekShu <108252944+AlekShu@users.noreply.github.com> Date: Wed, 14 Oct 2026 13:30:38 +0000 Subject: [PATCH 06/23] fix(session): label admission check failures on deferred wakes --- v2/src/session-service.ts | 41 ++++++++++++++++++++-------- v2/test/fast/session-service.test.ts | 4 +++ 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/v2/src/session-service.ts b/v2/src/session-service.ts index 3ec43a6b3..578f9234c 100644 --- a/v2/src/session-service.ts +++ b/v2/src/session-service.ts @@ -1412,6 +1412,16 @@ function isRestorableStatus(status: SessionStatus): boolean { return status === "running" || status === "stopped" || status === "paused"; } +type WakeAdmissionDenial = { kind: "denied" | "check_failed"; error: Error }; + +function wakeAdmissionReason(denial: WakeAdmissionDenial): string { + return denial.kind === "denied" ? "admission" : "admission_check_failed"; +} + +function wakeAdmissionLogLevel(denial: WakeAdmissionDenial): "warn" | "error" { + return denial.kind === "denied" ? "warn" : "error"; +} + // Interval or daily wake relaunch of a stopped or paused session. Stale-parked // and memory-shed sessions keep their own gate inside ensureSessionReadyForSend. function wakeRelaunchNeedsAdmission( @@ -5599,16 +5609,16 @@ export class SessionService { // Runs after the occurrence claim: a denial before it would leave the wake // due and retry every tick. An unexpected check failure denies only this // record: rethrowing would abort the wake loop for every later session. - private wakeAdmissionDenial(sessionId: string): Error | null { + private wakeAdmissionDenial(sessionId: string): WakeAdmissionDenial | null { const current = readSession(this.config.dataDir, sessionId); if (!current || !wakeRelaunchNeedsAdmission(current)) return null; try { this.assertAdmissible(current.project, "wake"); return null; } catch (error) { - if (error instanceof SessionAdmissionDeniedError) return error; + if (error instanceof SessionAdmissionDeniedError) return { kind: "denied", error }; const message = error instanceof Error ? error.message : String(error); - return new Error(`admission check failed: ${message}`); + return { kind: "check_failed", error: new Error(`admission check failed: ${message}`) }; } } @@ -5622,11 +5632,15 @@ export class SessionService { const denial = this.wakeAdmissionDenial(session.id); if (!denial) return false; this.logEvent("session.wake.interval_deferred", { - level: "warn", + level: wakeAdmissionLogLevel(denial), sessionId: session.id, projectId: session.project, - message: `Deferred interval wake for ${session.id}: ${denial.message}`, - details: { nextDueAt, intervalMs: intervalWake.intervalMs, reason: "admission" }, + message: `Deferred interval wake for ${session.id}: ${denial.error.message}`, + details: { + nextDueAt, + intervalMs: intervalWake.intervalMs, + reason: wakeAdmissionReason(denial), + }, }); return true; } @@ -5638,7 +5652,7 @@ export class SessionService { session: SessionRecord, dailyWake: SessionDailyWakeState, claimedNextDueAt: string, - denial: Error, + denial: WakeAdmissionDenial, now: number, ): Promise { const occurrence = dailyWake.admissionDeferredOccurrence ?? dailyWake.nextDueAt; @@ -5649,7 +5663,7 @@ export class SessionService { level: "warn", sessionId: session.id, projectId: session.project, - message: `Skipped daily wake occurrence ${occurrence} for ${session.id}: ${denial.message}`, + message: `Skipped daily wake occurrence ${occurrence} for ${session.id}: ${denial.error.message}`, details: { occurrence, nextDueAt: claimedNextDueAt, dailyAt: dailyWake.dailyAt }, }); return; @@ -5671,11 +5685,16 @@ export class SessionService { }); if (!deferred || dailyWake.admissionDeferredOccurrence !== undefined) return; this.logEvent("session.wake.daily_deferred", { - level: "warn", + level: wakeAdmissionLogLevel(denial), sessionId: session.id, projectId: session.project, - message: `Deferred daily wake for ${session.id}: ${denial.message}`, - details: { reason: "admission", occurrence, retryAt, dailyAt: dailyWake.dailyAt }, + message: `Deferred daily wake for ${session.id}: ${denial.error.message}`, + details: { + reason: wakeAdmissionReason(denial), + occurrence, + retryAt, + dailyAt: dailyWake.dailyAt, + }, }); } diff --git a/v2/test/fast/session-service.test.ts b/v2/test/fast/session-service.test.ts index 108ad5e72..b85aa5f87 100644 --- a/v2/test/fast/session-service.test.ts +++ b/v2/test/fast/session-service.test.ts @@ -49672,6 +49672,8 @@ describe("SessionService", () => { expect(logged[0]).toMatchObject({ event: "session.wake.interval_deferred", sessionId: "api-1", + level: "error", + details: { reason: "admission_check_failed" }, }); expect(logged[0]?.message).toContain("host memory read blew up"); expect(sessions.get("api-1")?.intervalWake?.nextDueAt).toBe("2026-03-18T10:05:59.000Z"); @@ -49707,6 +49709,8 @@ describe("SessionService", () => { expect(logged[0]).toMatchObject({ event: "session.wake.daily_deferred", sessionId: "api-1", + level: "error", + details: { reason: "admission_check_failed" }, }); expect(logged[0]?.message).toContain("host memory read blew up"); expect(sessions.get("api-1")?.dailyWake?.admissionDeferredOccurrence).toBe( From a141af8ce9ea1e0630df31f86817481edc787eca Mon Sep 17 00:00:00 2001 From: AlekShu <108252944+AlekShu@users.noreply.github.com> Date: Fri, 16 Oct 2026 04:59:40 +0000 Subject: [PATCH 07/23] fix(session): recover historical process probe errors --- v2/src/session-service.ts | 8 + v2/test/fast/session-service.test.ts | 297 +++++++++++++++++++++++---- 2 files changed, 266 insertions(+), 39 deletions(-) diff --git a/v2/src/session-service.ts b/v2/src/session-service.ts index 36118e5ea..60f28adee 100644 --- a/v2/src/session-service.ts +++ b/v2/src/session-service.ts @@ -617,6 +617,14 @@ function recoverableRuntimeError(session: SessionRecord): boolean { ) return false; if (session.errorSource === "runtime_probe") return true; + if (session.errorSource === "other") { + return ( + session.error === + "Session status detection failed: ps failed: malformed process table output" || + session.error === + "Session status detection failed: ps failed: Command failed: ps -eo pid,ppid,pgid,tpgid,tty,rss,args\n" + ); + } if (session.errorSource !== undefined) return false; const diagnostic = session.error.slice(STATUS_DETECTION_ERROR_PREFIX.length); return ( diff --git a/v2/test/fast/session-service.test.ts b/v2/test/fast/session-service.test.ts index 5b8efb0f4..7e8196edc 100644 --- a/v2/test/fast/session-service.test.ts +++ b/v2/test/fast/session-service.test.ts @@ -1449,6 +1449,7 @@ type SessionServiceInternals = { sessionBatch?: SessionRecord[], ): Promise<{ view: SessionView; classified: unknown }>; restoreWarmupUntil: Map; + restoreOwnsRuntimeRecovery(sessionId: string): boolean; withWorkspaceLifecycleLocks(sessionId: string, task: () => Promise): Promise; confirmAgentExited( session: Pick, @@ -18996,50 +18997,62 @@ describe("SessionService", () => { } }); - it("keeps codex working while a long exec_command tool call is still pending", async () => { - const now = new Date("2026-04-14T19:30:00.000Z"); - vi.setSystemTime(now); - const tenMinAgoMs = now.getTime() - 10 * 60_000; - readSessionMock.mockReturnValue({ - id: "spur-exec", - project: "sp", - agent: "codex", - prompt: "long exec", - branch: "feature/exec", - worktree: true, - worktreePath: "/tmp/spur-worktrees/sp/spur-exec", - tmuxSession: "spur-exec", - launchCommand: "codex --dangerously-bypass-approvals-and-sandbox", - status: "running", - createdAt: "2026-04-14T13:34:40.615Z", - updatedAt: "2026-04-14T19:20:00.000Z", - }); - readAgentHookStateMock.mockReturnValue({ - state: "working", - updatedAt: new Date(tenMinAgoMs).toISOString(), - hookEvent: "PreToolUse", - turnId: "019efdf7", - }); - readCodexRolloutStateMock.mockResolvedValue({ - rollout: { + it.each([ + undefined, + "Session status detection failed: ps failed: malformed process table output", + "Session status detection failed: ps failed: Command failed: ps -eo pid,ppid,pgid,tpgid,tty,rss,args\n", + ])( + "historical transport recovery: keeps codex working while a long exec_command tool call is still pending (%j)", + async (error) => { + const now = new Date("2026-04-14T19:30:00.000Z"); + vi.setSystemTime(now); + const tenMinAgoMs = now.getTime() - 10 * 60_000; + const sessions = createSessionStore(); + sessions.set("spur-exec", { + id: "spur-exec", + project: "sp", + agent: "codex", + prompt: "long exec", + branch: "feature/exec", + worktree: true, + worktreePath: "/tmp/spur-worktrees/sp/spur-exec", + tmuxSession: "spur-exec", + launchCommand: "codex --dangerously-bypass-approvals-and-sandbox", + status: "running", + createdAt: "2026-04-14T13:34:40.615Z", + updatedAt: "2026-04-14T19:20:00.000Z", + ...(error ? { error, errorSource: "other" } : {}), + }); + readAgentHookStateMock.mockReturnValue({ state: "working", - timestamp: new Date(tenMinAgoMs).toISOString(), - timestampMs: tenMinAgoMs, - filePath: "/tmp/spur-exec/rollout.jsonl", - reason: "function_call", - callId: "call_1", + updatedAt: new Date(tenMinAgoMs).toISOString(), + hookEvent: "PreToolUse", turnId: "019efdf7", - }, - rateLimit: null, - }); + }); + readCodexRolloutStateMock.mockResolvedValue({ + rollout: { + state: "working", + timestamp: new Date(tenMinAgoMs).toISOString(), + timestampMs: tenMinAgoMs, + filePath: "/tmp/spur-exec/rollout.jsonl", + reason: "function_call", + callId: "call_1", + turnId: "019efdf7", + }, + rateLimit: null, + }); - const { SessionService } = await loadSessionServiceModule(); - const service = new SessionService("/tmp/spur.yaml", "2026-04-14T13:34:40.615Z"); + const { SessionService } = await loadSessionServiceModule(); + const service = new SessionService("/tmp/spur.yaml", "2026-04-14T13:34:40.615Z"); - const result = await service.get("spur-exec"); + const result = await service.get("spur-exec"); - expect(result.state).toBe("working"); - }); + expect(result.state).toBe("working"); + expect(sessions.get("spur-exec")?.error).toBeUndefined(); + expect(sendMessageToTmuxMock).not.toHaveBeenCalled(); + expect(sendInterruptKeysToTmuxMock).not.toHaveBeenCalled(); + }, + ); it("flips codex working to waiting for a stale UserPromptSubmit hook (spur-7ce0)", async () => { const now = new Date("2026-06-25T15:00:00.000Z"); @@ -26532,6 +26545,116 @@ describe("SessionService", () => { }, ); + describe.each([ + "Session status detection failed: ps failed: malformed process table output", + "Session status detection failed: ps failed: Command failed: ps -eo pid,ppid,pgid,tpgid,tty,rss,args\n", + ])("historical transport recovery: %j", (error) => { + it.each(["running", "errored"] as const)("heals a healthy %s record", async (status) => { + const sessions = createSessionStore(); + const record = runningSession({ + id: "api-1", + status, + agentSessionId: "native-original", + agentLaunchId: "launch-original", + error, + errorSource: "other", + }); + sessions.set(record.id, record); + mockClaudeJsonlState("waiting"); + const service = await createDisposedSessionService(); + + const result = await service.get(record.id); + + expect(result.state).toBe("waiting"); + expect(sessions.get(record.id)).toMatchObject({ + id: record.id, + status: "running", + agentSessionId: "native-original", + agentLaunchId: "launch-original", + }); + expect(sessions.get(record.id)?.error).toBeUndefined(); + expect(sessions.get(record.id)?.errorSource).toBeUndefined(); + expect(sendMessageToTmuxMock).not.toHaveBeenCalled(); + }); + + it.each( + (["running", "errored"] as const).flatMap((status) => + ["unknown", "dead", "workspace", "server", "restore", "owner", "generation"].map( + (guard) => ({ status, guard }), + ), + ), + )("blocks $status recovery under $guard evidence", async ({ status, guard }) => { + const sessions = createSessionStore(); + const record = runningSession({ + id: "api-1", + status, + agentSessionId: "native-original", + agentLaunchId: "launch-original", + error, + errorSource: "other", + ...(guard === "server" ? { serverErrorAt: "2026-03-18T10:01:00.000Z" } : {}), + }); + sessions.set(record.id, record); + const service = await createDisposedSessionService(); + const internals = sessionServiceInternals(service); + vi.spyOn(internals, "readRuntimeSnapshot").mockImplementation(async () => { + if (guard === "owner" || guard === "generation") { + sessions.set(record.id, { + ...record, + ...(guard === "owner" + ? { agentSessionId: "native-replacement" } + : { agentLaunchId: "launch-replacement" }), + }); + } + return { + runtimeAlive: guard !== "dead", + paneUsable: guard !== "dead", + processAlive: guard !== "dead", + probeUnresponsive: guard === "unknown", + }; + }); + if (guard === "workspace") { + probeWorkspaceMock.mockReturnValue({ exists: false, missing: true }); + } + if (guard === "restore") { + vi.spyOn(internals, "restoreOwnsRuntimeRecovery").mockReturnValue(true); + } + + await internals.classifySessionRecord(record); + + expect(sessions.get(record.id)?.error).toBe( + guard === "unknown" + ? "Session status detection failed: runtime probe (tmux or ps) timed out; runtime state unknown" + : error, + ); + expect(sessions.get(record.id)?.errorSource).toBe( + guard === "unknown" ? "runtime_probe" : "other", + ); + expect(sendMessageToTmuxMock).not.toHaveBeenCalled(); + }); + }); + + it.each([ + "Session status detection failed: ps failed: malformed process table output\n", + "Session status detection failed: ps failed: Command failed: ps -eo pid,ppid,pgid,tpgid,tty,rss,args", + "Session status detection failed: ps failed: Command failed: ps -eo pid,ppid,pgid,tpgid,tty,rss,args\n\n", + "Session status detection failed: ps failed: Command failed: ps -eo pid,ppid,pgid,tpgid,tty,rss,args\npermission denied\n", + "Session status detection failed: ps failed: classifier", + "Session status detection failed: workspace failed", + "Session reporting detection failed: ps failed: malformed process table output", + "Provider request failed", + ])("historical transport recovery: retains unrelated complete error %j", async (error) => { + const sessions = createSessionStore(); + const record = runningSession({ id: "api-1", error, errorSource: "other" }); + sessions.set(record.id, record); + mockClaudeJsonlState("waiting"); + const service = await createDisposedSessionService(); + + expect((await service.get(record.id)).state).toBe("error"); + expect(sessions.get(record.id)?.error).toBe(error); + expect(sessions.get(record.id)?.errorSource).toBe("other"); + }); + it("keeps a workspace diagnostic after an unresponsive runtime probe recovers", async () => { const sessions = createSessionStore(); const record = runningSession({ id: "api-1" }); @@ -59391,6 +59514,102 @@ describe("SessionService", () => { const holdAgents = ["claude", "codex", "cursor", "opencode"] as const; const afterHold = Date.parse(PENDING_LAUNCH_AT) + 30_000; + it.each(["needs_input", "rate_limited", "unconfirmed"] as const)( + "historical transport recovery: blocks queued input under current %s", + async (guard) => { + timerPromisesSleepMock.mockImplementation( + (ms) => new Promise((resolve) => setTimeout(resolve, ms)), + ); + const sessions = createSessionStore(); + seedSendHold(sessions, "codex", { + error: "Session status detection failed: ps failed: malformed process table output", + errorSource: "other", + queuedMessages: { messages: ["A", "B"], awaitingPrompt: true }, + }); + mockAgentActivity("codex", Date.now(), "working"); + mockHoldAckScan(guard !== "unconfirmed"); + if (guard === "needs_input") { + readAgentHookStateMock.mockReturnValue({ + state: "needs_input", + updatedAt: new Date().toISOString(), + }); + } else if (guard === "rate_limited") { + readCodexRolloutStateMock.mockResolvedValue({ + rollout: null, + rateLimit: { limited: true, reason: "codex out of credits" }, + }); + } + const service = await deferredService(); + try { + const result = await service.get("api-1"); + expect(sessions.get("api-1")?.error).toBeUndefined(); + expect(result.state).toBe(guard === "unconfirmed" ? "working" : guard); + await vi.advanceTimersByTimeAsync(30_000); + expect(tmuxTexts()).toEqual([]); + expect(sessions.get("api-1")?.queuedMessages?.messages).toEqual(["A", "B"]); + if (guard === "unconfirmed") { + expect(sessions.get("api-1")?.submitUnconfirmedAt).toBe(PENDING_LAUNCH_AT); + } + expect(sendInterruptKeysToTmuxMock).not.toHaveBeenCalled(); + } finally { + service.dispose(); + } + }, + ); + + it.each([ + "Session status detection failed: ps failed: malformed process table output", + "Session status detection failed: ps failed: Command failed: ps -eo pid,ppid,pgid,tpgid,tty,rss,args\n", + ])("historical transport recovery: own Codex ACK resumes FIFO once for %j", async (error) => { + timerPromisesSleepMock.mockImplementation( + (ms) => new Promise((resolve) => setTimeout(resolve, ms)), + ); + const sessions = createSessionStore(); + seedSendHold(sessions, "codex", { + error, + errorSource: "other", + queuedMessages: { messages: ["A", "B"], awaitingPrompt: true }, + }); + mockAgentActivity("codex", afterHold, "working"); + const scan = mockHoldAckScan(true); + sendMessageToTmuxMock.mockImplementation(async () => { + mockAgentActivity("codex", Date.now(), "working"); + }); + const service = await deferredService(); + try { + await service.get("api-1"); + expect(scan).toHaveBeenCalledWith("typed"); + expect(sessions.get("api-1")?.submitUnconfirmedAt).toBeUndefined(); + expect(sessions.get("api-1")?.queuedMessageTyped).toBeUndefined(); + expect(sessions.get("api-1")?.error).toBeUndefined(); + await vi.advanceTimersByTimeAsync(60_000); + expect(tmuxTexts()).toEqual([]); + + mockAgentActivity("codex", Date.now(), "waiting"); + await vi.advanceTimersByTimeAsync(10_000); + expect(tmuxTexts()).toEqual(["A"]); + await vi.advanceTimersByTimeAsync(60_000); + expect(tmuxTexts()).toEqual(["A"]); + expect(sessions.get("api-1")?.queuedMessages?.messages).toEqual(["B"]); + + mockAgentActivity("codex", Date.now(), "waiting"); + await vi.advanceTimersByTimeAsync(10_000); + expect(tmuxTexts()).toEqual(["A", "B"]); + await vi.advanceTimersByTimeAsync(30_000); + expect(tmuxTexts()).toEqual(["A", "B"]); + expect(sessions.get("api-1")?.agentSessionId).toBe("ses_1"); + expect(eventsNamed("session.message.requeued")).toHaveLength(0); + expect(sendInterruptKeysToTmuxMock).not.toHaveBeenCalled(); + expect(createTmuxSessionMock).not.toHaveBeenCalled(); + expect(eventsNamed("session.message.sent")).toEqual([ + expect.objectContaining({ details: expect.objectContaining({ interrupt: false }) }), + expect.objectContaining({ details: expect.objectContaining({ interrupt: false }) }), + ]); + } finally { + service.dispose(); + } + }); + it.each(holdAgents)( "%s: a running turn's own records never clear a send's hold", async (agent) => { From 9743b7fe4a4f033b48f6e68ac912cfd6e25c5384 Mon Sep 17 00:00:00 2001 From: AlekShu <108252944+AlekShu@users.noreply.github.com> Date: Fri, 16 Oct 2026 06:03:10 +0000 Subject: [PATCH 08/23] docs(session): reference scheduled wake admission --- .agents/skills/spur/SKILL.md | 1 + .claude/skills/spur/SKILL.md | 1 + docs/configuration.md | 4 ++-- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.agents/skills/spur/SKILL.md b/.agents/skills/spur/SKILL.md index 8da85b762..fae3cba0c 100644 --- a/.agents/skills/spur/SKILL.md +++ b/.agents/skills/spur/SKILL.md @@ -20,6 +20,7 @@ INTERFACES Session variables: `env | grep '^SPUR_'`. Agent launch identity contract: `docs/commands.md#session-tools-and-environment`. Automatic reminder limits: `docs/configuration.md#automatic-reminders`. + Scheduled-wake admission: `docs/configuration.md#admission-control`. Spur ToDo: ledger starts empty, no code path seeds an item; the agent adds one item per step, before the step, and resolves it after. Empty or open/held work refuses an agent's own completion, self-destruct, and handoff — `todo_ledger_empty`/`todo_open_work` (409); a human `complete`/`handoff` from the CLI or UI is never blocked. Contract: `docs/commands.md#todo`. Telegram: agents send with `"$SPUR_SESSION_TOOL_DIR/spur" source reply`; users launch tasks and inspect attention cards. Binding, topic identity: `docs/configuration.md#telegram-binding`; replies: `docs/commands.md#source-reply`. Stop unwanted auto-pings with `spur auto-ping unsubscribe`. A thread that keeps coming back takes `--thread`; `--event` binds to one emitted occurrence only. Scopes and resume: `docs/commands.md#auto-ping`; API: `docs/daemon-api.md`. diff --git a/.claude/skills/spur/SKILL.md b/.claude/skills/spur/SKILL.md index 8da85b762..fae3cba0c 100644 --- a/.claude/skills/spur/SKILL.md +++ b/.claude/skills/spur/SKILL.md @@ -20,6 +20,7 @@ INTERFACES Session variables: `env | grep '^SPUR_'`. Agent launch identity contract: `docs/commands.md#session-tools-and-environment`. Automatic reminder limits: `docs/configuration.md#automatic-reminders`. + Scheduled-wake admission: `docs/configuration.md#admission-control`. Spur ToDo: ledger starts empty, no code path seeds an item; the agent adds one item per step, before the step, and resolves it after. Empty or open/held work refuses an agent's own completion, self-destruct, and handoff — `todo_ledger_empty`/`todo_open_work` (409); a human `complete`/`handoff` from the CLI or UI is never blocked. Contract: `docs/commands.md#todo`. Telegram: agents send with `"$SPUR_SESSION_TOOL_DIR/spur" source reply`; users launch tasks and inspect attention cards. Binding, topic identity: `docs/configuration.md#telegram-binding`; replies: `docs/commands.md#source-reply`. Stop unwanted auto-pings with `spur auto-ping unsubscribe`. A thread that keeps coming back takes `--thread`; `--event` binds to one emitted occurrence only. Scopes and resume: `docs/commands.md#auto-ping`; API: `docs/daemon-api.md`. diff --git a/docs/configuration.md b/docs/configuration.md index 8dd11465b..017de530b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -115,7 +115,7 @@ Type/constraint/default per key; full validation source `v2/src/config.ts`. ## Admission control -Cap limits concurrent live sessions at spawn, restore, the wake of a stale-parked or memory-shed session, and the interval or daily wake relaunch of a stopped or paused session. A denied interval wake waits for its next interval; a denied daily wake retries every 15 minutes until its next scheduled time, then that occurrence is skipped. Over cap: `429` naming stop candidates. Reporting route: [`GET /headroom`](daemon-api.md). CLI: [`spur doctor`](commands.md#doctor). Schema `v2/src/config.ts`, enforcement `v2/src/session-service.ts`. +Admission gates spawn, restore and wake relaunch; policy: `v2/src/session-service.ts`; capacity: [`GET /headroom`](daemon-api.md); diagnostics: [`spur doctor`](commands.md#doctor). The shed's session stop writes `status: "stopped"`, `stopReason: "memory_shed"`. Queued trigger batches for such a session are held, not dropped, and its wake passes the admission gate. @@ -164,7 +164,7 @@ Event names by source: - Webhook request: `POST ` with `Authorization: Bearer ` and `Content-Type: application/json`; runtime behavior and limits: `v2/src/event-sources/webhook.ts`. -Other event names (see source file for trigger conditions): `trigger.spawn.suppressed`, `source.poll.disabled`, `source.poll.enabled`, `gh.poll_cycle`, `gh.usage`, `gh.poll_budget_paused`, `session.message.{sent,delivery_recovered,delivery_failed,requeued,submit_unconfirmed,queued_ahead,queue_removed,stalled}`, `session.spawn.{orphan_stopped,launch_unconfirmed,launch_submitted}`, `session.controls.delivery_recovered`, `session.todo.{nudge_failed,nudge_disabled,nudge_exhausted}`, `session.{complete,pause,self_destruct,desk_complete,handoff}.{completed,failed}`, `session.wake.{failed,daily_failed,daily_exhausted,interval_failed,sent,daily_sent,interval_sent,interval_cancelled,daily_cancelled,suppressed}`, `session.wake.token_budget_blocked`, `session.token_budget.exhausted`, `session.token_budget.unenforced`, `session.token_budget.teardown_failed`, `session.attention_monitor.{failed,session_failed,slow}`, `session.runtime.{probe_unresponsive,pane_child_fallback}`, `session.agent_process.{pane_pid_unreadable,capture_unavailable,capture_blind,survivors}`, `session.{handoff,respawn}.startup_attachment_missing`, `session.server_error.reactivation_exhausted`, `session.sidecar.{start_noop,start_rejected,launch_reap_skipped}`, `session.submit.{timeout,confirmed,released,failure_dismissed}`, `session.subscription.{delivery_failed,spawn_failed}`, `session.recover.context_unconfirmed`, `trigger.send.{failed,suppressed_admission,suppressed_launch_pending}`, `host.disk.low`, `daemon.admission.startup`, `daemon.registry.{count,warning}`, `daemon.auto_update.{started,failed,retry,skipped,suppressed,paused,config_invalid,disarm_failed}`, `daemon.deploy_switch.{started,rejected}`, `cli.update.{started,rolled_back,abandoned}`. +Other event names (see source file for trigger conditions): `trigger.spawn.suppressed`, `source.poll.disabled`, `source.poll.enabled`, `gh.poll_cycle`, `gh.usage`, `gh.poll_budget_paused`, `session.message.{sent,delivery_recovered,delivery_failed,requeued,submit_unconfirmed,queued_ahead,queue_removed,stalled}`, `session.spawn.{orphan_stopped,launch_unconfirmed,launch_submitted}`, `session.controls.delivery_recovered`, `session.todo.{nudge_failed,nudge_disabled,nudge_exhausted}`, `session.{complete,pause,self_destruct,desk_complete,handoff}.{completed,failed}`, `session.wake.{failed,daily_failed,daily_deferred,daily_exhausted,interval_failed,interval_deferred,sent,daily_sent,interval_sent,interval_cancelled,daily_cancelled,suppressed}`, `session.wake.token_budget_blocked`, `session.token_budget.exhausted`, `session.token_budget.unenforced`, `session.token_budget.teardown_failed`, `session.attention_monitor.{failed,session_failed,slow}`, `session.runtime.{probe_unresponsive,pane_child_fallback}`, `session.agent_process.{pane_pid_unreadable,capture_unavailable,capture_blind,survivors}`, `session.{handoff,respawn}.startup_attachment_missing`, `session.server_error.reactivation_exhausted`, `session.sidecar.{start_noop,start_rejected,launch_reap_skipped}`, `session.submit.{timeout,confirmed,released,failure_dismissed}`, `session.subscription.{delivery_failed,spawn_failed}`, `session.recover.context_unconfirmed`, `trigger.send.{failed,suppressed_admission,suppressed_launch_pending}`, `host.disk.low`, `daemon.admission.startup`, `daemon.registry.{count,warning}`, `daemon.auto_update.{started,failed,retry,skipped,suppressed,paused,config_invalid,disarm_failed}`, `daemon.deploy_switch.{started,rejected}`, `cli.update.{started,rolled_back,abandoned}`. ## Auto update From bde05a0c0796ffc2c09c2a810bdc7723e4bc881b Mon Sep 17 00:00:00 2001 From: AlekShu <108252944+AlekShu@users.noreply.github.com> Date: Fri, 16 Oct 2026 07:05:17 +0000 Subject: [PATCH 09/23] fix(session): guard deferred recovery observations --- v2/src/server.ts | 2 +- v2/src/session-service.ts | 154 +++++++--- v2/test/fast/session-service.test.ts | 436 +++++++++++++++++++++++++-- 3 files changed, 527 insertions(+), 65 deletions(-) diff --git a/v2/src/server.ts b/v2/src/server.ts index 14d607623..b076c00fe 100644 --- a/v2/src/server.ts +++ b/v2/src/server.ts @@ -2345,7 +2345,7 @@ export async function startServer( throw error; } - let driftedSessions: { id: string; project: string }[] = []; + let driftedSessions: Parameters[0] = []; try { const { scanned, diff --git a/v2/src/session-service.ts b/v2/src/session-service.ts index 37b26e5d6..0e818c542 100644 --- a/v2/src/session-service.ts +++ b/v2/src/session-service.ts @@ -1356,6 +1356,13 @@ interface SessionRuntimeSnapshot { probeUnresponsive: boolean; diagnostic?: string; } +interface SessionObservation { + record: SessionRecord; + lifecycle: { id: string; snapshot: SessionLifecycleSnapshot }[]; +} +interface DailyWakeClaim extends SessionObservation { + record: SessionRecord & { dailyWake: SessionDailyWakeState }; +} interface SessionStateResult { session: SessionRecord; runtime: SessionRuntimeSnapshot; @@ -5455,9 +5462,9 @@ export class SessionService { continue; } const dailySchedule = withoutAdmissionDeferral(dailyWake); - const claimedDailyNextDueAt = await this.withWorkspaceLifecycleLocks( + const claimedDailyWake = await this.withWorkspaceLifecycleLocks( session.id, - async (): Promise => { + async (): Promise => { // Claim the due occurrence BEFORE sending: advance nextDueAt to the // next future scheduled time and persist it first. A slow or failing // send must not leave the wake due, or the `<= now` guard stays true @@ -5521,7 +5528,7 @@ export class SessionService { }); return null; } - const updated: SessionRecord = { + const updated = { ...currentDailyWakeSession, dailyWake: { ...dailySchedule, @@ -5530,18 +5537,25 @@ export class SessionService { updatedAt: nowIso(), }; writeSession(this.config.dataDir, updated); - return nextDueAt.toISOString(); + return { + record: updated, + lifecycle: this.lifecycleIdsFor(updated).map((id) => ({ + id, + snapshot: this.lifecycle.snapshot(id), + })), + }; } return null; }, ); - if (claimedDailyNextDueAt !== null) { + if (claimedDailyWake !== null) { + const claimedDailyNextDueAt = claimedDailyWake.record.dailyWake.nextDueAt; const admissionDenial = this.wakeAdmissionDenial(session.id); if (admissionDenial) { await this.deferDailyWakeForAdmission( session, dailyWake, - claimedDailyNextDueAt, + claimedDailyWake, admissionDenial, now, ); @@ -5659,31 +5673,51 @@ export class SessionService { private async deferDailyWakeForAdmission( session: SessionRecord, dailyWake: SessionDailyWakeState, - claimedNextDueAt: string, + claimed: DailyWakeClaim, denial: WakeAdmissionDenial, now: number, ): Promise { const occurrence = dailyWake.admissionDeferredOccurrence ?? dailyWake.nextDueAt; const windowEndMs = resolveNextDailyWakeAt(dailyWake.dailyAt, new Date(occurrence)).getTime(); const retryAtMs = now + DAILY_WAKE_ADMISSION_RETRY_MS; - if (retryAtMs >= windowEndMs) { - this.logEvent("session.wake.daily_exhausted", { - level: "warn", - sessionId: session.id, - projectId: session.project, - message: `Skipped daily wake occurrence ${occurrence} for ${session.id}: ${denial.error.message}`, - details: { occurrence, nextDueAt: claimedNextDueAt, dailyAt: dailyWake.dailyAt }, - }); - return; - } const retryAt = new Date(retryAtMs).toISOString(); const deferred = await this.withWorkspaceLifecycleLocks(session.id, async () => { const afterDenial = readSession(this.config.dataDir, session.id); - if (afterDenial?.dailyWake?.nextDueAt !== claimedNextDueAt) return false; + if ( + !afterDenial || + !sameSessionRuntimeOwner(afterDenial, claimed.record) || + afterDenial.agentLaunchId !== claimed.record.agentLaunchId || + !this.lifecycleStateMatches(afterDenial, this.lifecycleStamp(claimed.record)) || + !isDeepStrictEqual(afterDenial.dailyWake, claimed.record.dailyWake) || + claimed.lifecycle.some(({ id, snapshot }) => { + const current = this.lifecycle.snapshot(id); + return ( + snapshot.operation?.phase === "pending" || + current.operation?.phase === "pending" || + current.instanceId !== snapshot.instanceId || + current.revision !== snapshot.revision + ); + }) + ) + return false; + if (retryAtMs >= windowEndMs) { + this.logEvent("session.wake.daily_exhausted", { + level: "warn", + sessionId: session.id, + projectId: session.project, + message: `Skipped daily wake occurrence ${occurrence} for ${session.id}: ${denial.error.message}`, + details: { + occurrence, + nextDueAt: claimed.record.dailyWake.nextDueAt, + dailyAt: dailyWake.dailyAt, + }, + }); + return false; + } writeSession(this.config.dataDir, { ...afterDenial, dailyWake: { - ...afterDenial.dailyWake, + ...claimed.record.dailyWake, nextDueAt: retryAt, admissionDeferredOccurrence: occurrence, }, @@ -11214,8 +11248,9 @@ export class SessionService { return updated; } - async restoreRebootedSessions(drifted: { id: string; project: string }[]): Promise { - for (const [index, { id, project }] of drifted.entries()) { + async restoreRebootedSessions(drifted: SessionObservation[]): Promise { + for (const [index, { record, lifecycle }] of drifted.entries()) { + const { id, project } = record; const projectConfig = this.config.projects[project]; if (projectConfig?.restoreAfterReboot !== true) continue; const memory = readHostMemory(); @@ -11237,6 +11272,26 @@ export class SessionService { }); break; } + const current = readSession(this.config.dataDir, id); + if ( + !current || + !sameSessionRuntimeOwner(current, record) || + current.agentLaunchId !== record.agentLaunchId || + !this.lifecycleStateMatches(current, this.lifecycleStamp(record)) || + current.error !== record.error || + current.errorSource !== record.errorSource || + current.serverErrorAt !== record.serverErrorAt || + lifecycle.some(({ id: ownerId, snapshot }) => { + const latest = this.lifecycle.snapshot(ownerId); + return ( + snapshot.operation?.phase === "pending" || + latest.operation?.phase === "pending" || + latest.instanceId !== snapshot.instanceId || + latest.revision !== snapshot.revision + ); + }) + ) + continue; try { await this.restore(id); if (memory !== null) { @@ -11258,14 +11313,14 @@ export class SessionService { scanned: number; alive: number; drifted: number; - driftedSessions: { id: string; project: string }[]; + driftedSessions: SessionObservation[]; }> { const candidates = listSessions(this.config.dataDir).filter( (session) => session.status === "running" || session.status === "spawning", ); let alive = 0; let drifted = 0; - const driftedSessions: { id: string; project: string }[] = []; + const driftedSessions: SessionObservation[] = []; for (const session of candidates) { // A record that carried a probe error is stopped at boot but never @@ -11325,31 +11380,38 @@ export class SessionService { throw error; }); if (!reconciled) continue; + const usage = + (reconciled.session.status === "stopped" || reconciled.session.status === "errored") && + !hasRetainedSessionError(reconciled.session) && + !reconciled.runtime.probeUnresponsive && + !reconciled.runtime.diagnostic && + !workspace.diagnostic && + (!reconciled.runtime.paneUsable || !reconciled.runtime.processAlive) + ? await this.readFinalTokenUsage(session, reconciled.runtime) + : undefined; const latest = readSession(this.config.dataDir, session.id); if ( !latest || !sameObservationOwner(latest) || !sameObservationOwner(reconciled.session) || !this.lifecycleStateMatches(latest, this.lifecycleStamp(reconciled.session)) || + latest.error !== reconciled.session.error || + latest.errorSource !== reconciled.session.errorSource || + latest.serverErrorAt !== reconciled.session.serverErrorAt || latest.stopReason === "manual_pause" || latest.stopReason === "token_budget" ) continue; + let accepted = latest; + if (usage) { + const persisted = this.persistClassifiedTokenUsage(latest, usage); + if (persisted.status === "missing") continue; + accepted = persisted.session; + } if (reconciled.session.status === "stopped" || reconciled.session.status === "errored") { - if ( - !hasRetainedSessionError(reconciled.session) && - !reconciled.runtime.probeUnresponsive && - !reconciled.runtime.diagnostic && - !workspace.diagnostic && - (!reconciled.runtime.paneUsable || !reconciled.runtime.processAlive) - ) { - const usage = await this.readFinalTokenUsage(session, reconciled.runtime); - const persisted = this.persistClassifiedTokenUsage(session, usage); - if (persisted.status === "missing") continue; - } drifted += 1; if (reconciled.session.status === "stopped" && !healedProbeError) { - driftedSessions.push({ id: session.id, project: session.project }); + driftedSessions.push({ record: accepted, lifecycle }); } } else { alive += 1; @@ -17268,9 +17330,11 @@ export class SessionService { SessionRecord, "id" | "tmuxSession" | "agent" | "launchCommand" | "agentLaunchId" >, - options: { failOnSurvivors: boolean }, + options: { failOnSurvivors: boolean; isObservationCurrent?: () => boolean }, ): Promise { + if (options.isObservationCurrent?.() === false) return; const paneLookup = await this.lookupPanePidQuietly(session.tmuxSession, { fresh: true }); + if (options.isObservationCurrent?.() === false) return; if (paneLookup.status === "unavailable") { // tmux could not be read, so P1 has nothing to verify against. Record // it: the alternative is a silent pass that reads exactly like a clean @@ -17288,6 +17352,7 @@ export class SessionService { paneDead: paneLookup.status === "ok" ? paneLookup.paneDead : false, processMatchers: sessionProcessMatchers(session), }); + if (options.isObservationCurrent?.() === false) return; if (capture.status === "unavailable") { // The process table could not be read, so "no survivors" would be a // guess. Refusing a relaunch is recoverable; launching a duplicate is @@ -17328,6 +17393,7 @@ export class SessionService { }, { fresh: true }, ); + if (options.isObservationCurrent?.() === false) return; if (stillAlive) { const message = `Session ${session.id}: the agent process still reads alive but no owned process was captured; refusing to launch a replacement`; this.logEvent("session.agent_process.capture_blind", { @@ -17339,6 +17405,7 @@ export class SessionService { throw new Error(message); } } + if (options.isObservationCurrent?.() === false) return; await killTmuxSession(session.tmuxSession); const outcome = await terminateAgentProcesses(capture.status === "ok" ? capture.processes : []); if (outcome.status === "clear") { @@ -21838,7 +21905,10 @@ export class SessionService { // reached it, so its pane is stopped; the caller then reconciles the record // to "stopped" like a dead orphan, queue kept, and restore or the next send // relaunches it with the task prompt. - private async stopOrphanedSpawnPane(session: SessionRecord): Promise { + private async stopOrphanedSpawnPane( + session: SessionRecord, + isObservationCurrent: () => boolean, + ): Promise { this.logEvent("session.spawn.orphan_stopped", { level: "warn", sessionId: session.id, @@ -21849,7 +21919,10 @@ export class SessionService { queuedCount: queuedMessages(session).length, }, }); - await this.killAgentPaneAndConfirmExit(session, { failOnSurvivors: false }); + await this.killAgentPaneAndConfirmExit(session, { + failOnSurvivors: false, + isObservationCurrent, + }); return this.readRuntimeSnapshot(session, { fresh: true }); } @@ -21965,7 +22038,10 @@ export class SessionService { readSession(this.config.dataDir, session.id)?.status === "spawning" && !this.spawnsInFlight.has(session.id) ) { - confirmedRuntime = await this.stopOrphanedSpawnPane(session); + confirmedRuntime = await this.stopOrphanedSpawnPane(session, () => readObservation().valid); + const afterOrphan = readObservation(); + if (!afterOrphan.valid) + return { session: afterOrphan.latest ?? session, runtime: confirmedRuntime }; } if (live(confirmedRuntime)) { return { session, runtime: confirmedRuntime }; diff --git a/v2/test/fast/session-service.test.ts b/v2/test/fast/session-service.test.ts index 0144cc38b..567f3cdd7 100644 --- a/v2/test/fast/session-service.test.ts +++ b/v2/test/fast/session-service.test.ts @@ -1492,6 +1492,24 @@ async function createDisposedSessionService(options?: { realPaneGeneration?: boo return service; } +function rebootCandidates( + service: Awaited>, + ids: string[], +): Parameters[0] { + const lifecycle = (service as unknown as { lifecycle: SessionLifecycleRegister }).lifecycle; + return ids.map((id) => { + const record = readSessionMock(TEST_DATA_DIR, id) as SessionRecord | null; + if (!record) throw new Error(`Missing reboot fixture: ${id}`); + return { + record, + lifecycle: [...new Set([id, record.workspaceId ?? id])].map((ownerId) => ({ + id: ownerId, + snapshot: lifecycle.snapshot(ownerId), + })), + }; + }); +} + // Only `revision`, `items[].status` and `items[].latestTransition.blocker` are // read by maybeNudgeTodo's human-held branch; the rest is shape filler. function heldProjection(revision: string, requiredAction: string): TodoProjection { @@ -19590,7 +19608,11 @@ describe("SessionService", () => { scanned: 1, alive: 0, drifted: 1, - driftedSessions: [{ id: "api-1", project: "api" }], + driftedSessions: [ + expect.objectContaining({ + record: expect.objectContaining({ id: "api-1", project: "api" }), + }), + ], }); expect(sessions.get("api-1")).toMatchObject({ status: "stopped", @@ -19656,7 +19678,11 @@ describe("SessionService", () => { expect(sessions.get("api-1")).not.toHaveProperty("errorSource"); expect(sessions.get("api-2")?.status).toBe("stopped"); expect(result.drifted).toBe(2); - expect(result.driftedSessions).toEqual([{ id: "api-2", project: "api" }]); + expect(result.driftedSessions).toEqual([ + expect.objectContaining({ + record: expect.objectContaining({ id: "api-2", project: "api" }), + }), + ]); const restoreSpy = vi.spyOn(service, "restore").mockResolvedValue({} as never); await service.restoreRebootedSessions(result.driftedSessions); @@ -19681,7 +19707,11 @@ describe("SessionService", () => { await vi.advanceTimersByTimeAsync(1_000); const result = await pending; - expect(result.driftedSessions).toEqual([{ id: "api-2", project: "api" }]); + expect(result.driftedSessions).toEqual([ + expect.objectContaining({ + record: expect.objectContaining({ id: "api-2", project: "api" }), + }), + ]); }); }); @@ -19942,10 +19972,7 @@ describe("SessionService", () => { const service = await createDisposedSessionService(); const restoreSpy = vi.spyOn(service, "restore").mockResolvedValue({} as never); - await service.restoreRebootedSessions([ - { id: "api-1", project: "api" }, - { id: "web-1", project: "web" }, - ]); + await service.restoreRebootedSessions(rebootCandidates(service, ["api-1", "web-1"])); expect(restoreSpy).toHaveBeenCalledWith("api-1"); expect(restoreSpy).not.toHaveBeenCalledWith("web-1"); @@ -20016,7 +20043,7 @@ describe("SessionService", () => { ); const duplicateAutoStart = vi.spyOn(internals, "startAutoStartSidecars"); - const rebootRestore = service.restoreRebootedSessions([{ id: "api-2", project: "api" }]); + const rebootRestore = service.restoreRebootedSessions(rebootCandidates(service, ["api-2"])); await restoreStartedBarrier; const stop = service.stopSidecar("api-1", "proxy"); await Promise.resolve(); @@ -20059,10 +20086,7 @@ describe("SessionService", () => { .mockRejectedValueOnce(new Error("boom")) .mockResolvedValueOnce({} as never); - await service.restoreRebootedSessions([ - { id: "api-1", project: "api" }, - { id: "api-2", project: "api" }, - ]); + await service.restoreRebootedSessions(rebootCandidates(service, ["api-1", "api-2"])); expect(restoreSpy).toHaveBeenCalledTimes(2); expect(logSpurEventMock).toHaveBeenCalledWith( @@ -20075,6 +20099,9 @@ describe("SessionService", () => { }); it("restoreRebootedSessions keeps iterating and logs reboot.restore.failed after a denied restore (no narrowed catch)", async () => { + const sessions = createSessionStore(); + for (const id of ["api-1", "api-2"]) + sessions.set(id, runningSession({ id, status: "stopped" })); loadConfigMock.mockReturnValue({ ...baseConfig(), projects: { @@ -20103,10 +20130,7 @@ describe("SessionService", () => { ), ); - await service.restoreRebootedSessions([ - { id: "api-1", project: "api" }, - { id: "api-2", project: "api" }, - ]); + await service.restoreRebootedSessions(rebootCandidates(service, ["api-1", "api-2"])); expect(restoreSpy).toHaveBeenCalledTimes(2); expect(logSpurEventMock).toHaveBeenCalledWith( @@ -20120,6 +20144,9 @@ describe("SessionService", () => { }); it("restoreRebootedSessions stops before the next session below the restore floor", async () => { + const sessions = createSessionStore(); + for (const id of ["api-1", "api-2"]) + sessions.set(id, runningSession({ id, status: "stopped" })); const config = baseConfig(); loadConfigMock.mockReturnValue({ ...config, @@ -20136,10 +20163,7 @@ describe("SessionService", () => { const service = await createDisposedSessionService(); const restoreSpy = vi.spyOn(service, "restore"); - await service.restoreRebootedSessions([ - { id: "api-1", project: "api" }, - { id: "api-2", project: "api" }, - ]); + await service.restoreRebootedSessions(rebootCandidates(service, ["api-1", "api-2"])); expect(restoreSpy).not.toHaveBeenCalled(); expect(logSpurEventMock).toHaveBeenCalledWith( @@ -20149,6 +20173,9 @@ describe("SessionService", () => { }); it("restoreRebootedSessions rechecks the floor after each restore settles", async () => { + const sessions = createSessionStore(); + for (const id of ["api-1", "api-2"]) + sessions.set(id, runningSession({ id, status: "stopped" })); const config = baseConfig(); loadConfigMock.mockReturnValue({ ...config, @@ -20173,10 +20200,7 @@ describe("SessionService", () => { const service = await createDisposedSessionService(); const restoreSpy = vi.spyOn(service, "restore").mockResolvedValue({} as never); - await service.restoreRebootedSessions([ - { id: "api-1", project: "api" }, - { id: "api-2", project: "api" }, - ]); + await service.restoreRebootedSessions(rebootCandidates(service, ["api-1", "api-2"])); expect(restoreSpy).toHaveBeenCalledOnce(); expect(timerPromisesSleepMock).toHaveBeenCalledOnce(); @@ -20190,6 +20214,7 @@ describe("SessionService", () => { }); it("restoreRebootedSessions does not add a settle delay without a memory sample", async () => { + createSessionStore(runningSession({ status: "stopped" })); const config = baseConfig(); loadConfigMock.mockReturnValue({ ...config, @@ -20201,7 +20226,7 @@ describe("SessionService", () => { const service = await createDisposedSessionService(); vi.spyOn(service, "restore").mockResolvedValue({} as never); - await service.restoreRebootedSessions([{ id: "api-1", project: "api" }]); + await service.restoreRebootedSessions(rebootCandidates(service, ["api-1"])); expect(timerPromisesSleepMock).not.toHaveBeenCalled(); }); @@ -27046,6 +27071,367 @@ describe("SessionService", () => { }); }); + describe("pr1197 review repair:", () => { + const dead = { + runtimeAlive: false, + paneUsable: false, + processAlive: false, + probeUnresponsive: false, + }; + const live = { ...dead, runtimeAlive: true, paneUsable: true, processAlive: true }; + const unknown = { + ...dead, + probeUnresponsive: true, + diagnostic: "ps failed: old probe timeout", + }; + type Observation = ReturnType[number]; + type DailyClaim = Observation & { + record: SessionRecord & { dailyWake: NonNullable }; + }; + function repairInternals(service: unknown) { + return service as SessionServiceInternals & { + lifecycle: SessionLifecycleRegister; + lookupPanePidQuietly(): Promise<{ status: "ok"; panePid: number; paneDead: boolean }>; + readFinalTokenUsage(): Promise; + reconcileUnexpectedStop( + record: SessionRecord, + runtime: typeof dead, + reason: "runtime_check", + missing: boolean, + ): Promise<{ session: SessionRecord }>; + deferDailyWakeForAdmission( + record: SessionRecord, + daily: NonNullable, + claim: DailyClaim, + denial: { kind: "denied"; error: Error }, + now: number, + ): Promise; + }; + } + + it.each(["lookup", "capture", "lifecycle"])( + "binds orphan pane effects through %s", + async (boundary) => { + const sessions = createSessionStore(); + const service = await createDisposedSessionService(); + const original = runningSession({ status: "spawning", agentLaunchId: "original" }); + const replacement = + boundary === "lifecycle" + ? original + : { + ...original, + status: "running" as const, + agentLaunchId: "replacement", + }; + sessions.set(original.id, original); + const internals = repairInternals(service); + const replace = () => { + sessions.set(original.id, replacement); + if (boundary === "lifecycle") internals.lifecycle.begin("restore", [original.id]); + }; + vi.spyOn(internals, "lookupPanePidQuietly").mockImplementation(async () => { + if (boundary === "lookup") replace(); + return { status: "ok", panePid: 101, paneDead: false }; + }); + capturePaneAgentProcessesMock.mockImplementation(async () => { + if (boundary !== "lookup") replace(); + return { status: "ok", processes: [] }; + }); + vi.spyOn(internals, "readRuntimeSnapshot").mockResolvedValue(dead); + await internals.reconcileUnexpectedStop(original, live, "runtime_check", false); + expect(sessions.get(original.id)).toEqual(replacement); + expect(killTmuxSessionMock).not.toHaveBeenCalled(); + expect(terminateAgentProcessesMock).not.toHaveBeenCalled(); + }, + ); + + it("keeps a legitimate old kill without stamping the postkill replacement", async () => { + const sessions = createSessionStore(); + const service = await createDisposedSessionService(); + const original = runningSession({ status: "spawning", agentLaunchId: "original" }); + const replacement = { ...original, status: "running" as const, agentLaunchId: "replacement" }; + sessions.set(original.id, original); + const internals = repairInternals(service); + vi.spyOn(internals, "lookupPanePidQuietly").mockResolvedValue({ + status: "ok", + panePid: 101, + paneDead: false, + }); + vi.spyOn(internals, "readRuntimeSnapshot").mockImplementation(async () => { + sessions.set(original.id, replacement); + return unknown; + }); + await internals.reconcileUnexpectedStop(original, live, "runtime_check", false); + expect(killTmuxSessionMock).toHaveBeenCalledOnce(); + expect(sessions.get(original.id)).toEqual(replacement); + }); + + it.each(["reject", "unknown"])( + "retains original C1 protection during fresh %s", + async (mode) => { + const sessions = createSessionStore(); + const service = await createDisposedSessionService(); + const original = runningSession(); + const replacement = { ...original, agentSessionId: "replacement" }; + sessions.set(original.id, original); + const internals = repairInternals(service); + vi.spyOn(internals, "readRuntimeSnapshot").mockImplementation(async () => { + sessions.set(original.id, replacement); + if (mode === "reject") throw new Error("old probe failed"); + return unknown; + }); + const pending = internals.reconcileUnexpectedStop(original, dead, "runtime_check", false); + await vi.advanceTimersByTimeAsync(1_000); + await pending; + expect(sessions.get(original.id)).toEqual(replacement); + expect(killTmuxSessionMock).not.toHaveBeenCalled(); + }, + ); + + it.each(["native", "launch", "pause", "pending", "settled"])( + "rejects boot attribution after %s during usage", + async (change) => { + const sessions = createSessionStore(); + const service = await createDisposedSessionService(); + const original = runningSession({ + agent: "codex", + agentSessionId: "original", + agentLaunchId: "original", + }); + sessions.set(original.id, original); + const internals = repairInternals(service); + vi.spyOn(internals, "readRuntimeSnapshot").mockResolvedValue(dead); + let replacement = original; + vi.spyOn(internals, "readFinalTokenUsage").mockImplementation(async () => { + const current = sessions.get(original.id); + if (!current) throw new Error("Missing boot usage fixture"); + replacement = { + ...(change === "pending" || change === "settled" ? current : original), + ...(change === "native" ? { agentSessionId: "replacement" } : {}), + ...(change === "launch" ? { agentLaunchId: "replacement" } : {}), + ...(change === "pause" + ? { status: "stopped" as const, stopReason: "manual_pause" as const } + : {}), + }; + sessions.set(original.id, replacement); + if (change === "pending" || change === "settled") { + const owner = internals.lifecycle.begin("restore", [original.id]); + if (change === "settled") + internals.lifecycle.settle( + owner, + true, + () => "succeeded", + () => true, + ); + } + return { + tokenUsage: { + provider: "codex", + generationId: "original", + inputTokens: 7, + outputTokens: 3, + totalTokens: 10, + }, + }; + }); + const pending = service.reconcileStoppedSessions(); + await vi.advanceTimersByTimeAsync(1_000); + const result = await pending; + expect(sessions.get(original.id)).toEqual(replacement); + expect(result.driftedSessions).toEqual([]); + expect(result.drifted).toBe(0); + const restore = vi.spyOn(service, "restore").mockResolvedValue({} as never); + await service.restoreRebootedSessions(result.driftedSessions); + expect(restore).not.toHaveBeenCalled(); + }, + ); + + it("keeps a postusage queue append eligible for reboot restore", async () => { + const config = baseConfig(); + loadConfigMock.mockReturnValue({ + ...config, + projects: { api: { ...config.projects.api, restoreAfterReboot: true } }, + }); + readHostMemoryMock.mockReturnValue(null); + const sessions = createSessionStore(); + const service = await createDisposedSessionService(); + sessions.set("api-1", runningSession()); + const internals = repairInternals(service); + vi.spyOn(internals, "readRuntimeSnapshot").mockResolvedValue(dead); + vi.spyOn(internals, "readFinalTokenUsage").mockImplementation(async () => { + const current = sessions.get("api-1"); + if (!current) throw new Error("Missing stopped boot fixture"); + sessions.set(current.id, { + ...current, + updatedAt: "2026-03-18T10:06:00.000Z", + queuedMessages: { messages: ["new"], awaitingPrompt: true }, + }); + return {}; + }); + const pending = service.reconcileStoppedSessions(); + await vi.advanceTimersByTimeAsync(1_000); + const result = await pending; + expect(result.driftedSessions).toHaveLength(1); + expect(result.driftedSessions[0]?.record.queuedMessages?.messages).toEqual(["new"]); + const restore = vi.spyOn(service, "restore").mockResolvedValue({} as never); + await service.restoreRebootedSessions(result.driftedSessions); + expect(restore).toHaveBeenCalledExactlyOnceWith("api-1"); + }); + + it.each(["native", "launch", "pause", "pending", "settled", "queue"])( + "rechecks the boot candidate at consumer after %s", + async (change) => { + const config = baseConfig(); + loadConfigMock.mockReturnValue({ + ...config, + projects: { api: { ...config.projects.api, restoreAfterReboot: true } }, + }); + readHostMemoryMock.mockReturnValue(null); + const sessions = createSessionStore(); + const service = await createDisposedSessionService(); + const original = runningSession({ + status: "stopped", + agentSessionId: "original", + agentLaunchId: "original", + }); + sessions.set(original.id, original); + const candidates = rebootCandidates(service, [original.id]); + sessions.set(original.id, { + ...original, + ...(change === "native" ? { agentSessionId: "replacement" } : {}), + ...(change === "launch" ? { agentLaunchId: "replacement" } : {}), + ...(change === "pause" ? { stopReason: "manual_pause" as const } : {}), + ...(change === "queue" + ? { queuedMessages: { messages: ["new"], awaitingPrompt: true } } + : {}), + }); + const internals = repairInternals(service); + if (change === "pending" || change === "settled") { + const owner = internals.lifecycle.begin("restore", [original.id]); + if (change === "settled") + internals.lifecycle.settle( + owner, + true, + () => "succeeded", + () => true, + ); + } + const restore = vi.spyOn(service, "restore").mockResolvedValue({} as never); + await service.restoreRebootedSessions(candidates); + expect(restore).toHaveBeenCalledTimes(change === "queue" ? 1 : 0); + }, + ); + + it("rejects a later boot row replaced during an earlier restore", async () => { + const config = baseConfig(); + loadConfigMock.mockReturnValue({ + ...config, + projects: { api: { ...config.projects.api, restoreAfterReboot: true } }, + }); + readHostMemoryMock.mockReturnValue(null); + const sessions = createSessionStore(); + const service = await createDisposedSessionService(); + for (const id of ["api-1", "api-2"]) + sessions.set(id, runningSession({ id, status: "stopped" })); + const candidates = rebootCandidates(service, ["api-1", "api-2"]); + const restore = vi.spyOn(service, "restore").mockImplementation(async () => { + sessions.set( + "api-2", + runningSession({ id: "api-2", status: "stopped", stopReason: "manual_pause" }), + ); + return {} as never; + }); + await service.restoreRebootedSessions(candidates); + expect(restore.mock.calls.map(([id]) => id)).toEqual(["api-1"]); + }); + + it.each([ + "dailyAt", + "message", + "stopCondition", + "marker", + "native", + "launch", + "pending", + "settled", + "queue", + "unchanged", + "exhausted", + "stale-exhausted", + ])("qualifies the full daily claim before %s denial", async (change) => { + const sessions = createSessionStore(); + const service = await createDisposedSessionService(); + const internals = repairInternals(service); + const daily = { + dailyAt: ["09:00", "09:40"], + nextDueAt: "2026-10-16T09:00:00.000Z", + message: "old", + stopCondition: "done", + }; + const claimedTime = "2026-10-16T09:40:00.000Z"; + const original = runningSession({ + status: "stopped", + agentSessionId: "original", + agentLaunchId: "original", + dailyWake: { ...daily, nextDueAt: claimedTime }, + }); + sessions.set(original.id, original); + const claim = rebootCandidates(service, [original.id])[0] as DailyClaim; + let replacement = original; + vi.spyOn(internals, "withWorkspaceLifecycleLocks").mockImplementation(async (_id, action) => { + replacement = { + ...original, + dailyWake: { + ...daily, + nextDueAt: claimedTime, + ...(change === "dailyAt" ? { dailyAt: ["09:40"] } : {}), + ...(change === "message" || change === "stale-exhausted" ? { message: "new" } : {}), + ...(change === "stopCondition" ? { stopCondition: "new" } : {}), + ...(change === "marker" + ? { admissionDeferredOccurrence: "2026-10-16T08:00:00.000Z" } + : {}), + }, + ...(change === "native" ? { agentSessionId: "replacement" } : {}), + ...(change === "launch" ? { agentLaunchId: "replacement" } : {}), + ...(change === "queue" + ? { queuedMessages: { messages: ["new"], awaitingPrompt: true } } + : {}), + }; + sessions.set(original.id, replacement); + if (change === "pending" || change === "settled") { + const owner = internals.lifecycle.begin("restore", [original.id]); + if (change === "settled") + internals.lifecycle.settle( + owner, + true, + () => "succeeded", + () => true, + ); + } + return action(); + }); + const exhausted = change === "exhausted" || change === "stale-exhausted"; + await internals.deferDailyWakeForAdmission( + original, + daily, + claim, + { kind: "denied", error: new Error("capacity") }, + Date.parse(exhausted ? "2026-10-16T09:25:00.000Z" : "2026-10-16T09:00:01.000Z"), + ); + if (change === "unchanged" || change === "queue") { + expect(sessions.get(original.id)?.dailyWake?.nextDueAt).toBe("2026-10-16T09:15:01.000Z"); + expect(sessions.get(original.id)?.queuedMessages).toEqual(replacement.queuedMessages); + } else expect(sessions.get(original.id)).toEqual(replacement); + const events = logSpurEventMock.mock.calls.map(([, entry]) => entry.event); + expect(events.filter((event) => event === "session.wake.daily_exhausted")).toHaveLength( + change === "exhausted" ? 1 : 0, + ); + expect(events.filter((event) => event === "session.wake.daily_deferred")).toHaveLength( + change === "unchanged" || change === "queue" ? 1 : 0, + ); + }); + }); + describe("runtime probe errors on non-running and dead records", () => { const PROBE_ERROR = "Session status detection failed: tmux list-windows failed: "; From d9a3cb9bd261f7fd0d7170e84fa36c568ee0413e Mon Sep 17 00:00:00 2001 From: AlekShu <108252944+AlekShu@users.noreply.github.com> Date: Fri, 16 Oct 2026 07:30:47 +0000 Subject: [PATCH 10/23] fix(session): fence rejected orphan observations --- v2/src/session-service.ts | 12 ++++- v2/test/fast/session-service.test.ts | 76 +++++++++++++++++++--------- 2 files changed, 62 insertions(+), 26 deletions(-) diff --git a/v2/src/session-service.ts b/v2/src/session-service.ts index 0e818c542..e72f089a6 100644 --- a/v2/src/session-service.ts +++ b/v2/src/session-service.ts @@ -22038,7 +22038,17 @@ export class SessionService { readSession(this.config.dataDir, session.id)?.status === "spawning" && !this.spawnsInFlight.has(session.id) ) { - confirmedRuntime = await this.stopOrphanedSpawnPane(session, () => readObservation().valid); + try { + confirmedRuntime = await this.stopOrphanedSpawnPane( + session, + () => readObservation().valid, + ); + } catch (error) { + const afterOrphan = readObservation(); + if (!afterOrphan.valid) + return { session: afterOrphan.latest ?? session, runtime: confirmedRuntime }; + throw error; + } const afterOrphan = readObservation(); if (!afterOrphan.valid) return { session: afterOrphan.latest ?? session, runtime: confirmedRuntime }; diff --git a/v2/test/fast/session-service.test.ts b/v2/test/fast/session-service.test.ts index 567f3cdd7..0a9b002f9 100644 --- a/v2/test/fast/session-service.test.ts +++ b/v2/test/fast/session-service.test.ts @@ -27166,6 +27166,48 @@ describe("SessionService", () => { expect(sessions.get(original.id)).toEqual(replacement); }); + it.each(["replacement", "current"])( + "handles rejected orphan snapshots through classifier for %s owner", + async (owner) => { + const sessions = createSessionStore(); + const service = await createDisposedSessionService(); + const original = runningSession({ + status: "spawning", + agentSessionId: "original", + agentLaunchId: "original", + queuedMessages: { messages: ["A", "B"], awaitingPrompt: true }, + }); + const replacement = { + ...original, + status: "running" as const, + agentLaunchId: "replacement", + }; + sessions.set(original.id, original); + const internals = repairInternals(service); + vi.spyOn(internals, "lookupPanePidQuietly").mockResolvedValue({ + status: "ok", + panePid: 101, + paneDead: false, + }); + let reads = 0; + vi.spyOn(internals, "readRuntimeSnapshot").mockImplementation(async () => { + if (reads++ === 0) return live; + if (owner === "replacement") sessions.set(original.id, replacement); + throw new Error("old orphan post-kill probe rejected"); + }); + await internals.classifySessionRecord(original); + expect(killTmuxSessionMock).toHaveBeenCalledOnce(); + if (owner === "replacement") expect(sessions.get(original.id)).toEqual(replacement); + else + expect(sessions.get(original.id)).toMatchObject({ + agentLaunchId: "original", + error: "Session status detection failed: old orphan post-kill probe rejected", + errorSource: "other", + queuedMessages: original.queuedMessages, + }); + }, + ); + it.each(["reject", "unknown"])( "retains original C1 protection during fresh %s", async (mode) => { @@ -53432,38 +53474,22 @@ describe("SessionService", () => { }); it("refuses to launch a replacement when the pane still reads alive but nothing was captured", async () => { - // capturePaneAgentProcessesMock stays at its ok+empty default: the - // matchers found nothing owned. isProcessRunningInTmuxMock disagrees - // with itself on purpose: classification (no `fresh`) and - // reconcileUnexpectedStop's own confirmation re-probe (its own - // `{fresh:true}` call, unrelated to this guard) must both still read - // dead so restore() proceeds past the "not restorable" gate at all — - // only the SECOND `fresh` probe, the one inside - // killAgentPaneAndConfirmExit's new ownership-blind check, reads alive, - // simulating a process the ownership matchers cannot see. + // Only the liveness check after an empty ownership capture sees the + // foreign process; earlier classification remains dead. mockClaudeJsonlState("waiting"); findAgentSessionIdMock.mockResolvedValueOnce(null).mockResolvedValue("session-uuid"); createSessionStore(runningSession({ id: "api-1" })); mockExitedThenRestoredProcess(); agentLaunchUsesForeignBinaryMock.mockReturnValue(true); - // The first fresh lookup (restore's own) resolves unavailable; every - // later lookup, including the kill path's fresh one, resolves an ok pane - // without a pid. - let freshLookups = 0; - lookupTmuxPanePidMock.mockImplementation(async (_name, options) => { - if (options?.fresh) { - freshLookups += 1; - if (freshLookups === 1) return { status: "unavailable" }; - } - return { status: "ok", panePid: null, paneDead: false }; + lookupTmuxPanePidMock.mockResolvedValue({ status: "ok", panePid: null, paneDead: false }); + let emptyOwnershipCaptured = false; + capturePaneAgentProcessesMock.mockImplementation(async () => { + emptyOwnershipCaptured = true; + return { status: "ok", processes: [] }; }); - let freshProbes = 0; isProcessRunningInTmuxMock.mockImplementation( - async (_tmuxSession: string, _matchers: string[], opts?: { fresh?: boolean }) => { - if (opts?.fresh !== true) return false; - freshProbes += 1; - return freshProbes >= 2; - }, + async (_tmuxSession: string, _matchers: string[], opts?: { fresh?: boolean }) => + opts?.fresh === true && emptyOwnershipCaptured, ); const service = await createDisposedSessionService(); From c31d594d110e44327ce9aa73ce5014406334be8c Mon Sep 17 00:00:00 2001 From: AlekShu <108252944+AlekShu@users.noreply.github.com> Date: Fri, 16 Oct 2026 10:58:20 +0000 Subject: [PATCH 11/23] fix(session): admit exact legacy diagnostics through probe recovery --- v2/src/session-service.ts | 20 +- v2/test/fast/session-service.test.ts | 535 ++++++++++++++++++++++----- 2 files changed, 445 insertions(+), 110 deletions(-) diff --git a/v2/src/session-service.ts b/v2/src/session-service.ts index e72f089a6..4ab30db77 100644 --- a/v2/src/session-service.ts +++ b/v2/src/session-service.ts @@ -626,19 +626,11 @@ function recoverableRuntimeError(session: SessionRecord): boolean { (session.errorSource === undefined && session.launchCommand !== "") ); } - if (session.errorSource === "other") { - return ( - session.error === - "Session status detection failed: ps failed: malformed process table output" || - session.error === - "Session status detection failed: ps failed: Command failed: ps -eo pid,ppid,pgid,tpgid,tty,rss,args\n" - ); - } return isRuntimeProbeError(session); } -// Retained error that only records a failed runtime probe (runtime_probe -// source, or a legacy unsourced tmux/ps diagnostic): void once a probe answers. +// Failed runtime probes: runtime_probe, legacy unsourced tmux/ps, or the +// two exact historical other-source ps diagnostics; void once a probe answers. function isRuntimeProbeError(session: SessionRecord): boolean { if ( !session.error?.startsWith(STATUS_DETECTION_ERROR_PREFIX) || @@ -646,6 +638,14 @@ function isRuntimeProbeError(session: SessionRecord): boolean { ) return false; if (session.errorSource === "runtime_probe") return true; + if (session.errorSource === "other") { + return ( + session.error === + "Session status detection failed: ps failed: malformed process table output" || + session.error === + "Session status detection failed: ps failed: Command failed: ps -eo pid,ppid,pgid,tpgid,tty,rss,args\n" + ); + } if (session.errorSource !== undefined) return false; const diagnostic = session.error.slice(STATUS_DETECTION_ERROR_PREFIX.length); return ( diff --git a/v2/test/fast/session-service.test.ts b/v2/test/fast/session-service.test.ts index 0a9b002f9..043afc583 100644 --- a/v2/test/fast/session-service.test.ts +++ b/v2/test/fast/session-service.test.ts @@ -1,6 +1,7 @@ import type * as cryptoModule from "node:crypto"; import { randomUUID } from "node:crypto"; import type * as timersPromisesModule from "node:timers/promises"; +import type * as childProcessModule from "node:child_process"; import { spawn as spawnChildProcess } from "node:child_process"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { @@ -26680,7 +26681,7 @@ describe("SessionService", () => { (guard) => ({ status, guard }), ), ), - )("blocks $status recovery under $guard evidence", async ({ status, guard }) => { + )("preserves $status guarded outcome under $guard evidence", async ({ status, guard }) => { const sessions = createSessionStore(); const record = runningSession({ id: "api-1", @@ -26717,15 +26718,25 @@ describe("SessionService", () => { vi.spyOn(internals, "restoreOwnsRuntimeRecovery").mockReturnValue(true); } - await internals.classifySessionRecord(record); + const pending = internals.classifySessionRecord(record); + if (guard === "dead" && status === "running") await vi.advanceTimersByTimeAsync(1_000); + await pending; expect(sessions.get(record.id)?.error).toBe( guard === "unknown" && status === "running" ? "Session status detection failed: runtime probe (tmux or ps) timed out; runtime state unknown" - : error, + : status === "running" && guard === "dead" + ? undefined + : status === "running" && guard === "workspace" + ? "Agent worktree is missing." + : error, ); expect(sessions.get(record.id)?.errorSource).toBe( - guard === "unknown" && status === "running" ? "runtime_probe" : "other", + guard === "unknown" && status === "running" + ? "runtime_probe" + : guard === "dead" && status === "running" + ? undefined + : "other", ); expect(sendMessageToTmuxMock).not.toHaveBeenCalled(); }); @@ -26878,6 +26889,137 @@ describe("SessionService", () => { }); } + it.each( + historicalErrors.flatMap((error, index) => + ["prewrite", "postwrite", "postwrite dead"].map((stage) => ({ error, index, stage })), + ), + )( + "canonical historical probe: unit producer to retaining send catch ($stage $error)", + async ({ error, index, stage }) => { + const originalRuntime = await import("../../src/runtime-tmux.js"); + const { promisify } = await import("node:util"); + const execFileAsync = vi.fn(async (file: string, args: string[]) => { + if (file === "tmux" && args.includes("list-panes")) { + return { stdout: "api-1 1 1 0 100 /dev/pts/1", stderr: "" }; + } + if (file === "ps") { + if (index === 0) return { stdout: "malformed row", stderr: "" }; + throw new Error("Command failed: ps -eo pid,ppid,pgid,tpgid,tty,rss,args\n"); + } + throw new Error(`unexpected unit subprocess: ${file}`); + }); + vi.doMock("node:child_process", async () => ({ + ...(await vi.importActual("node:child_process")), + execFile: Object.assign(vi.fn(), { [promisify.custom]: execFileAsync }), + })); + try { + vi.resetModules(); + const runtime = await vi.importActual( + "../../src/runtime-tmux.js", + ); + const generated: unknown = await runtime + .isProcessRunningInTmux("api-1", ["claude"], { fresh: true }) + .catch((failure: unknown) => failure); + expect(generated).toBeInstanceOf(runtime.TmuxProbeUnknownError); + expect((generated as Error).message).toBe( + error.slice("Session status detection failed: ".length), + ); + expect(execFileAsync).toHaveBeenCalledWith( + "ps", + ["-eo", "pid,ppid,pgid,tpgid,tty,rss,args"], + expect.anything(), + ); + vi.doMock("../../src/runtime-tmux.js", () => ({ + ...originalRuntime, + TmuxProbeUnknownError: runtime.TmuxProbeUnknownError, + })); + const record = runningSession({ + agentSessionId: "native-original", + agentLaunchId: "launch-original", + }); + if (stage !== "prewrite") { + record.queuedMessages = { messages: ["current input", "tail"], awaitingPrompt: false }; + } + const sessions = createSessionStore(); + mockClaudeJsonlState("waiting"); + const service = await createDisposedSessionService(); + sessions.set(record.id, record); + let written: SessionRecord | undefined; + if (stage === "prewrite") { + vi.spyOn( + service as unknown as { capturePaneGeneration(): Promise }, + "capturePaneGeneration", + ).mockRejectedValue(generated); + } else { + createAgentSubmitAckBindingMock.mockResolvedValue({ + baseline: { agent: "claude", file: "prior.jsonl", size: 123 }, + scan: vi.fn(), + }); + vi.spyOn(sessionServiceInternals(service), "waitForSubmitAck").mockImplementation( + async () => { + written = structuredClone(sessions.get(record.id)); + throw generated; + }, + ); + } + if (stage === "prewrite") { + await expect( + service.send(record.id, { message: "current input", queue: false }), + ).rejects.toBe(generated); + } else { + const result = await sessionServiceInternals(service).tryDeliverQueuedMessage( + record.id, + ); + expect(result).toBe(true); + } + expect(sessions.get(record.id)).toMatchObject({ + agentSessionId: record.agentSessionId, + agentLaunchId: record.agentLaunchId, + error, + errorSource: "other", + }); + expect(sendMessageToTmuxMock).toHaveBeenCalledTimes(stage === "prewrite" ? 0 : 1); + if (stage !== "prewrite") { + expect(written?.queuedMessageTyped).toMatchObject({ + message: "current input", + ackBaseline: { agent: "claude", file: "prior.jsonl", size: 123 }, + }); + expect(sessions.get(record.id)?.queuedMessageTyped).toBeUndefined(); + expect(sessions.get(record.id)?.queuedMessages).toEqual({ + messages: ["current input", "tail"], + awaitingPrompt: false, + }); + } + if (stage === "postwrite dead") { + const current = sessions.get(record.id); + if (!current) throw new Error("missing actual postcatch fixture"); + const internals = internalsFor(service); + vi.spyOn(internals, "readRuntimeSnapshot").mockResolvedValue(dead); + const pending = internals.reconcileUnexpectedStop( + current, + dead, + "runtime_check", + false, + ); + await vi.advanceTimersByTimeAsync(1_000); + await pending; + expect(sessions.get(record.id)).toMatchObject({ + status: "stopped", + queuedMessages: current.queuedMessages, + }); + expect(sessions.get(record.id)?.error).toBeUndefined(); + expect(sessions.get(record.id)?.queuedMessageTyped).toBeUndefined(); + expect(sendMessageToTmuxMock).toHaveBeenCalledOnce(); + expect(createTmuxSessionMock).not.toHaveBeenCalled(); + } + expect(sendInterruptKeysToTmuxMock).not.toHaveBeenCalled(); + } finally { + vi.doUnmock("node:child_process"); + vi.doMock("../../src/runtime-tmux.js", () => originalRuntime); + } + }, + ); + it.each(historicalErrors)("heals healthy historical live evidence %j", async (error) => { const sessions = createSessionStore(runningSession({ error, errorSource: "other" })); mockClaudeJsonlState("waiting"); @@ -26886,21 +27028,182 @@ describe("SessionService", () => { expect(sessions.get("api-1")?.error).toBeUndefined(); }); - it.each(historicalErrors)( - "keeps raw historical dead and boot evidence excluded %j", - async (error) => { + it.each( + historicalErrors.flatMap((error) => + [false, true].map((panePresent) => ({ error, panePresent })), + ), + )( + "canonical historical probe: stops confirmed unavailable terminal ($panePresent $error)", + async ({ error, panePresent }) => { const record = observationRecord(error, "other"); const sessions = createSessionStore(); const service = await createDisposedSessionService(); sessions.set(record.id, record); const internals = internalsFor(service); - vi.spyOn(internals, "readRuntimeSnapshot").mockResolvedValue(dead); - expect( - (await internals.reconcileUnexpectedStop(record, dead, "runtime_check", false)).session, - ).toEqual(record); - const result = await service.reconcileStoppedSessions(); - expect(sessions.get("api-1")).toEqual(record); - expect(result.driftedSessions).toEqual([]); + const runtime = { ...dead, runtimeAlive: panePresent }; + const read = vi.spyOn(internals, "readRuntimeSnapshot").mockResolvedValue({ ...runtime }); + const ports = vi.spyOn(internals, "sessionWithReleasedSidecarPorts"); + const teardown = vi.spyOn(internals, "teardownSessionSidecars").mockResolvedValue(); + const pending = internals.reconcileUnexpectedStop(record, runtime, "runtime_check", false); + expect(read).not.toHaveBeenCalled(); + expect(sessions.get(record.id)).toEqual(record); + await vi.advanceTimersByTimeAsync(1_000); + await pending; + expect(read).toHaveBeenCalledExactlyOnceWith(record, { fresh: true }); + expect(sessions.get(record.id)).toMatchObject({ + status: "stopped", + agentSessionId: record.agentSessionId, + agentLaunchId: record.agentLaunchId, + queuedMessages: record.queuedMessages, + queuedMessageTyped: record.queuedMessageTyped, + }); + expect(sessions.get(record.id)?.error).toBeUndefined(); + expect(sessions.get(record.id)?.errorSource).toBeUndefined(); + expect(ports).toHaveBeenCalledOnce(); + expect(teardown).toHaveBeenCalledOnce(); + expect(sendMessageToTmuxMock).not.toHaveBeenCalled(); + expect(createTmuxSessionMock).not.toHaveBeenCalled(); + }, + ); + + it("canonical historical probe: actual boot output excludes both diagnostics and restores plain drift once", async () => { + const config = baseConfig(); + loadConfigMock.mockReturnValue({ + ...config, + projects: { ...config.projects, api: { ...config.projects.api, restoreAfterReboot: true } }, + }); + const sessions = createSessionStore(); + historicalErrors.forEach((error, index) => { + const record = { ...observationRecord(error, "other"), id: `api-${index + 1}` }; + sessions.set(record.id, record); + }); + sessions.set("api-3", runningSession({ id: "api-3" })); + const service = await createDisposedSessionService(); + vi.spyOn(internalsFor(service), "readRuntimeSnapshot").mockResolvedValue(dead); + const pending = service.reconcileStoppedSessions(); + await vi.advanceTimersByTimeAsync(3_000); + const result = await pending; + for (const id of ["api-1", "api-2"]) { + expect(sessions.get(id)?.status).toBe("stopped"); + expect(sessions.get(id)?.error).toBeUndefined(); + expect(sessions.get(id)?.errorSource).toBeUndefined(); + } + expect(result.driftedSessions.map(({ record }) => record.id)).toEqual(["api-3"]); + const restore = vi.spyOn(service, "restore").mockResolvedValue({} as never); + await service.restoreRebootedSessions(result.driftedSessions); + expect(restore).toHaveBeenCalledExactlyOnceWith("api-3"); + expect(sendMessageToTmuxMock).not.toHaveBeenCalled(); + expect(createTmuxSessionMock).not.toHaveBeenCalled(); + }); + + it.each( + historicalErrors.flatMap((error) => + ["initial unknown", "fresh unknown", "fresh rejection"].map((mode) => ({ error, mode })), + ), + )( + "canonical historical probe: preserves identical diagnostic under $mode ($error)", + async ({ error, mode }) => { + const record = { + ...observationRecord(error, "other"), + submitUnconfirmedAt: "2026-03-18T10:04:00.000Z", + }; + const sessions = createSessionStore(record); + const service = await createDisposedSessionService(); + const internals = internalsFor(service); + const diagnostic = error.slice("Session status detection failed: ".length); + const runtime = { ...unknown, diagnostic }; + const read = vi.spyOn(internals, "readRuntimeSnapshot"); + if (mode === "fresh rejection") read.mockRejectedValue(new Error(diagnostic)); + else read.mockResolvedValue(runtime); + const ports = vi.spyOn(internals, "sessionWithReleasedSidecarPorts"); + const teardown = vi.spyOn(internals, "teardownSessionSidecars"); + const pending = internals.reconcileUnexpectedStop( + record, + mode === "initial unknown" ? runtime : dead, + "runtime_check", + false, + ); + await vi.advanceTimersByTimeAsync(1_000); + await pending; + expect(sessions.get(record.id)).toEqual(record); + expect(ports).not.toHaveBeenCalled(); + expect(teardown).not.toHaveBeenCalled(); + expect(sendMessageToTmuxMock).not.toHaveBeenCalled(); + expect(sendInterruptKeysToTmuxMock).not.toHaveBeenCalled(); + }, + ); + + it.each( + historicalErrors.flatMap((error) => [false, true].map((missing) => ({ error, missing }))), + )( + "canonical historical probe: preserves terminal error distinction (missing=$missing $error)", + async ({ error, missing }) => { + const record = observationRecord(error, "other"); + const sessions = createSessionStore(); + const service = await createDisposedSessionService(); + sessions.set(record.id, record); + const internals = internalsFor(service); + const runtime = { ...dead, runtimeAlive: true, paneUsable: true }; + vi.spyOn(internals, "readRuntimeSnapshot").mockResolvedValue(runtime); + const pending = internals.reconcileUnexpectedStop( + record, + runtime, + "runtime_check", + missing, + ); + await vi.advanceTimersByTimeAsync(1_000); + await pending; + expect(sessions.get(record.id)).toMatchObject({ + status: "errored", + error: missing ? "Agent worktree is missing." : "Agent runtime exited unexpectedly.", + errorSource: missing ? "other" : "inferred_runtime_exit", + queuedMessages: record.queuedMessages, + queuedMessageTyped: record.queuedMessageTyped, + }); + expect(sendMessageToTmuxMock).not.toHaveBeenCalled(); + expect(createTmuxSessionMock).not.toHaveBeenCalled(); + }, + ); + + it.each( + historicalErrors.flatMap((error) => + [ + { status: "stopped" as const, stopReason: "manual_pause" as const }, + { status: "stopped" as const, stopReason: "token_budget" as const }, + { status: "stopped" as const, stopReason: undefined }, + { status: "errored" as const, stopReason: undefined }, + { status: "completed" as const, stopReason: undefined }, + { status: "killed" as const, stopReason: undefined }, + ].map((state) => ({ error, ...state })), + ), + )( + "canonical historical probe: nonrunning status=$status reason=$stopReason ($error)", + async ({ error, status, stopReason }) => { + const record = { + ...observationRecord(error, "other"), + status, + ...(stopReason ? { stopReason } : {}), + }; + const sessions = createSessionStore(record); + const service = await createDisposedSessionService(); + vi.spyOn(internalsFor(service), "readRuntimeSnapshot").mockResolvedValue(dead); + await internalsFor(service).classifySessionRecord(record); + expect(sessions.get(record.id)?.status).toBe(status); + expect(sessions.get(record.id)?.stopReason).toBe(stopReason); + expect(sessions.get(record.id)?.error).toBe(status === "stopped" ? undefined : error); + expect(sendMessageToTmuxMock).not.toHaveBeenCalled(); + expect(createTmuxSessionMock).not.toHaveBeenCalled(); + }, + ); + + it.each(historicalErrors)( + "canonical historical probe: skips spawning boot %j", + async (error) => { + const record = { ...observationRecord(error, "other"), status: "spawning" as const }; + const sessions = createSessionStore(record); + const service = await createDisposedSessionService(); + await internalsFor(service).reconcileUnexpectedStop(record, dead, "boot", false); + expect(sessions.get(record.id)).toEqual(record); expect(sendMessageToTmuxMock).not.toHaveBeenCalled(); expect(createTmuxSessionMock).not.toHaveBeenCalled(); }, @@ -26966,6 +27269,8 @@ describe("SessionService", () => { "project", "workspace", "pause", + "status", + "source", "provider", "server", "session-pending", @@ -26978,97 +27283,127 @@ describe("SessionService", () => { ] as const; it.each( (["fresh", "boot", "boot-fresh"] as const).flatMap((boundary) => - races.map((race) => ({ boundary, race })), - ), - )("binds $boundary observation under $race", async ({ boundary, race }) => { - const record = observationRecord(); - const sessions = createSessionStore(); - const service = await createDisposedSessionService(); - sessions.set(record.id, record); - const internals = internalsFor(service); - const releasePorts = vi.spyOn(internals, "sessionWithReleasedSidecarPorts"); - const teardown = vi.spyOn(internals, "teardownSessionSidecars"); - let replacement = record; - let reads = 0; - vi.spyOn(internals, "readRuntimeSnapshot").mockImplementation(async () => { - if (boundary === "boot-fresh" && reads++ === 0) return dead; - const changes: Partial = - race === "native" - ? { agentSessionId: "replacement" } - : race === "launch" - ? { agentLaunchId: "replacement" } - : race === "tmux" - ? { tmuxSession: "replacement" } - : race === "command" - ? { launchCommand: "replacement" } - : race === "path" - ? { worktreePath: "/replacement" } - : race === "project" - ? { project: "replacement" } - : race === "workspace" - ? { workspaceId: "replacement" } - : race === "pause" - ? { status: "stopped", stopReason: "manual_pause" } - : race === "provider" - ? { error: "Provider request failed", errorSource: "other" } - : race === "server" - ? { serverErrorAt: "2026-03-18T10:04:00.000Z" } - : {}; - replacement = { ...record, ...changes }; - sessions.set(record.id, replacement); - if (race === "instance") internals.lifecycle = new SessionLifecycleRegister(); - if (race.includes("pending") || race.includes("settled") || race.includes("reopen")) { - const owner = internals.lifecycle.begin( - race.includes("reopen") ? "reopen" : "restore", - race.includes("workspace") ? [record.workspaceId ?? record.id] : [record.id], + races.flatMap((race) => [ + { boundary, race, error: historicalErrors[0], errorSource: "runtime_probe" as const }, + ...historicalErrors.map((error) => ({ + boundary, race, - ); - if (race.includes("settled")) - internals.lifecycle.settle( - owner, - true, - () => "succeeded", - () => true, + error, + errorSource: "other" as const, + })), + ]), + ), + )( + "binds $boundary observation under $race ($errorSource $error)", + async ({ boundary, race, error, errorSource }) => { + const record = observationRecord(error, errorSource); + const sessions = createSessionStore(); + const service = await createDisposedSessionService(); + sessions.set(record.id, record); + const internals = internalsFor(service); + const releasePorts = vi.spyOn(internals, "sessionWithReleasedSidecarPorts"); + const teardown = vi.spyOn(internals, "teardownSessionSidecars"); + let replacement = record; + let reads = 0; + vi.spyOn(internals, "readRuntimeSnapshot").mockImplementation(async () => { + if (boundary === "boot-fresh" && reads++ === 0) return dead; + const changes: Partial = + race === "native" + ? { agentSessionId: "replacement" } + : race === "launch" + ? { agentLaunchId: "replacement" } + : race === "tmux" + ? { tmuxSession: "replacement" } + : race === "command" + ? { launchCommand: "replacement" } + : race === "path" + ? { worktreePath: "/replacement" } + : race === "project" + ? { project: "replacement" } + : race === "workspace" + ? { workspaceId: "replacement" } + : race === "pause" + ? { status: "stopped", stopReason: "manual_pause" } + : race === "status" + ? { status: "errored" } + : race === "source" + ? { + errorSource: + errorSource === "other" ? "runtime_probe" : "other", + } + : race === "provider" + ? { error: "Provider request failed", errorSource: "other" } + : race === "server" + ? { serverErrorAt: "2026-03-18T10:04:00.000Z" } + : {}; + replacement = { ...record, ...changes }; + sessions.set(record.id, replacement); + if (race === "instance") internals.lifecycle = new SessionLifecycleRegister(); + if (race.includes("pending") || race.includes("settled") || race.includes("reopen")) { + const owner = internals.lifecycle.begin( + race.includes("reopen") ? "reopen" : "restore", + race.includes("workspace") ? [record.workspaceId ?? record.id] : [record.id], + race, ); - } - return dead; - }); - const pending = - boundary !== "fresh" - ? service.reconcileStoppedSessions() - : internals.reconcileUnexpectedStop(record, dead, "runtime_check", false); - await vi.advanceTimersByTimeAsync(1_000); - await pending; - expect(sessions.get(record.id), `${boundary}/${race}`).toEqual(replacement); - expect(releasePorts, `${boundary}/${race}`).not.toHaveBeenCalled(); - expect(teardown, `${boundary}/${race}`).not.toHaveBeenCalled(); - expect(sendMessageToTmuxMock).not.toHaveBeenCalled(); - expect(createTmuxSessionMock).not.toHaveBeenCalled(); - }); + if (race.includes("settled")) + internals.lifecycle.settle( + owner, + true, + () => "succeeded", + () => true, + ); + } + return dead; + }); + const pending = + boundary !== "fresh" + ? service.reconcileStoppedSessions() + : internals.reconcileUnexpectedStop(record, dead, "runtime_check", false); + await vi.advanceTimersByTimeAsync(1_000); + await pending; + expect(sessions.get(record.id), `${boundary}/${race}`).toEqual(replacement); + expect(releasePorts, `${boundary}/${race}`).not.toHaveBeenCalled(); + expect(teardown, `${boundary}/${race}`).not.toHaveBeenCalled(); + expect(sendMessageToTmuxMock).not.toHaveBeenCalled(); + expect(createTmuxSessionMock).not.toHaveBeenCalled(); + }, + ); - it("preserves a concurrent queue append with unchanged owner", async () => { - const record = observationRecord(); - const sessions = createSessionStore(); - const service = await createDisposedSessionService(); - sessions.set(record.id, record); - const internals = internalsFor(service); - vi.spyOn(internals, "readRuntimeSnapshot").mockImplementation(async () => { - sessions.set(record.id, { - ...record, - updatedAt: "2026-03-18T10:05:00.000Z", + it.each([ + { error: historicalErrors[0], errorSource: "runtime_probe" as const }, + ...historicalErrors.map((error) => ({ error, errorSource: "other" as const })), + ])( + "canonical historical probe: preserves a concurrent queue append ($errorSource $error)", + async ({ error, errorSource }) => { + const record = { + ...observationRecord(error, errorSource), + submitUnconfirmedAt: "2026-03-18T10:04:00.000Z", + }; + const sessions = createSessionStore(); + const service = await createDisposedSessionService(); + sessions.set(record.id, record); + const internals = internalsFor(service); + vi.spyOn(internals, "readRuntimeSnapshot").mockImplementation(async () => { + sessions.set(record.id, { + ...record, + updatedAt: "2026-03-18T10:05:00.000Z", + queuedMessages: { messages: ["A", "B", "C"], awaitingPrompt: true }, + }); + return dead; + }); + const pending = internals.reconcileUnexpectedStop(record, dead, "runtime_check", false); + await vi.advanceTimersByTimeAsync(1_000); + await pending; + expect(sessions.get(record.id)).toMatchObject({ + status: "stopped", queuedMessages: { messages: ["A", "B", "C"], awaitingPrompt: true }, + queuedMessageTyped: record.queuedMessageTyped, + submitUnconfirmedAt: record.submitUnconfirmedAt, }); - return dead; - }); - const pending = internals.reconcileUnexpectedStop(record, dead, "runtime_check", false); - await vi.advanceTimersByTimeAsync(1_000); - await pending; - expect(sessions.get(record.id)).toMatchObject({ - status: "stopped", - queuedMessages: { messages: ["A", "B", "C"], awaitingPrompt: true }, - queuedMessageTyped: record.queuedMessageTyped, - }); - }); + expect(sendMessageToTmuxMock).not.toHaveBeenCalled(); + expect(createTmuxSessionMock).not.toHaveBeenCalled(); + }, + ); }); describe("pr1197 review repair:", () => { From 34b373b8e9e1ecc2436d2ccac6d3de9f3f03de55 Mon Sep 17 00:00:00 2001 From: AlekShu <108252944+AlekShu@users.noreply.github.com> Date: Fri, 16 Oct 2026 13:59:18 +0000 Subject: [PATCH 12/23] fix(session): serialize daily denial with wake claims --- v2/src/session-service.ts | 125 ++++++++++++--------------- v2/test/fast/session-service.test.ts | 77 +++++++++++++---- 2 files changed, 117 insertions(+), 85 deletions(-) diff --git a/v2/src/session-service.ts b/v2/src/session-service.ts index 4ab30db77..1e107c2b0 100644 --- a/v2/src/session-service.ts +++ b/v2/src/session-service.ts @@ -5537,30 +5537,25 @@ export class SessionService { updatedAt: nowIso(), }; writeSession(this.config.dataDir, updated); - return { + const claim: DailyWakeClaim = { record: updated, lifecycle: this.lifecycleIdsFor(updated).map((id) => ({ id, snapshot: this.lifecycle.snapshot(id), })), }; + const admissionDenial = this.wakeAdmissionDenial(session.id); + if (admissionDenial) { + this.deferDailyWakeForAdmission(session, dailyWake, claim, admissionDenial, now); + return null; + } + return claim; } return null; }, ); if (claimedDailyWake !== null) { const claimedDailyNextDueAt = claimedDailyWake.record.dailyWake.nextDueAt; - const admissionDenial = this.wakeAdmissionDenial(session.id); - if (admissionDenial) { - await this.deferDailyWakeForAdmission( - session, - dailyWake, - claimedDailyWake, - admissionDenial, - now, - ); - continue; - } try { await this.send(session.id, { message: this.formatDailyWakeMessage( @@ -5670,62 +5665,60 @@ export class SessionService { // Denied daily occurrence retries every DAILY_WAKE_ADMISSION_RETRY_MS until // its next scheduled slot, where the claim already points; past that it is // skipped. The occurrence is persisted so a restart keeps the window. - private async deferDailyWakeForAdmission( + // Called synchronously under the claim's workspace lifecycle locks. + private deferDailyWakeForAdmission( session: SessionRecord, dailyWake: SessionDailyWakeState, claimed: DailyWakeClaim, denial: WakeAdmissionDenial, now: number, - ): Promise { + ): void { const occurrence = dailyWake.admissionDeferredOccurrence ?? dailyWake.nextDueAt; const windowEndMs = resolveNextDailyWakeAt(dailyWake.dailyAt, new Date(occurrence)).getTime(); const retryAtMs = now + DAILY_WAKE_ADMISSION_RETRY_MS; const retryAt = new Date(retryAtMs).toISOString(); - const deferred = await this.withWorkspaceLifecycleLocks(session.id, async () => { - const afterDenial = readSession(this.config.dataDir, session.id); - if ( - !afterDenial || - !sameSessionRuntimeOwner(afterDenial, claimed.record) || - afterDenial.agentLaunchId !== claimed.record.agentLaunchId || - !this.lifecycleStateMatches(afterDenial, this.lifecycleStamp(claimed.record)) || - !isDeepStrictEqual(afterDenial.dailyWake, claimed.record.dailyWake) || - claimed.lifecycle.some(({ id, snapshot }) => { - const current = this.lifecycle.snapshot(id); - return ( - snapshot.operation?.phase === "pending" || - current.operation?.phase === "pending" || - current.instanceId !== snapshot.instanceId || - current.revision !== snapshot.revision - ); - }) - ) - return false; - if (retryAtMs >= windowEndMs) { - this.logEvent("session.wake.daily_exhausted", { - level: "warn", - sessionId: session.id, - projectId: session.project, - message: `Skipped daily wake occurrence ${occurrence} for ${session.id}: ${denial.error.message}`, - details: { - occurrence, - nextDueAt: claimed.record.dailyWake.nextDueAt, - dailyAt: dailyWake.dailyAt, - }, - }); - return false; - } - writeSession(this.config.dataDir, { - ...afterDenial, - dailyWake: { - ...claimed.record.dailyWake, - nextDueAt: retryAt, - admissionDeferredOccurrence: occurrence, + const afterDenial = readSession(this.config.dataDir, session.id); + if ( + !afterDenial || + !sameSessionRuntimeOwner(afterDenial, claimed.record) || + afterDenial.agentLaunchId !== claimed.record.agentLaunchId || + !this.lifecycleStateMatches(afterDenial, this.lifecycleStamp(claimed.record)) || + !isDeepStrictEqual(afterDenial.dailyWake, claimed.record.dailyWake) || + claimed.lifecycle.some(({ id, snapshot }) => { + const current = this.lifecycle.snapshot(id); + return ( + snapshot.operation?.phase === "pending" || + current.operation?.phase === "pending" || + current.instanceId !== snapshot.instanceId || + current.revision !== snapshot.revision + ); + }) + ) + return; + if (retryAtMs >= windowEndMs) { + this.logEvent("session.wake.daily_exhausted", { + level: "warn", + sessionId: session.id, + projectId: session.project, + message: `Skipped daily wake occurrence ${occurrence} for ${session.id}: ${denial.error.message}`, + details: { + occurrence, + nextDueAt: claimed.record.dailyWake.nextDueAt, + dailyAt: dailyWake.dailyAt, }, - updatedAt: nowIso(), }); - return true; + return; + } + writeSession(this.config.dataDir, { + ...afterDenial, + dailyWake: { + ...claimed.record.dailyWake, + nextDueAt: retryAt, + admissionDeferredOccurrence: occurrence, + }, + updatedAt: nowIso(), }); - if (!deferred || dailyWake.admissionDeferredOccurrence !== undefined) return; + if (dailyWake.admissionDeferredOccurrence !== undefined) return; this.logEvent("session.wake.daily_deferred", { level: wakeAdmissionLogLevel(denial), sessionId: session.id, @@ -21940,10 +21933,11 @@ export class SessionService { })); const readObservation = () => { const latest = readSession(this.config.dataDir, observed.id); + if (latest === null) + throw new SessionResourceNotFoundError(`Session not found: ${observed.id}`); return { latest, valid: - latest !== null && sameSessionRuntimeOwner(latest, observed) && latest.agentLaunchId === observed.agentLaunchId && this.lifecycleStateMatches(latest, stamp) && @@ -21962,7 +21956,7 @@ export class SessionService { }; }; const initial = readObservation(); - if (!initial.valid) return { session: initial.latest ?? session, runtime }; + if (!initial.valid) return { session: initial.latest, runtime }; if (this.isDeliveryStopped()) return { session, runtime }; if (session.status !== "running" && session.status !== "spawning") { return { session, runtime }; @@ -22011,8 +22005,7 @@ export class SessionService { confirmedRuntime = await this.readRuntimeSnapshot(session, { fresh: true }); } catch (error) { const observation = readObservation(); - if (!observation.valid) - return { session: observation.latest ?? session, runtime: confirmedRuntime }; + if (!observation.valid) return { session: observation.latest, runtime: confirmedRuntime }; const diagnostic = error instanceof Error ? error.message : String(error); const retained = this.retainDetectionError(session, diagnostic, "runtime_probe"); return { @@ -22030,8 +22023,7 @@ export class SessionService { if (this.isDeliveryStopped()) return { session, runtime: confirmedRuntime }; } const observation = readObservation(); - if (!observation.valid) - return { session: observation.latest ?? session, runtime: confirmedRuntime }; + if (!observation.valid) return { session: observation.latest, runtime: confirmedRuntime }; if ( live(confirmedRuntime) && session.status === "spawning" && @@ -22045,13 +22037,11 @@ export class SessionService { ); } catch (error) { const afterOrphan = readObservation(); - if (!afterOrphan.valid) - return { session: afterOrphan.latest ?? session, runtime: confirmedRuntime }; + if (!afterOrphan.valid) return { session: afterOrphan.latest, runtime: confirmedRuntime }; throw error; } const afterOrphan = readObservation(); - if (!afterOrphan.valid) - return { session: afterOrphan.latest ?? session, runtime: confirmedRuntime }; + if (!afterOrphan.valid) return { session: afterOrphan.latest, runtime: confirmedRuntime }; } if (live(confirmedRuntime)) { return { session, runtime: confirmedRuntime }; @@ -22084,8 +22074,7 @@ export class SessionService { const observation = readObservation(); const latest = observation.latest; - if (!observation.valid || !latest) - return { session: latest ?? session, runtime: confirmedRuntime }; + if (!observation.valid) return { session: latest, runtime: confirmedRuntime }; if (hasRetainedSessionError(latest) && !this.probeErrorHealable(latest)) return { session: latest, runtime: confirmedRuntime }; diff --git a/v2/test/fast/session-service.test.ts b/v2/test/fast/session-service.test.ts index 043afc583..a28b71f9d 100644 --- a/v2/test/fast/session-service.test.ts +++ b/v2/test/fast/session-service.test.ts @@ -12444,6 +12444,7 @@ describe("SessionService", () => { // Reconcile-shaped drift: status flips to "stopped" with no stopReason, // the pipeline is left running untouched (matches the terminalUnavailable // branch at session-service.ts's reconcile, :14520-14530). + await waitForRealTime(() => expect(timerPromisesSleepMock).toHaveBeenCalledWith(1_000)); const parked = sessions.get("api-1"); if (!parked) throw new Error("expected api-1 to exist"); sessions.set("api-1", { @@ -12655,6 +12656,7 @@ describe("SessionService", () => { // Reconcile's errored branch (session-service.ts:14531-14538): same // unintended wedge, no stopReason, different status. + await waitForRealTime(() => expect(timerPromisesSleepMock).toHaveBeenCalledWith(1_000)); const parked = sessions.get("api-1"); if (!parked) throw new Error("expected api-1 to exist"); sessions.set("api-1", { @@ -12708,6 +12710,7 @@ describe("SessionService", () => { const run = sessionServiceInternals(service).deliveryRuns.get("api-1"); expect(run).toBeDefined(); + await waitForRealTime(() => expect(timerPromisesSleepMock).toHaveBeenCalledWith(1_000)); const parked = sessions.get("api-1"); if (!parked) throw new Error("expected api-1 to exist"); sessions.set("api-1", { @@ -12834,6 +12837,7 @@ describe("SessionService", () => { const run = sessionServiceInternals(service).deliveryRuns.get("api-1"); expect(run).toBeDefined(); + await waitForRealTime(() => expect(timerPromisesSleepMock).toHaveBeenCalledWith(1_000)); const parked = sessions.get("api-1"); if (!parked) throw new Error("expected api-1 to exist"); sessions.set("api-1", { @@ -27440,7 +27444,7 @@ describe("SessionService", () => { claim: DailyClaim, denial: { kind: "denied"; error: Error }, now: number, - ): Promise; + ): void; }; } @@ -27754,8 +27758,8 @@ describe("SessionService", () => { }); sessions.set(original.id, original); const claim = rebootCandidates(service, [original.id])[0] as DailyClaim; - let replacement = original; - vi.spyOn(internals, "withWorkspaceLifecycleLocks").mockImplementation(async (_id, action) => { + let replacement: SessionRecord; + { replacement = { ...original, dailyWake: { @@ -27785,8 +27789,7 @@ describe("SessionService", () => { () => true, ); } - return action(); - }); + } const exhausted = change === "exhausted" || change === "stale-exhausted"; await internals.deferDailyWakeForAdmission( original, @@ -51054,6 +51057,49 @@ describe("SessionService", () => { return { sessions, panes, freeSlot, takeSlot }; } + it.each(["daily-api-1", "new-message"])( + "serializes a denied claim before supported rearm with %s", + async (message) => { + const { sessions } = seedDaily(); + vi.setSystemTime(at(9, 0, 1)); + const service = await createDisposedSessionService(); + const internals = service as unknown as { + assertAdmissible(...args: unknown[]): unknown; + }; + const original = internals.assertAdmissible.bind(service); + let rearm: Promise | undefined; + vi.spyOn(internals, "assertAdmissible").mockImplementationOnce((...args) => { + vi.setSystemTime(at(9, 0, 2)); + rearm = service.scheduleWake("api-1", { + dailyAt: ["09:00", "09:40"], + message, + stopCondition: "Stop condition", + }); + return original(...args); + }); + await tick(service); + expect(rearm).toBeDefined(); + await rearm; + expect(sessions.get("api-1")?.dailyWake).toEqual({ + dailyAt: ["09:00", "09:40"], + nextDueAt: iso(at(9, 40)), + message, + stopCondition: "Stop condition", + }); + expect(wakeSends(message)).toBe(0); + const writes = writeSessionMock.mock.calls + .map(([, record]) => record) + .filter((record) => record.id === "api-1"); + expect(writes.map((record) => record.dailyWake?.nextDueAt)).toEqual([ + iso(at(9, 40)), + iso(at(9, 15, 1)), + iso(at(9, 40)), + ]); + expect(events("session.wake.daily_deferred")).toHaveLength(1); + service.dispose(); + }, + ); + it("retries a denied occurrence until admitted, then sends it once", async () => { const { sessions, freeSlot } = seedDaily(); vi.setSystemTime(at(9, 0, 1)); @@ -53884,20 +53930,17 @@ describe("SessionService", () => { createTmuxSessionMock.mockImplementation(async () => { restoredTmuxCreated = true; }); - // Same disagreement shape as the sibling above: classification's own - // reads (including reconcileUnexpectedStop's `{fresh:true}` confirm) - // must stay dead so restore() proceeds; only a SECOND `fresh` probe — - // reachable only if the paneLookup.status gate were ever dropped — - // would read alive. With the gate intact, killAgentPaneAndConfirmExit - // never issues that second probe at all. - let freshProbes = 0; + // Bind liveness to capture and creation, not classification probe count. + // A forbidden fresh capture probe reads alive; readiness after creation + // also reads alive, while every pre-capture classification reads dead. + let captured = false; + capturePaneAgentProcessesMock.mockImplementation(async () => { + captured = true; + return { status: "ok", processes: [] }; + }); isProcessRunningInTmuxMock.mockImplementation( async (_tmuxSession: string, _matchers: string[], opts?: { fresh?: boolean }) => { - if (opts?.fresh === true) { - freshProbes += 1; - return freshProbes >= 2; - } - return restoredTmuxCreated; + return restoredTmuxCreated || (opts?.fresh === true && captured); }, ); From 8ab94958404850ab4c90cbf577c07875b3b2c611 Mon Sep 17 00:00:00 2001 From: AlekShu <108252944+AlekShu@users.noreply.github.com> Date: Tue, 20 Oct 2026 08:48:00 +0000 Subject: [PATCH 13/23] fix(session): skip runtime reconcile for unpersisted records Classification of a record with no persisted file reached the observation fence, which threw not-found and failed cursor classification and desk member enrichment. --- v2/src/session-service.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/v2/src/session-service.ts b/v2/src/session-service.ts index 58e9f1ccd..aa0fe9d1d 100644 --- a/v2/src/session-service.ts +++ b/v2/src/session-service.ts @@ -23358,6 +23358,8 @@ export class SessionService { }), }; }; + // An unpersisted record has nothing to reconcile. + if (readSession(this.config.dataDir, observed.id) === null) return { session, runtime }; const initial = readObservation(); if (!initial.valid) return { session: initial.latest, runtime }; if (this.isDeliveryStopped()) return { session, runtime }; From d6e3453fbd4545b0a133f3f0db92b1b942304eb4 Mon Sep 17 00:00:00 2001 From: AlekShu <108252944+AlekShu@users.noreply.github.com> Date: Tue, 20 Oct 2026 09:04:00 +0000 Subject: [PATCH 14/23] fix(session): read final usage for boot-healed probe errors --- v2/src/session-service.ts | 2 +- v2/test/fast/session-service.test.ts | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/v2/src/session-service.ts b/v2/src/session-service.ts index aa0fe9d1d..cb5d99506 100644 --- a/v2/src/session-service.ts +++ b/v2/src/session-service.ts @@ -11899,7 +11899,7 @@ export class SessionService { if (!reconciled) continue; const usage = (reconciled.session.status === "stopped" || reconciled.session.status === "errored") && - !hasRetainedSessionError(current) && + (healedProbeError || !hasRetainedSessionError(current)) && !current.serverErrorAt && !reconciled.runtime.probeUnresponsive && !reconciled.runtime.diagnostic && diff --git a/v2/test/fast/session-service.test.ts b/v2/test/fast/session-service.test.ts index d9ac11045..60d4a618d 100644 --- a/v2/test/fast/session-service.test.ts +++ b/v2/test/fast/session-service.test.ts @@ -21983,6 +21983,31 @@ describe("SessionService", () => { expect(restoreSpy).not.toHaveBeenCalledWith("api-1"); }); + it("records final usage for a probe-error record healed at boot", async () => { + const sessions = seedRestoreAfterRebootFleet(); + getTmuxSessionPresenceMock.mockResolvedValue({ present: false, unresponsive: false }); + mockClaudeJsonlState("waiting", { + tokenUsage: { + provider: "claude", + generationId: "final-message", + inputTokens: 7, + outputTokens: 3, + totalTokens: 10, + }, + }); + const service = await createDisposedSessionService(); + + const pending = service.reconcileStoppedSessions(); + await vi.advanceTimersByTimeAsync(1_000); + await pending; + + expect(sessions.get("api-1")).toMatchObject({ + status: "stopped", + tokenUsage: { totalTokens: 10 }, + }); + expect(sessions.get("api-1")).not.toHaveProperty("error"); + }); + it("keeps a record healed between list and read out of reboot restore", async () => { const sessions = seedRestoreAfterRebootFleet(); getTmuxSessionPresenceMock.mockImplementation(async (name: string) => { From edc81a8a0100a713dd5e4768236044b27e587241 Mon Sep 17 00:00:00 2001 From: AlekShu <108252944+AlekShu@users.noreply.github.com> Date: Fri, 6 Nov 2026 10:43:50 +0000 Subject: [PATCH 15/23] test(runtime): synchronize isolated fixture readiness --- v2/test/fast/isolated-daemon-owner.test.ts | 39 ++++++++++++++++++---- v2/test/fast/tmux-exit-net.fixture.test.ts | 3 ++ v2/test/fast/tmux-exit-net.test.ts | 8 ++++- 3 files changed, 43 insertions(+), 7 deletions(-) diff --git a/v2/test/fast/isolated-daemon-owner.test.ts b/v2/test/fast/isolated-daemon-owner.test.ts index e0dabf41a..7f1db2e10 100644 --- a/v2/test/fast/isolated-daemon-owner.test.ts +++ b/v2/test/fast/isolated-daemon-owner.test.ts @@ -584,12 +584,39 @@ describe("isolated owner control", () => { } }); it("preserves a dummy native-client descendant across restart and terminal owner stop", async () => { - const code = `const fs=require('node:fs');const path=require('node:path');const pidFile=path.join(__dirname,'native.pid');if(!fs.existsSync(pidFile)){const client=require('node:child_process').spawn(process.execPath,['-e','setInterval(()=>{},1000)'],{detached:false,stdio:'ignore'});fs.writeFileSync(pidFile,String(client.pid));client.unref();}setInterval(()=>{},1000);`; - const fixture = await launch(code); - await expect - .poll(async () => Number(await readFile(join(fixture.directory, "native.pid"), "utf8"))) - .toBeGreaterThan(0); - const nativePid = Number(await readFile(join(fixture.directory, "native.pid"), "utf8")); + const childProcess = await import("node:child_process"); + const original = await vi.importActual("node:child_process"); + let fixtureChild: Cp.ChildProcess | undefined; + let nativeReady: Promise | undefined; + const spawn = vi + .spyOn(childProcess, "spawn") + .mockImplementationOnce((command, args, options) => { + const child = original.spawn(command, args, { + ...options, + stdio: ["inherit", "inherit", "inherit", "ipc"], + }); + fixtureChild = child; + nativeReady = new Promise((resolve, reject) => { + child.once("message", resolve); + child.once("error", reject); + child.once("exit", () => reject(new Error("Fixture exited before native readiness"))); + }); + void nativeReady.catch(() => undefined); + return child; + }); + const code = `const fs=require('node:fs');const path=require('node:path');const pidFile=path.join(__dirname,'native.pid');if(process.send){process.once('message',()=>{const client=require('node:child_process').spawn(process.execPath,['-e','setInterval(()=>{},1000)'],{detached:false,stdio:'ignore'});fs.writeFileSync(pidFile,String(client.pid));client.unref();process.send(client.pid);process.disconnect();});}setInterval(()=>{},1000);`; + // Mocked daemon readiness precedes this real child's native-client setup. + const fixture = await launch(code).finally(() => spawn.mockRestore()); + if (!nativeReady || !fixtureChild) throw new Error("Fixture did not spawn"); + await expect(readFile(join(fixture.directory, "native.pid"))).rejects.toMatchObject({ + code: "ENOENT", + }); + fixtureChild.send("start-native"); + const nativePid = await nativeReady; + expect(nativePid).toBeGreaterThan(0); + if (typeof nativePid !== "number" || !Number.isSafeInteger(nativePid)) + throw new Error("Invalid fixture native PID"); + expect(Number(await readFile(join(fixture.directory, "native.pid"), "utf8"))).toBe(nativePid); const nativeStarttime = await readLiveProcessStarttime(nativePid); try { expect(nativeStarttime).not.toBeNull(); diff --git a/v2/test/fast/tmux-exit-net.fixture.test.ts b/v2/test/fast/tmux-exit-net.fixture.test.ts index 5b014b4b2..12cde0d55 100644 --- a/v2/test/fast/tmux-exit-net.fixture.test.ts +++ b/v2/test/fast/tmux-exit-net.fixture.test.ts @@ -53,6 +53,9 @@ describe("tmux-exit-net fixture", () => { const markerPath = process.env["TMUX_EXIT_NET_MARKER_FILE"]; if (markerPath) { writeFileSync(markerPath, JSON.stringify({ socketA, socketB })); + // Registered after the exit net: observing this proves its sweep finished + // in this worker, beyond the supervising Vitest CLI's exit. + process.once("exit", () => writeFileSync(`${markerPath}.exited`, "exited")); } // Deliberately no cleanup()/kill-server call and no afterAll — the exit diff --git a/v2/test/fast/tmux-exit-net.test.ts b/v2/test/fast/tmux-exit-net.test.ts index 662861b44..257ed7648 100644 --- a/v2/test/fast/tmux-exit-net.test.ts +++ b/v2/test/fast/tmux-exit-net.test.ts @@ -12,7 +12,7 @@ // `const socketName = activeTmuxSocketName` closure) leaves the SECOND // socket alive after this test's fixture exits. import { execFile } from "node:child_process"; -import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { promisify } from "node:util"; @@ -36,6 +36,7 @@ describe.skipIf(!tmuxOk)("tmux exit net (abnormal-exit safety)", () => { const vitestBin = join(process.cwd(), "node_modules", ".bin", "vitest"); const markerDir = mkdtempSync(join(tmpdir(), "spur-tmux-exit-net-marker-")); const markerPath = join(markerDir, "marker.json"); + let sockets: string[] = []; try { await execFileAsync( @@ -51,10 +52,15 @@ describe.skipIf(!tmuxOk)("tmux exit net (abnormal-exit safety)", () => { socketA: string; socketB: string; }; + sockets = [marker.socketA, marker.socketB]; + await expect.poll(() => existsSync(`${markerPath}.exited`)).toBe(true); expect(await tmuxHasSession(marker.socketA)).toBe(false); expect(await tmuxHasSession(marker.socketB)).toBe(false); } finally { + for (const socket of sockets) { + await execFileAsync("tmux", ["-L", socket, "kill-server"]).catch(() => undefined); + } rmSync(markerDir, { recursive: true, force: true }); } }, 60_000); From f3c10dea98aee5c4d064af33e4d0f93f1f5d188a Mon Sep 17 00:00:00 2001 From: AlekShu <108252944+AlekShu@users.noreply.github.com> Date: Fri, 6 Nov 2026 15:25:02 +0000 Subject: [PATCH 16/23] fix(session): preserve reboot siblings and reserve scheduled wake admission --- docs/configuration.md | 2 +- docs/daemon-api.md | 2 +- v2/src/session-service.ts | 333 +++++++++++++++++--------- v2/test/fast/mcp-contracts.test.ts | 2 +- v2/test/fast/session-service.test.ts | 338 +++++++++++++++++++++++++++ 5 files changed, 566 insertions(+), 111 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 7f395f7c7..c575ae034 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -126,7 +126,7 @@ Type/constraint/default per key; full validation source `v2/src/config.ts`. ## Admission control -Admission gates spawn, restore and wake relaunch; policy: `v2/src/session-service.ts`; capacity: [`GET /headroom`](daemon-api.md); diagnostics: [`spur doctor`](commands.md#doctor). +Admission gates spawn and restore. Background interval/daily wakes gate stopped/paused, stale-parked and memory-shed relaunches; manual interval/daily dispatch gates only stale-parked and memory-shed relaunches. Policy: `v2/src/session-service.ts`; capacity: [`GET /headroom`](daemon-api.md); diagnostics: [`spur doctor`](commands.md#doctor). `stopReason: "runtime_lost"`: confirmed runtime loss becomes restorable `stopped`, retains error evidence, releases capacity; unknown probes retain capacity (`v2/src/session-service.ts`). diff --git a/docs/daemon-api.md b/docs/daemon-api.md index 019f1b302..246ff2ecb 100644 --- a/docs/daemon-api.md +++ b/docs/daemon-api.md @@ -41,7 +41,7 @@ - `POST /sessions/background` — `409 spawn_in_progress|spawn_committed|spawn_failed` body `{ code, sessionId, error }` when `preflightBatchId` is already claimed; `sessionId` is the claiming session; `v2/src/session-service.ts`. - `POST /shepherd/spawn` - `POST /sidecars/sweep` — body `{ reap?: boolean }`, default `false`. Always `200`. Response `{ supported, leaked, reaped }`: `supported` is `false` when the process table or procfs is unreadable, in which case `leaked` and `reaped` are both `[]`. Each `leaked` row carries `kind: "worktree-tree" | "orphan-daemon"`; a `worktree-tree` row's `reapable` reflects proven Spur provenance and carries `rootPid`, `pgid`, `ageSeconds`, `worktreePath`, `args`, `sidecarName`, `tree` (descendant pids, root first), `treeRssKb`; an `orphan-daemon` row is always `reapable: false` and carries `configPath`/`cliEntryPath` instead of a `sidecarName`, plus `port: number | null` and `liveness: "serving" | "not-serving" | "unknown"`. `reap: true` signals only `reapable` rows — never an `orphan-daemon` row, serving or not — and populates `reaped`, one entry per `reapable` leaked tree it signaled: `{ sessionName, panePid, survivors, blindKill }` — `survivors` is the pids still alive after the SIGKILL confirmation window, and `blindKill` is `true` when tmux was killed with no verified process-tree signal (no pane pid, or an unusable snapshot, and no identity fallback confirmed a reap either) — a `survivors: []` alongside `blindKill: true` is not proof anything died, only that nothing was checked. See [Sidecars](commands.md#sidecars) -- `GET /sessions/:id` — current row, `sidecars[].url` ready endpoint, `sidecars[].ports[].listening` host TCP listener true|false|null; Cursor `queuedMessageTyped.ackBaseline.deliveryId` optional UUIDv4, correlated `file:null` requires `size:0` and valid ID, malformed IDs drop baseline, legacy string-file floors retained; optional read-only `intervalWake.dispatchClaimId`: opaque correlation, absent on legacy records, ignored on input; `v2/src/types.ts`, `v2/src/metadata.ts`; ephemeral `lifecycle` receipt; `v2/src/session-lifecycle.ts`. +- `GET /sessions/:id` — current row, `sidecars[].url` ready endpoint, `sidecars[].ports[].listening` host TCP listener true|false|null; Cursor `queuedMessageTyped.ackBaseline.deliveryId` optional UUIDv4, correlated `file:null` requires `size:0` and valid ID, malformed IDs drop baseline, legacy string-file floors retained; optional read-only `intervalWake.dispatchClaimId`: opaque correlation, absent on legacy records, ignored on input; optional `dailyWake.admissionDeferredOccurrence`: persisted retry occurrence; `v2/src/types.ts`, `v2/src/metadata.ts`; ephemeral `lifecycle` receipt; `v2/src/session-lifecycle.ts`. - `GET /sessions/:id/slash-commands` - `GET /sessions/:id/conversation` — optional `view=messages&limit=<1..100>` (default 50): bounded visible-message page; `v2/src/server.ts`, `v2/src/conversation-page.ts`; omit view for legacy response. - `GET /sessions/:id/user-actions` diff --git a/v2/src/session-service.ts b/v2/src/session-service.ts index 9cfd9bd21..a9294cea9 100644 --- a/v2/src/session-service.ts +++ b/v2/src/session-service.ts @@ -20,6 +20,7 @@ import { isDeepStrictEqual } from "node:util"; import { annotateLifecycleError, createRestoreOperation, + lifecycleErrorReceipt, RestoreOperationError, type RestoreOperationContext, SessionLifecycleError, @@ -1284,6 +1285,7 @@ type ScheduledWakeClaim = { revision: number; manual: boolean; stage: "unattempted" | "input-started" | "accepted"; + admissionReservation?: symbol; }; type AgentMessageTarget = Pick< @@ -1363,6 +1365,7 @@ type SessionPreparationOptions = { operation?: RestoreOperationContext; preparedObservation?: PreparedSessionObservation; observePreparedSession?: (session: SessionRecord) => void; + wakeAdmissionReservation?: symbol; }; type ForegroundSpawnReceipt = { @@ -1444,6 +1447,7 @@ interface SessionObservation { } interface DailyWakeClaim extends SessionObservation { record: SessionRecord & { dailyWake: SessionDailyWakeState }; + admissionReservation?: symbol; } // Plan-time facts the reap executor re-proves under the workspace lifecycle lock. interface SidecarReapEntryContext { @@ -1524,6 +1528,7 @@ function isRestorableStatus(status: SessionStatus): boolean { } type WakeAdmissionDenial = { kind: "denied" | "check_failed"; error: Error }; +type WakeAdmissionClaim = { reservation?: symbol; denial?: WakeAdmissionDenial }; function wakeAdmissionReason(denial: WakeAdmissionDenial): string { return denial.kind === "denied" ? "admission" : "admission_check_failed"; @@ -4180,7 +4185,7 @@ export class SessionService { this.deliveryStopped = true; this.codexPaneLimitObservedAt.clear(); this.codexRateLimitPoller.dispose(); - this.scheduledWakeClaims.clear(); + for (const claim of this.scheduledWakeClaims.values()) this.releaseScheduledWakeClaim(claim); this.scheduledWakeLastFailure.clear(); this.sidecarEndpoints.clear(); this.sidecarObservation = undefined; @@ -5675,72 +5680,85 @@ export class SessionService { return null; }, ); - if ( - claimedNextDueAt !== null && - !this.deferIntervalWakeForAdmission(session, intervalWake, claimedNextDueAt) - ) { + if (claimedNextDueAt !== null) { const nextDueAt = claimedNextDueAt; - try { - await this.send(session.id, { - message: this.formatIntervalWakeMessage( + const admission = this.claimWakeAdmission(session.id); + if (admission.denial) { + this.deferIntervalWakeForAdmission( + session, + intervalWake, + claimedNextDueAt, + admission.denial, + ); + } else { + try { + await this.sendScheduledWake( session.id, - intervalWake.message, - intervalWake.stopCondition, - ), - }); - this.logEvent("session.wake.interval_sent", { - level: "info", - sessionId: session.id, - projectId: session.project, - message: `Sent interval wake to ${session.id}`, - details: { - nextDueAt, - intervalMs: intervalWake.intervalMs, - }, - }); - } catch (error) { - if (error instanceof SessionAdmissionDeniedError) { - // Same reasoning as the scheduled-wake branch above: this - // occurrence was already claimed (nextDueAt advanced and - // persisted) before the send attempt. Roll nextDueAt back to - // the occurrence that was just refused (CAS: only if nothing - // else re-armed it meanwhile) so the next poll tick retries - // it promptly instead of waiting a full interval. - await this.withWorkspaceLifecycleLocks(session.id, async () => { - const afterDenial = readSession(this.config.dataDir, session.id); - if (afterDenial?.intervalWake?.nextDueAt === nextDueAt) { - writeSession(this.config.dataDir, { - ...afterDenial, - intervalWake: { - ...afterDenial.intervalWake, - nextDueAt: intervalWake.nextDueAt, - }, - updatedAt: nowIso(), - }); - } - }); - this.logEvent("session.wake.interval_deferred", { - level: "warn", - sessionId: session.id, - projectId: session.project, - message: `Deferred interval wake for ${session.id}: ${error.message}`, - details: { - nextDueAt, - intervalMs: intervalWake.intervalMs, + { + message: this.formatIntervalWakeMessage( + session.id, + intervalWake.message, + intervalWake.stopCondition, + ), }, - }); - } else { - const message = error instanceof Error ? error.message : String(error); - this.logEvent("session.wake.interval_failed", { - level: "error", + admission.reservation, + ); + this.logEvent("session.wake.interval_sent", { + level: "info", sessionId: session.id, projectId: session.project, - message: `Failed to send interval wake to ${session.id}: ${message}`, + message: `Sent interval wake to ${session.id}`, details: { nextDueAt, intervalMs: intervalWake.intervalMs, }, }); + } catch (error) { + if (error instanceof SessionAdmissionDeniedError) { + // Same reasoning as the scheduled-wake branch above: this + // occurrence was already claimed (nextDueAt advanced and + // persisted) before the send attempt. Roll nextDueAt back to + // the occurrence that was just refused (CAS: only if nothing + // else re-armed it meanwhile) so the next poll tick retries + // it promptly instead of waiting a full interval. + await this.withWorkspaceLifecycleLocks(session.id, async () => { + const afterDenial = readSession(this.config.dataDir, session.id); + if (afterDenial?.intervalWake?.nextDueAt === nextDueAt) { + writeSession(this.config.dataDir, { + ...afterDenial, + intervalWake: { + ...afterDenial.intervalWake, + nextDueAt: intervalWake.nextDueAt, + }, + updatedAt: nowIso(), + }); + } + }); + this.logEvent("session.wake.interval_deferred", { + level: "warn", + sessionId: session.id, + projectId: session.project, + message: `Deferred interval wake for ${session.id}: ${error.message}`, + details: { + nextDueAt, + intervalMs: intervalWake.intervalMs, + }, + }); + } else { + const message = error instanceof Error ? error.message : String(error); + this.logEvent("session.wake.interval_failed", { + level: "error", + sessionId: session.id, + projectId: session.project, + message: `Failed to send interval wake to ${session.id}: ${message}`, + details: { + nextDueAt, + intervalMs: intervalWake.intervalMs, + }, + }); + } + } finally { + this.releaseAdmissionReservation(admission.reservation); } } } @@ -6018,6 +6036,12 @@ export class SessionService { ) { return null; } + const lifecycle = this.lifecycleIdsFor(currentDailyWakeSession).map((id) => ({ + id, + snapshot: this.lifecycle.snapshot(id), + })); + if (lifecycle.some(({ snapshot }) => snapshot.operation?.phase === "pending")) + return null; // CAS: only claim if the wake is unchanged (no concurrent re-arm/cancel). const dailyWakeClaimed = currentDailyWakeSession.dailyWake?.nextDueAt === dailyWake.nextDueAt && @@ -6074,16 +6098,14 @@ export class SessionService { writeSession(this.config.dataDir, updated); const claim: DailyWakeClaim = { record: updated, - lifecycle: this.lifecycleIdsFor(updated).map((id) => ({ - id, - snapshot: this.lifecycle.snapshot(id), - })), + lifecycle, }; - const admissionDenial = this.wakeAdmissionDenial(session.id); - if (admissionDenial) { - this.deferDailyWakeForAdmission(session, dailyWake, claim, admissionDenial, now); + const admission = this.claimWakeAdmission(session.id); + if (admission.denial) { + this.deferDailyWakeForAdmission(session, dailyWake, claim, admission.denial, now); return null; } + claim.admissionReservation = admission.reservation; return claim; } return null; @@ -6092,13 +6114,17 @@ export class SessionService { if (claimedDailyWake !== null) { const claimedDailyNextDueAt = claimedDailyWake.record.dailyWake.nextDueAt; try { - await this.send(session.id, { - message: this.formatDailyWakeMessage( - session.id, - dailyWake.message, - dailyWake.stopCondition, - ), - }); + await this.sendScheduledWake( + session.id, + { + message: this.formatDailyWakeMessage( + session.id, + dailyWake.message, + dailyWake.stopCondition, + ), + }, + claimedDailyWake.admissionReservation, + ); this.logEvent("session.wake.daily_sent", { level: "info", sessionId: session.id, @@ -6150,6 +6176,8 @@ export class SessionService { }, }); } + } finally { + this.releaseAdmissionReservation(claimedDailyWake.admissionReservation); } } } @@ -6161,16 +6189,18 @@ export class SessionService { // Runs after the occurrence claim: a denial before it would leave the wake // due and retry every tick. An unexpected check failure denies only this // record: rethrowing would abort the wake loop for every later session. - private wakeAdmissionDenial(sessionId: string): WakeAdmissionDenial | null { + private claimWakeAdmission(sessionId: string): WakeAdmissionClaim { const current = readSession(this.config.dataDir, sessionId); - if (!current || !wakeRelaunchNeedsAdmission(current)) return null; + if (!current || !wakeRelaunchNeedsAdmission(current)) return {}; try { - this.assertAdmissible(current.project, "wake"); - return null; + return { reservation: this.reserveAdmission(current.project, "wake") }; } catch (error) { - if (error instanceof SessionAdmissionDeniedError) return { kind: "denied", error }; + if (error instanceof SessionAdmissionDeniedError) + return { denial: { kind: "denied", error } }; const message = error instanceof Error ? error.message : String(error); - return { kind: "check_failed", error: new Error(`admission check failed: ${message}`) }; + return { + denial: { kind: "check_failed", error: new Error(`admission check failed: ${message}`) }, + }; } } @@ -6180,9 +6210,8 @@ export class SessionService { session: SessionRecord, intervalWake: SessionIntervalWakeState, nextDueAt: string, - ): boolean { - const denial = this.wakeAdmissionDenial(session.id); - if (!denial) return false; + denial: WakeAdmissionDenial, + ): void { this.logEvent("session.wake.interval_deferred", { level: wakeAdmissionLogLevel(denial), sessionId: session.id, @@ -6194,7 +6223,6 @@ export class SessionService { reason: wakeAdmissionReason(denial), }, }); - return true; } // Denied daily occurrence retries every DAILY_WAKE_ADMISSION_RETRY_MS until @@ -7805,8 +7833,8 @@ export class SessionService { // runDashboardCacheTick owns their pruning. private pruneSessionScopedState(liveIds: ReadonlySet): void { this.codexRateLimitPoller.prune(liveIds); - for (const sessionId of this.scheduledWakeClaims.keys()) { - if (!liveIds.has(sessionId)) this.scheduledWakeClaims.delete(sessionId); + for (const [sessionId, claim] of this.scheduledWakeClaims) { + if (!liveIds.has(sessionId)) this.releaseScheduledWakeClaim(claim); } for (const sessionId of this.scheduledWakeLastFailure.keys()) { if (!liveIds.has(sessionId)) this.scheduledWakeLastFailure.delete(sessionId); @@ -12226,6 +12254,12 @@ export class SessionService { } async restoreRebootedSessions(drifted: SessionObservation[]): Promise { + const loopOwnedLifecycle = new Map(); + const rememberRestoreReceipt = (receipt: SessionLifecycleSnapshot | undefined): void => { + const operation = receipt?.operation; + if (operation?.action !== "restore" || operation.phase === "pending") return; + for (const id of operation.targetIds) loopOwnedLifecycle.set(id, receipt); + }; for (const [index, { record, lifecycle }] of drifted.entries()) { const { id, project } = record; const projectConfig = this.config.projects[project]; @@ -12250,6 +12284,16 @@ export class SessionService { break; } const current = readSession(this.config.dataDir, id); + const fencedLifecycleIds = lifecycle.flatMap(({ id: ownerId, snapshot }) => { + const expected = loopOwnedLifecycle.get(ownerId) ?? snapshot; + const latest = this.lifecycle.snapshot(ownerId); + return snapshot.operation?.phase === "pending" || + latest.operation?.phase === "pending" || + latest.instanceId !== expected.instanceId || + latest.revision !== expected.revision + ? [ownerId] + : []; + }); if ( !current || !sameSessionRuntimeOwner(current, record) || @@ -12258,23 +12302,27 @@ export class SessionService { current.error !== record.error || current.errorSource !== record.errorSource || current.serverErrorAt !== record.serverErrorAt || - lifecycle.some(({ id: ownerId, snapshot }) => { - const latest = this.lifecycle.snapshot(ownerId); - return ( - snapshot.operation?.phase === "pending" || - latest.operation?.phase === "pending" || - latest.instanceId !== snapshot.instanceId || - latest.revision !== snapshot.revision - ); - }) - ) + fencedLifecycleIds.length > 0 + ) { + if (fencedLifecycleIds.length > 0) { + this.logEvent("session.reboot.restore.aborted", { + level: "warn", + sessionId: id, + projectId: project, + message: `Reboot restore skipped for ${id}: lifecycle ownership changed`, + details: { reason: "lifecycle_changed", lifecycleIds: fencedLifecycleIds }, + }); + } continue; + } try { - await this.restore(id); + const restored = await this.restore(id); + rememberRestoreReceipt(restored.lifecycle); if (memory !== null) { await sleep(RESTORE_SETTLE_MS); } } catch (error) { + rememberRestoreReceipt(lifecycleErrorReceipt(error)); const message = error instanceof Error ? error.message : String(error); this.logEvent("session.reboot.restore.failed", { level: "warn", @@ -15276,7 +15324,8 @@ export class SessionService { } writeSession(this.config.dataDir, updated); if (request.target === "scheduled") { - this.scheduledWakeClaims.delete(sessionId); + const claim = this.scheduledWakeClaims.get(sessionId); + if (claim) this.releaseScheduledWakeClaim(claim); this.scheduledWakeLastFailure.delete(sessionId); } return this.enrich(updated); @@ -15371,8 +15420,16 @@ export class SessionService { this.lifecycle.snapshot(session.id).operation?.phase === "pending" ) return null; + const previousClaim = this.scheduledWakeClaims.get(session.id); + const previousReservation = previousClaim?.admissionReservation; + let admissionReservation = previousReservation; if (isStaleParked(current) || current.stopReason === "memory_shed") { - this.assertWakeAdmissible(current, manual ? undefined : wake); + admissionReservation = this.checkWakeAdmission( + current, + manual ? undefined : wake, + admissionReservation, + true, + ); } const claim: ScheduledWakeClaim = { session: current, @@ -15380,11 +15437,21 @@ export class SessionService { manual, revision: this.lifecycle.snapshot(session.id).revision, stage: "unattempted", + ...(admissionReservation ? { admissionReservation } : {}), }; this.scheduledWakeClaims.set(session.id, claim); - if (manual) { - const { scheduledWake: _wake, ...base } = current; - writeSession(this.config.dataDir, { ...base, updatedAt: nowIso() }); + if (previousClaim && previousReservation) { + previousClaim.admissionReservation = undefined; + if (!claim.admissionReservation) this.admissionReservations.delete(previousReservation); + } + try { + if (manual) { + const { scheduledWake: _wake, ...base } = current; + writeSession(this.config.dataDir, { ...base, updatedAt: nowIso() }); + } + } catch (error) { + this.releaseScheduledWakeClaim(claim); + throw error; } return claim; }); @@ -15413,8 +15480,20 @@ export class SessionService { } private releaseScheduledWakeClaim(claim: ScheduledWakeClaim): void { - if (this.scheduledWakeClaims.get(claim.session.id) === claim) + if (this.scheduledWakeClaims.get(claim.session.id) === claim) { this.scheduledWakeClaims.delete(claim.session.id); + this.releaseScheduledWakeAdmission(claim); + } + } + + private releaseScheduledWakeAdmission(claim: ScheduledWakeClaim): void { + if (!claim.admissionReservation) return; + this.releaseAdmissionReservation(claim.admissionReservation); + claim.admissionReservation = undefined; + } + + private releaseAdmissionReservation(reservation?: symbol): void { + if (reservation) this.admissionReservations.delete(reservation); } private logScheduledWakeFailure( @@ -15436,12 +15515,20 @@ export class SessionService { }); } - private assertWakeAdmissible( + private checkWakeAdmission( session: SessionRecord, scheduledWake?: NonNullable, - ): void { + admissionReservation?: symbol, + reserve = false, + ): symbol | undefined { try { + if (admissionReservation) { + this.assertAdmissible(session.project, "wake", { admissionReservation }); + return admissionReservation; + } + if (reserve) return this.reserveAdmission(session.project, "wake"); this.assertAdmissible(session.project, "wake"); + return undefined; } catch (error) { // Record the actual refusal before releasing ownership to a replacement. if (scheduledWake && error instanceof SessionAdmissionDeniedError) @@ -15713,7 +15800,8 @@ export class SessionService { updatedAt: nowIso(), }; writeSession(this.config.dataDir, updated); - this.scheduledWakeClaims.delete(sessionId); + const claim = this.scheduledWakeClaims.get(sessionId); + if (claim) this.releaseScheduledWakeClaim(claim); this.scheduledWakeLastFailure.delete(sessionId); this.logEvent("session.wake.scheduled", { level: "info", @@ -16251,12 +16339,33 @@ export class SessionService { } async send(sessionId: string, request: SendMessageRequest): Promise { + return this.sendAndWait(sessionId, request); + } + + private async sendScheduledWake( + sessionId: string, + request: SendMessageRequest, + wakeAdmissionReservation?: symbol, + ): Promise { + return this.sendAndWait(sessionId, request, wakeAdmissionReservation); + } + + private async sendAndWait( + sessionId: string, + request: SendMessageRequest, + wakeAdmissionReservation?: symbol, + ): Promise { const appended = request.queue !== false ? this.appendBehindTypingDelivery(sessionId, request) : null; if (appended) { return this.enrich(appended); } - const sent = await this.sendWithAttempt(sessionId, request); + const sent = await this.sendWithAttempt( + sessionId, + request, + undefined, + wakeAdmissionReservation, + ); // The typed marker also covers an attempt that wrote before registration. const { view, paneWrite } = { view: sent.view, @@ -16301,6 +16410,7 @@ export class SessionService { sessionId: string, request: SendMessageRequest, wakeClaim?: ScheduledWakeClaim, + wakeAdmissionReservation?: symbol, ): Promise<{ view: SessionView; attempt: Promise | null }> { const session = readSession(this.config.dataDir, sessionId); if (!session) { @@ -16354,6 +16464,7 @@ export class SessionService { await this.ensureReadyForQueuedSend(sessionId, activationStamp, { ...(operation ? { operation } : {}), ...(wakeClaim ? { wakeClaim } : {}), + ...(wakeAdmissionReservation ? { wakeAdmissionReservation } : {}), }); } finally { operation?.dispose(); @@ -20094,9 +20205,10 @@ export class SessionService { isStaleParked(session) || (session.status === "stopped" && session.stopReason === "memory_shed") ) { - this.assertWakeAdmissible( + this.checkWakeAdmission( session, options?.wakeClaim && !options.wakeClaim.manual ? options.wakeClaim.wake : undefined, + options?.wakeClaim?.admissionReservation, ); } @@ -20135,8 +20247,13 @@ export class SessionService { this.restoreWarmupUntil.delete(session.id); this.launchesInFlight.delete(session.id); } - operation?.assertActive(); - recovered.session = this.writeRecoverySession(recovered.session, options?.wakeClaim, options); + try { + operation?.assertActive(); + recovered.session = this.writeRecoverySession(recovered.session, options?.wakeClaim, options); + } finally { + if (options?.wakeClaim) this.releaseScheduledWakeAdmission(options.wakeClaim); + this.releaseAdmissionReservation(options?.wakeAdmissionReservation); + } // After the running record is on disk: a project sidecar can take tens of // seconds to come up, and the reaper's running|spawning filter must cover // that whole window without relying on the warmup cleared just above. @@ -23838,7 +23955,7 @@ export class SessionService { private reserveAdmission( projectId: string, - context: "spawn" | "restore", + context: "spawn" | "restore" | "wake", opts?: { replacingSessionId?: string }, ): symbol { this.assertAdmissible(projectId, context, opts); diff --git a/v2/test/fast/mcp-contracts.test.ts b/v2/test/fast/mcp-contracts.test.ts index 2b80a3fb9..1ff4e6501 100644 --- a/v2/test/fast/mcp-contracts.test.ts +++ b/v2/test/fast/mcp-contracts.test.ts @@ -24,7 +24,7 @@ const reviewedFingerprints = { "server.ts": "9dd7816d37109f50a291c0289801dec461d6560ee81e3322a0553def1cb54047", "types.ts": "9bcc3d6d539777f1f6e663c82567d4918b27c31c0a755e5f9cd3b865da43aedc", "cron-job-editor.ts": "33fc7b2993819693ef511d9312f9eb8f4260feb2f263421aec2d4e5ac613ec06", - "session-service.ts": "dc983073764769ef9aad3337fe0b41a548d3e0bceefcb3125b2ea59bbef1fea8", + "session-service.ts": "1c5c74104ffe47e330f41e4998a333d9e70519e403d68fa7d5900787fa8b5ecf", "session-lifecycle.ts": "11ed4e84c26d589f92627e9d09fc89141f273ae99199a0263d6c2a3b49acb72c", "session-memory.ts": "1d06c2d904fa46b95ef8bed971c5767fad76663482f22822f9a541b345f161ff", "shared-memory.ts": "f5804ebc0592f40c37cf30a1545d88b37075cd6e7e63b3c9e1817bf42c93de67", diff --git a/v2/test/fast/session-service.test.ts b/v2/test/fast/session-service.test.ts index 1865453a8..66e83586e 100644 --- a/v2/test/fast/session-service.test.ts +++ b/v2/test/fast/session-service.test.ts @@ -34196,6 +34196,99 @@ describe("SessionService", () => { expect(restore.mock.calls.map(([id]) => id)).toEqual(["api-1"]); }); + it.each([ + ["owner first", ["api-1", "api-2"]], + ["member first", ["api-2", "api-1"]], + ] as const)("restores shared-workspace reboot rows when listed %s", async (_order, ids) => { + const config = baseConfig(); + loadConfigMock.mockReturnValue({ + ...config, + projects: { api: { ...config.projects.api, restoreAfterReboot: true } }, + }); + readHostMemoryMock.mockReturnValue(null); + const sessions = createSessionStore( + runningSession({ id: "api-1", workspaceId: "api-1", status: "stopped" }), + runningSession({ id: "api-2", workspaceId: "api-1", status: "stopped" }), + ); + const service = await createDisposedSessionService(); + const internals = repairInternals(service); + const restore = vi.spyOn(service, "restore").mockImplementation(async (id) => { + const owner = internals.lifecycle.begin("restore", [id]); + const settled = internals.lifecycle.settle( + owner, + true, + () => "succeeded", + () => true, + ); + const current = sessions.get(id); + if (current) sessions.set(id, { ...current, status: "running" }); + return { lifecycle: settled } as never; + }); + + await service.restoreRebootedSessions(rebootCandidates(service, [...ids])); + + expect(restore.mock.calls.map(([id]) => id)).toEqual(ids); + expect(logSpurEventMock).not.toHaveBeenCalledWith( + TEST_DATA_DIR, + expect.objectContaining({ + event: "session.reboot.restore.aborted", + details: expect.objectContaining({ reason: "lifecycle_changed" }), + }), + ); + service.dispose(); + }); + + it("logs a shared-workspace sibling skipped by an external lifecycle operation", async () => { + const config = baseConfig(); + loadConfigMock.mockReturnValue({ + ...config, + projects: { api: { ...config.projects.api, restoreAfterReboot: true } }, + }); + readHostMemoryMock.mockReturnValue(null); + const sessions = createSessionStore( + runningSession({ id: "api-1", workspaceId: "api-1", status: "stopped" }), + runningSession({ id: "api-2", workspaceId: "api-1", status: "stopped" }), + ); + const service = await createDisposedSessionService(); + const internals = repairInternals(service); + const restore = vi.spyOn(service, "restore").mockImplementation(async (id) => { + const owner = internals.lifecycle.begin("restore", [id]); + const settled = internals.lifecycle.settle( + owner, + true, + () => "succeeded", + () => true, + ); + if (id === "api-1") { + const external = internals.lifecycle.begin("stop", ["api-1"]); + internals.lifecycle.settle( + external, + true, + () => "succeeded", + () => true, + ); + } + return { lifecycle: settled } as never; + }); + + await service.restoreRebootedSessions(rebootCandidates(service, ["api-1", "api-2"])); + + expect(restore.mock.calls.map(([id]) => id)).toEqual(["api-1"]); + expect(logSpurEventMock).toHaveBeenCalledWith( + TEST_DATA_DIR, + expect.objectContaining({ + event: "session.reboot.restore.aborted", + level: "warn", + sessionId: "api-2", + details: expect.objectContaining({ + reason: "lifecycle_changed", + lifecycleIds: ["api-1"], + }), + }), + ); + service.dispose(); + }); + it.each([ "dailyAt", "message", @@ -61804,6 +61897,217 @@ describe("SessionService", () => { return WATCHED.reduce((total, event) => total + events(event).length, 0); } + it("reserves the parked wake slot while its runtime probe is pending", async () => { + capAt(1); + trackRuntimes([]); + mockClaudeJsonlState("waiting"); + createSessionStore( + stoppedRecord("api-1", { + stopReason: "stale_timeout", + scheduledWake: { + dueAt: "2026-03-18T10:00:00.000Z", + message: "scheduled wake", + }, + }), + ); + const service = await createDisposedSessionService(); + const internals = service as unknown as { + readRuntimeSnapshot(session: SessionRecord): Promise<{ + runtimeAlive: boolean; + paneUsable: boolean; + processAlive: boolean; + tmuxActivityAt: Date | null; + probeUnresponsive: boolean; + diagnostic?: string; + }>; + reserveAdmission(projectId: string, context: "spawn"): symbol; + admissionReservations: Map; + }; + const readRuntimeSnapshot = internals.readRuntimeSnapshot.bind(service); + let releaseProbe!: () => void; + const probeBarrier = new Promise((resolve) => { + releaseProbe = resolve; + }); + let signalProbe!: () => void; + const probeStarted = new Promise((resolve) => { + signalProbe = resolve; + }); + vi.spyOn(internals, "readRuntimeSnapshot").mockImplementation(async (session) => { + if (session.id === "api-1") { + signalProbe(); + await probeBarrier; + } + return readRuntimeSnapshot(session); + }); + const pendingWake = tick(service); + await probeStarted; + + expect(internals.admissionReservations.size).toBe(1); + let spawnDenial: unknown; + try { + internals.reserveAdmission("api", "spawn"); + } catch (error) { + spawnDenial = error; + } + expect(spawnDenial).toBeInstanceOf(Error); + expect(spawnDenial).toMatchObject({ statusCode: 429, reason: "cap" }); + + releaseProbe(); + await pendingWake; + expect(internals.admissionReservations.size).toBe(0); + expect(createTmuxSessionMock).toHaveBeenCalledTimes(1); + service.dispose(); + }); + + type RuntimeSnapshot = { + runtimeAlive: boolean; + paneUsable: boolean; + processAlive: boolean; + tmuxActivityAt: Date | null; + probeUnresponsive: boolean; + diagnostic?: string; + }; + + function holdWakeProbe(service: unknown, fail = false) { + const internals = service as unknown as { + readRuntimeSnapshot(session: SessionRecord): Promise; + admissionReservations: Map; + countLiveSessions(): { total: number }; + }; + const readRuntimeSnapshot = internals.readRuntimeSnapshot.bind(service); + let releaseProbe!: () => void; + const probeBarrier = new Promise((resolve) => { + releaseProbe = resolve; + }); + let signalProbe!: () => void; + const probeStarted = new Promise((resolve) => { + signalProbe = resolve; + }); + let reservationCountAtProbe = 0; + let probeHeld = false; + vi.spyOn(internals, "readRuntimeSnapshot").mockImplementation(async (session) => { + if (session.id === "api-1" && !probeHeld) { + probeHeld = true; + reservationCountAtProbe = internals.admissionReservations.size; + signalProbe(); + await probeBarrier; + if (fail) throw new Error("held wake runtime probe failed"); + } + return readRuntimeSnapshot(session); + }); + return { + internals, + probeStarted, + releaseProbe, + reservationCount: () => reservationCountAtProbe, + pendingWake: tick(service), + }; + } + + const ordinaryWakeCases = [ + ["stopped", "interval"], + ["stopped", "daily"], + ["paused", "interval"], + ["paused", "daily"], + ] as const; + + it.each(ordinaryWakeCases)( + "reserves an ordinary %s %s wake slot across the runtime probe and counts the live result", + async (status, kind) => { + capAt(1); + trackRuntimes([]); + mockClaudeJsonlState("waiting"); + const due = "2026-03-18T10:00:00.000Z"; + const wakeOverrides: Partial = { status }; + if (status === "paused") wakeOverrides.stopReason = "manual_pause"; + if (kind === "interval") { + wakeOverrides.intervalWake = { + nextDueAt: due, + intervalMs: 60_000, + message: "interval wake", + stopCondition: "Stop condition", + }; + } else { + wakeOverrides.dailyWake = { + dailyAt: ["10:00"], + nextDueAt: due, + message: "daily wake", + stopCondition: "Stop condition", + }; + } + const sessions = createSessionStore(stoppedRecord("api-1", wakeOverrides)); + const service = await createDisposedSessionService(); + const held = holdWakeProbe(service); + await held.probeStarted; + + let spawnError: unknown; + try { + await service.spawn({ project: "api", prompt: "concurrent spawn" }); + } catch (error) { + spawnError = error; + } finally { + held.releaseProbe(); + } + await held.pendingWake; + + expect(held.reservationCount()).toBe(1); + expect(held.internals.countLiveSessions().total).toBe(1); + expect(held.internals.admissionReservations.size).toBe(0); + expect(spawnError).toMatchObject({ statusCode: 429, reason: "cap" }); + expect(sessions.get("api-1")?.status).toBe("running"); + expect(createTmuxSessionMock).toHaveBeenCalledTimes(1); + expect( + kind === "interval" + ? events("session.wake.interval_sent", "api-1") + : events("session.wake.daily_sent", "api-1"), + ).toHaveLength(1); + await expect( + service.spawn({ project: "api", prompt: "after wake" }), + ).rejects.toMatchObject({ statusCode: 429, reason: "cap" }); + service.dispose(); + }, + ); + + it.each(ordinaryWakeCases)( + "releases an ordinary %s %s wake reservation when the runtime probe fails", + async (status, kind) => { + capAt(1); + trackRuntimes([]); + mockClaudeJsonlState("waiting"); + const due = "2026-03-18T10:00:00.000Z"; + const wakeOverrides: Partial = { status }; + if (status === "paused") wakeOverrides.stopReason = "manual_pause"; + if (kind === "interval") { + wakeOverrides.intervalWake = { + nextDueAt: due, + intervalMs: 60_000, + message: "interval wake", + stopCondition: "Stop condition", + }; + } else { + wakeOverrides.dailyWake = { + dailyAt: ["10:00"], + nextDueAt: due, + message: "daily wake", + stopCondition: "Stop condition", + }; + } + const sessions = createSessionStore(stoppedRecord("api-1", wakeOverrides)); + const service = await createDisposedSessionService(); + const held = holdWakeProbe(service, true); + await held.probeStarted; + const reservationCountAtProbe = held.reservationCount(); + held.releaseProbe(); + await held.pendingWake; + + expect(reservationCountAtProbe).toBe(1); + expect(held.internals.admissionReservations.size).toBe(0); + expect(held.internals.countLiveSessions().total).toBe(0); + expect(sessions.get("api-1")?.status).not.toBe("running"); + service.dispose(); + }, + ); + describe("unexpected admission check failure", () => { function failFirstCheck(service: unknown): void { vi.spyOn( @@ -62142,6 +62446,40 @@ describe("SessionService", () => { service.dispose(); }); + it("leaves a due occurrence armed while a lifecycle operation is pending", async () => { + const { sessions } = seedDaily(); + vi.setSystemTime(at(9, 0, 1)); + const service = await createDisposedSessionService(); + const lifecycle = (service as unknown as { lifecycle: SessionLifecycleRegister }) + .lifecycle; + const pending = lifecycle.begin("restore", ["api-1"]); + + await tick(service); + + expect(sessions.get("api-1")?.dailyWake).toEqual({ + dailyAt: ["09:00", "09:40"], + nextDueAt: iso(at(9, 0)), + message: "daily-api-1", + stopCondition: "Stop condition", + }); + expect(events("session.admission.denied")).toHaveLength(0); + expect(events("session.wake.daily_deferred")).toHaveLength(0); + + lifecycle.settle( + pending, + true, + () => "succeeded", + () => true, + ); + await tick(service, at(9, 0, 2)); + expect(sessions.get("api-1")?.dailyWake).toMatchObject({ + nextDueAt: iso(at(9, 15, 2)), + admissionDeferredOccurrence: iso(at(9, 0)), + }); + expect(events("session.wake.daily_deferred")).toHaveLength(1); + service.dispose(); + }); + it("skips the occurrence once its retry window reaches the next slot", async () => { const { sessions } = seedDaily(); vi.setSystemTime(at(9, 0, 1)); From aa5920a6950452c6d335b65b219bc165a4fcee75 Mon Sep 17 00:00:00 2001 From: AlekShu <108252944+AlekShu@users.noreply.github.com> Date: Fri, 6 Nov 2026 16:03:51 +0000 Subject: [PATCH 17/23] fix(session): omit absent wake admission reservations --- v2/src/session-service.ts | 8 ++++---- v2/test/fast/mcp-contracts.test.ts | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/v2/src/session-service.ts b/v2/src/session-service.ts index a9294cea9..8fc51e8c0 100644 --- a/v2/src/session-service.ts +++ b/v2/src/session-service.ts @@ -6105,7 +6105,7 @@ export class SessionService { this.deferDailyWakeForAdmission(session, dailyWake, claim, admission.denial, now); return null; } - claim.admissionReservation = admission.reservation; + if (admission.reservation) claim.admissionReservation = admission.reservation; return claim; } return null; @@ -12257,7 +12257,7 @@ export class SessionService { const loopOwnedLifecycle = new Map(); const rememberRestoreReceipt = (receipt: SessionLifecycleSnapshot | undefined): void => { const operation = receipt?.operation; - if (operation?.action !== "restore" || operation.phase === "pending") return; + if (!receipt || operation?.action !== "restore" || operation.phase === "pending") return; for (const id of operation.targetIds) loopOwnedLifecycle.set(id, receipt); }; for (const [index, { record, lifecycle }] of drifted.entries()) { @@ -15441,7 +15441,7 @@ export class SessionService { }; this.scheduledWakeClaims.set(session.id, claim); if (previousClaim && previousReservation) { - previousClaim.admissionReservation = undefined; + delete previousClaim.admissionReservation; if (!claim.admissionReservation) this.admissionReservations.delete(previousReservation); } try { @@ -15489,7 +15489,7 @@ export class SessionService { private releaseScheduledWakeAdmission(claim: ScheduledWakeClaim): void { if (!claim.admissionReservation) return; this.releaseAdmissionReservation(claim.admissionReservation); - claim.admissionReservation = undefined; + delete claim.admissionReservation; } private releaseAdmissionReservation(reservation?: symbol): void { diff --git a/v2/test/fast/mcp-contracts.test.ts b/v2/test/fast/mcp-contracts.test.ts index f4a7f43df..f3d02139a 100644 --- a/v2/test/fast/mcp-contracts.test.ts +++ b/v2/test/fast/mcp-contracts.test.ts @@ -24,7 +24,7 @@ const reviewedFingerprints = { "server.ts": "74a508c1d567d59c726787581b0533f5f929339f681a74a80a883596804bffd9", "types.ts": "9bcc3d6d539777f1f6e663c82567d4918b27c31c0a755e5f9cd3b865da43aedc", "cron-job-editor.ts": "33fc7b2993819693ef511d9312f9eb8f4260feb2f263421aec2d4e5ac613ec06", - "session-service.ts": "1c5c74104ffe47e330f41e4998a333d9e70519e403d68fa7d5900787fa8b5ecf", + "session-service.ts": "ad54ebef0284dcdfdba69a2c87fb28c673f218df2b4ea9ea7af4f93ac7e7d1d3", "session-lifecycle.ts": "11ed4e84c26d589f92627e9d09fc89141f273ae99199a0263d6c2a3b49acb72c", "session-memory.ts": "1d06c2d904fa46b95ef8bed971c5767fad76663482f22822f9a541b345f161ff", "shared-memory.ts": "f5804ebc0592f40c37cf30a1545d88b37075cd6e7e63b3c9e1817bf42c93de67", From 8ea306ca07c1e01315249f8b393f316de5ab7b90 Mon Sep 17 00:00:00 2001 From: AlekShu <108252944+AlekShu@users.noreply.github.com> Date: Fri, 6 Nov 2026 16:27:14 +0000 Subject: [PATCH 18/23] fix(session): hand wake reservations to counted warmup --- docs/configuration.md | 2 +- v2/src/session-service.ts | 12 +-- v2/test/fast/mcp-contracts.test.ts | 2 +- v2/test/fast/session-service.test.ts | 139 ++++++++++++++++++++++++++- 4 files changed, 145 insertions(+), 10 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index c575ae034..3e1e89622 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -126,7 +126,7 @@ Type/constraint/default per key; full validation source `v2/src/config.ts`. ## Admission control -Admission gates spawn and restore. Background interval/daily wakes gate stopped/paused, stale-parked and memory-shed relaunches; manual interval/daily dispatch gates only stale-parked and memory-shed relaunches. Policy: `v2/src/session-service.ts`; capacity: [`GET /headroom`](daemon-api.md); diagnostics: [`spur doctor`](commands.md#doctor). +Cap limits concurrent live sessions at spawn, restore, and the wake of a stale-parked or memory-shed session (including one-shot wakes and relaunching sends). Ordinary stopped/paused interval/daily relaunches: background gated, manual ungated. Over cap: `429` naming stop candidates. Reporting route: [`GET /headroom`](daemon-api.md). CLI: [`spur doctor`](commands.md#doctor). Schema `v2/src/config.ts`, enforcement `v2/src/session-service.ts`. `stopReason: "runtime_lost"`: confirmed runtime loss becomes restorable `stopped`, retains error evidence, releases capacity; unknown probes retain capacity (`v2/src/session-service.ts`). diff --git a/v2/src/session-service.ts b/v2/src/session-service.ts index 8fc51e8c0..6b58df244 100644 --- a/v2/src/session-service.ts +++ b/v2/src/session-service.ts @@ -20223,6 +20223,9 @@ export class SessionService { // "working" the way a genuine post-restore warmup intentionally does. this.restoreWarmupUntil.set(session.id, Date.now() + RESTORE_WARMUP_MS); this.launchesInFlight.add(session.id); + // Transfer the reservation to the counted warmup without yielding. + if (options?.wakeClaim) this.releaseScheduledWakeAdmission(options.wakeClaim); + this.releaseAdmissionReservation(options?.wakeAdmissionReservation); let recovered: ReadySessionForSend; try { recovered = await this.relaunchSessionInPlace(session, project, { @@ -20247,13 +20250,8 @@ export class SessionService { this.restoreWarmupUntil.delete(session.id); this.launchesInFlight.delete(session.id); } - try { - operation?.assertActive(); - recovered.session = this.writeRecoverySession(recovered.session, options?.wakeClaim, options); - } finally { - if (options?.wakeClaim) this.releaseScheduledWakeAdmission(options.wakeClaim); - this.releaseAdmissionReservation(options?.wakeAdmissionReservation); - } + operation?.assertActive(); + recovered.session = this.writeRecoverySession(recovered.session, options?.wakeClaim, options); // After the running record is on disk: a project sidecar can take tens of // seconds to come up, and the reaper's running|spawning filter must cover // that whole window without relying on the warmup cleared just above. diff --git a/v2/test/fast/mcp-contracts.test.ts b/v2/test/fast/mcp-contracts.test.ts index f3d02139a..f0c24a5db 100644 --- a/v2/test/fast/mcp-contracts.test.ts +++ b/v2/test/fast/mcp-contracts.test.ts @@ -24,7 +24,7 @@ const reviewedFingerprints = { "server.ts": "74a508c1d567d59c726787581b0533f5f929339f681a74a80a883596804bffd9", "types.ts": "9bcc3d6d539777f1f6e663c82567d4918b27c31c0a755e5f9cd3b865da43aedc", "cron-job-editor.ts": "33fc7b2993819693ef511d9312f9eb8f4260feb2f263421aec2d4e5ac613ec06", - "session-service.ts": "ad54ebef0284dcdfdba69a2c87fb28c673f218df2b4ea9ea7af4f93ac7e7d1d3", + "session-service.ts": "f86bb90e040038cb5e041f30f60ac8edb27de29d21fd67da64cbe942f66e0ff8", "session-lifecycle.ts": "11ed4e84c26d589f92627e9d09fc89141f273ae99199a0263d6c2a3b49acb72c", "session-memory.ts": "1d06c2d904fa46b95ef8bed971c5767fad76663482f22822f9a541b345f161ff", "shared-memory.ts": "f5804ebc0592f40c37cf30a1545d88b37075cd6e7e63b3c9e1817bf42c93de67", diff --git a/v2/test/fast/session-service.test.ts b/v2/test/fast/session-service.test.ts index 66e83586e..2594bb24b 100644 --- a/v2/test/fast/session-service.test.ts +++ b/v2/test/fast/session-service.test.ts @@ -60153,7 +60153,7 @@ describe("SessionService", () => { entered = resolve; }); const send = vi - .spyOn(service, "send") + .spyOn(service as unknown as { sendAndWait: typeof service.send }, "sendAndWait") .mockImplementationOnce(async () => { entered(); await gate; @@ -62011,6 +62011,143 @@ describe("SessionService", () => { ["paused", "daily"], ] as const; + const warmingWakeCases = [ + ...ordinaryWakeCases, + ["parked", "scheduled"], + ["shed", "scheduled"], + ] as const; + + function warmingWakeRecord( + status: (typeof warmingWakeCases)[number][0], + kind: (typeof warmingWakeCases)[number][1], + ): SessionRecord { + const due = "2026-03-18T10:00:00.000Z"; + return stoppedRecord("api-1", { + status: status === "paused" ? "paused" : "stopped", + ...(status === "paused" + ? { stopReason: "manual_pause" } + : status === "parked" + ? { stopReason: "stale_timeout" } + : status === "shed" + ? { stopReason: "memory_shed" } + : {}), + ...(kind === "interval" + ? { + intervalWake: { + nextDueAt: due, + intervalMs: 60_000, + message: "warming interval", + stopCondition: "Done", + }, + } + : kind === "daily" + ? { + dailyWake: { + dailyAt: ["10:00"], + nextDueAt: due, + message: "warming daily", + stopCondition: "Done", + }, + } + : { scheduledWake: { dueAt: due, message: "warming scheduled" } }), + }); + } + + function holdWakeLaunch(service: unknown, panes: Set, fail = false) { + let releaseLaunch!: () => void; + const barrier = new Promise((resolve) => { + releaseLaunch = resolve; + }); + let signalLaunch!: () => void; + const launchStarted = new Promise((resolve) => { + signalLaunch = resolve; + }); + createTmuxSessionMock.mockImplementation( + async ({ sessionName }: { sessionName: string }) => { + if (sessionName === "api-1") { + signalLaunch(); + await barrier; + if (fail) throw new Error("held wake launch failed"); + } + panes.add(sessionName); + }, + ); + return { launchStarted, releaseLaunch, pendingWake: tick(service) }; + } + + describe.each([2, 3])("wake warmup at cap %i", (cap) => { + it.each(warmingWakeCases)( + "counts the %s %s wake once while admitting the remaining slot", + async (status, kind) => { + capAt(cap); + const panes = trackRuntimes(cap === 3 ? ["api-2"] : []); + mockClaudeJsonlState("waiting"); + reserveNextSessionIdMock.mockResolvedValue("api-3"); + const sessions = createSessionStore( + warmingWakeRecord(status, kind), + ...(cap === 3 ? [runningSession({ id: "api-2" })] : []), + ); + const service = await createDisposedSessionService(); + const internals = service as unknown as { + admissionReservations: Map; + countLiveSessions(): { total: number }; + }; + const held = holdWakeLaunch(service, panes); + try { + await held.launchStarted; + expect(internals.countLiveSessions().total).toBe(cap - 1); + const spawned = await service.spawn({ project: "api", prompt: "remaining slot" }); + expect(spawned.id).toBe("api-3"); + expect(internals.countLiveSessions().total).toBe(cap); + expect(internals.admissionReservations.size).toBe(0); + await expect( + service.spawn({ project: "api", prompt: "over cap" }), + ).rejects.toMatchObject({ statusCode: 429, reason: "cap" }); + held.releaseLaunch(); + await held.pendingWake; + expect(sessions.get("api-1")?.status).toBe("running"); + expect(internals.countLiveSessions().total).toBe(cap); + expect(internals.admissionReservations.size).toBe(0); + } finally { + held.releaseLaunch(); + await held.pendingWake; + service.dispose(); + } + }, + ); + }); + + it.each(warmingWakeCases)( + "releases the %s %s wake slot after a held launch failure", + async (status, kind) => { + capAt(2); + const panes = trackRuntimes([]); + mockClaudeJsonlState("waiting"); + reserveNextSessionIdMock.mockResolvedValue("api-3"); + createSessionStore(warmingWakeRecord(status, kind)); + const service = await createDisposedSessionService(); + const internals = service as unknown as { + admissionReservations: Map; + countLiveSessions(): { total: number }; + }; + const held = holdWakeLaunch(service, panes, true); + try { + await held.launchStarted; + expect(internals.countLiveSessions().total).toBe(1); + held.releaseLaunch(); + await held.pendingWake; + expect(internals.countLiveSessions().total).toBe(0); + expect(internals.admissionReservations.size).toBe(0); + const spawned = await service.spawn({ project: "api", prompt: "after launch failure" }); + expect(spawned.id).toBe("api-3"); + } finally { + held.releaseLaunch(); + await held.pendingWake; + service.dispose(); + } + }, + ); + it.each(ordinaryWakeCases)( "reserves an ordinary %s %s wake slot across the runtime probe and counts the live result", async (status, kind) => { From 09f9d0aeaad7723d0b5706cce5b94e737b1b5bba Mon Sep 17 00:00:00 2001 From: AlekShu <108252944+AlekShu@users.noreply.github.com> Date: Fri, 6 Nov 2026 16:36:14 +0000 Subject: [PATCH 19/23] fix(session): retain wake slot throughout pending launch --- v2/src/session-service.ts | 3 ++- v2/test/fast/mcp-contracts.test.ts | 2 +- v2/test/fast/session-service.test.ts | 16 +++++++++++----- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/v2/src/session-service.ts b/v2/src/session-service.ts index 6b58df244..9197dafa4 100644 --- a/v2/src/session-service.ts +++ b/v2/src/session-service.ts @@ -20221,7 +20221,8 @@ export class SessionService { // (send, tryDeliverQueuedMessage, switchAuth) classify the session's real // state right after this returns and must not have that forced to // "working" the way a genuine post-restore warmup intentionally does. - this.restoreWarmupUntil.set(session.id, Date.now() + RESTORE_WARMUP_MS); + // This frame clears the marker in finally; a pending launch must not age out of capacity. + this.restoreWarmupUntil.set(session.id, Number.POSITIVE_INFINITY); this.launchesInFlight.add(session.id); // Transfer the reservation to the counted warmup without yielding. if (options?.wakeClaim) this.releaseScheduledWakeAdmission(options.wakeClaim); diff --git a/v2/test/fast/mcp-contracts.test.ts b/v2/test/fast/mcp-contracts.test.ts index f0c24a5db..fe185ac81 100644 --- a/v2/test/fast/mcp-contracts.test.ts +++ b/v2/test/fast/mcp-contracts.test.ts @@ -24,7 +24,7 @@ const reviewedFingerprints = { "server.ts": "74a508c1d567d59c726787581b0533f5f929339f681a74a80a883596804bffd9", "types.ts": "9bcc3d6d539777f1f6e663c82567d4918b27c31c0a755e5f9cd3b865da43aedc", "cron-job-editor.ts": "33fc7b2993819693ef511d9312f9eb8f4260feb2f263421aec2d4e5ac613ec06", - "session-service.ts": "f86bb90e040038cb5e041f30f60ac8edb27de29d21fd67da64cbe942f66e0ff8", + "session-service.ts": "7c5b0522412f94dedf546eeddd7b5d890e15efa1900dc0d6c59620f8c6b18e35", "session-lifecycle.ts": "11ed4e84c26d589f92627e9d09fc89141f273ae99199a0263d6c2a3b49acb72c", "session-memory.ts": "1d06c2d904fa46b95ef8bed971c5767fad76663482f22822f9a541b345f161ff", "shared-memory.ts": "f5804ebc0592f40c37cf30a1545d88b37075cd6e7e63b3c9e1817bf42c93de67", diff --git a/v2/test/fast/session-service.test.ts b/v2/test/fast/session-service.test.ts index 2594bb24b..acca8e65b 100644 --- a/v2/test/fast/session-service.test.ts +++ b/v2/test/fast/session-service.test.ts @@ -62075,14 +62075,19 @@ describe("SessionService", () => { return { launchStarted, releaseLaunch, pendingWake: tick(service) }; } - describe.each([2, 3])("wake warmup at cap %i", (cap) => { + describe.each([ + [2, 0], + [3, 0], + [2, 31_000], + [3, 31_000], + ] as const)("wake warmup at cap %i after %ims", (cap, elapsed) => { it.each(warmingWakeCases)( "counts the %s %s wake once while admitting the remaining slot", async (status, kind) => { capAt(cap); const panes = trackRuntimes(cap === 3 ? ["api-2"] : []); mockClaudeJsonlState("waiting"); - reserveNextSessionIdMock.mockResolvedValue("api-3"); + reserveNextSessionIdMock.mockResolvedValueOnce("api-3").mockResolvedValue("api-4"); const sessions = createSessionStore( warmingWakeRecord(status, kind), ...(cap === 3 ? [runningSession({ id: "api-2" })] : []), @@ -62095,14 +62100,14 @@ describe("SessionService", () => { const held = holdWakeLaunch(service, panes); try { await held.launchStarted; - expect(internals.countLiveSessions().total).toBe(cap - 1); + vi.setSystemTime(Date.now() + elapsed); const spawned = await service.spawn({ project: "api", prompt: "remaining slot" }); expect(spawned.id).toBe("api-3"); - expect(internals.countLiveSessions().total).toBe(cap); - expect(internals.admissionReservations.size).toBe(0); await expect( service.spawn({ project: "api", prompt: "over cap" }), ).rejects.toMatchObject({ statusCode: 429, reason: "cap" }); + expect(internals.countLiveSessions().total).toBe(cap); + expect(internals.admissionReservations.size).toBe(0); held.releaseLaunch(); await held.pendingWake; expect(sessions.get("api-1")?.status).toBe("running"); @@ -62133,6 +62138,7 @@ describe("SessionService", () => { const held = holdWakeLaunch(service, panes, true); try { await held.launchStarted; + vi.setSystemTime(Date.now() + 31_000); expect(internals.countLiveSessions().total).toBe(1); held.releaseLaunch(); await held.pendingWake; From 7d91e4771358c896211f82325ff4506fa280680a Mon Sep 17 00:00:00 2001 From: AlekShu <108252944+AlekShu@users.noreply.github.com> Date: Fri, 6 Nov 2026 18:33:39 +0000 Subject: [PATCH 20/23] fix(session): reconcile boot before background checks --- v2/src/server.ts | 3 +- v2/test/fast/mcp-contracts.test.ts | 2 +- v2/test/fast/server-restore-request.test.ts | 147 +++++++++++++++++++- v2/test/fast/session-service.test.ts | 2 +- 4 files changed, 148 insertions(+), 6 deletions(-) diff --git a/v2/src/server.ts b/v2/src/server.ts index 3a935bd8b..f4efeb322 100644 --- a/v2/src/server.ts +++ b/v2/src/server.ts @@ -897,7 +897,6 @@ export async function startServer( setGhEventSink(cfg.dataDir); }; applyLogConfigs(service.config); - service.startBackgroundLoops(); const bus = new EventBus(); let triggers: TriggerGroupController | null = null; let sources: Awaited> | null = null; @@ -2694,6 +2693,8 @@ export async function startServer( }); } + service.startBackgroundLoops(); + try { await startAutomation(); } catch (error) { diff --git a/v2/test/fast/mcp-contracts.test.ts b/v2/test/fast/mcp-contracts.test.ts index fe185ac81..a7e0f327f 100644 --- a/v2/test/fast/mcp-contracts.test.ts +++ b/v2/test/fast/mcp-contracts.test.ts @@ -21,7 +21,7 @@ const source = (name: string) => readFile(new URL(`../../src/${name}`, import.me // Review contract metadata when these parser/DTO implementations change; never regenerate blindly. const reviewedFingerprints = { - "server.ts": "74a508c1d567d59c726787581b0533f5f929339f681a74a80a883596804bffd9", + "server.ts": "6e82c0a548f18350ba9179083f45976cdfb40e33dd4b7fc7fcd7eec396336a2b", "types.ts": "9bcc3d6d539777f1f6e663c82567d4918b27c31c0a755e5f9cd3b865da43aedc", "cron-job-editor.ts": "33fc7b2993819693ef511d9312f9eb8f4260feb2f263421aec2d4e5ac613ec06", "session-service.ts": "7c5b0522412f94dedf546eeddd7b5d890e15efa1900dc0d6c59620f8c6b18e35", diff --git a/v2/test/fast/server-restore-request.test.ts b/v2/test/fast/server-restore-request.test.ts index e062d5e8c..a35c43f93 100644 --- a/v2/test/fast/server-restore-request.test.ts +++ b/v2/test/fast/server-restore-request.test.ts @@ -1,4 +1,4 @@ -import { mkdir, mkdtemp, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, it, vi } from "vitest"; @@ -15,9 +15,9 @@ import { } from "../../src/session-lifecycle.js"; import { SessionService } from "../../src/session-service.js"; import * as eventSources from "../../src/event-sources/index.js"; -import { writeSession } from "../../src/metadata.js"; +import { readSession, writeSession } from "../../src/metadata.js"; import type { SessionRecord, SessionView } from "../../src/types.js"; -import { findFreePort } from "../helpers/common.js"; +import { createTempDir, findFreePort } from "../helpers/common.js"; describe("parseRestoreSessionRequest", () => { it.each([parseCompleteSessionRequest, parseRestoreSessionRequest])( @@ -58,6 +58,147 @@ describe("parseRestoreSessionRequest", () => { // A8: the override reaches the daemon over HTTP and is forwarded unchanged // to the service call — this is the only body either route reads. describe("POST /sessions/:id/restore and /reopen forward the force override", () => { + it("restores owner-first shared-workspace boot rows before background checks consume them", async () => { + const root = await createTempDir("spur-shared-reboot-"); + const repoDir = join(root, "repo"); + const dataDir = join(root, "data"); + const configPath = join(root, "spur.yaml"); + const ownerId = "demo-owner"; + const memberId = "demo-member"; + await mkdir(repoDir); + await writeFile( + configPath, + [ + "server:", + " host: 127.0.0.1", + ` port: ${await findFreePort()}`, + `dataDir: ${dataDir}`, + `worktreeDir: ${join(root, "worktrees")}`, + "admission:", + " maxLiveSessions: 2", + " memoryGuard:", + " enforce: false", + " enforceFloors: false", + "projects:", + " demo:", + ` path: ${repoDir}`, + " restoreAfterReboot: true", + ].join("\n"), + ); + for (const [index, id] of [ownerId, memberId].entries()) { + writeSession(dataDir, { + id, + project: "demo", + agent: "claude", + prompt: "unit reboot", + branch: "main", + worktree: false, + worktreePath: repoDir, + workspaceId: ownerId, + tmuxSession: `shared-reboot-${id}`, + launchCommand: "claude", + agentSessionId: `fixture-native-${id}`, + status: "running", + createdAt: `2026-03-18T10:0${1 - index}:00.000Z`, + updatedAt: "2026-03-18T10:02:00.000Z", + }); + } + const dead = { + runtimeAlive: false, + paneUsable: false, + processAlive: false, + tmuxActivityAt: null, + probeUnresponsive: false, + }; + const live = { ...dead, runtimeAlive: true, paneUsable: true, processAlive: true }; + const seam = SessionService.prototype as unknown as { + runDashboardCacheTick(): Promise; + readRuntimeSnapshot(record: SessionRecord): Promise; + restoreLocked(id: string): Promise; + }; + const originalTick = seam.runDashboardCacheTick; + const originalReconcile = SessionService.prototype.reconcileStoppedSessions; + const restored = new Set(); + let enterBoot!: () => void; + const bootEntered = new Promise((resolve) => { + enterBoot = resolve; + }); + let reconciling = false; + let dashboardRunning = false; + let bootProbeStarted = false; + const dashboard = vi.spyOn(seam, "runDashboardCacheTick").mockImplementation(async function ( + this: SessionService, + ) { + await bootEntered; + dashboardRunning = true; + try { + await originalTick.call(this); + } finally { + dashboardRunning = false; + } + }); + const probe = vi.spyOn(seam, "readRuntimeSnapshot").mockImplementation(async function ( + this: SessionService, + record, + ) { + if (reconciling && !dashboardRunning && !bootProbeStarted) { + bootProbeStarted = true; + expect(record.id).toBe(ownerId); + enterBoot(); + const cache = (this as unknown as { dashboardCacheReady: Promise | null }) + .dashboardCacheReady; + if (cache) { + await cache; + expect(readSession(dataDir, memberId)?.status).toBe("stopped"); + } + return dead; + } + return restored.has(record.id) || (dashboardRunning && reconciling && record.id === ownerId) + ? live + : dead; + }); + const reconcile = vi + .spyOn(SessionService.prototype, "reconcileStoppedSessions") + .mockImplementation(async function (this: SessionService) { + reconciling = true; + try { + return await originalReconcile.call(this); + } finally { + reconciling = false; + } + }); + const restoreBody = vi.spyOn(seam, "restoreLocked").mockImplementation(async (id) => { + const current = readSession(dataDir, id); + if (!current) throw new Error("Missing shared-workspace boot fixture"); + const { stopReason: _stopped, ...base } = current; + const running = { ...base, status: "running" as const }; + writeSession(dataDir, running); + restored.add(id); + return running as SessionView; + }); + const restore = vi.spyOn(SessionService.prototype, "restore"); + let server: StartedServer | undefined; + try { + server = await startServer(configPath, { info: () => undefined, warn: () => undefined }); + expect(reconcile).toHaveBeenCalledOnce(); + expect(restore.mock.calls.map(([id]) => id)).toEqual([ownerId, memberId]); + expect(readSession(dataDir, ownerId)?.status).toBe("running"); + expect(readSession(dataDir, memberId)?.status).toBe("running"); + } finally { + enterBoot(); + try { + await server?.stop(); + } finally { + restore.mockRestore(); + restoreBody.mockRestore(); + reconcile.mockRestore(); + probe.mockRestore(); + dashboard.mockRestore(); + await rm(root, { recursive: true, force: true }); + } + } + }); + it("starts source automation only after startup reconcile while routes remain unready", async () => { let reconciled = false; const original = SessionService.prototype.reconcileStoppedSessions; diff --git a/v2/test/fast/session-service.test.ts b/v2/test/fast/session-service.test.ts index acca8e65b..7de4a1437 100644 --- a/v2/test/fast/session-service.test.ts +++ b/v2/test/fast/session-service.test.ts @@ -34245,7 +34245,7 @@ describe("SessionService", () => { projects: { api: { ...config.projects.api, restoreAfterReboot: true } }, }); readHostMemoryMock.mockReturnValue(null); - const sessions = createSessionStore( + createSessionStore( runningSession({ id: "api-1", workspaceId: "api-1", status: "stopped" }), runningSession({ id: "api-2", workspaceId: "api-1", status: "stopped" }), ); From acfa9d08d2daa69ce7fe8900539c9fa13c1ce703 Mon Sep 17 00:00:00 2001 From: AlekShu <108252944+AlekShu@users.noreply.github.com> Date: Fri, 6 Nov 2026 19:44:16 +0000 Subject: [PATCH 21/23] test(session): use valid external lifecycle action --- v2/test/fast/session-service.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/test/fast/session-service.test.ts b/v2/test/fast/session-service.test.ts index 7de4a1437..068f9589d 100644 --- a/v2/test/fast/session-service.test.ts +++ b/v2/test/fast/session-service.test.ts @@ -34260,7 +34260,7 @@ describe("SessionService", () => { () => true, ); if (id === "api-1") { - const external = internals.lifecycle.begin("stop", ["api-1"]); + const external = internals.lifecycle.begin("complete", ["api-1"]); internals.lifecycle.settle( external, true, From aeb62f7e33a245591c00605bc0f4a41bacf1ce15 Mon Sep 17 00:00:00 2001 From: AlekShu <108252944+AlekShu@users.noreply.github.com> Date: Fri, 6 Nov 2026 20:24:46 +0000 Subject: [PATCH 22/23] test(server): align startup events with boot reconciliation --- v2/test/fast/server.test.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/v2/test/fast/server.test.ts b/v2/test/fast/server.test.ts index 1962aba80..d0e29cec7 100644 --- a/v2/test/fast/server.test.ts +++ b/v2/test/fast/server.test.ts @@ -529,14 +529,21 @@ describe("startServer", () => { event: "daemon.registry.count", details: { read: 1, worktreeInternalDropped: 0 }, }); - expect(events[1]).toMatchObject({ + const reconciledIndex = events.findIndex((entry) => entry.event === "daemon.startup.reconciled"); + const attentionPollIndex = events.findIndex((entry) => entry.event === "gh.poll_cycle"); + expect(events[reconciledIndex]).toMatchObject({ + event: "daemon.startup.reconciled", + details: { scanned: 0, alive: 0, drifted: 0 }, + }); + expect(attentionPollIndex).toBeGreaterThan(reconciledIndex); + expect(events[attentionPollIndex]).toMatchObject({ event: "gh.poll_cycle", details: { cycle: "attention", calls: 0 }, }); expect(events.map((entry) => entry.event)).toEqual([ "daemon.registry.count", - "gh.poll_cycle", "daemon.startup.reconciled", + "gh.poll_cycle", "daemon.admission.startup", "daemon.started", "http.route.not_found", From cfd827dad98b58ccb1f2a3abb0c0a5a66a9d63e2 Mon Sep 17 00:00:00 2001 From: AlekShu <108252944+AlekShu@users.noreply.github.com> Date: Fri, 6 Nov 2026 21:07:59 +0000 Subject: [PATCH 23/23] test(runtime): format startup event test --- v2/test/fast/server.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/v2/test/fast/server.test.ts b/v2/test/fast/server.test.ts index d0e29cec7..f7e628416 100644 --- a/v2/test/fast/server.test.ts +++ b/v2/test/fast/server.test.ts @@ -529,7 +529,9 @@ describe("startServer", () => { event: "daemon.registry.count", details: { read: 1, worktreeInternalDropped: 0 }, }); - const reconciledIndex = events.findIndex((entry) => entry.event === "daemon.startup.reconciled"); + const reconciledIndex = events.findIndex( + (entry) => entry.event === "daemon.startup.reconciled", + ); const attentionPollIndex = events.findIndex((entry) => entry.event === "gh.poll_cycle"); expect(events[reconciledIndex]).toMatchObject({ event: "daemon.startup.reconciled",