diff --git a/.agents/skills/spur/SKILL.md b/.agents/skills/spur/SKILL.md index 955aa907a..812538cd1 100644 --- a/.agents/skills/spur/SKILL.md +++ b/.agents/skills/spur/SKILL.md @@ -21,6 +21,7 @@ INTERFACES Agent launch identity contract: `docs/commands.md#session-tools-and-environment`. Manual-wake response fields: `docs/daemon-api.md`. Automatic reminder limits: `docs/configuration.md#automatic-reminders`. + Scheduled-wake admission: `docs/configuration.md#admission-control`. Edit scheduled jobs through footer Cron viewer or MCP; config behavior: docs/configuration.md; operations: docs/daemon-api.md. Spur ToDo: ledger starts empty, no code path seeds an item; the agent adds one item per step, before the step, and resolves it after. Empty or open/held work refuses an agent's own completion, self-destruct, and handoff — `todo_ledger_empty`/`todo_open_work` (409); a human `complete`/`handoff` from the CLI or UI is never blocked. Contract: `docs/commands.md#todo`. Telegram: bot DM for commands/spawn; agent communication in assigned forum topics; completion deletes owned topics. Source identity: `docs/commands.md#source`; attachments/binding: `docs/configuration.md#telegram-binding`; sends: `docs/commands.md#source-reply`. diff --git a/.claude/skills/spur/SKILL.md b/.claude/skills/spur/SKILL.md index 955aa907a..812538cd1 100644 --- a/.claude/skills/spur/SKILL.md +++ b/.claude/skills/spur/SKILL.md @@ -21,6 +21,7 @@ INTERFACES Agent launch identity contract: `docs/commands.md#session-tools-and-environment`. Manual-wake response fields: `docs/daemon-api.md`. Automatic reminder limits: `docs/configuration.md#automatic-reminders`. + Scheduled-wake admission: `docs/configuration.md#admission-control`. Edit scheduled jobs through footer Cron viewer or MCP; config behavior: docs/configuration.md; operations: docs/daemon-api.md. Spur ToDo: ledger starts empty, no code path seeds an item; the agent adds one item per step, before the step, and resolves it after. Empty or open/held work refuses an agent's own completion, self-destruct, and handoff — `todo_ledger_empty`/`todo_open_work` (409); a human `complete`/`handoff` from the CLI or UI is never blocked. Contract: `docs/commands.md#todo`. Telegram: bot DM for commands/spawn; agent communication in assigned forum topics; completion deletes owned topics. Source identity: `docs/commands.md#source`; attachments/binding: `docs/configuration.md#telegram-binding`; sends: `docs/commands.md#source-reply`. diff --git a/docs/configuration.md b/docs/configuration.md index 62e6a6f8a..3e1e89622 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -126,7 +126,7 @@ Type/constraint/default per key; full validation source `v2/src/config.ts`. ## Admission control -Cap limits concurrent live sessions at spawn, restore, and the wake of a stale-parked or memory-shed session. Over cap: `429` naming stop candidates. Reporting route: [`GET /headroom`](daemon-api.md). CLI: [`spur doctor`](commands.md#doctor). Schema `v2/src/config.ts`, enforcement `v2/src/session-service.ts`. +Cap limits concurrent live sessions at spawn, restore, and the wake of a stale-parked or memory-shed session (including one-shot wakes and relaunching sends). Ordinary stopped/paused interval/daily relaunches: background gated, manual ungated. Over cap: `429` naming stop candidates. Reporting route: [`GET /headroom`](daemon-api.md). CLI: [`spur doctor`](commands.md#doctor). Schema `v2/src/config.ts`, enforcement `v2/src/session-service.ts`. `stopReason: "runtime_lost"`: confirmed runtime loss becomes restorable `stopped`, retains error evidence, releases capacity; unknown probes retain capacity (`v2/src/session-service.ts`). @@ -185,7 +185,7 @@ Event names by source: - Webhook request: `POST ` with `Authorization: Bearer ` and `Content-Type: application/json`; runtime behavior and limits: `v2/src/event-sources/webhook.ts`. -Other event names (see source file for trigger conditions): `trigger.spawn.suppressed`, `source.poll.disabled`, `source.poll.enabled`, `source.poll_cycle.error`, `source.telegram.{callback_dropped,choice_rejected}`, `gh.poll_cycle`, `gh.usage`, `gh.poll_budget_paused`, `session.message.{sent,delivery_recovered,delivery_failed,requeued,submit_unconfirmed,queued_ahead,queue_removed,stalled}`, `session.spawn.{orphan_stopped,launch_unconfirmed,launch_submitted}`, `auto-ping.{muted,unmuted,migrated}`, `session.todo.{nudge_failed,nudge_disabled,nudge_exhausted}`, `session.{complete,pause,self_destruct,desk_complete,handoff}.{completed,failed}`, `session.wake.{failed,daily_failed,interval_failed,sent,daily_sent,interval_sent,interval_cancelled,daily_cancelled,suppressed}`, `session.wake.token_budget_blocked`, `session.token_budget.exhausted`, `session.token_budget.unenforced`, `session.token_budget.teardown_failed`, `session.attention_monitor.{failed,session_failed,slow}`, `session.runtime.{probe_unresponsive,pane_child_fallback}`, `session.agent_process.{pane_pid_unreadable,capture_unavailable,capture_blind,survivors}`, `session.{handoff,respawn}.startup_attachment_missing`, `session.server_error.reactivation_exhausted`, `session.sidecar.{start_noop,start_rejected,launch_reap_skipped,owner_missing}`, `session.list.enrich_failed`, `session.submit.{timeout,confirmed,released,failure_dismissed}`, `session.subscription.{delivery_failed,spawn_failed}`, `session.recover.context_unconfirmed`, `session.record.invalid`, `trigger.send.{failed,suppressed,suppressed_admission,suppressed_launch_pending}`, `host.disk.low`, `daemon.admission.startup`, `daemon.registry.{count,warning}`, `daemon.auto_update.{started,failed,retry,skipped,suppressed,paused,config_invalid,disarm_failed}`, `daemon.deploy_switch.{started,rejected}`, `cli.update.{started,rolled_back,abandoned}`. +Other event names (see source file for trigger conditions): `trigger.spawn.suppressed`, `source.poll.disabled`, `source.poll.enabled`, `source.poll_cycle.error`, `source.telegram.{callback_dropped,choice_rejected}`, `gh.poll_cycle`, `gh.usage`, `gh.poll_budget_paused`, `session.message.{sent,delivery_recovered,delivery_failed,requeued,submit_unconfirmed,queued_ahead,queue_removed,stalled}`, `session.spawn.{orphan_stopped,launch_unconfirmed,launch_submitted}`, `auto-ping.{muted,unmuted,migrated}`, `session.todo.{nudge_failed,nudge_disabled,nudge_exhausted}`, `session.{complete,pause,self_destruct,desk_complete,handoff}.{completed,failed}`, `session.wake.{failed,daily_failed,daily_deferred,daily_exhausted,interval_failed,interval_deferred,sent,daily_sent,interval_sent,interval_cancelled,daily_cancelled,suppressed}`, `session.wake.token_budget_blocked`, `session.token_budget.exhausted`, `session.token_budget.unenforced`, `session.token_budget.teardown_failed`, `session.attention_monitor.{failed,session_failed,slow}`, `session.runtime.{probe_unresponsive,pane_child_fallback}`, `session.agent_process.{pane_pid_unreadable,capture_unavailable,capture_blind,survivors}`, `session.{handoff,respawn}.startup_attachment_missing`, `session.server_error.reactivation_exhausted`, `session.sidecar.{start_noop,start_rejected,launch_reap_skipped,owner_missing}`, `session.list.enrich_failed`, `session.submit.{timeout,confirmed,released,failure_dismissed}`, `session.subscription.{delivery_failed,spawn_failed}`, `session.recover.context_unconfirmed`, `session.record.invalid`, `trigger.send.{failed,suppressed,suppressed_admission,suppressed_launch_pending}`, `host.disk.low`, `daemon.admission.startup`, `daemon.registry.{count,warning}`, `daemon.auto_update.{started,failed,retry,skipped,suppressed,paused,config_invalid,disarm_failed}`, `daemon.deploy_switch.{started,rejected}`, `cli.update.{started,rolled_back,abandoned}`. ## Auto update diff --git a/docs/daemon-api.md b/docs/daemon-api.md index 2c8f81321..15dba3653 100644 --- a/docs/daemon-api.md +++ b/docs/daemon-api.md @@ -42,7 +42,7 @@ - `POST /sessions/background` — `409 spawn_in_progress|spawn_committed|spawn_failed` body `{ code, sessionId, error }` when `preflightBatchId` is already claimed; `sessionId` is the claiming session; `v2/src/session-service.ts`. - `POST /shepherd/spawn` - `POST /sidecars/sweep` — body `{ reap?: boolean }`, default `false`. Always `200`. Response `{ supported, leaked, reaped }`: `supported` is `false` when the process table or procfs is unreadable, in which case `leaked` and `reaped` are both `[]`. Each `leaked` row carries `kind: "worktree-tree" | "orphan-daemon"`; a `worktree-tree` row's `reapable` reflects proven Spur provenance and carries `rootPid`, `pgid`, `ageSeconds`, `worktreePath`, `args`, `sidecarName`, `tree` (descendant pids, root first), `treeRssKb`; an `orphan-daemon` row is always `reapable: false` and carries `configPath`/`cliEntryPath` instead of a `sidecarName`, plus `port: number | null` and `liveness: "serving" | "not-serving" | "unknown"`. `reap: true` signals only `reapable` rows — never an `orphan-daemon` row, serving or not — and populates `reaped`, one entry per `reapable` leaked tree it signaled: `{ sessionName, panePid, survivors, blindKill }` — `survivors` is the pids still alive after the SIGKILL confirmation window, and `blindKill` is `true` when tmux was killed with no verified process-tree signal (no pane pid, or an unusable snapshot, and no identity fallback confirmed a reap either) — a `survivors: []` alongside `blindKill: true` is not proof anything died, only that nothing was checked. See [Sidecars](commands.md#sidecars) -- `GET /sessions/:id` — current row, `sidecars[].url` ready endpoint, `sidecars[].ports[].listening` host TCP listener true|false|null; Cursor `queuedMessageTyped.ackBaseline.deliveryId` optional UUIDv4, correlated `file:null` requires `size:0` and valid ID, malformed IDs drop baseline, legacy string-file floors retained; optional read-only `intervalWake.dispatchClaimId`: opaque correlation, absent on legacy records, ignored on input; `v2/src/types.ts`, `v2/src/metadata.ts`; ephemeral `lifecycle` receipt; `v2/src/session-lifecycle.ts`. +- `GET /sessions/:id` — current row, `sidecars[].url` ready endpoint, `sidecars[].ports[].listening` host TCP listener true|false|null; Cursor `queuedMessageTyped.ackBaseline.deliveryId` optional UUIDv4, correlated `file:null` requires `size:0` and valid ID, malformed IDs drop baseline, legacy string-file floors retained; optional read-only `intervalWake.dispatchClaimId`: opaque correlation, absent on legacy records, ignored on input; optional `dailyWake.admissionDeferredOccurrence`: persisted retry occurrence; `v2/src/types.ts`, `v2/src/metadata.ts`; ephemeral `lifecycle` receipt; `v2/src/session-lifecycle.ts`. - `GET /sessions/:id/slash-commands` - `GET /sessions/:id/conversation` — optional `view=messages&limit=<1..100>` (default 50): bounded visible-message page; `v2/src/server.ts`, `v2/src/conversation-page.ts`; omit view for legacy response. - `GET /sessions/:id/user-actions` diff --git a/v2/src/server.ts b/v2/src/server.ts index 0fb0a3a2d..f4efeb322 100644 --- a/v2/src/server.ts +++ b/v2/src/server.ts @@ -897,7 +897,6 @@ export async function startServer( setGhEventSink(cfg.dataDir); }; applyLogConfigs(service.config); - service.startBackgroundLoops(); const bus = new EventBus(); let triggers: TriggerGroupController | null = null; let sources: Awaited> | null = null; @@ -2672,7 +2671,7 @@ export async function startServer( throw error; } - let driftedSessions: { id: string; project: string }[] = []; + let driftedSessions: Parameters[0] = []; try { const { scanned, @@ -2694,6 +2693,8 @@ export async function startServer( }); } + service.startBackgroundLoops(); + try { await startAutomation(); } catch (error) { diff --git a/v2/src/session-service.ts b/v2/src/session-service.ts index 45b44ea0e..9197dafa4 100644 --- a/v2/src/session-service.ts +++ b/v2/src/session-service.ts @@ -20,6 +20,7 @@ import { isDeepStrictEqual } from "node:util"; import { annotateLifecycleError, createRestoreOperation, + lifecycleErrorReceipt, RestoreOperationError, type RestoreOperationContext, SessionLifecycleError, @@ -535,6 +536,8 @@ import { type SharedMemoryScope, type StartSidecarRequest, type SessionRecord, + type SessionDailyWakeState, + type SessionIntervalWakeState, type CursorRestoreBoundary, type SessionSlots, type SessionStatus, @@ -632,6 +635,19 @@ const LEGACY_RUNTIME_DIAGNOSTICS = [ "tmux process failed:", ]; +// Daily wake schedule without the admission retry marker: every claim write +// drops it, the claimed occurrence is the one now in flight. +function withoutAdmissionDeferral(dailyWake: SessionDailyWakeState): SessionDailyWakeState { + const { admissionDeferredOccurrence: _deferredOccurrence, ...schedule } = dailyWake; + return schedule; +} + +// Status whose record claims a live agent runtime. Only these carry a runtime +// probe failure as a session error; any other status ignores it. +function expectsLiveRuntime(status: SessionStatus): boolean { + return status === "running" || status === "spawning"; +} + function recoverableRuntimeError(session: SessionRecord): boolean { if (session.error === INFERRED_RUNTIME_EXIT_ERROR) { return ( @@ -639,6 +655,12 @@ function recoverableRuntimeError(session: SessionRecord): boolean { (session.errorSource === undefined && session.launchCommand !== "") ); } + return isRuntimeProbeError(session); +} + +// Failed runtime probes: runtime_probe, legacy unsourced tmux/ps, or the +// two exact historical other-source ps diagnostics; void once a probe answers. +function isRuntimeProbeError(session: SessionRecord): boolean { if ( !session.error?.startsWith(STATUS_DETECTION_ERROR_PREFIX) || session.error.startsWith(REPORTING_DETECTION_ERROR_PREFIX) @@ -666,6 +688,7 @@ const CLAUDE_SERVER_ERROR_REACTIVATION_PROMPT = const CLAUDE_SERVER_ERROR_REACTIVATION_MS = 30 * 60 * 1000; const PIPELINE_POLL_INTERVAL_MS = 1_000; const SCHEDULED_WAKE_POLL_INTERVAL_MS = 1_000; +const DAILY_WAKE_ADMISSION_RETRY_MS = 15 * 60_000; const SIDECAR_REAPER_INTERVAL_MS = 60_000; const MEMORY_SHED_INTERVAL_MS = 1_000; // Consecutive 1s ticks above the restore floor plus margin required before @@ -1262,6 +1285,7 @@ type ScheduledWakeClaim = { revision: number; manual: boolean; stage: "unattempted" | "input-started" | "accepted"; + admissionReservation?: symbol; }; type AgentMessageTarget = Pick< @@ -1341,6 +1365,7 @@ type SessionPreparationOptions = { operation?: RestoreOperationContext; preparedObservation?: PreparedSessionObservation; observePreparedSession?: (session: SessionRecord) => void; + wakeAdmissionReservation?: symbol; }; type ForegroundSpawnReceipt = { @@ -1416,6 +1441,14 @@ interface SessionRuntimeSnapshot { probeUnresponsive: boolean; diagnostic?: string; } +interface SessionObservation { + record: SessionRecord; + lifecycle: { id: string; snapshot: SessionLifecycleSnapshot }[]; +} +interface DailyWakeClaim extends SessionObservation { + record: SessionRecord & { dailyWake: SessionDailyWakeState }; + admissionReservation?: symbol; +} // Plan-time facts the reap executor re-proves under the workspace lifecycle lock. interface SidecarReapEntryContext { identity: SidecarProcessIdentity | undefined; @@ -1494,6 +1527,28 @@ function isRestorableStatus(status: SessionStatus): boolean { return status === "running" || status === "stopped" || status === "paused"; } +type WakeAdmissionDenial = { kind: "denied" | "check_failed"; error: Error }; +type WakeAdmissionClaim = { reservation?: symbol; denial?: WakeAdmissionDenial }; + +function wakeAdmissionReason(denial: WakeAdmissionDenial): string { + return denial.kind === "denied" ? "admission" : "admission_check_failed"; +} + +function wakeAdmissionLogLevel(denial: WakeAdmissionDenial): "warn" | "error" { + return denial.kind === "denied" ? "warn" : "error"; +} + +// Interval or daily wake relaunch of a stopped or paused session. Stale-parked +// and memory-shed sessions keep their own gate inside ensureSessionReadyForSend. +function wakeRelaunchNeedsAdmission( + session: Pick, +): boolean { + if (session.status === "paused") return true; + return ( + session.status === "stopped" && session.stopReason !== "memory_shed" && !isStaleParked(session) + ); +} + // A launch or Send now whose submit never confirmed: the prompt may still sit // unsubmitted in the composer, and any typed send (its line clear) would erase // it. Every path that types into the pane checks this one predicate. @@ -4130,7 +4185,7 @@ export class SessionService { this.deliveryStopped = true; this.codexPaneLimitObservedAt.clear(); this.codexRateLimitPoller.dispose(); - this.scheduledWakeClaims.clear(); + for (const claim of this.scheduledWakeClaims.values()) this.releaseScheduledWakeClaim(claim); this.scheduledWakeLastFailure.clear(); this.sidecarEndpoints.clear(); this.sidecarObservation = undefined; @@ -5627,67 +5682,83 @@ export class SessionService { ); if (claimedNextDueAt !== null) { const nextDueAt = claimedNextDueAt; - try { - await this.send(session.id, { - message: this.formatIntervalWakeMessage( + const admission = this.claimWakeAdmission(session.id); + if (admission.denial) { + this.deferIntervalWakeForAdmission( + session, + intervalWake, + claimedNextDueAt, + admission.denial, + ); + } else { + try { + await this.sendScheduledWake( session.id, - intervalWake.message, - intervalWake.stopCondition, - ), - }); - this.logEvent("session.wake.interval_sent", { - level: "info", - sessionId: session.id, - projectId: session.project, - message: `Sent interval wake to ${session.id}`, - details: { - nextDueAt, - intervalMs: intervalWake.intervalMs, - }, - }); - } catch (error) { - if (error instanceof SessionAdmissionDeniedError) { - // Same reasoning as the scheduled-wake branch above: this - // occurrence was already claimed (nextDueAt advanced and - // persisted) before the send attempt. Roll nextDueAt back to - // the occurrence that was just refused (CAS: only if nothing - // else re-armed it meanwhile) so the next poll tick retries - // it promptly instead of waiting a full interval. - await this.withWorkspaceLifecycleLocks(session.id, async () => { - const afterDenial = readSession(this.config.dataDir, session.id); - if (afterDenial?.intervalWake?.nextDueAt === nextDueAt) { - writeSession(this.config.dataDir, { - ...afterDenial, - intervalWake: { - ...afterDenial.intervalWake, - nextDueAt: intervalWake.nextDueAt, - }, - updatedAt: nowIso(), - }); - } - }); - this.logEvent("session.wake.interval_deferred", { - level: "warn", - sessionId: session.id, - projectId: session.project, - message: `Deferred interval wake for ${session.id}: ${error.message}`, - details: { - nextDueAt, - intervalMs: intervalWake.intervalMs, + { + message: this.formatIntervalWakeMessage( + session.id, + intervalWake.message, + intervalWake.stopCondition, + ), }, - }); - } else { - const message = error instanceof Error ? error.message : String(error); - this.logEvent("session.wake.interval_failed", { - level: "error", + admission.reservation, + ); + this.logEvent("session.wake.interval_sent", { + level: "info", sessionId: session.id, projectId: session.project, - message: `Failed to send interval wake to ${session.id}: ${message}`, + message: `Sent interval wake to ${session.id}`, details: { nextDueAt, intervalMs: intervalWake.intervalMs, }, }); + } catch (error) { + if (error instanceof SessionAdmissionDeniedError) { + // Same reasoning as the scheduled-wake branch above: this + // occurrence was already claimed (nextDueAt advanced and + // persisted) before the send attempt. Roll nextDueAt back to + // the occurrence that was just refused (CAS: only if nothing + // else re-armed it meanwhile) so the next poll tick retries + // it promptly instead of waiting a full interval. + await this.withWorkspaceLifecycleLocks(session.id, async () => { + const afterDenial = readSession(this.config.dataDir, session.id); + if (afterDenial?.intervalWake?.nextDueAt === nextDueAt) { + writeSession(this.config.dataDir, { + ...afterDenial, + intervalWake: { + ...afterDenial.intervalWake, + nextDueAt: intervalWake.nextDueAt, + }, + updatedAt: nowIso(), + }); + } + }); + this.logEvent("session.wake.interval_deferred", { + level: "warn", + sessionId: session.id, + projectId: session.project, + message: `Deferred interval wake for ${session.id}: ${error.message}`, + details: { + nextDueAt, + intervalMs: intervalWake.intervalMs, + }, + }); + } else { + const message = error instanceof Error ? error.message : String(error); + this.logEvent("session.wake.interval_failed", { + level: "error", + sessionId: session.id, + projectId: session.project, + message: `Failed to send interval wake to ${session.id}: ${message}`, + details: { + nextDueAt, + intervalMs: intervalWake.intervalMs, + }, + }); + } + } finally { + this.releaseAdmissionReservation(admission.reservation); } } } @@ -5943,9 +6014,10 @@ export class SessionService { ) { continue; } - const claimedDailyNextDueAt = await this.withWorkspaceLifecycleLocks( + const dailySchedule = withoutAdmissionDeferral(dailyWake); + const claimedDailyWake = await this.withWorkspaceLifecycleLocks( session.id, - async (): Promise => { + async (): Promise => { // Claim the due occurrence BEFORE sending: advance nextDueAt to the // next future scheduled time and persist it first. A slow or failing // send must not leave the wake due, or the `<= now` guard stays true @@ -5964,6 +6036,12 @@ export class SessionService { ) { return null; } + const lifecycle = this.lifecycleIdsFor(currentDailyWakeSession).map((id) => ({ + id, + snapshot: this.lifecycle.snapshot(id), + })); + if (lifecycle.some(({ snapshot }) => snapshot.operation?.phase === "pending")) + return null; // CAS: only claim if the wake is unchanged (no concurrent re-arm/cancel). const dailyWakeClaimed = currentDailyWakeSession.dailyWake?.nextDueAt === dailyWake.nextDueAt && @@ -5991,7 +6069,7 @@ export class SessionService { const skipped: SessionRecord = { ...currentDailyWakeSession, dailyWake: { - ...dailyWake, + ...dailySchedule, nextDueAt: new Date(now + 24 * 60 * 60 * 1000).toISOString(), }, updatedAt: nowIso(), @@ -6009,36 +6087,51 @@ export class SessionService { }); return null; } - const updated: SessionRecord = { + const updated = { ...currentDailyWakeSession, dailyWake: { - ...dailyWake, + ...dailySchedule, nextDueAt: nextDueAt.toISOString(), }, updatedAt: nowIso(), }; writeSession(this.config.dataDir, updated); - return nextDueAt.toISOString(); + const claim: DailyWakeClaim = { + record: updated, + lifecycle, + }; + const admission = this.claimWakeAdmission(session.id); + if (admission.denial) { + this.deferDailyWakeForAdmission(session, dailyWake, claim, admission.denial, now); + return null; + } + if (admission.reservation) claim.admissionReservation = admission.reservation; + return claim; } return null; }, ); - if (claimedDailyNextDueAt !== null) { + if (claimedDailyWake !== null) { + const claimedDailyNextDueAt = claimedDailyWake.record.dailyWake.nextDueAt; try { - await this.send(session.id, { - message: this.formatDailyWakeMessage( - session.id, - dailyWake.message, - dailyWake.stopCondition, - ), - }); + await this.sendScheduledWake( + session.id, + { + message: this.formatDailyWakeMessage( + session.id, + dailyWake.message, + dailyWake.stopCondition, + ), + }, + claimedDailyWake.admissionReservation, + ); this.logEvent("session.wake.daily_sent", { level: "info", sessionId: session.id, projectId: session.project, message: `Sent daily wake to ${session.id}`, details: { - nextDueAt: dailyWake.nextDueAt, + nextDueAt: dailyWake.admissionDeferredOccurrence ?? dailyWake.nextDueAt, dailyAt: dailyWake.dailyAt, }, }); @@ -6083,6 +6176,8 @@ export class SessionService { }, }); } + } finally { + this.releaseAdmissionReservation(claimedDailyWake.admissionReservation); } } } @@ -6091,6 +6186,116 @@ export class SessionService { } } + // Runs after the occurrence claim: a denial before it would leave the wake + // due and retry every tick. An unexpected check failure denies only this + // record: rethrowing would abort the wake loop for every later session. + private claimWakeAdmission(sessionId: string): WakeAdmissionClaim { + const current = readSession(this.config.dataDir, sessionId); + if (!current || !wakeRelaunchNeedsAdmission(current)) return {}; + try { + return { reservation: this.reserveAdmission(current.project, "wake") }; + } catch (error) { + if (error instanceof SessionAdmissionDeniedError) + return { denial: { kind: "denied", error } }; + const message = error instanceof Error ? error.message : String(error); + return { + denial: { kind: "check_failed", error: new Error(`admission check failed: ${message}`) }, + }; + } + } + + // Denied interval occurrence stays consumed: the claim already advanced + // nextDueAt to the next interval. + private deferIntervalWakeForAdmission( + session: SessionRecord, + intervalWake: SessionIntervalWakeState, + nextDueAt: string, + denial: WakeAdmissionDenial, + ): void { + this.logEvent("session.wake.interval_deferred", { + level: wakeAdmissionLogLevel(denial), + sessionId: session.id, + projectId: session.project, + message: `Deferred interval wake for ${session.id}: ${denial.error.message}`, + details: { + nextDueAt, + intervalMs: intervalWake.intervalMs, + reason: wakeAdmissionReason(denial), + }, + }); + } + + // Denied daily occurrence retries every DAILY_WAKE_ADMISSION_RETRY_MS until + // its next scheduled slot, where the claim already points; past that it is + // skipped. The occurrence is persisted so a restart keeps the window. + // Called synchronously under the claim's workspace lifecycle locks. + private deferDailyWakeForAdmission( + session: SessionRecord, + dailyWake: SessionDailyWakeState, + claimed: DailyWakeClaim, + denial: WakeAdmissionDenial, + now: number, + ): void { + const occurrence = dailyWake.admissionDeferredOccurrence ?? dailyWake.nextDueAt; + const windowEndMs = resolveNextDailyWakeAt(dailyWake.dailyAt, new Date(occurrence)).getTime(); + const retryAtMs = now + DAILY_WAKE_ADMISSION_RETRY_MS; + const retryAt = new Date(retryAtMs).toISOString(); + const afterDenial = readSession(this.config.dataDir, session.id); + if ( + !afterDenial || + !sameSessionRuntimeOwner(afterDenial, claimed.record) || + afterDenial.agentLaunchId !== claimed.record.agentLaunchId || + !this.lifecycleStateMatches(afterDenial, this.lifecycleStamp(claimed.record)) || + !isDeepStrictEqual(afterDenial.dailyWake, claimed.record.dailyWake) || + claimed.lifecycle.some(({ id, snapshot }) => { + const current = this.lifecycle.snapshot(id); + return ( + snapshot.operation?.phase === "pending" || + current.operation?.phase === "pending" || + current.instanceId !== snapshot.instanceId || + current.revision !== snapshot.revision + ); + }) + ) + return; + if (retryAtMs >= windowEndMs) { + this.logEvent("session.wake.daily_exhausted", { + level: "warn", + sessionId: session.id, + projectId: session.project, + message: `Skipped daily wake occurrence ${occurrence} for ${session.id}: ${denial.error.message}`, + details: { + occurrence, + nextDueAt: claimed.record.dailyWake.nextDueAt, + dailyAt: dailyWake.dailyAt, + }, + }); + return; + } + writeSession(this.config.dataDir, { + ...afterDenial, + dailyWake: { + ...claimed.record.dailyWake, + nextDueAt: retryAt, + admissionDeferredOccurrence: occurrence, + }, + updatedAt: nowIso(), + }); + if (dailyWake.admissionDeferredOccurrence !== undefined) return; + this.logEvent("session.wake.daily_deferred", { + level: wakeAdmissionLogLevel(denial), + sessionId: session.id, + projectId: session.project, + message: `Deferred daily wake for ${session.id}: ${denial.error.message}`, + details: { + reason: wakeAdmissionReason(denial), + occurrence, + retryAt, + dailyAt: dailyWake.dailyAt, + }, + }); + } + previewConfigConnect(configPath: string): { config: AppConfig; registryPaths: string[]; @@ -7628,8 +7833,8 @@ export class SessionService { // runDashboardCacheTick owns their pruning. private pruneSessionScopedState(liveIds: ReadonlySet): void { this.codexRateLimitPoller.prune(liveIds); - for (const sessionId of this.scheduledWakeClaims.keys()) { - if (!liveIds.has(sessionId)) this.scheduledWakeClaims.delete(sessionId); + for (const [sessionId, claim] of this.scheduledWakeClaims) { + if (!liveIds.has(sessionId)) this.releaseScheduledWakeClaim(claim); } for (const sessionId of this.scheduledWakeLastFailure.keys()) { if (!liveIds.has(sessionId)) this.scheduledWakeLastFailure.delete(sessionId); @@ -12048,8 +12253,15 @@ export class SessionService { return awake; } - async restoreRebootedSessions(drifted: { id: string; project: string }[]): Promise { - for (const [index, { id, project }] of drifted.entries()) { + async restoreRebootedSessions(drifted: SessionObservation[]): Promise { + const loopOwnedLifecycle = new Map(); + const rememberRestoreReceipt = (receipt: SessionLifecycleSnapshot | undefined): void => { + const operation = receipt?.operation; + if (!receipt || operation?.action !== "restore" || operation.phase === "pending") return; + for (const id of operation.targetIds) loopOwnedLifecycle.set(id, receipt); + }; + for (const [index, { record, lifecycle }] of drifted.entries()) { + const { id, project } = record; const projectConfig = this.config.projects[project]; if (projectConfig?.restoreAfterReboot !== true) continue; const memory = readHostMemory(); @@ -12071,12 +12283,46 @@ export class SessionService { }); break; } + const current = readSession(this.config.dataDir, id); + const fencedLifecycleIds = lifecycle.flatMap(({ id: ownerId, snapshot }) => { + const expected = loopOwnedLifecycle.get(ownerId) ?? snapshot; + const latest = this.lifecycle.snapshot(ownerId); + return snapshot.operation?.phase === "pending" || + latest.operation?.phase === "pending" || + latest.instanceId !== expected.instanceId || + latest.revision !== expected.revision + ? [ownerId] + : []; + }); + if ( + !current || + !sameSessionRuntimeOwner(current, record) || + current.agentLaunchId !== record.agentLaunchId || + !this.lifecycleStateMatches(current, this.lifecycleStamp(record)) || + current.error !== record.error || + current.errorSource !== record.errorSource || + current.serverErrorAt !== record.serverErrorAt || + fencedLifecycleIds.length > 0 + ) { + if (fencedLifecycleIds.length > 0) { + this.logEvent("session.reboot.restore.aborted", { + level: "warn", + sessionId: id, + projectId: project, + message: `Reboot restore skipped for ${id}: lifecycle ownership changed`, + details: { reason: "lifecycle_changed", lifecycleIds: fencedLifecycleIds }, + }); + } + continue; + } try { - await this.restore(id); + const restored = await this.restore(id); + rememberRestoreReceipt(restored.lifecycle); if (memory !== null) { await sleep(RESTORE_SETTLE_MS); } } catch (error) { + rememberRestoreReceipt(lifecycleErrorReceipt(error)); const message = error instanceof Error ? error.message : String(error); this.logEvent("session.reboot.restore.failed", { level: "warn", @@ -12092,20 +12338,58 @@ export class SessionService { scanned: number; alive: number; drifted: number; - driftedSessions: { id: string; project: string }[]; + driftedSessions: SessionObservation[]; }> { const candidates = listSessions(this.config.dataDir).filter( (session) => session.status === "running" || session.status === "spawning", ); let alive = 0; let drifted = 0; - const driftedSessions: { id: string; project: string }[] = []; + const driftedSessions: SessionObservation[] = []; for (const session of candidates) { + // A record that carried a probe error is stopped at boot but never + // reboot-restored. Read from the listed snapshot: a concurrent heal + // between list and read must not turn it into a plain drifted record. + const healedProbeError = this.probeErrorHealable(session); + const observed = { ...session }; + const stamp = this.lifecycleStamp(observed); + const lifecycle = this.lifecycleIdsFor(session).map((id) => ({ + id, + snapshot: this.lifecycle.snapshot(id), + })); + const sameObservationOwner = (record: SessionRecord) => + sameSessionRuntimeOwner(record, observed) && + record.agentLaunchId === observed.agentLaunchId && + lifecycle.every(({ id, snapshot }) => { + const latest = this.lifecycle.snapshot(id); + return ( + snapshot.operation?.phase !== "pending" && + latest.operation?.phase !== "pending" && + latest.instanceId === snapshot.instanceId && + latest.revision === snapshot.revision + ); + }); const probeRevision = this.lifecycle.snapshot(session.id).revision; const runtime = await this.readRuntimeSnapshot(session); const current = readSession(this.config.dataDir, session.id); if (!current) continue; + const concurrentlyHealed = + healedProbeError && + current.status === "stopped" && + !hasRetainedSessionError(current) && + !current.serverErrorAt && + current.stopReason !== "manual_pause" && + current.stopReason !== "token_budget"; + if ( + !sameObservationOwner(current) || + (!this.lifecycleStateMatches(current, stamp) && !concurrentlyHealed) || + (!concurrentlyHealed && + (current.error !== observed.error || + current.errorSource !== observed.errorSource || + current.serverErrorAt !== observed.serverErrorAt)) + ) + continue; const workspace = probeWorkspace(current.worktreePath); const reconciled = await this.reconcileUnexpectedStop( session, @@ -12123,28 +12407,45 @@ export class SessionService { throw error; }); if (!reconciled) continue; - if (!readSession(this.config.dataDir, session.id)) continue; + const usage = + (reconciled.session.status === "stopped" || reconciled.session.status === "errored") && + (healedProbeError || !hasRetainedSessionError(current)) && + !current.serverErrorAt && + !reconciled.runtime.probeUnresponsive && + !reconciled.runtime.diagnostic && + !workspace.diagnostic && + workspace.exists && + (!reconciled.runtime.paneUsable || !reconciled.runtime.processAlive) + ? await this.readFinalTokenUsage(session, reconciled.runtime) + : undefined; + const latest = readSession(this.config.dataDir, session.id); + if ( + !latest || + !sameObservationOwner(latest) || + !sameObservationOwner(reconciled.session) || + !this.lifecycleStateMatches(latest, this.lifecycleStamp(reconciled.session)) || + latest.error !== reconciled.session.error || + latest.errorSource !== reconciled.session.errorSource || + latest.serverErrorAt !== reconciled.session.serverErrorAt || + latest.stopReason === "manual_pause" || + latest.stopReason === "token_budget" + ) + continue; + let accepted = latest; + if (usage) { + const persisted = this.persistClassifiedTokenUsage(latest, usage); + if (persisted.status === "missing") continue; + accepted = persisted.session; + } if (reconciled.session.status === "stopped" || reconciled.session.status === "errored") { - if ( - !hasRetainedSessionError(current) && - !current.serverErrorAt && - !reconciled.runtime.probeUnresponsive && - !reconciled.runtime.diagnostic && - !workspace.diagnostic && - workspace.exists && - (!reconciled.runtime.paneUsable || !reconciled.runtime.processAlive) - ) { - const usage = await this.readFinalTokenUsage(session, reconciled.runtime); - const persisted = this.persistClassifiedTokenUsage(session, usage); - if (persisted.status === "missing") continue; - } drifted += 1; if ( - reconciled.session.status === "stopped" || - (reconciled.session.errorSource === "inferred_runtime_exit" && - !reconciled.session.serverErrorAt) + !healedProbeError && + (reconciled.session.status === "stopped" || + (reconciled.session.errorSource === "inferred_runtime_exit" && + !reconciled.session.serverErrorAt)) ) { - driftedSessions.push({ id: session.id, project: session.project }); + driftedSessions.push({ record: accepted, lifecycle }); } } else { alive += 1; @@ -15023,7 +15324,8 @@ export class SessionService { } writeSession(this.config.dataDir, updated); if (request.target === "scheduled") { - this.scheduledWakeClaims.delete(sessionId); + const claim = this.scheduledWakeClaims.get(sessionId); + if (claim) this.releaseScheduledWakeClaim(claim); this.scheduledWakeLastFailure.delete(sessionId); } return this.enrich(updated); @@ -15118,8 +15420,16 @@ export class SessionService { this.lifecycle.snapshot(session.id).operation?.phase === "pending" ) return null; + const previousClaim = this.scheduledWakeClaims.get(session.id); + const previousReservation = previousClaim?.admissionReservation; + let admissionReservation = previousReservation; if (isStaleParked(current) || current.stopReason === "memory_shed") { - this.assertWakeAdmissible(current, manual ? undefined : wake); + admissionReservation = this.checkWakeAdmission( + current, + manual ? undefined : wake, + admissionReservation, + true, + ); } const claim: ScheduledWakeClaim = { session: current, @@ -15127,11 +15437,21 @@ export class SessionService { manual, revision: this.lifecycle.snapshot(session.id).revision, stage: "unattempted", + ...(admissionReservation ? { admissionReservation } : {}), }; this.scheduledWakeClaims.set(session.id, claim); - if (manual) { - const { scheduledWake: _wake, ...base } = current; - writeSession(this.config.dataDir, { ...base, updatedAt: nowIso() }); + if (previousClaim && previousReservation) { + delete previousClaim.admissionReservation; + if (!claim.admissionReservation) this.admissionReservations.delete(previousReservation); + } + try { + if (manual) { + const { scheduledWake: _wake, ...base } = current; + writeSession(this.config.dataDir, { ...base, updatedAt: nowIso() }); + } + } catch (error) { + this.releaseScheduledWakeClaim(claim); + throw error; } return claim; }); @@ -15160,8 +15480,20 @@ export class SessionService { } private releaseScheduledWakeClaim(claim: ScheduledWakeClaim): void { - if (this.scheduledWakeClaims.get(claim.session.id) === claim) + if (this.scheduledWakeClaims.get(claim.session.id) === claim) { this.scheduledWakeClaims.delete(claim.session.id); + this.releaseScheduledWakeAdmission(claim); + } + } + + private releaseScheduledWakeAdmission(claim: ScheduledWakeClaim): void { + if (!claim.admissionReservation) return; + this.releaseAdmissionReservation(claim.admissionReservation); + delete claim.admissionReservation; + } + + private releaseAdmissionReservation(reservation?: symbol): void { + if (reservation) this.admissionReservations.delete(reservation); } private logScheduledWakeFailure( @@ -15183,12 +15515,20 @@ export class SessionService { }); } - private assertWakeAdmissible( + private checkWakeAdmission( session: SessionRecord, scheduledWake?: NonNullable, - ): void { + admissionReservation?: symbol, + reserve = false, + ): symbol | undefined { try { + if (admissionReservation) { + this.assertAdmissible(session.project, "wake", { admissionReservation }); + return admissionReservation; + } + if (reserve) return this.reserveAdmission(session.project, "wake"); this.assertAdmissible(session.project, "wake"); + return undefined; } catch (error) { // Record the actual refusal before releasing ownership to a replacement. if (scheduledWake && error instanceof SessionAdmissionDeniedError) @@ -15327,7 +15667,7 @@ export class SessionService { } writeSession(this.config.dataDir, { ...current, - dailyWake: { ...dailyWake, nextDueAt: nextDueAt.toISOString() }, + dailyWake: { ...withoutAdmissionDeferral(dailyWake), nextDueAt: nextDueAt.toISOString() }, updatedAt: nowIso(), }); claimedNextDueAt = nextDueAt.toISOString(); @@ -15460,7 +15800,8 @@ export class SessionService { updatedAt: nowIso(), }; writeSession(this.config.dataDir, updated); - this.scheduledWakeClaims.delete(sessionId); + const claim = this.scheduledWakeClaims.get(sessionId); + if (claim) this.releaseScheduledWakeClaim(claim); this.scheduledWakeLastFailure.delete(sessionId); this.logEvent("session.wake.scheduled", { level: "info", @@ -15998,12 +16339,33 @@ export class SessionService { } async send(sessionId: string, request: SendMessageRequest): Promise { + return this.sendAndWait(sessionId, request); + } + + private async sendScheduledWake( + sessionId: string, + request: SendMessageRequest, + wakeAdmissionReservation?: symbol, + ): Promise { + return this.sendAndWait(sessionId, request, wakeAdmissionReservation); + } + + private async sendAndWait( + sessionId: string, + request: SendMessageRequest, + wakeAdmissionReservation?: symbol, + ): Promise { const appended = request.queue !== false ? this.appendBehindTypingDelivery(sessionId, request) : null; if (appended) { return this.enrich(appended); } - const sent = await this.sendWithAttempt(sessionId, request); + const sent = await this.sendWithAttempt( + sessionId, + request, + undefined, + wakeAdmissionReservation, + ); // The typed marker also covers an attempt that wrote before registration. const { view, paneWrite } = { view: sent.view, @@ -16048,6 +16410,7 @@ export class SessionService { sessionId: string, request: SendMessageRequest, wakeClaim?: ScheduledWakeClaim, + wakeAdmissionReservation?: symbol, ): Promise<{ view: SessionView; attempt: Promise | null }> { const session = readSession(this.config.dataDir, sessionId); if (!session) { @@ -16101,6 +16464,7 @@ export class SessionService { await this.ensureReadyForQueuedSend(sessionId, activationStamp, { ...(operation ? { operation } : {}), ...(wakeClaim ? { wakeClaim } : {}), + ...(wakeAdmissionReservation ? { wakeAdmissionReservation } : {}), }); } finally { operation?.dispose(); @@ -19841,9 +20205,10 @@ export class SessionService { isStaleParked(session) || (session.status === "stopped" && session.stopReason === "memory_shed") ) { - this.assertWakeAdmissible( + this.checkWakeAdmission( session, options?.wakeClaim && !options.wakeClaim.manual ? options.wakeClaim.wake : undefined, + options?.wakeClaim?.admissionReservation, ); } @@ -19856,8 +20221,12 @@ export class SessionService { // (send, tryDeliverQueuedMessage, switchAuth) classify the session's real // state right after this returns and must not have that forced to // "working" the way a genuine post-restore warmup intentionally does. - this.restoreWarmupUntil.set(session.id, Date.now() + RESTORE_WARMUP_MS); + // This frame clears the marker in finally; a pending launch must not age out of capacity. + this.restoreWarmupUntil.set(session.id, Number.POSITIVE_INFINITY); this.launchesInFlight.add(session.id); + // Transfer the reservation to the counted warmup without yielding. + if (options?.wakeClaim) this.releaseScheduledWakeAdmission(options.wakeClaim); + this.releaseAdmissionReservation(options?.wakeAdmissionReservation); let recovered: ReadySessionForSend; try { recovered = await this.relaunchSessionInPlace(session, project, { @@ -23585,7 +23954,7 @@ export class SessionService { private reserveAdmission( projectId: string, - context: "spawn" | "restore", + context: "spawn" | "restore" | "wake", opts?: { replacingSessionId?: string }, ): symbol { this.assertAdmissible(projectId, context, opts); @@ -23961,6 +24330,40 @@ export class SessionService { if (this.isDeliveryStopped() || this.staleParksInFlight.has(session.id)) { return { session, runtime }; } + const observed = { ...session }; + const stamp = this.lifecycleStamp(observed); + const lifecycle = this.lifecycleIdsFor(observed).map((id) => ({ + id, + snapshot: this.lifecycle.snapshot(id), + })); + const readObservation = () => { + const latest = readSession(this.config.dataDir, observed.id); + if (latest === null) + throw new SessionResourceNotFoundError(`Session not found: ${observed.id}`); + return { + latest, + valid: + sameSessionRuntimeOwner(latest, observed) && + latest.agentLaunchId === observed.agentLaunchId && + this.lifecycleStateMatches(latest, stamp) && + latest.error === observed.error && + latest.errorSource === observed.errorSource && + latest.serverErrorAt === observed.serverErrorAt && + lifecycle.every(({ id, snapshot }) => { + const current = this.lifecycle.snapshot(id); + return ( + snapshot.operation?.phase !== "pending" && + current.operation?.phase !== "pending" && + current.instanceId === snapshot.instanceId && + current.revision === snapshot.revision + ); + }), + }; + }; + // An unpersisted record has nothing to reconcile. + if (readSession(this.config.dataDir, observed.id) === null) return { session, runtime }; + const initial = readObservation(); + if (!initial.valid) return { session: initial.latest, runtime }; if (session.status !== "running" && session.status !== "spawning") { return { session, runtime }; } @@ -24003,6 +24406,8 @@ export class SessionService { try { confirmedRuntime = await this.readRuntimeSnapshot(session, { fresh: true }); } catch (error) { + const observation = readObservation(); + if (!observation.valid) return { session: observation.latest, runtime: confirmedRuntime }; const diagnostic = error instanceof Error ? error.message : String(error); const retained = this.retainDetectionError( session, @@ -24026,6 +24431,8 @@ export class SessionService { return { session, runtime: confirmedRuntime }; } } + const observation = readObservation(); + if (!observation.valid) return { session: observation.latest, runtime: confirmedRuntime }; if ( live(confirmedRuntime) && session.status === "spawning" && @@ -24044,7 +24451,12 @@ export class SessionService { async (ownedPane) => { const assertCurrent = () => { const latest = readSession(this.config.dataDir, session.id); - if (!latest || protectedRuntime(ownedPane) || !runtimeOwnerMatches(latest)) { + if ( + !latest || + protectedRuntime(ownedPane) || + !runtimeOwnerMatches(latest) || + !readObservation().valid + ) { throw superseded; } }; @@ -24059,6 +24471,8 @@ export class SessionService { runtime: confirmedRuntime, }; } + const afterOrphan = readObservation(); + if (!afterOrphan.valid) return { session: afterOrphan.latest, runtime: confirmedRuntime }; } if (live(confirmedRuntime)) { return { session, runtime: confirmedRuntime }; @@ -24090,10 +24504,9 @@ export class SessionService { } } - const latest = readSession(this.config.dataDir, session.id); - if (!latest) { - return { session, runtime }; - } + const observation = readObservation(); + const latest = observation.latest; + if (!observation.valid) return { session: latest, runtime: confirmedRuntime }; if (protectedRuntime() || !runtimeOwnerMatches(latest)) { return { session: latest, runtime }; } @@ -24103,21 +24516,26 @@ export class SessionService { const updatedAt = nowIso(); // Runtime absence does not prove sidecar absence. Keep ownership evidence // for the existing sidecar reaper; reconciliation performs no teardown. + // A retained runtime-probe error is void once the runtime answers dead: + // this write replaces it. + const { error: _probeError, errorSource: _probeErrorSource, ...withoutError } = latest; + const base: SessionRecord = + this.probeErrorHealable(latest) && !latest.serverErrorAt ? withoutError : latest; let updated: SessionRecord; if (terminalUnavailable) { updated = { - ...latest, + ...base, status: "stopped", stopReason: "runtime_lost", updatedAt, }; } else { updated = { - ...latest, + ...base, status: "errored", ...(workspaceGone ? { error: "Agent worktree is missing.", errorSource: "other" as const } - : hasSessionErrorEvidence(latest) + : hasSessionErrorEvidence(base) ? {} : { error: INFERRED_RUNTIME_EXIT_ERROR, @@ -24306,6 +24724,12 @@ export class SessionService { ); } + // A retained runtime-probe error does not block reconciling a runtime that + // answers dead: the stop/errored write replaces it in one write. + private probeErrorHealable(record: SessionRecord): boolean { + return isRuntimeProbeError(record) && !this.restoreOwnsRuntimeRecovery(record.id); + } + private async readFinalTokenUsage( session: SessionRecord, runtime: SessionRuntimeSnapshot, @@ -24386,7 +24810,7 @@ export class SessionService { probeUnresponsive: true, diagnostic, }, - state: "error", + state: expectsLiveRuntime(retained.status) ? "error" : statusFallbackState(retained), source: "status", workspacePresent: false, serverError: false, @@ -24402,7 +24826,6 @@ export class SessionService { ): SessionRecord { const latest = readSession(this.config.dataDir, session.id); if (!latest) throw new SessionResourceNotFoundError(`Session not found: ${session.id}`); - if (isTerminalSessionStatus(latest.status)) return latest; if ( !sameSessionRuntimeOwner(latest, session) || latest.agentLaunchId !== session.agentLaunchId || @@ -24413,6 +24836,13 @@ export class SessionService { ) return latest; const message = STATUS_DETECTION_ERROR_PREFIX + diagnostic; + // Resource readout keeps stamping any non-terminal owner: it has its own + // clear path and its retained error protects sidecars from the reaper. + if ( + isTerminalSessionStatus(latest.status) || + (!expectsLiveRuntime(latest.status) && !message.startsWith(REPORTING_DETECTION_ERROR_PREFIX)) + ) + return latest; if (latest.error?.startsWith(REPORTING_DETECTION_ERROR_PREFIX)) return latest; if (latest.error?.startsWith(STATUS_DETECTION_ERROR_PREFIX) && !recoverableRuntimeError(latest)) return latest; @@ -24632,7 +25062,17 @@ export class SessionService { } let effectiveSession = session; let runtimeOwnerChanged = false; - if (runtime.probeUnresponsive || workspace.diagnostic) { + const probeFailed = runtime.probeUnresponsive || workspace.diagnostic !== undefined; + const runtimeLive = + workspace.exists && runtime.runtimeAlive && runtime.paneUsable && runtime.processAlive; + // A live runtime voids any recoverable error. A dead one voids a probe + // error only off running/spawning: a dead running record heals inside + // reconcileUnexpectedStop, in the same write that stops it. + const clearableError = (record: SessionRecord): boolean => + runtimeLive + ? recoverableRuntimeError(record) + : !expectsLiveRuntime(record.status) && isRuntimeProbeError(record); + if (probeFailed && expectsLiveRuntime(session.status)) { effectiveSession = this.retainDetectionError( session, runtime.diagnostic ?? @@ -24644,13 +25084,10 @@ export class SessionService { probeRevision, ); } else if ( + !probeFailed && session.status !== "errored" && - workspace.exists && - runtime.runtimeAlive && - runtime.paneUsable && - runtime.processAlive && !session.serverErrorAt && - recoverableRuntimeError(session) + clearableError(session) ) { const latest = readSession(this.config.dataDir, session.id); if (!latest) throw new SessionResourceNotFoundError(`Session not found: ${session.id}`); @@ -24659,11 +25096,12 @@ export class SessionService { !sameSessionRuntimeOwner(latest, session) || latest.agentLaunchId !== session.agentLaunchId; if ( !isTerminalSessionStatus(latest.status) && + latest.status !== "errored" && !runtimeOwnerChanged && latest.error === session.error && latest.errorSource === session.errorSource && !latest.serverErrorAt && - recoverableRuntimeError(latest) && + clearableError(latest) && !this.restoreOwnsRuntimeRecovery(session.id) ) { const { error: _ignoredError, errorSource: _ignoredErrorSource, ...recovered } = latest; @@ -24828,8 +25266,9 @@ export class SessionService { } } if ( - (runtime.probeUnresponsive && codexProbeRecovery === undefined) || - workspace.diagnostic || + (probeFailed && + expectsLiveRuntime(effectiveSession.status) && + codexProbeRecovery === undefined) || (hasRetainedSessionError(effectiveSession) && codexProbeRecovery === undefined) ) { state = "error"; diff --git a/v2/src/types.ts b/v2/src/types.ts index cbeb98f0d..8b5d801c4 100644 --- a/v2/src/types.ts +++ b/v2/src/types.ts @@ -1137,6 +1137,9 @@ export interface SessionDailyWakeState { nextDueAt: string; message: string; stopCondition: string; + // Scheduled occurrence retried after an admission denial; nextDueAt is then + // the next retry. Cleared by the next claim. + admissionDeferredOccurrence?: string; } export interface SessionStateSubscription { diff --git a/v2/test/fast/isolated-daemon-owner.test.ts b/v2/test/fast/isolated-daemon-owner.test.ts index e0dabf41a..7f1db2e10 100644 --- a/v2/test/fast/isolated-daemon-owner.test.ts +++ b/v2/test/fast/isolated-daemon-owner.test.ts @@ -584,12 +584,39 @@ describe("isolated owner control", () => { } }); it("preserves a dummy native-client descendant across restart and terminal owner stop", async () => { - const code = `const fs=require('node:fs');const path=require('node:path');const pidFile=path.join(__dirname,'native.pid');if(!fs.existsSync(pidFile)){const client=require('node:child_process').spawn(process.execPath,['-e','setInterval(()=>{},1000)'],{detached:false,stdio:'ignore'});fs.writeFileSync(pidFile,String(client.pid));client.unref();}setInterval(()=>{},1000);`; - const fixture = await launch(code); - await expect - .poll(async () => Number(await readFile(join(fixture.directory, "native.pid"), "utf8"))) - .toBeGreaterThan(0); - const nativePid = Number(await readFile(join(fixture.directory, "native.pid"), "utf8")); + const childProcess = await import("node:child_process"); + const original = await vi.importActual("node:child_process"); + let fixtureChild: Cp.ChildProcess | undefined; + let nativeReady: Promise | undefined; + const spawn = vi + .spyOn(childProcess, "spawn") + .mockImplementationOnce((command, args, options) => { + const child = original.spawn(command, args, { + ...options, + stdio: ["inherit", "inherit", "inherit", "ipc"], + }); + fixtureChild = child; + nativeReady = new Promise((resolve, reject) => { + child.once("message", resolve); + child.once("error", reject); + child.once("exit", () => reject(new Error("Fixture exited before native readiness"))); + }); + void nativeReady.catch(() => undefined); + return child; + }); + const code = `const fs=require('node:fs');const path=require('node:path');const pidFile=path.join(__dirname,'native.pid');if(process.send){process.once('message',()=>{const client=require('node:child_process').spawn(process.execPath,['-e','setInterval(()=>{},1000)'],{detached:false,stdio:'ignore'});fs.writeFileSync(pidFile,String(client.pid));client.unref();process.send(client.pid);process.disconnect();});}setInterval(()=>{},1000);`; + // Mocked daemon readiness precedes this real child's native-client setup. + const fixture = await launch(code).finally(() => spawn.mockRestore()); + if (!nativeReady || !fixtureChild) throw new Error("Fixture did not spawn"); + await expect(readFile(join(fixture.directory, "native.pid"))).rejects.toMatchObject({ + code: "ENOENT", + }); + fixtureChild.send("start-native"); + const nativePid = await nativeReady; + expect(nativePid).toBeGreaterThan(0); + if (typeof nativePid !== "number" || !Number.isSafeInteger(nativePid)) + throw new Error("Invalid fixture native PID"); + expect(Number(await readFile(join(fixture.directory, "native.pid"), "utf8"))).toBe(nativePid); const nativeStarttime = await readLiveProcessStarttime(nativePid); try { expect(nativeStarttime).not.toBeNull(); diff --git a/v2/test/fast/mcp-contracts.test.ts b/v2/test/fast/mcp-contracts.test.ts index 88345eb2d..a7e0f327f 100644 --- a/v2/test/fast/mcp-contracts.test.ts +++ b/v2/test/fast/mcp-contracts.test.ts @@ -21,10 +21,10 @@ const source = (name: string) => readFile(new URL(`../../src/${name}`, import.me // Review contract metadata when these parser/DTO implementations change; never regenerate blindly. const reviewedFingerprints = { - "server.ts": "8f66b35bef9a76216def99b03d811ffa2cce4507d7384bc4a8dcfa3c08234a88", - "types.ts": "107adafc49c533009d550e2de862b93797fa80b7db94be70134d0f9fd6209238", + "server.ts": "6e82c0a548f18350ba9179083f45976cdfb40e33dd4b7fc7fcd7eec396336a2b", + "types.ts": "9bcc3d6d539777f1f6e663c82567d4918b27c31c0a755e5f9cd3b865da43aedc", "cron-job-editor.ts": "33fc7b2993819693ef511d9312f9eb8f4260feb2f263421aec2d4e5ac613ec06", - "session-service.ts": "7f2cdc6b66e94159a6c27a896d71888b3d7786afd3a8f3a68814e905832a6ccb", + "session-service.ts": "7c5b0522412f94dedf546eeddd7b5d890e15efa1900dc0d6c59620f8c6b18e35", "session-lifecycle.ts": "11ed4e84c26d589f92627e9d09fc89141f273ae99199a0263d6c2a3b49acb72c", "session-memory.ts": "1d06c2d904fa46b95ef8bed971c5767fad76663482f22822f9a541b345f161ff", "shared-memory.ts": "f5804ebc0592f40c37cf30a1545d88b37075cd6e7e63b3c9e1817bf42c93de67", diff --git a/v2/test/fast/metadata.test.ts b/v2/test/fast/metadata.test.ts index a707f155b..f89f46049 100644 --- a/v2/test/fast/metadata.test.ts +++ b/v2/test/fast/metadata.test.ts @@ -1995,6 +1995,7 @@ describe("session metadata PR migration", () => { nextDueAt: "2026-03-19T09:30:00.000Z", message: "Check daily state", stopCondition: "Daily checks done", + admissionDeferredOccurrence: "2026-03-19T09:30:00.000Z", }, }; diff --git a/v2/test/fast/server-restore-request.test.ts b/v2/test/fast/server-restore-request.test.ts index e062d5e8c..a35c43f93 100644 --- a/v2/test/fast/server-restore-request.test.ts +++ b/v2/test/fast/server-restore-request.test.ts @@ -1,4 +1,4 @@ -import { mkdir, mkdtemp, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, it, vi } from "vitest"; @@ -15,9 +15,9 @@ import { } from "../../src/session-lifecycle.js"; import { SessionService } from "../../src/session-service.js"; import * as eventSources from "../../src/event-sources/index.js"; -import { writeSession } from "../../src/metadata.js"; +import { readSession, writeSession } from "../../src/metadata.js"; import type { SessionRecord, SessionView } from "../../src/types.js"; -import { findFreePort } from "../helpers/common.js"; +import { createTempDir, findFreePort } from "../helpers/common.js"; describe("parseRestoreSessionRequest", () => { it.each([parseCompleteSessionRequest, parseRestoreSessionRequest])( @@ -58,6 +58,147 @@ describe("parseRestoreSessionRequest", () => { // A8: the override reaches the daemon over HTTP and is forwarded unchanged // to the service call — this is the only body either route reads. describe("POST /sessions/:id/restore and /reopen forward the force override", () => { + it("restores owner-first shared-workspace boot rows before background checks consume them", async () => { + const root = await createTempDir("spur-shared-reboot-"); + const repoDir = join(root, "repo"); + const dataDir = join(root, "data"); + const configPath = join(root, "spur.yaml"); + const ownerId = "demo-owner"; + const memberId = "demo-member"; + await mkdir(repoDir); + await writeFile( + configPath, + [ + "server:", + " host: 127.0.0.1", + ` port: ${await findFreePort()}`, + `dataDir: ${dataDir}`, + `worktreeDir: ${join(root, "worktrees")}`, + "admission:", + " maxLiveSessions: 2", + " memoryGuard:", + " enforce: false", + " enforceFloors: false", + "projects:", + " demo:", + ` path: ${repoDir}`, + " restoreAfterReboot: true", + ].join("\n"), + ); + for (const [index, id] of [ownerId, memberId].entries()) { + writeSession(dataDir, { + id, + project: "demo", + agent: "claude", + prompt: "unit reboot", + branch: "main", + worktree: false, + worktreePath: repoDir, + workspaceId: ownerId, + tmuxSession: `shared-reboot-${id}`, + launchCommand: "claude", + agentSessionId: `fixture-native-${id}`, + status: "running", + createdAt: `2026-03-18T10:0${1 - index}:00.000Z`, + updatedAt: "2026-03-18T10:02:00.000Z", + }); + } + const dead = { + runtimeAlive: false, + paneUsable: false, + processAlive: false, + tmuxActivityAt: null, + probeUnresponsive: false, + }; + const live = { ...dead, runtimeAlive: true, paneUsable: true, processAlive: true }; + const seam = SessionService.prototype as unknown as { + runDashboardCacheTick(): Promise; + readRuntimeSnapshot(record: SessionRecord): Promise; + restoreLocked(id: string): Promise; + }; + const originalTick = seam.runDashboardCacheTick; + const originalReconcile = SessionService.prototype.reconcileStoppedSessions; + const restored = new Set(); + let enterBoot!: () => void; + const bootEntered = new Promise((resolve) => { + enterBoot = resolve; + }); + let reconciling = false; + let dashboardRunning = false; + let bootProbeStarted = false; + const dashboard = vi.spyOn(seam, "runDashboardCacheTick").mockImplementation(async function ( + this: SessionService, + ) { + await bootEntered; + dashboardRunning = true; + try { + await originalTick.call(this); + } finally { + dashboardRunning = false; + } + }); + const probe = vi.spyOn(seam, "readRuntimeSnapshot").mockImplementation(async function ( + this: SessionService, + record, + ) { + if (reconciling && !dashboardRunning && !bootProbeStarted) { + bootProbeStarted = true; + expect(record.id).toBe(ownerId); + enterBoot(); + const cache = (this as unknown as { dashboardCacheReady: Promise | null }) + .dashboardCacheReady; + if (cache) { + await cache; + expect(readSession(dataDir, memberId)?.status).toBe("stopped"); + } + return dead; + } + return restored.has(record.id) || (dashboardRunning && reconciling && record.id === ownerId) + ? live + : dead; + }); + const reconcile = vi + .spyOn(SessionService.prototype, "reconcileStoppedSessions") + .mockImplementation(async function (this: SessionService) { + reconciling = true; + try { + return await originalReconcile.call(this); + } finally { + reconciling = false; + } + }); + const restoreBody = vi.spyOn(seam, "restoreLocked").mockImplementation(async (id) => { + const current = readSession(dataDir, id); + if (!current) throw new Error("Missing shared-workspace boot fixture"); + const { stopReason: _stopped, ...base } = current; + const running = { ...base, status: "running" as const }; + writeSession(dataDir, running); + restored.add(id); + return running as SessionView; + }); + const restore = vi.spyOn(SessionService.prototype, "restore"); + let server: StartedServer | undefined; + try { + server = await startServer(configPath, { info: () => undefined, warn: () => undefined }); + expect(reconcile).toHaveBeenCalledOnce(); + expect(restore.mock.calls.map(([id]) => id)).toEqual([ownerId, memberId]); + expect(readSession(dataDir, ownerId)?.status).toBe("running"); + expect(readSession(dataDir, memberId)?.status).toBe("running"); + } finally { + enterBoot(); + try { + await server?.stop(); + } finally { + restore.mockRestore(); + restoreBody.mockRestore(); + reconcile.mockRestore(); + probe.mockRestore(); + dashboard.mockRestore(); + await rm(root, { recursive: true, force: true }); + } + } + }); + it("starts source automation only after startup reconcile while routes remain unready", async () => { let reconciled = false; const original = SessionService.prototype.reconcileStoppedSessions; diff --git a/v2/test/fast/server.test.ts b/v2/test/fast/server.test.ts index 1962aba80..f7e628416 100644 --- a/v2/test/fast/server.test.ts +++ b/v2/test/fast/server.test.ts @@ -529,14 +529,23 @@ describe("startServer", () => { event: "daemon.registry.count", details: { read: 1, worktreeInternalDropped: 0 }, }); - expect(events[1]).toMatchObject({ + const reconciledIndex = events.findIndex( + (entry) => entry.event === "daemon.startup.reconciled", + ); + const attentionPollIndex = events.findIndex((entry) => entry.event === "gh.poll_cycle"); + expect(events[reconciledIndex]).toMatchObject({ + event: "daemon.startup.reconciled", + details: { scanned: 0, alive: 0, drifted: 0 }, + }); + expect(attentionPollIndex).toBeGreaterThan(reconciledIndex); + expect(events[attentionPollIndex]).toMatchObject({ event: "gh.poll_cycle", details: { cycle: "attention", calls: 0 }, }); expect(events.map((entry) => entry.event)).toEqual([ "daemon.registry.count", - "gh.poll_cycle", "daemon.startup.reconciled", + "gh.poll_cycle", "daemon.admission.startup", "daemon.started", "http.route.not_found", diff --git a/v2/test/fast/session-service.test.ts b/v2/test/fast/session-service.test.ts index c2507b440..068f9589d 100644 --- a/v2/test/fast/session-service.test.ts +++ b/v2/test/fast/session-service.test.ts @@ -272,7 +272,7 @@ import { type StateSource, type TodoProjection, } from "../../src/types.js"; -import type { SessionLifecycleRegister } from "../../src/session-lifecycle.js"; +import { SessionLifecycleRegister } from "../../src/session-lifecycle.js"; import type { ProviderTokenUsageSample } from "../../src/token-usage.js"; const __dirname = dirname(fileURLToPath(import.meta.url)); @@ -507,6 +507,24 @@ async function createDisposedSessionService(options?: { realPaneGeneration?: boo return service; } +function rebootCandidates( + service: Awaited>, + ids: string[], +): Parameters[0] { + const lifecycle = (service as unknown as { lifecycle: SessionLifecycleRegister }).lifecycle; + return ids.map((id) => { + const record = readSessionMock(TEST_DATA_DIR, id) as SessionRecord | null; + if (!record) throw new Error(`Missing reboot fixture: ${id}`); + return { + record, + lifecycle: [...new Set([id, record.workspaceId ?? id])].map((ownerId) => ({ + id: ownerId, + snapshot: lifecycle.snapshot(ownerId), + })), + }; + }); +} + // Only `revision`, `items[].status` and `items[].latestTransition.blocker` are // read by maybeNudgeTodo's human-held branch; the rest is shape filler. function heldProjection(revision: string, requiredAction: string): TodoProjection { @@ -13897,6 +13915,7 @@ describe("SessionService", () => { // Reconcile-shaped drift: status flips to "stopped" with no stopReason, // the pipeline is left running untouched (matches the terminalUnavailable // branch at session-service.ts's reconcile, :14520-14530). + await waitForRealTime(() => expect(timerPromisesSleepMock).toHaveBeenCalledWith(1_000)); const parked = sessions.get("api-1"); if (!parked) throw new Error("expected api-1 to exist"); sessions.set("api-1", { @@ -14108,6 +14127,7 @@ describe("SessionService", () => { // Reconcile's errored branch (session-service.ts:14531-14538): same // unintended wedge, no stopReason, different status. + await waitForRealTime(() => expect(timerPromisesSleepMock).toHaveBeenCalledWith(1_000)); const parked = sessions.get("api-1"); if (!parked) throw new Error("expected api-1 to exist"); sessions.set("api-1", { @@ -14161,6 +14181,7 @@ describe("SessionService", () => { const run = sessionServiceInternals(service).deliveryRuns.get("api-1"); expect(run).toBeDefined(); + await waitForRealTime(() => expect(timerPromisesSleepMock).toHaveBeenCalledWith(1_000)); const parked = sessions.get("api-1"); if (!parked) throw new Error("expected api-1 to exist"); sessions.set("api-1", { @@ -14287,6 +14308,7 @@ describe("SessionService", () => { const run = sessionServiceInternals(service).deliveryRuns.get("api-1"); expect(run).toBeDefined(); + await waitForRealTime(() => expect(timerPromisesSleepMock).toHaveBeenCalledWith(1_000)); const parked = sessions.get("api-1"); if (!parked) throw new Error("expected api-1 to exist"); sessions.set("api-1", { @@ -22768,50 +22790,62 @@ describe("SessionService", () => { } }); - it("keeps codex working while a long exec_command tool call is still pending", async () => { - const now = new Date("2026-04-14T19:30:00.000Z"); - vi.setSystemTime(now); - const tenMinAgoMs = now.getTime() - 10 * 60_000; - readSessionMock.mockReturnValue({ - id: "spur-exec", - project: "sp", - agent: "codex", - prompt: "long exec", - branch: "feature/exec", - worktree: true, - worktreePath: "/tmp/spur-worktrees/sp/spur-exec", - tmuxSession: "spur-exec", - launchCommand: "codex --dangerously-bypass-approvals-and-sandbox", - status: "running", - createdAt: "2026-04-14T13:34:40.615Z", - updatedAt: "2026-04-14T19:20:00.000Z", - }); - readAgentHookStateMock.mockReturnValue({ - state: "working", - updatedAt: new Date(tenMinAgoMs).toISOString(), - hookEvent: "PreToolUse", - turnId: "019efdf7", - }); - readCodexRolloutStateMock.mockResolvedValue({ - rollout: { + it.each([ + undefined, + "Session status detection failed: ps failed: malformed process table output", + "Session status detection failed: ps failed: Command failed: ps -eo pid,ppid,pgid,tpgid,tty,rss,args\n", + ])( + "historical transport recovery: keeps codex working while a long exec_command tool call is still pending (%j)", + async (error) => { + const now = new Date("2026-04-14T19:30:00.000Z"); + vi.setSystemTime(now); + const tenMinAgoMs = now.getTime() - 10 * 60_000; + const sessions = createSessionStore(); + sessions.set("spur-exec", { + id: "spur-exec", + project: "sp", + agent: "codex", + prompt: "long exec", + branch: "feature/exec", + worktree: true, + worktreePath: "/tmp/spur-worktrees/sp/spur-exec", + tmuxSession: "spur-exec", + launchCommand: "codex --dangerously-bypass-approvals-and-sandbox", + status: "running", + createdAt: "2026-04-14T13:34:40.615Z", + updatedAt: "2026-04-14T19:20:00.000Z", + ...(error ? { error, errorSource: "other" } : {}), + }); + readAgentHookStateMock.mockReturnValue({ state: "working", - timestamp: new Date(tenMinAgoMs).toISOString(), - timestampMs: tenMinAgoMs, - filePath: "/tmp/spur-exec/rollout.jsonl", - reason: "function_call", - callId: "call_1", + updatedAt: new Date(tenMinAgoMs).toISOString(), + hookEvent: "PreToolUse", turnId: "019efdf7", - }, - rateLimit: null, - }); + }); + readCodexRolloutStateMock.mockResolvedValue({ + rollout: { + state: "working", + timestamp: new Date(tenMinAgoMs).toISOString(), + timestampMs: tenMinAgoMs, + filePath: "/tmp/spur-exec/rollout.jsonl", + reason: "function_call", + callId: "call_1", + turnId: "019efdf7", + }, + rateLimit: null, + }); - const { SessionService } = await loadSessionServiceModule(); - const service = new SessionService("/tmp/spur.yaml", "2026-04-14T13:34:40.615Z"); + const { SessionService } = await loadSessionServiceModule(); + const service = new SessionService("/tmp/spur.yaml", "2026-04-14T13:34:40.615Z"); - const result = await service.get("spur-exec"); + const result = await service.get("spur-exec"); - expect(result.state).toBe("working"); - }); + expect(result.state).toBe("working"); + expect(sessions.get("spur-exec")?.error).toBeUndefined(); + expect(sendMessageToTmuxMock).not.toHaveBeenCalled(); + expect(sendInterruptKeysToTmuxMock).not.toHaveBeenCalled(); + }, + ); it("flips codex working to waiting for a stale UserPromptSubmit hook (spur-7ce0)", async () => { const now = new Date("2026-06-25T15:00:00.000Z"); @@ -23520,7 +23554,11 @@ describe("SessionService", () => { scanned: 1, alive: 0, drifted: 1, - driftedSessions: [{ id: "api-1", project: "api" }], + driftedSessions: [ + expect.objectContaining({ + record: expect.objectContaining({ id: "api-1", project: "api" }), + }), + ], }); expect(sessions.get("api-1")).toMatchObject({ status: "stopped", @@ -23544,7 +23582,116 @@ describe("SessionService", () => { errorSource: "inferred_runtime_exit", }); expect(result.drifted).toBe(1); - expect(result.driftedSessions).toEqual([{ id: "api-1", project: "api" }]); + expect(result.driftedSessions).toEqual([ + expect.objectContaining({ + record: expect.objectContaining({ id: "api-1", project: "api" }), + }), + ]); + }); + + describe("boot heal of a runtime-probe error", () => { + const PROBE_ERROR = "Session status detection failed: tmux list-windows failed: "; + + function seedRestoreAfterRebootFleet() { + loadConfigMock.mockReturnValue({ + ...baseConfig(), + projects: { + api: { + path: "/repo/api", + defaultBranch: "main", + sessionPrefix: "api", + worktree: true, + restoreAfterReboot: true, + symlinks: [], + sidecars: {}, + sources: {}, + triggers: {}, + }, + }, + }); + const sessions = createSessionStore(); + sessions.set( + "api-1", + runningSession({ id: "api-1", error: PROBE_ERROR, errorSource: "runtime_probe" }), + ); + sessions.set("api-2", runningSession({ id: "api-2" })); + return sessions; + } + + it("stops a probe-error record at boot without handing it to reboot restore", async () => { + const sessions = seedRestoreAfterRebootFleet(); + getTmuxSessionPresenceMock.mockResolvedValue({ present: false, unresponsive: false }); + const service = await createDisposedSessionService(); + + const pending = service.reconcileStoppedSessions(); + await vi.advanceTimersByTimeAsync(1_000); + const result = await pending; + + expect(sessions.get("api-1")?.status).toBe("stopped"); + expect(sessions.get("api-1")).not.toHaveProperty("error"); + expect(sessions.get("api-1")).not.toHaveProperty("errorSource"); + expect(sessions.get("api-2")?.status).toBe("stopped"); + expect(result.drifted).toBe(2); + expect(result.driftedSessions).toEqual([ + expect.objectContaining({ + record: expect.objectContaining({ id: "api-2", project: "api" }), + }), + ]); + + const restoreSpy = vi.spyOn(service, "restore").mockResolvedValue({} as never); + await service.restoreRebootedSessions(result.driftedSessions); + expect(restoreSpy).toHaveBeenCalledTimes(1); + expect(restoreSpy).toHaveBeenCalledWith("api-2"); + expect(restoreSpy).not.toHaveBeenCalledWith("api-1"); + }); + + it("records final usage for a probe-error record healed at boot", async () => { + const sessions = seedRestoreAfterRebootFleet(); + getTmuxSessionPresenceMock.mockResolvedValue({ present: false, unresponsive: false }); + mockClaudeJsonlState("waiting", { + tokenUsage: { + provider: "claude", + generationId: "final-message", + inputTokens: 7, + outputTokens: 3, + totalTokens: 10, + }, + }); + const service = await createDisposedSessionService(); + + const pending = service.reconcileStoppedSessions(); + await vi.advanceTimersByTimeAsync(1_000); + await pending; + + expect(sessions.get("api-1")).toMatchObject({ + status: "stopped", + tokenUsage: { totalTokens: 10 }, + }); + expect(sessions.get("api-1")).not.toHaveProperty("error"); + }); + + it("keeps a record healed between list and read out of reboot restore", async () => { + const sessions = seedRestoreAfterRebootFleet(); + getTmuxSessionPresenceMock.mockImplementation(async (name: string) => { + const listed = sessions.get("api-1"); + if (name === "api-1" && listed?.error) { + const { error: _error, errorSource: _errorSource, ...healed } = listed; + sessions.set("api-1", { ...healed, status: "stopped" }); + } + return { present: false, unresponsive: false }; + }); + const service = await createDisposedSessionService(); + + const pending = service.reconcileStoppedSessions(); + await vi.advanceTimersByTimeAsync(1_000); + const result = await pending; + + expect(result.driftedSessions).toEqual([ + expect.objectContaining({ + record: expect.objectContaining({ id: "api-2", project: "api" }), + }), + ]); + }); }); it.each([ @@ -23563,6 +23710,11 @@ describe("SessionService", () => { error: "Session status detection failed: ps failed: controlled", errorSource: "runtime_probe" as const, }, + // A dead runtime voids the probe error; the exit replaces it. + stored: { + error: "Agent runtime exited unexpectedly.", + errorSource: "inferred_runtime_exit" as const, + }, eligible: false, }, { @@ -23597,9 +23749,18 @@ describe("SessionService", () => { const result = await service.reconcileStoppedSessions(); await service.restoreRebootedSessions(result.driftedSessions); - expect(sessions.get("api-1")).toMatchObject({ status: "errored", ...scenario.evidence }); + expect(sessions.get("api-1")).toMatchObject({ + status: "errored", + ...("stored" in scenario ? scenario.stored : scenario.evidence), + }); expect(result.driftedSessions).toEqual( - scenario.eligible ? [{ id: "api-1", project: "api" }] : [], + scenario.eligible + ? [ + expect.objectContaining({ + record: expect.objectContaining({ id: "api-1", project: "api" }), + }), + ] + : [], ); if (scenario.eligible && scenario.restoreAfterReboot) { expect(restoreSpy).toHaveBeenCalledExactlyOnceWith("api-1"); @@ -23924,10 +24085,7 @@ describe("SessionService", () => { const service = await createDisposedSessionService(); const restoreSpy = vi.spyOn(service, "restore").mockResolvedValue({} as never); - await service.restoreRebootedSessions([ - { id: "api-1", project: "api" }, - { id: "web-1", project: "web" }, - ]); + await service.restoreRebootedSessions(rebootCandidates(service, ["api-1", "web-1"])); expect(restoreSpy).toHaveBeenCalledWith("api-1"); expect(restoreSpy).not.toHaveBeenCalledWith("web-1"); @@ -23998,7 +24156,7 @@ describe("SessionService", () => { ); const duplicateAutoStart = vi.spyOn(internals, "startAutoStartSidecars"); - const rebootRestore = service.restoreRebootedSessions([{ id: "api-2", project: "api" }]); + const rebootRestore = service.restoreRebootedSessions(rebootCandidates(service, ["api-2"])); await restoreStartedBarrier; const stop = service.stopSidecar("api-1", "proxy"); await Promise.resolve(); @@ -24041,10 +24199,7 @@ describe("SessionService", () => { .mockRejectedValueOnce(new Error("boom")) .mockResolvedValueOnce({} as never); - await service.restoreRebootedSessions([ - { id: "api-1", project: "api" }, - { id: "api-2", project: "api" }, - ]); + await service.restoreRebootedSessions(rebootCandidates(service, ["api-1", "api-2"])); expect(restoreSpy).toHaveBeenCalledTimes(2); expect(logSpurEventMock).toHaveBeenCalledWith( @@ -24057,6 +24212,9 @@ describe("SessionService", () => { }); it("restoreRebootedSessions keeps iterating and logs reboot.restore.failed after a denied restore (no narrowed catch)", async () => { + const sessions = createSessionStore(); + for (const id of ["api-1", "api-2"]) + sessions.set(id, runningSession({ id, status: "stopped" })); loadConfigMock.mockReturnValue({ ...baseConfig(), projects: { @@ -24085,10 +24243,7 @@ describe("SessionService", () => { ), ); - await service.restoreRebootedSessions([ - { id: "api-1", project: "api" }, - { id: "api-2", project: "api" }, - ]); + await service.restoreRebootedSessions(rebootCandidates(service, ["api-1", "api-2"])); expect(restoreSpy).toHaveBeenCalledTimes(2); expect(logSpurEventMock).toHaveBeenCalledWith( @@ -24102,6 +24257,9 @@ describe("SessionService", () => { }); it("restoreRebootedSessions stops before the next session below the restore floor", async () => { + const sessions = createSessionStore(); + for (const id of ["api-1", "api-2"]) + sessions.set(id, runningSession({ id, status: "stopped" })); const config = baseConfig(); loadConfigMock.mockReturnValue({ ...config, @@ -24118,10 +24276,7 @@ describe("SessionService", () => { const service = await createDisposedSessionService(); const restoreSpy = vi.spyOn(service, "restore"); - await service.restoreRebootedSessions([ - { id: "api-1", project: "api" }, - { id: "api-2", project: "api" }, - ]); + await service.restoreRebootedSessions(rebootCandidates(service, ["api-1", "api-2"])); expect(restoreSpy).not.toHaveBeenCalled(); expect(logSpurEventMock).toHaveBeenCalledWith( @@ -24131,6 +24286,9 @@ describe("SessionService", () => { }); it("restoreRebootedSessions rechecks the floor after each restore settles", async () => { + const sessions = createSessionStore(); + for (const id of ["api-1", "api-2"]) + sessions.set(id, runningSession({ id, status: "stopped" })); const config = baseConfig(); loadConfigMock.mockReturnValue({ ...config, @@ -24155,10 +24313,7 @@ describe("SessionService", () => { const service = await createDisposedSessionService(); const restoreSpy = vi.spyOn(service, "restore").mockResolvedValue({} as never); - await service.restoreRebootedSessions([ - { id: "api-1", project: "api" }, - { id: "api-2", project: "api" }, - ]); + await service.restoreRebootedSessions(rebootCandidates(service, ["api-1", "api-2"])); expect(restoreSpy).toHaveBeenCalledOnce(); expect(timerPromisesSleepMock).toHaveBeenCalledOnce(); @@ -24172,6 +24327,7 @@ describe("SessionService", () => { }); it("restoreRebootedSessions does not add a settle delay without a memory sample", async () => { + createSessionStore(runningSession({ status: "stopped" })); const config = baseConfig(); loadConfigMock.mockReturnValue({ ...config, @@ -24183,7 +24339,7 @@ describe("SessionService", () => { const service = await createDisposedSessionService(); vi.spyOn(service, "restore").mockResolvedValue({} as never); - await service.restoreRebootedSessions([{ id: "api-1", project: "api" }]); + await service.restoreRebootedSessions(rebootCandidates(service, ["api-1"])); expect(timerPromisesSleepMock).not.toHaveBeenCalled(); }); @@ -27020,7 +27176,11 @@ describe("SessionService", () => { scanned: 1, alive: 0, drifted: 1, - driftedSessions: [{ id: "api-1", project: "api" }], + driftedSessions: [ + expect.objectContaining({ + record: expect.objectContaining({ id: "api-1", project: "api" }), + }), + ], }); expect(sessions.get("api-1")).toMatchObject({ status: "stopped", @@ -32962,6 +33122,126 @@ describe("SessionService", () => { }, ); + describe.each([ + "Session status detection failed: ps failed: malformed process table output", + "Session status detection failed: ps failed: Command failed: ps -eo pid,ppid,pgid,tpgid,tty,rss,args\n", + ])("historical transport recovery: %j", (error) => { + it.each(["running", "errored"] as const)("heals a healthy %s record", async (status) => { + const sessions = createSessionStore(); + const record = runningSession({ + id: "api-1", + status, + agentSessionId: "native-original", + agentLaunchId: "launch-original", + error, + errorSource: "other", + }); + sessions.set(record.id, record); + mockClaudeJsonlState("waiting"); + const service = await createDisposedSessionService(); + + const result = await service.get(record.id); + + expect(result.state).toBe("waiting"); + expect(sessions.get(record.id)).toMatchObject({ + id: record.id, + status: "running", + agentSessionId: "native-original", + agentLaunchId: "launch-original", + }); + expect(sessions.get(record.id)?.error).toBeUndefined(); + expect(sessions.get(record.id)?.errorSource).toBeUndefined(); + expect(sendMessageToTmuxMock).not.toHaveBeenCalled(); + }); + + it.each( + (["running", "errored"] as const).flatMap((status) => + ["unknown", "dead", "workspace", "server", "restore", "owner", "generation"].map( + (guard) => ({ status, guard }), + ), + ), + )("preserves $status guarded outcome under $guard evidence", async ({ status, guard }) => { + const sessions = createSessionStore(); + const record = runningSession({ + id: "api-1", + status, + agentSessionId: "native-original", + agentLaunchId: "launch-original", + error, + errorSource: "other", + ...(guard === "server" ? { serverErrorAt: "2026-03-18T10:01:00.000Z" } : {}), + }); + sessions.set(record.id, record); + const service = await createDisposedSessionService(); + const internals = sessionServiceInternals(service); + vi.spyOn(internals, "readRuntimeSnapshot").mockImplementation(async () => { + if (guard === "owner" || guard === "generation") { + sessions.set(record.id, { + ...record, + ...(guard === "owner" + ? { agentSessionId: "native-replacement" } + : { agentLaunchId: "launch-replacement" }), + }); + } + return { + runtimeAlive: guard !== "dead", + paneUsable: guard !== "dead", + processAlive: guard !== "dead", + probeUnresponsive: guard === "unknown", + }; + }); + if (guard === "workspace") { + probeWorkspaceMock.mockReturnValue({ exists: false, missing: true }); + } + if (guard === "restore") { + vi.spyOn(internals, "restoreOwnsRuntimeRecovery").mockReturnValue(true); + } + + const pending = internals.classifySessionRecord(record); + if (guard === "dead" && status === "running") await vi.advanceTimersByTimeAsync(1_000); + await pending; + + expect(sessions.get(record.id)?.error).toBe( + guard === "unknown" && status === "running" + ? "Session status detection failed: runtime probe (tmux or ps) timed out; runtime state unknown" + : status === "running" && guard === "dead" + ? undefined + : status === "running" && guard === "workspace" + ? "Agent worktree is missing." + : error, + ); + expect(sessions.get(record.id)?.errorSource).toBe( + guard === "unknown" && status === "running" + ? "runtime_probe" + : guard === "dead" && status === "running" + ? undefined + : "other", + ); + expect(sendMessageToTmuxMock).not.toHaveBeenCalled(); + }); + }); + + it.each([ + "Session status detection failed: ps failed: malformed process table output\n", + "Session status detection failed: ps failed: Command failed: ps -eo pid,ppid,pgid,tpgid,tty,rss,args", + "Session status detection failed: ps failed: Command failed: ps -eo pid,ppid,pgid,tpgid,tty,rss,args\n\n", + "Session status detection failed: ps failed: Command failed: ps -eo pid,ppid,pgid,tpgid,tty,rss,args\npermission denied\n", + "Session status detection failed: ps failed: classifier", + "Session status detection failed: workspace failed", + "Session reporting detection failed: ps failed: malformed process table output", + "Provider request failed", + ])("historical transport recovery: retains unrelated complete error %j", async (error) => { + const sessions = createSessionStore(); + const record = runningSession({ id: "api-1", error, errorSource: "other" }); + sessions.set(record.id, record); + mockClaudeJsonlState("waiting"); + const service = await createDisposedSessionService(); + + expect((await service.get(record.id)).state).toBe("error"); + expect(sessions.get(record.id)?.error).toBe(error); + expect(sessions.get(record.id)?.errorSource).toBe("other"); + }); + it("keeps a workspace diagnostic after an unresponsive runtime probe recovers", async () => { const sessions = createSessionStore(); const record = runningSession({ id: "api-1" }); @@ -33041,6 +33321,1468 @@ describe("SessionService", () => { expect(sessions.get(record.id)?.errorSource).toBe(patch.errorSource); }); + describe("pr1197 historical preservation:", () => { + const historicalErrors = [ + "Session status detection failed: ps failed: malformed process table output", + "Session status detection failed: ps failed: Command failed: ps -eo pid,ppid,pgid,tpgid,tty,rss,args\n", + ] as const; + const dead = { + runtimeAlive: false, + paneUsable: false, + processAlive: false, + probeUnresponsive: false, + }; + const unknown = { ...dead, probeUnresponsive: true }; + + function internalsFor(service: unknown) { + return service as SessionServiceInternals & { + lifecycle: SessionLifecycleRegister; + sessionWithReleasedSidecarPorts(record: SessionRecord): SessionRecord; + teardownSessionSidecars(record: SessionRecord): Promise; + reconcileUnexpectedStop( + record: SessionRecord, + runtime: typeof dead, + reason: "boot" | "runtime_check", + workspaceMissing: boolean, + ): Promise<{ session: SessionRecord; runtime: typeof dead }>; + }; + } + + function observationRecord( + error: string = historicalErrors[0], + errorSource: "other" | "runtime_probe" = "runtime_probe", + ) { + return runningSession({ + agentSessionId: "native-original", + agentLaunchId: "launch-original", + workspaceId: "workspace-original", + error, + errorSource, + queuedMessages: { messages: ["A", "B"], awaitingPrompt: true }, + queuedMessageTyped: { + message: "original", + typedAt: "2026-03-18T10:00:00.000Z", + ackBaseline: { agent: "codex", offsets: { "/t/original.jsonl": 10 } }, + }, + sidecarPorts: { test: { http: 4200 } }, + }); + } + + it.each( + historicalErrors.flatMap((error, index) => + ["prewrite", "postwrite", "postwrite dead"].map((stage) => ({ error, index, stage })), + ), + )( + "canonical historical probe: unit producer to retaining send catch ($stage $error)", + async ({ error, index, stage }) => { + const originalRuntime = await import("../../src/runtime-tmux.js"); + const { promisify } = await import("node:util"); + const execFileAsync = vi.fn(async (file: string, args: string[]) => { + if (file === "tmux" && args.includes("list-panes")) { + return { stdout: "api-1 1 1 0 100 /dev/pts/1", stderr: "" }; + } + if (file === "ps") { + if (index === 0) return { stdout: "malformed row", stderr: "" }; + throw new Error("Command failed: ps -eo pid,ppid,pgid,tpgid,tty,rss,args\n"); + } + throw new Error(`unexpected unit subprocess: ${file}`); + }); + vi.doMock("node:child_process", async () => ({ + ...(await vi.importActual("node:child_process")), + execFile: Object.assign(vi.fn(), { [promisify.custom]: execFileAsync }), + })); + try { + vi.resetModules(); + const runtime = await vi.importActual( + "../../src/runtime-tmux.js", + ); + const generated: unknown = await runtime + .isProcessRunningInTmux("api-1", ["claude"], { fresh: true }) + .catch((failure: unknown) => failure); + expect(generated).toBeInstanceOf(runtime.TmuxProbeUnknownError); + // The current producer extends the historical malformed-table text. + expect((generated as Error).message).toContain( + error.slice("Session status detection failed: ".length), + ); + expect(execFileAsync).toHaveBeenCalledWith( + "ps", + ["-eo", "pid,ppid,pgid,tpgid,tty,rss,args"], + expect.anything(), + ); + vi.doMock("../../src/runtime-tmux.js", () => ({ + ...originalRuntime, + TmuxProbeUnknownError: runtime.TmuxProbeUnknownError, + })); + const record = runningSession({ + agentSessionId: "native-original", + agentLaunchId: "launch-original", + }); + if (stage !== "prewrite") { + record.queuedMessages = { messages: ["current input", "tail"], awaitingPrompt: false }; + } + const sessions = createSessionStore(); + mockClaudeJsonlState("waiting"); + const service = await createDisposedSessionService(); + sessions.set(record.id, record); + let written: SessionRecord | undefined; + if (stage === "prewrite") { + vi.spyOn( + service as unknown as { capturePaneGeneration(): Promise }, + "capturePaneGeneration", + ).mockRejectedValue(generated); + } else { + createAgentSubmitAckBindingMock.mockResolvedValue({ + baseline: { agent: "claude", file: "prior.jsonl", size: 123 }, + scan: vi.fn(), + }); + vi.spyOn(sessionServiceInternals(service), "waitForSubmitAck").mockImplementation( + async () => { + written = structuredClone(sessions.get(record.id)); + throw generated; + }, + ); + } + if (stage === "prewrite") { + await expect( + service.send(record.id, { message: "current input", queue: false }), + ).rejects.toBe(generated); + } else { + const result = await sessionServiceInternals(service).tryDeliverQueuedMessage( + record.id, + ); + expect(result).toBe(true); + } + expect(sessions.get(record.id)).toMatchObject({ + agentSessionId: record.agentSessionId, + agentLaunchId: record.agentLaunchId, + error: `Session status detection failed: ${(generated as Error).message}`, + errorSource: "runtime_probe", + }); + expect(sendMessageToTmuxMock).toHaveBeenCalledTimes(stage === "prewrite" ? 0 : 1); + if (stage !== "prewrite") { + expect(written?.queuedMessageTyped).toMatchObject({ + message: "current input", + ackBaseline: { agent: "claude", file: "prior.jsonl", size: 123 }, + }); + // A probe error leaves the typed marker for its ACK to settle. + expect(sessions.get(record.id)?.queuedMessageTyped).toMatchObject({ + message: "current input", + ackBaseline: { agent: "claude", file: "prior.jsonl", size: 123 }, + }); + expect(sessions.get(record.id)?.queuedMessages).toEqual({ + messages: ["tail"], + awaitingPrompt: true, + }); + } + if (stage === "postwrite dead") { + const current = sessions.get(record.id); + if (!current) throw new Error("missing actual postcatch fixture"); + const internals = internalsFor(service); + vi.spyOn(internals, "readRuntimeSnapshot").mockResolvedValue(dead); + const pending = internals.reconcileUnexpectedStop( + current, + dead, + "runtime_check", + false, + ); + await vi.advanceTimersByTimeAsync(1_000); + await pending; + expect(sessions.get(record.id)).toMatchObject({ + status: "stopped", + queuedMessages: current.queuedMessages, + }); + expect(sessions.get(record.id)?.error).toBeUndefined(); + expect(sessions.get(record.id)?.queuedMessageTyped).toEqual(current.queuedMessageTyped); + expect(sendMessageToTmuxMock).toHaveBeenCalledOnce(); + expect(createTmuxSessionMock).not.toHaveBeenCalled(); + } + expect(sendInterruptKeysToTmuxMock).not.toHaveBeenCalled(); + } finally { + vi.doUnmock("node:child_process"); + vi.doMock("../../src/runtime-tmux.js", () => originalRuntime); + } + }, + ); + + it.each(historicalErrors)("heals healthy historical live evidence %j", async (error) => { + const sessions = createSessionStore(runningSession({ error, errorSource: "other" })); + mockClaudeJsonlState("waiting"); + const service = await createDisposedSessionService(); + expect((await service.get("api-1")).state).toBe("waiting"); + expect(sessions.get("api-1")?.error).toBeUndefined(); + }); + + it.each( + historicalErrors.flatMap((error) => + [false, true].map((panePresent) => ({ error, panePresent })), + ), + )( + "canonical historical probe: stops confirmed unavailable terminal ($panePresent $error)", + async ({ error, panePresent }) => { + const record = observationRecord(error, "other"); + const sessions = createSessionStore(); + const service = await createDisposedSessionService(); + sessions.set(record.id, record); + const internals = internalsFor(service); + const runtime = { ...dead, runtimeAlive: panePresent }; + const read = vi.spyOn(internals, "readRuntimeSnapshot").mockResolvedValue({ ...runtime }); + const ports = vi.spyOn(internals, "sessionWithReleasedSidecarPorts"); + const teardown = vi.spyOn(internals, "teardownSessionSidecars").mockResolvedValue(); + const pending = internals.reconcileUnexpectedStop(record, runtime, "runtime_check", false); + expect(read).not.toHaveBeenCalled(); + expect(sessions.get(record.id)).toEqual(record); + await vi.advanceTimersByTimeAsync(1_000); + await pending; + expect(read).toHaveBeenCalledExactlyOnceWith(record, { fresh: true }); + expect(sessions.get(record.id)).toMatchObject({ + status: "stopped", + stopReason: "runtime_lost", + agentSessionId: record.agentSessionId, + agentLaunchId: record.agentLaunchId, + queuedMessages: record.queuedMessages, + queuedMessageTyped: record.queuedMessageTyped, + sidecarPorts: record.sidecarPorts, + }); + expect(sessions.get(record.id)?.error).toBeUndefined(); + expect(sessions.get(record.id)?.errorSource).toBeUndefined(); + // Reconciliation keeps sidecar ownership for the reaper. + expect(ports).not.toHaveBeenCalled(); + expect(teardown).not.toHaveBeenCalled(); + expect(sendMessageToTmuxMock).not.toHaveBeenCalled(); + expect(createTmuxSessionMock).not.toHaveBeenCalled(); + }, + ); + + it("canonical historical probe: actual boot output excludes both diagnostics and restores plain drift once", async () => { + const config = baseConfig(); + loadConfigMock.mockReturnValue({ + ...config, + projects: { ...config.projects, api: { ...config.projects.api, restoreAfterReboot: true } }, + }); + const sessions = createSessionStore(); + historicalErrors.forEach((error, index) => { + // Untyped: startup typed-message recovery rewrites a typed record, + // and that write supersedes the boot observation. + const { queuedMessageTyped: _typed, ...untyped } = observationRecord(error, "other"); + const record = { ...untyped, id: `api-${index + 1}` }; + sessions.set(record.id, record); + }); + sessions.set("api-3", runningSession({ id: "api-3" })); + const service = await createDisposedSessionService(); + vi.spyOn(internalsFor(service), "readRuntimeSnapshot").mockResolvedValue(dead); + const pending = service.reconcileStoppedSessions(); + await vi.advanceTimersByTimeAsync(3_000); + const result = await pending; + for (const id of ["api-1", "api-2"]) { + expect(sessions.get(id)?.status).toBe("stopped"); + expect(sessions.get(id)?.error).toBeUndefined(); + expect(sessions.get(id)?.errorSource).toBeUndefined(); + } + expect(result.driftedSessions.map(({ record }) => record.id)).toEqual(["api-3"]); + const restore = vi.spyOn(service, "restore").mockResolvedValue({} as never); + await service.restoreRebootedSessions(result.driftedSessions); + expect(restore).toHaveBeenCalledExactlyOnceWith("api-3"); + expect(sendMessageToTmuxMock).not.toHaveBeenCalled(); + expect(createTmuxSessionMock).not.toHaveBeenCalled(); + }); + + it.each( + historicalErrors.flatMap((error) => + ["initial unknown", "fresh unknown", "fresh rejection"].map((mode) => ({ error, mode })), + ), + )( + "canonical historical probe: preserves identical diagnostic under $mode ($error)", + async ({ error, mode }) => { + const record = { + ...observationRecord(error, "other"), + submitUnconfirmedAt: "2026-03-18T10:04:00.000Z", + }; + const sessions = createSessionStore(record); + const service = await createDisposedSessionService(); + const internals = internalsFor(service); + const diagnostic = error.slice("Session status detection failed: ".length); + const runtime = { ...unknown, diagnostic }; + const read = vi.spyOn(internals, "readRuntimeSnapshot"); + if (mode === "fresh rejection") read.mockRejectedValue(new Error(diagnostic)); + else read.mockResolvedValue(runtime); + const ports = vi.spyOn(internals, "sessionWithReleasedSidecarPorts"); + const teardown = vi.spyOn(internals, "teardownSessionSidecars"); + const pending = internals.reconcileUnexpectedStop( + record, + mode === "initial unknown" ? runtime : dead, + "runtime_check", + false, + ); + await vi.advanceTimersByTimeAsync(1_000); + await pending; + expect(sessions.get(record.id)).toEqual(record); + expect(ports).not.toHaveBeenCalled(); + expect(teardown).not.toHaveBeenCalled(); + expect(sendMessageToTmuxMock).not.toHaveBeenCalled(); + expect(sendInterruptKeysToTmuxMock).not.toHaveBeenCalled(); + }, + ); + + it.each( + historicalErrors.flatMap((error) => [false, true].map((missing) => ({ error, missing }))), + )( + "canonical historical probe: preserves terminal error distinction (missing=$missing $error)", + async ({ error, missing }) => { + const record = observationRecord(error, "other"); + const sessions = createSessionStore(); + const service = await createDisposedSessionService(); + sessions.set(record.id, record); + const internals = internalsFor(service); + const runtime = { ...dead, runtimeAlive: true, paneUsable: true }; + vi.spyOn(internals, "readRuntimeSnapshot").mockResolvedValue(runtime); + const pending = internals.reconcileUnexpectedStop( + record, + runtime, + "runtime_check", + missing, + ); + await vi.advanceTimersByTimeAsync(1_000); + await pending; + expect(sessions.get(record.id)).toMatchObject({ + status: "errored", + error: missing ? "Agent worktree is missing." : "Agent runtime exited unexpectedly.", + errorSource: missing ? "other" : "inferred_runtime_exit", + queuedMessages: record.queuedMessages, + queuedMessageTyped: record.queuedMessageTyped, + }); + expect(sendMessageToTmuxMock).not.toHaveBeenCalled(); + expect(createTmuxSessionMock).not.toHaveBeenCalled(); + }, + ); + + it.each( + historicalErrors.flatMap((error) => + [ + { status: "stopped" as const, stopReason: "manual_pause" as const }, + { status: "stopped" as const, stopReason: "token_budget" as const }, + { status: "stopped" as const, stopReason: undefined }, + { status: "errored" as const, stopReason: undefined }, + { status: "completed" as const, stopReason: undefined }, + { status: "killed" as const, stopReason: undefined }, + ].map((state) => ({ error, ...state })), + ), + )( + "canonical historical probe: nonrunning status=$status reason=$stopReason ($error)", + async ({ error, status, stopReason }) => { + const record = { + ...observationRecord(error, "other"), + status, + ...(stopReason ? { stopReason } : {}), + }; + const sessions = createSessionStore(record); + const service = await createDisposedSessionService(); + vi.spyOn(internalsFor(service), "readRuntimeSnapshot").mockResolvedValue(dead); + await internalsFor(service).classifySessionRecord(record); + expect(sessions.get(record.id)?.status).toBe(status); + expect(sessions.get(record.id)?.stopReason).toBe(stopReason); + expect(sessions.get(record.id)?.error).toBe(status === "stopped" ? undefined : error); + expect(sendMessageToTmuxMock).not.toHaveBeenCalled(); + expect(createTmuxSessionMock).not.toHaveBeenCalled(); + }, + ); + + it.each( + historicalErrors.flatMap((error) => + [true, false].flatMap((diagnostic) => + ["boot", "runtime_check"].map((reason) => ({ + error, + diagnostic, + reason: reason as "boot" | "runtime_check", + })), + ), + ), + )( + "replaces historical evidence with current unknown then fresh dead ($diagnostic $reason $error)", + async ({ error, diagnostic, reason }) => { + const record = observationRecord(error, "other"); + const sessions = createSessionStore(); + const service = await createDisposedSessionService(); + sessions.set(record.id, record); + const internals = internalsFor(service); + vi.spyOn(internals, "readRuntimeSnapshot").mockResolvedValue({ + ...unknown, + ...(diagnostic ? { diagnostic: "ps failed: current timeout" } : {}), + }); + await internals.classifySessionRecord(record); + const current = sessions.get("api-1"); + if (!current) throw new Error("Missing current unit fixture"); + expect(current.errorSource).toBe("runtime_probe"); + expect(current.error).toBe( + `Session status detection failed: ${diagnostic ? "ps failed: current timeout" : "runtime probe (tmux or ps) timed out; runtime state unknown"}`, + ); + expect(current.status).toBe("running"); + vi.mocked(internals.readRuntimeSnapshot).mockResolvedValue(dead); + const pending = + reason === "boot" + ? service.reconcileStoppedSessions() + : internals.reconcileUnexpectedStop(current, dead, reason, false); + await vi.advanceTimersByTimeAsync(1_000); + const result = await pending; + expect(sessions.get("api-1")).toMatchObject({ + status: "stopped", + agentSessionId: record.agentSessionId, + agentLaunchId: record.agentLaunchId, + queuedMessages: record.queuedMessages, + queuedMessageTyped: record.queuedMessageTyped, + }); + expect(sessions.get("api-1")?.error).toBeUndefined(); + if ("driftedSessions" in result) expect(result.driftedSessions).toEqual([]); + expect(sendMessageToTmuxMock).not.toHaveBeenCalled(); + expect(createTmuxSessionMock).not.toHaveBeenCalled(); + }, + ); + + const races = [ + "native", + "launch", + "tmux", + "command", + "path", + "project", + "workspace", + "pause", + "status", + "source", + "provider", + "server", + "session-pending", + "workspace-pending", + "session-settled", + "workspace-settled", + "session-reopen", + "workspace-reopen", + "instance", + ] as const; + it.each( + (["fresh", "boot", "boot-fresh"] as const).flatMap((boundary) => + races.flatMap((race) => [ + { boundary, race, error: historicalErrors[0], errorSource: "runtime_probe" as const }, + ...historicalErrors.map((error) => ({ + boundary, + race, + error, + errorSource: "other" as const, + })), + ]), + ), + )( + "binds $boundary observation under $race ($errorSource $error)", + async ({ boundary, race, error, errorSource }) => { + const record = observationRecord(error, errorSource); + const sessions = createSessionStore(); + const service = await createDisposedSessionService(); + sessions.set(record.id, record); + const internals = internalsFor(service); + const releasePorts = vi.spyOn(internals, "sessionWithReleasedSidecarPorts"); + const teardown = vi.spyOn(internals, "teardownSessionSidecars"); + let replacement = record; + let reads = 0; + vi.spyOn(internals, "readRuntimeSnapshot").mockImplementation(async () => { + if (boundary === "boot-fresh" && reads++ === 0) return dead; + const changes: Partial = + race === "native" + ? { agentSessionId: "replacement" } + : race === "launch" + ? { agentLaunchId: "replacement" } + : race === "tmux" + ? { tmuxSession: "replacement" } + : race === "command" + ? { launchCommand: "replacement" } + : race === "path" + ? { worktreePath: "/replacement" } + : race === "project" + ? { project: "replacement" } + : race === "workspace" + ? { workspaceId: "replacement" } + : race === "pause" + ? { status: "stopped", stopReason: "manual_pause" } + : race === "status" + ? { status: "errored" } + : race === "source" + ? { + errorSource: + errorSource === "other" ? "runtime_probe" : "other", + } + : race === "provider" + ? { error: "Provider request failed", errorSource: "other" } + : race === "server" + ? { serverErrorAt: "2026-03-18T10:04:00.000Z" } + : {}; + replacement = { ...record, ...changes }; + sessions.set(record.id, replacement); + if (race === "instance") internals.lifecycle = new SessionLifecycleRegister(); + if (race.includes("pending") || race.includes("settled") || race.includes("reopen")) { + const owner = internals.lifecycle.begin( + race.includes("reopen") ? "reopen" : "restore", + race.includes("workspace") ? [record.workspaceId ?? record.id] : [record.id], + race, + ); + if (race.includes("settled")) + internals.lifecycle.settle( + owner, + true, + () => "succeeded", + () => true, + ); + } + return dead; + }); + const pending = + boundary !== "fresh" + ? service.reconcileStoppedSessions() + : internals.reconcileUnexpectedStop(record, dead, "runtime_check", false); + await vi.advanceTimersByTimeAsync(1_000); + await pending; + expect(sessions.get(record.id), `${boundary}/${race}`).toEqual(replacement); + expect(releasePorts, `${boundary}/${race}`).not.toHaveBeenCalled(); + expect(teardown, `${boundary}/${race}`).not.toHaveBeenCalled(); + expect(sendMessageToTmuxMock).not.toHaveBeenCalled(); + expect(createTmuxSessionMock).not.toHaveBeenCalled(); + }, + ); + + it.each([ + { error: historicalErrors[0], errorSource: "runtime_probe" as const }, + ...historicalErrors.map((error) => ({ error, errorSource: "other" as const })), + ])( + "canonical historical probe: preserves a concurrent queue append ($errorSource $error)", + async ({ error, errorSource }) => { + const record = { + ...observationRecord(error, errorSource), + submitUnconfirmedAt: "2026-03-18T10:04:00.000Z", + }; + const sessions = createSessionStore(); + const service = await createDisposedSessionService(); + sessions.set(record.id, record); + const internals = internalsFor(service); + const appended = { + ...record, + updatedAt: "2026-03-18T10:05:00.000Z", + queuedMessages: { messages: ["A", "B", "C"], awaitingPrompt: true }, + }; + vi.spyOn(internals, "readRuntimeSnapshot").mockImplementation(async () => { + sessions.set(record.id, appended); + return dead; + }); + const pending = internals.reconcileUnexpectedStop(record, dead, "runtime_check", false); + await vi.advanceTimersByTimeAsync(1_000); + await pending; + // The append supersedes this observation; the next check reconciles. + expect(sessions.get(record.id)).toEqual(appended); + expect(sendMessageToTmuxMock).not.toHaveBeenCalled(); + expect(createTmuxSessionMock).not.toHaveBeenCalled(); + }, + ); + }); + + describe("pr1197 review repair:", () => { + const dead = { + runtimeAlive: false, + paneUsable: false, + processAlive: false, + probeUnresponsive: false, + }; + const live = { ...dead, runtimeAlive: true, paneUsable: true, processAlive: true }; + const unknown = { + ...dead, + probeUnresponsive: true, + diagnostic: "ps failed: old probe timeout", + }; + type Observation = ReturnType[number]; + type DailyClaim = Observation & { + record: SessionRecord & { dailyWake: NonNullable }; + }; + function repairInternals(service: unknown) { + return service as SessionServiceInternals & { + lifecycle: SessionLifecycleRegister; + lookupPanePidQuietly(): Promise<{ status: "ok"; panePid: number; paneDead: boolean }>; + readFinalTokenUsage(): Promise; + reconcileUnexpectedStop( + record: SessionRecord, + runtime: typeof dead, + reason: "runtime_check", + missing: boolean, + ): Promise<{ session: SessionRecord }>; + deferDailyWakeForAdmission( + record: SessionRecord, + daily: NonNullable, + claim: DailyClaim, + denial: { kind: "denied"; error: Error }, + now: number, + ): void; + }; + } + + it.each(["lookup", "capture", "lifecycle"])( + "binds orphan pane effects through %s", + async (boundary) => { + const sessions = createSessionStore(); + const service = await createDisposedSessionService(); + const original = runningSession({ status: "spawning", agentLaunchId: "original" }); + const replacement = + boundary === "lifecycle" + ? original + : { + ...original, + status: "running" as const, + agentLaunchId: "replacement", + }; + sessions.set(original.id, original); + const internals = repairInternals(service); + const replace = () => { + sessions.set(original.id, replacement); + if (boundary === "lifecycle") internals.lifecycle.begin("restore", [original.id]); + }; + vi.spyOn(internals, "lookupPanePidQuietly").mockImplementation(async () => { + if (boundary === "lookup") replace(); + return { status: "ok", panePid: 101, paneDead: false }; + }); + capturePaneAgentProcessesMock.mockImplementation(async () => { + if (boundary !== "lookup") replace(); + return { status: "ok", processes: [] }; + }); + vi.spyOn(internals, "readRuntimeSnapshot").mockResolvedValue(dead); + await internals.reconcileUnexpectedStop(original, live, "runtime_check", false); + expect(sessions.get(original.id)).toEqual(replacement); + expect(killTmuxSessionMock).not.toHaveBeenCalled(); + expect(terminateAgentProcessesMock).not.toHaveBeenCalled(); + }, + ); + + it("keeps a legitimate old kill without stamping the postkill replacement", async () => { + const sessions = createSessionStore(); + const service = await createDisposedSessionService(); + const original = runningSession({ status: "spawning", agentLaunchId: "original" }); + const replacement = { ...original, status: "running" as const, agentLaunchId: "replacement" }; + sessions.set(original.id, original); + const internals = repairInternals(service); + vi.spyOn(internals, "lookupPanePidQuietly").mockResolvedValue({ + status: "ok", + panePid: 101, + paneDead: false, + }); + vi.spyOn(internals, "readRuntimeSnapshot").mockImplementation(async () => { + sessions.set(original.id, replacement); + return unknown; + }); + await internals.reconcileUnexpectedStop(original, live, "runtime_check", false); + expect(killTmuxSessionMock).toHaveBeenCalledOnce(); + expect(sessions.get(original.id)).toEqual(replacement); + }); + + it.each(["replacement", "current"])( + "handles rejected orphan snapshots through classifier for %s owner", + async (owner) => { + const sessions = createSessionStore(); + const service = await createDisposedSessionService(); + const original = runningSession({ + status: "spawning", + agentSessionId: "original", + agentLaunchId: "original", + queuedMessages: { messages: ["A", "B"], awaitingPrompt: true }, + }); + const replacement = { + ...original, + status: "running" as const, + agentLaunchId: "replacement", + }; + sessions.set(original.id, original); + const internals = repairInternals(service); + vi.spyOn(internals, "lookupPanePidQuietly").mockResolvedValue({ + status: "ok", + panePid: 101, + paneDead: false, + }); + let reads = 0; + vi.spyOn(internals, "readRuntimeSnapshot").mockImplementation(async () => { + if (reads++ === 0) return live; + if (owner === "replacement") sessions.set(original.id, replacement); + throw new Error("old orphan post-kill probe rejected"); + }); + await internals.classifySessionRecord(original); + expect(killTmuxSessionMock).toHaveBeenCalledOnce(); + if (owner === "replacement") expect(sessions.get(original.id)).toEqual(replacement); + else + expect(sessions.get(original.id)).toMatchObject({ + agentLaunchId: "original", + error: "Session status detection failed: old orphan post-kill probe rejected", + errorSource: "other", + queuedMessages: original.queuedMessages, + }); + }, + ); + + it.each(["reject", "unknown"])( + "retains original C1 protection during fresh %s", + async (mode) => { + const sessions = createSessionStore(); + const service = await createDisposedSessionService(); + const original = runningSession(); + const replacement = { ...original, agentSessionId: "replacement" }; + sessions.set(original.id, original); + const internals = repairInternals(service); + vi.spyOn(internals, "readRuntimeSnapshot").mockImplementation(async () => { + sessions.set(original.id, replacement); + if (mode === "reject") throw new Error("old probe failed"); + return unknown; + }); + const pending = internals.reconcileUnexpectedStop(original, dead, "runtime_check", false); + await vi.advanceTimersByTimeAsync(1_000); + await pending; + expect(sessions.get(original.id)).toEqual(replacement); + expect(killTmuxSessionMock).not.toHaveBeenCalled(); + }, + ); + + it.each(["native", "launch", "pause", "pending", "settled"])( + "rejects boot attribution after %s during usage", + async (change) => { + const sessions = createSessionStore(); + const service = await createDisposedSessionService(); + const original = runningSession({ + agent: "codex", + agentSessionId: "original", + agentLaunchId: "original", + }); + sessions.set(original.id, original); + const internals = repairInternals(service); + vi.spyOn(internals, "readRuntimeSnapshot").mockResolvedValue(dead); + let replacement = original; + vi.spyOn(internals, "readFinalTokenUsage").mockImplementation(async () => { + const current = sessions.get(original.id); + if (!current) throw new Error("Missing boot usage fixture"); + replacement = { + ...(change === "pending" || change === "settled" ? current : original), + ...(change === "native" ? { agentSessionId: "replacement" } : {}), + ...(change === "launch" ? { agentLaunchId: "replacement" } : {}), + ...(change === "pause" + ? { status: "stopped" as const, stopReason: "manual_pause" as const } + : {}), + }; + sessions.set(original.id, replacement); + if (change === "pending" || change === "settled") { + const owner = internals.lifecycle.begin("restore", [original.id]); + if (change === "settled") + internals.lifecycle.settle( + owner, + true, + () => "succeeded", + () => true, + ); + } + return { + tokenUsage: { + provider: "codex", + generationId: "original", + inputTokens: 7, + outputTokens: 3, + totalTokens: 10, + }, + }; + }); + const pending = service.reconcileStoppedSessions(); + await vi.advanceTimersByTimeAsync(1_000); + const result = await pending; + expect(sessions.get(original.id)).toEqual(replacement); + expect(result.driftedSessions).toEqual([]); + expect(result.drifted).toBe(0); + const restore = vi.spyOn(service, "restore").mockResolvedValue({} as never); + await service.restoreRebootedSessions(result.driftedSessions); + expect(restore).not.toHaveBeenCalled(); + }, + ); + + it("keeps a postusage queue append eligible for reboot restore", async () => { + const config = baseConfig(); + loadConfigMock.mockReturnValue({ + ...config, + projects: { api: { ...config.projects.api, restoreAfterReboot: true } }, + }); + readHostMemoryMock.mockReturnValue(null); + const sessions = createSessionStore(); + const service = await createDisposedSessionService(); + sessions.set("api-1", runningSession()); + const internals = repairInternals(service); + vi.spyOn(internals, "readRuntimeSnapshot").mockResolvedValue(dead); + vi.spyOn(internals, "readFinalTokenUsage").mockImplementation(async () => { + const current = sessions.get("api-1"); + if (!current) throw new Error("Missing stopped boot fixture"); + sessions.set(current.id, { + ...current, + updatedAt: "2026-03-18T10:06:00.000Z", + queuedMessages: { messages: ["new"], awaitingPrompt: true }, + }); + return {}; + }); + const pending = service.reconcileStoppedSessions(); + await vi.advanceTimersByTimeAsync(1_000); + const result = await pending; + expect(result.driftedSessions).toHaveLength(1); + expect(result.driftedSessions[0]?.record.queuedMessages?.messages).toEqual(["new"]); + const restore = vi.spyOn(service, "restore").mockResolvedValue({} as never); + await service.restoreRebootedSessions(result.driftedSessions); + expect(restore).toHaveBeenCalledExactlyOnceWith("api-1"); + }); + + it.each(["native", "launch", "pause", "pending", "settled", "queue"])( + "rechecks the boot candidate at consumer after %s", + async (change) => { + const config = baseConfig(); + loadConfigMock.mockReturnValue({ + ...config, + projects: { api: { ...config.projects.api, restoreAfterReboot: true } }, + }); + readHostMemoryMock.mockReturnValue(null); + const sessions = createSessionStore(); + const service = await createDisposedSessionService(); + const original = runningSession({ + status: "stopped", + agentSessionId: "original", + agentLaunchId: "original", + }); + sessions.set(original.id, original); + const candidates = rebootCandidates(service, [original.id]); + sessions.set(original.id, { + ...original, + ...(change === "native" ? { agentSessionId: "replacement" } : {}), + ...(change === "launch" ? { agentLaunchId: "replacement" } : {}), + ...(change === "pause" ? { stopReason: "manual_pause" as const } : {}), + ...(change === "queue" + ? { queuedMessages: { messages: ["new"], awaitingPrompt: true } } + : {}), + }); + const internals = repairInternals(service); + if (change === "pending" || change === "settled") { + const owner = internals.lifecycle.begin("restore", [original.id]); + if (change === "settled") + internals.lifecycle.settle( + owner, + true, + () => "succeeded", + () => true, + ); + } + const restore = vi.spyOn(service, "restore").mockResolvedValue({} as never); + await service.restoreRebootedSessions(candidates); + expect(restore).toHaveBeenCalledTimes(change === "queue" ? 1 : 0); + }, + ); + + it("rejects a later boot row replaced during an earlier restore", async () => { + const config = baseConfig(); + loadConfigMock.mockReturnValue({ + ...config, + projects: { api: { ...config.projects.api, restoreAfterReboot: true } }, + }); + readHostMemoryMock.mockReturnValue(null); + const sessions = createSessionStore(); + const service = await createDisposedSessionService(); + for (const id of ["api-1", "api-2"]) + sessions.set(id, runningSession({ id, status: "stopped" })); + const candidates = rebootCandidates(service, ["api-1", "api-2"]); + const restore = vi.spyOn(service, "restore").mockImplementation(async () => { + sessions.set( + "api-2", + runningSession({ id: "api-2", status: "stopped", stopReason: "manual_pause" }), + ); + return {} as never; + }); + await service.restoreRebootedSessions(candidates); + expect(restore.mock.calls.map(([id]) => id)).toEqual(["api-1"]); + }); + + it.each([ + ["owner first", ["api-1", "api-2"]], + ["member first", ["api-2", "api-1"]], + ] as const)("restores shared-workspace reboot rows when listed %s", async (_order, ids) => { + const config = baseConfig(); + loadConfigMock.mockReturnValue({ + ...config, + projects: { api: { ...config.projects.api, restoreAfterReboot: true } }, + }); + readHostMemoryMock.mockReturnValue(null); + const sessions = createSessionStore( + runningSession({ id: "api-1", workspaceId: "api-1", status: "stopped" }), + runningSession({ id: "api-2", workspaceId: "api-1", status: "stopped" }), + ); + const service = await createDisposedSessionService(); + const internals = repairInternals(service); + const restore = vi.spyOn(service, "restore").mockImplementation(async (id) => { + const owner = internals.lifecycle.begin("restore", [id]); + const settled = internals.lifecycle.settle( + owner, + true, + () => "succeeded", + () => true, + ); + const current = sessions.get(id); + if (current) sessions.set(id, { ...current, status: "running" }); + return { lifecycle: settled } as never; + }); + + await service.restoreRebootedSessions(rebootCandidates(service, [...ids])); + + expect(restore.mock.calls.map(([id]) => id)).toEqual(ids); + expect(logSpurEventMock).not.toHaveBeenCalledWith( + TEST_DATA_DIR, + expect.objectContaining({ + event: "session.reboot.restore.aborted", + details: expect.objectContaining({ reason: "lifecycle_changed" }), + }), + ); + service.dispose(); + }); + + it("logs a shared-workspace sibling skipped by an external lifecycle operation", async () => { + const config = baseConfig(); + loadConfigMock.mockReturnValue({ + ...config, + projects: { api: { ...config.projects.api, restoreAfterReboot: true } }, + }); + readHostMemoryMock.mockReturnValue(null); + createSessionStore( + runningSession({ id: "api-1", workspaceId: "api-1", status: "stopped" }), + runningSession({ id: "api-2", workspaceId: "api-1", status: "stopped" }), + ); + const service = await createDisposedSessionService(); + const internals = repairInternals(service); + const restore = vi.spyOn(service, "restore").mockImplementation(async (id) => { + const owner = internals.lifecycle.begin("restore", [id]); + const settled = internals.lifecycle.settle( + owner, + true, + () => "succeeded", + () => true, + ); + if (id === "api-1") { + const external = internals.lifecycle.begin("complete", ["api-1"]); + internals.lifecycle.settle( + external, + true, + () => "succeeded", + () => true, + ); + } + return { lifecycle: settled } as never; + }); + + await service.restoreRebootedSessions(rebootCandidates(service, ["api-1", "api-2"])); + + expect(restore.mock.calls.map(([id]) => id)).toEqual(["api-1"]); + expect(logSpurEventMock).toHaveBeenCalledWith( + TEST_DATA_DIR, + expect.objectContaining({ + event: "session.reboot.restore.aborted", + level: "warn", + sessionId: "api-2", + details: expect.objectContaining({ + reason: "lifecycle_changed", + lifecycleIds: ["api-1"], + }), + }), + ); + service.dispose(); + }); + + it.each([ + "dailyAt", + "message", + "stopCondition", + "marker", + "native", + "launch", + "pending", + "settled", + "queue", + "unchanged", + "exhausted", + "stale-exhausted", + ])("qualifies the full daily claim before %s denial", async (change) => { + const sessions = createSessionStore(); + const service = await createDisposedSessionService(); + const internals = repairInternals(service); + const daily = { + dailyAt: ["09:00", "09:40"], + nextDueAt: "2026-10-16T09:00:00.000Z", + message: "old", + stopCondition: "done", + }; + const claimedTime = "2026-10-16T09:40:00.000Z"; + const original = runningSession({ + status: "stopped", + agentSessionId: "original", + agentLaunchId: "original", + dailyWake: { ...daily, nextDueAt: claimedTime }, + }); + sessions.set(original.id, original); + const claim = rebootCandidates(service, [original.id])[0] as DailyClaim; + let replacement: SessionRecord; + { + replacement = { + ...original, + dailyWake: { + ...daily, + nextDueAt: claimedTime, + ...(change === "dailyAt" ? { dailyAt: ["09:40"] } : {}), + ...(change === "message" || change === "stale-exhausted" ? { message: "new" } : {}), + ...(change === "stopCondition" ? { stopCondition: "new" } : {}), + ...(change === "marker" + ? { admissionDeferredOccurrence: "2026-10-16T08:00:00.000Z" } + : {}), + }, + ...(change === "native" ? { agentSessionId: "replacement" } : {}), + ...(change === "launch" ? { agentLaunchId: "replacement" } : {}), + ...(change === "queue" + ? { queuedMessages: { messages: ["new"], awaitingPrompt: true } } + : {}), + }; + sessions.set(original.id, replacement); + if (change === "pending" || change === "settled") { + const owner = internals.lifecycle.begin("restore", [original.id]); + if (change === "settled") + internals.lifecycle.settle( + owner, + true, + () => "succeeded", + () => true, + ); + } + } + const exhausted = change === "exhausted" || change === "stale-exhausted"; + await internals.deferDailyWakeForAdmission( + original, + daily, + claim, + { kind: "denied", error: new Error("capacity") }, + Date.parse(exhausted ? "2026-10-16T09:25:00.000Z" : "2026-10-16T09:00:01.000Z"), + ); + if (change === "unchanged" || change === "queue") { + expect(sessions.get(original.id)?.dailyWake?.nextDueAt).toBe("2026-10-16T09:15:01.000Z"); + expect(sessions.get(original.id)?.queuedMessages).toEqual(replacement.queuedMessages); + } else expect(sessions.get(original.id)).toEqual(replacement); + const events = logSpurEventMock.mock.calls.map(([, entry]) => entry.event); + expect(events.filter((event) => event === "session.wake.daily_exhausted")).toHaveLength( + change === "exhausted" ? 1 : 0, + ); + expect(events.filter((event) => event === "session.wake.daily_deferred")).toHaveLength( + change === "unchanged" || change === "queue" ? 1 : 0, + ); + }); + }); + + describe("runtime probe errors on non-running and dead records", () => { + const PROBE_ERROR = "Session status detection failed: tmux list-windows failed: "; + + function failProbe(): void { + getTmuxSessionPresenceMock.mockResolvedValue({ + present: false, + unresponsive: true, + diagnostic: "tmux list-windows failed: ", + }); + } + + function answerDead(): void { + getTmuxSessionPresenceMock.mockResolvedValue({ present: false, unresponsive: false }); + } + + async function getSettled(service: { get(id: string): Promise }, id: string) { + const pending = service.get(id); + await vi.advanceTimersByTimeAsync(1_000); + return pending; + } + + function spyNotify(service: unknown) { + return vi + .spyOn( + service as { notifyAttention(view: SessionView, state: "error"): Promise }, + "notifyAttention", + ) + .mockResolvedValue(); + } + + it.each([ + { status: "stopped" as const, state: "stopped" }, + { status: "paused" as const, state: "stopped" }, + { status: "budget_limited" as const, state: "stopped" }, + { + status: "errored" as const, + state: "error", + error: "Agent runtime exited unexpectedly.", + errorSource: "inferred_runtime_exit" as const, + }, + ])("leaves a $status record untouched on a failed probe", async ({ state, ...patch }) => { + const sessions = createSessionStore(runningSession(patch)); + const before = clone(sessions.get("api-1")); + failProbe(); + const service = await createDisposedSessionService(); + + const result = await service.get("api-1"); + + expect(result.state).toBe(state); + expect(sessions.get("api-1")).toEqual(before); + expect(writeSessionMock).not.toHaveBeenCalled(); + }); + + it("leaves a stopped record untouched when the runtime probe throws", async () => { + const sessions = createSessionStore(runningSession({ status: "stopped" })); + const before = clone(sessions.get("api-1")); + tmuxSessionExistsMock.mockRejectedValue(new Error("boom")); + const service = await createDisposedSessionService(); + + expect((await service.get("api-1")).state).toBe("stopped"); + expect(sessions.get("api-1")).toEqual(before); + expect(writeSessionMock).not.toHaveBeenCalled(); + }); + + it("fires no attention notification for a stopped record under a failed probe", async () => { + createSessionStore(runningSession({ status: "stopped" })); + failProbe(); + const service = await createDisposedSessionService(); + const notify = spyNotify(service); + const internals = sessionServiceInternals(service); + + await internals.pollAttentionStates(false); + await internals.pollAttentionStates(false); + + expect(notify).not.toHaveBeenCalled(); + expect(internals.attentionStates.has("api-1")).toBe(false); + }); + + it("still stamps a running record on a failed probe", async () => { + const sessions = createSessionStore(runningSession()); + failProbe(); + const service = await createDisposedSessionService(); + + expect((await service.get("api-1")).state).toBe("error"); + expect(sessions.get("api-1")?.error).toBe(PROBE_ERROR); + expect(sessions.get("api-1")?.errorSource).toBe("runtime_probe"); + }); + + it("stops a dead running record and drops its probe error in one write", async () => { + const sessions = createSessionStore(runningSession()); + failProbe(); + const service = await createDisposedSessionService(); + const notify = spyNotify(service); + expect((await service.get("api-1")).state).toBe("error"); + expect(sessions.get("api-1")?.errorSource).toBe("runtime_probe"); + writeSessionMock.mockClear(); + + answerDead(); + const result = await getSettled(service, "api-1"); + + expect(result.state).toBe("stopped"); + expect(result.status).toBe("stopped"); + const { isRestorableSession } = await loadSessionServiceModule(); + expect(isRestorableSession({ ...result, workspaceExists: true })).toBe(true); + expect(sessions.get("api-1")?.status).toBe("stopped"); + expect(sessions.get("api-1")?.error).toBeUndefined(); + expect(sessions.get("api-1")?.errorSource).toBeUndefined(); + const intermediate = writeSessionMock.mock.calls + .map((call) => call[1] as SessionRecord) + .filter( + (record) => + record.id === "api-1" && record.status === "running" && record.error === undefined, + ); + expect(intermediate).toEqual([]); + expect(createTmuxSessionMock).not.toHaveBeenCalled(); + const internals = sessionServiceInternals(service); + await internals.pollAttentionStates(false); + expect(notify).not.toHaveBeenCalled(); + expect(sendDesktopNotificationMock).not.toHaveBeenCalled(); + }); + + it("errors a running probe-error record whose pane lives but agent process died", async () => { + const sessions = createSessionStore( + runningSession({ error: PROBE_ERROR, errorSource: "runtime_probe" }), + ); + tmuxSessionExistsMock.mockResolvedValue(true); + tmuxPaneDeadMock.mockResolvedValue(false); + isProcessRunningInTmuxMock.mockResolvedValue(false); + const service = await createDisposedSessionService(); + const notify = spyNotify(service); + + await getSettled(service, "api-1"); + + expect(sessions.get("api-1")).toMatchObject({ + status: "errored", + error: "Agent runtime exited unexpectedly.", + errorSource: "inferred_runtime_exit", + }); + const internals = sessionServiceInternals(service); + await internals.pollAttentionStates(false); + await internals.pollAttentionStates(false); + expect(notify).toHaveBeenCalledTimes(1); + expect(notify).toHaveBeenCalledWith(expect.objectContaining({ state: "error" }), "error"); + expect(createTmuxSessionMock).not.toHaveBeenCalled(); + }); + + it("errors a running probe-error record whose worktree is missing", async () => { + const sessions = createSessionStore( + runningSession({ error: PROBE_ERROR, errorSource: "runtime_probe" }), + ); + answerDead(); + probeWorkspaceMock.mockReturnValue({ exists: false, missing: true }); + const service = await createDisposedSessionService(); + const notify = spyNotify(service); + + await service.get("api-1"); + + expect(sessions.get("api-1")).toMatchObject({ + status: "errored", + error: "Agent worktree is missing.", + errorSource: "other", + }); + expect(createTmuxSessionMock).not.toHaveBeenCalled(); + const internals = sessionServiceInternals(service); + await internals.pollAttentionStates(false); + await internals.pollAttentionStates(false); + expect(notify).toHaveBeenCalledTimes(1); + const settled = clone(sessions.get("api-1")); + await service.get("api-1"); + expect(sessions.get("api-1")).toEqual(settled); + }); + + it("leaves a dead probe-error record untouched without runtime reconcile", async () => { + const sessions = createSessionStore( + runningSession({ error: PROBE_ERROR, errorSource: "runtime_probe" }), + ); + const before = clone(sessions.get("api-1")); + answerDead(); + const service = await createDisposedSessionService(); + const classify = sessionServiceInternals(service).classifySessionRecord as ( + session: SessionRecord, + options: { scanPane: boolean; reconcileRuntime: boolean }, + ) => Promise; + + await classify.call(service, clone(before) as SessionRecord, { + scanPane: false, + reconcileRuntime: false, + }); + + expect(sessions.get("api-1")).toEqual(before); + expect(createTmuxSessionMock).not.toHaveBeenCalled(); + }); + + it.each([ + { status: "stopped" as const, expected: "stopped" as const }, + { status: "running" as const, expected: "stopped" as const }, + ])("heals a legacy unsourced probe error on a $status record", async ({ status, expected }) => { + const sessions = createSessionStore(runningSession({ status, error: PROBE_ERROR })); + answerDead(); + const service = await createDisposedSessionService(); + + const result = await getSettled(service, "api-1"); + + expect(result.state).toBe("stopped"); + expect(sessions.get("api-1")?.status).toBe(expected); + expect(sessions.get("api-1")?.error).toBeUndefined(); + }); + + it.each([ + { + error: "Agent runtime exited unexpectedly.", + errorSource: "inferred_runtime_exit" as const, + }, + { + error: "Session status detection failed: workspace stat failed: permission denied", + errorSource: "other" as const, + }, + { error: "Session status detection failed: classifier unavailable" }, + { error: "Provider request failed" }, + { + error: PROBE_ERROR, + errorSource: "runtime_probe" as const, + serverErrorAt: "2026-03-18T10:01:00.000Z", + }, + { error: "Session status detection failed: Resource readout api-1--dev: x" }, + { status: "errored" as const, error: PROBE_ERROR, errorSource: "runtime_probe" as const }, + ])("keeps a real error on a dead record: $error ($status)", async (patch) => { + const sessions = createSessionStore(runningSession(patch)); + const before = sessions.get("api-1"); + answerDead(); + const service = await createDisposedSessionService(); + + await getSettled(service, "api-1"); + + // A dead running record stops; its real error stays. + expect(sessions.get("api-1")?.status).toBe( + before?.status === "running" ? "stopped" : before?.status, + ); + expect(sessions.get("api-1")?.error).toBe(before?.error); + expect(sessions.get("api-1")?.errorSource).toBe(before?.errorSource); + }); + + it("never relaunches or delivers queued work while a dead probe-error record heals", async () => { + const sessions = createSessionStore( + runningSession({ + error: PROBE_ERROR, + errorSource: "runtime_probe", + queuedMessages: { messages: ["held message"], awaitingPrompt: false }, + }), + ); + answerDead(); + const service = await createDisposedSessionService(); + const internals = sessionServiceInternals(service) as SessionServiceInternals & { + ensureDeliveryRunner(sessionId: string): void; + }; + + internals.ensureDeliveryRunner("api-1"); + await vi.advanceTimersByTimeAsync(1_000); + await getSettled(service, "api-1"); + await waitForRealTime(() => expect(internals.deliveryRuns.has("api-1")).toBe(false)); + + expect(sessions.get("api-1")?.status).toBe("stopped"); + expect(sessions.get("api-1")?.queuedMessages?.messages).toEqual(["held message"]); + expect(createTmuxSessionMock).not.toHaveBeenCalled(); + expect(sendMessageToTmuxMock).not.toHaveBeenCalled(); + const events = logSpurEventMock.mock.calls.map(([, entry]) => entry.event); + expect(events).not.toContain("session.recover.completed"); + expect(events).not.toContain("session.message.sent"); + }); + + it("keeps a dead probe-error record while restore owns its recovery", async () => { + const sessions = createSessionStore( + runningSession({ error: PROBE_ERROR, errorSource: "runtime_probe" }), + ); + answerDead(); + const service = await createDisposedSessionService(); + const lifecycle = (service as unknown as { lifecycle: SessionLifecycleRegister }).lifecycle; + lifecycle.begin("restore", ["api-1"]); + + await getSettled(service, "api-1"); + + expect(sessions.get("api-1")).toMatchObject({ + status: "running", + error: PROBE_ERROR, + errorSource: "runtime_probe", + }); + }); + + it("keeps a stopped probe error that carries serverErrorAt", async () => { + const sessions = createSessionStore( + runningSession({ + status: "stopped", + error: PROBE_ERROR, + errorSource: "runtime_probe", + serverErrorAt: "2026-03-18T10:01:00.000Z", + }), + ); + const before = clone(sessions.get("api-1")); + answerDead(); + const service = await createDisposedSessionService(); + + await service.get("api-1"); + + expect(sessions.get("api-1")).toEqual(before); + }); + + it("keeps a stopped probe error while restore owns its recovery", async () => { + const sessions = createSessionStore( + runningSession({ status: "stopped", error: PROBE_ERROR, errorSource: "runtime_probe" }), + ); + const before = clone(sessions.get("api-1")); + answerDead(); + const service = await createDisposedSessionService(); + const lifecycle = (service as unknown as { lifecycle: SessionLifecycleRegister }).lifecycle; + lifecycle.begin("restore", ["api-1"]); + + await getSettled(service, "api-1"); + + expect(sessions.get("api-1")).toEqual(before); + }); + + it.each(["paused", "budget_limited"] as const)( + "drops a probe error from a %s record once the probe answers", + async (status) => { + const sessions = createSessionStore( + runningSession({ status, error: PROBE_ERROR, errorSource: "runtime_probe" }), + ); + answerDead(); + const service = await createDisposedSessionService(); + + await getSettled(service, "api-1"); + + expect(sessions.get("api-1")?.status).toBe(status); + expect(sessions.get("api-1")?.error).toBeUndefined(); + expect(sessions.get("api-1")?.errorSource).toBeUndefined(); + }, + ); + + it.each([ + "runtime state unknown", + "runtime probe (tmux or ps) timed out; runtime state unknown", + "tmux list-windows failed: x", + "tmux list-panes failed: x", + "ps failed: x", + "tmux process failed: x", + ])("heals a legacy unsourced stopped record carrying: %s", async (diagnostic) => { + const sessions = createSessionStore( + runningSession({ + status: "stopped", + error: `Session status detection failed: ${diagnostic}`, + }), + ); + answerDead(); + const service = await createDisposedSessionService(); + + await getSettled(service, "api-1"); + + expect(sessions.get("api-1")?.status).toBe("stopped"); + expect(sessions.get("api-1")?.error).toBeUndefined(); + }); + + it("keeps stamping a stopped owner on a Resource readout failure", async () => { + const sessions = createSessionStore( + runningSession({ status: "stopped", sidecarNames: ["dev"] }), + ); + answerDead(); + const { TmuxProbeUnknownError } = await import("../../src/runtime-tmux.js"); + sidecarTmuxAliveMock.mockRejectedValue(new TmuxProbeUnknownError("resource unavailable")); + const service = await createDisposedSessionService(); + + await service.get("api-1"); + + expect(sessions.get("api-1")?.error).toMatch( + /^Session status detection failed: Resource readout /, + ); + expect(sessions.get("api-1")?.errorSource).toBe("other"); + }); + + it("refuses a send to a stopped record under a failed probe without stamping it", async () => { + const sessions = createSessionStore(runningSession({ status: "stopped" })); + failProbe(); + const service = await createDisposedSessionService(); + const { TmuxProbeUnknownError } = await import("../../src/runtime-tmux.js"); + + await expect(service.send("api-1", { message: "x" })).rejects.toBeInstanceOf( + TmuxProbeUnknownError, + ); + + expect(sessions.get("api-1")?.error).toBeUndefined(); + expect(createTmuxSessionMock).not.toHaveBeenCalled(); + }); + + it("heals a stopped probe-error record into the plain stopped record", async () => { + const plain = runningSession({ status: "stopped" }); + const sessions = createSessionStore({ + ...plain, + error: PROBE_ERROR, + errorSource: "runtime_probe", + }); + answerDead(); + const service = await createDisposedSessionService(); + + expect((await service.get("api-1")).state).toBe("stopped"); + + expect({ ...sessions.get("api-1"), updatedAt: plain.updatedAt }).toEqual(plain); + }); + }); + it("does not promote a stale errored record when the pane is dead", async () => { const sessions = createSessionStore(); sessions.set("api-1", { @@ -58411,7 +60153,7 @@ describe("SessionService", () => { entered = resolve; }); const send = vi - .spyOn(service, "send") + .spyOn(service as unknown as { sendAndWait: typeof service.send }, "sendAndWait") .mockImplementationOnce(async () => { entered(); await gate; @@ -60082,6 +61824,960 @@ describe("SessionService", () => { service.dispose(); }); + describe("wake relaunch admission", () => { + type WakeInternals = { processScheduledWakes(): Promise }; + + function capAt(maxLiveSessions: number): void { + loadConfigMock.mockReturnValue({ + ...baseConfig(), + admission: { ...baseConfig().admission, maxLiveSessions }, + }); + } + + // Per-pane runtime: a pane exists and runs its agent once it is live or + // relaunched; everything else answers dead. + function trackRuntimes(live: string[]): Set { + const panes = new Set(live); + createTmuxSessionMock.mockImplementation( + async ({ sessionName }: { sessionName: string }) => { + panes.add(sessionName); + }, + ); + tmuxSessionExistsMock.mockImplementation(async (name: string) => panes.has(name)); + isProcessRunningInTmuxMock.mockImplementation(async (name: string) => panes.has(name)); + return panes; + } + + function stoppedRecord(id: string, overrides: Partial = {}): SessionRecord { + return runningSession({ + id, + status: "stopped", + agentSessionId: `native-${id}`, + ...overrides, + }); + } + + function events(event: string, sessionId?: string) { + return logSpurEventMock.mock.calls + .map(([, entry]) => entry) + .filter( + (entry) => + entry.event === event && (sessionId === undefined || entry.sessionId === sessionId), + ); + } + + function wakeSends(text: string): number { + return sendMessageToTmuxMock.mock.calls.filter( + ([, message]) => typeof message === "string" && message.includes(text), + ).length; + } + + async function tick(service: unknown, atMs?: number): Promise { + if (atMs !== undefined) vi.setSystemTime(atMs); + await (service as WakeInternals).processScheduledWakes(); + } + + async function idleTicks(service: unknown, count: number): Promise { + for (let i = 0; i < count; i += 1) { + vi.setSystemTime(Date.now() + 1_000); + await tick(service); + } + } + + const WATCHED = [ + "session.wake.interval_deferred", + "session.wake.interval_sent", + "session.wake.daily_deferred", + "session.wake.daily_exhausted", + "session.wake.daily_sent", + "session.recover.completed", + "session.admission.denied", + ]; + function watchedCount(): number { + return WATCHED.reduce((total, event) => total + events(event).length, 0); + } + + it("reserves the parked wake slot while its runtime probe is pending", async () => { + capAt(1); + trackRuntimes([]); + mockClaudeJsonlState("waiting"); + createSessionStore( + stoppedRecord("api-1", { + stopReason: "stale_timeout", + scheduledWake: { + dueAt: "2026-03-18T10:00:00.000Z", + message: "scheduled wake", + }, + }), + ); + const service = await createDisposedSessionService(); + const internals = service as unknown as { + readRuntimeSnapshot(session: SessionRecord): Promise<{ + runtimeAlive: boolean; + paneUsable: boolean; + processAlive: boolean; + tmuxActivityAt: Date | null; + probeUnresponsive: boolean; + diagnostic?: string; + }>; + reserveAdmission(projectId: string, context: "spawn"): symbol; + admissionReservations: Map; + }; + const readRuntimeSnapshot = internals.readRuntimeSnapshot.bind(service); + let releaseProbe!: () => void; + const probeBarrier = new Promise((resolve) => { + releaseProbe = resolve; + }); + let signalProbe!: () => void; + const probeStarted = new Promise((resolve) => { + signalProbe = resolve; + }); + vi.spyOn(internals, "readRuntimeSnapshot").mockImplementation(async (session) => { + if (session.id === "api-1") { + signalProbe(); + await probeBarrier; + } + return readRuntimeSnapshot(session); + }); + const pendingWake = tick(service); + await probeStarted; + + expect(internals.admissionReservations.size).toBe(1); + let spawnDenial: unknown; + try { + internals.reserveAdmission("api", "spawn"); + } catch (error) { + spawnDenial = error; + } + expect(spawnDenial).toBeInstanceOf(Error); + expect(spawnDenial).toMatchObject({ statusCode: 429, reason: "cap" }); + + releaseProbe(); + await pendingWake; + expect(internals.admissionReservations.size).toBe(0); + expect(createTmuxSessionMock).toHaveBeenCalledTimes(1); + service.dispose(); + }); + + type RuntimeSnapshot = { + runtimeAlive: boolean; + paneUsable: boolean; + processAlive: boolean; + tmuxActivityAt: Date | null; + probeUnresponsive: boolean; + diagnostic?: string; + }; + + function holdWakeProbe(service: unknown, fail = false) { + const internals = service as unknown as { + readRuntimeSnapshot(session: SessionRecord): Promise; + admissionReservations: Map; + countLiveSessions(): { total: number }; + }; + const readRuntimeSnapshot = internals.readRuntimeSnapshot.bind(service); + let releaseProbe!: () => void; + const probeBarrier = new Promise((resolve) => { + releaseProbe = resolve; + }); + let signalProbe!: () => void; + const probeStarted = new Promise((resolve) => { + signalProbe = resolve; + }); + let reservationCountAtProbe = 0; + let probeHeld = false; + vi.spyOn(internals, "readRuntimeSnapshot").mockImplementation(async (session) => { + if (session.id === "api-1" && !probeHeld) { + probeHeld = true; + reservationCountAtProbe = internals.admissionReservations.size; + signalProbe(); + await probeBarrier; + if (fail) throw new Error("held wake runtime probe failed"); + } + return readRuntimeSnapshot(session); + }); + return { + internals, + probeStarted, + releaseProbe, + reservationCount: () => reservationCountAtProbe, + pendingWake: tick(service), + }; + } + + const ordinaryWakeCases = [ + ["stopped", "interval"], + ["stopped", "daily"], + ["paused", "interval"], + ["paused", "daily"], + ] as const; + + const warmingWakeCases = [ + ...ordinaryWakeCases, + ["parked", "scheduled"], + ["shed", "scheduled"], + ] as const; + + function warmingWakeRecord( + status: (typeof warmingWakeCases)[number][0], + kind: (typeof warmingWakeCases)[number][1], + ): SessionRecord { + const due = "2026-03-18T10:00:00.000Z"; + return stoppedRecord("api-1", { + status: status === "paused" ? "paused" : "stopped", + ...(status === "paused" + ? { stopReason: "manual_pause" } + : status === "parked" + ? { stopReason: "stale_timeout" } + : status === "shed" + ? { stopReason: "memory_shed" } + : {}), + ...(kind === "interval" + ? { + intervalWake: { + nextDueAt: due, + intervalMs: 60_000, + message: "warming interval", + stopCondition: "Done", + }, + } + : kind === "daily" + ? { + dailyWake: { + dailyAt: ["10:00"], + nextDueAt: due, + message: "warming daily", + stopCondition: "Done", + }, + } + : { scheduledWake: { dueAt: due, message: "warming scheduled" } }), + }); + } + + function holdWakeLaunch(service: unknown, panes: Set, fail = false) { + let releaseLaunch!: () => void; + const barrier = new Promise((resolve) => { + releaseLaunch = resolve; + }); + let signalLaunch!: () => void; + const launchStarted = new Promise((resolve) => { + signalLaunch = resolve; + }); + createTmuxSessionMock.mockImplementation( + async ({ sessionName }: { sessionName: string }) => { + if (sessionName === "api-1") { + signalLaunch(); + await barrier; + if (fail) throw new Error("held wake launch failed"); + } + panes.add(sessionName); + }, + ); + return { launchStarted, releaseLaunch, pendingWake: tick(service) }; + } + + describe.each([ + [2, 0], + [3, 0], + [2, 31_000], + [3, 31_000], + ] as const)("wake warmup at cap %i after %ims", (cap, elapsed) => { + it.each(warmingWakeCases)( + "counts the %s %s wake once while admitting the remaining slot", + async (status, kind) => { + capAt(cap); + const panes = trackRuntimes(cap === 3 ? ["api-2"] : []); + mockClaudeJsonlState("waiting"); + reserveNextSessionIdMock.mockResolvedValueOnce("api-3").mockResolvedValue("api-4"); + const sessions = createSessionStore( + warmingWakeRecord(status, kind), + ...(cap === 3 ? [runningSession({ id: "api-2" })] : []), + ); + const service = await createDisposedSessionService(); + const internals = service as unknown as { + admissionReservations: Map; + countLiveSessions(): { total: number }; + }; + const held = holdWakeLaunch(service, panes); + try { + await held.launchStarted; + vi.setSystemTime(Date.now() + elapsed); + const spawned = await service.spawn({ project: "api", prompt: "remaining slot" }); + expect(spawned.id).toBe("api-3"); + await expect( + service.spawn({ project: "api", prompt: "over cap" }), + ).rejects.toMatchObject({ statusCode: 429, reason: "cap" }); + expect(internals.countLiveSessions().total).toBe(cap); + expect(internals.admissionReservations.size).toBe(0); + held.releaseLaunch(); + await held.pendingWake; + expect(sessions.get("api-1")?.status).toBe("running"); + expect(internals.countLiveSessions().total).toBe(cap); + expect(internals.admissionReservations.size).toBe(0); + } finally { + held.releaseLaunch(); + await held.pendingWake; + service.dispose(); + } + }, + ); + }); + + it.each(warmingWakeCases)( + "releases the %s %s wake slot after a held launch failure", + async (status, kind) => { + capAt(2); + const panes = trackRuntimes([]); + mockClaudeJsonlState("waiting"); + reserveNextSessionIdMock.mockResolvedValue("api-3"); + createSessionStore(warmingWakeRecord(status, kind)); + const service = await createDisposedSessionService(); + const internals = service as unknown as { + admissionReservations: Map; + countLiveSessions(): { total: number }; + }; + const held = holdWakeLaunch(service, panes, true); + try { + await held.launchStarted; + vi.setSystemTime(Date.now() + 31_000); + expect(internals.countLiveSessions().total).toBe(1); + held.releaseLaunch(); + await held.pendingWake; + expect(internals.countLiveSessions().total).toBe(0); + expect(internals.admissionReservations.size).toBe(0); + const spawned = await service.spawn({ project: "api", prompt: "after launch failure" }); + expect(spawned.id).toBe("api-3"); + } finally { + held.releaseLaunch(); + await held.pendingWake; + service.dispose(); + } + }, + ); + + it.each(ordinaryWakeCases)( + "reserves an ordinary %s %s wake slot across the runtime probe and counts the live result", + async (status, kind) => { + capAt(1); + trackRuntimes([]); + mockClaudeJsonlState("waiting"); + const due = "2026-03-18T10:00:00.000Z"; + const wakeOverrides: Partial = { status }; + if (status === "paused") wakeOverrides.stopReason = "manual_pause"; + if (kind === "interval") { + wakeOverrides.intervalWake = { + nextDueAt: due, + intervalMs: 60_000, + message: "interval wake", + stopCondition: "Stop condition", + }; + } else { + wakeOverrides.dailyWake = { + dailyAt: ["10:00"], + nextDueAt: due, + message: "daily wake", + stopCondition: "Stop condition", + }; + } + const sessions = createSessionStore(stoppedRecord("api-1", wakeOverrides)); + const service = await createDisposedSessionService(); + const held = holdWakeProbe(service); + await held.probeStarted; + + let spawnError: unknown; + try { + await service.spawn({ project: "api", prompt: "concurrent spawn" }); + } catch (error) { + spawnError = error; + } finally { + held.releaseProbe(); + } + await held.pendingWake; + + expect(held.reservationCount()).toBe(1); + expect(held.internals.countLiveSessions().total).toBe(1); + expect(held.internals.admissionReservations.size).toBe(0); + expect(spawnError).toMatchObject({ statusCode: 429, reason: "cap" }); + expect(sessions.get("api-1")?.status).toBe("running"); + expect(createTmuxSessionMock).toHaveBeenCalledTimes(1); + expect( + kind === "interval" + ? events("session.wake.interval_sent", "api-1") + : events("session.wake.daily_sent", "api-1"), + ).toHaveLength(1); + await expect( + service.spawn({ project: "api", prompt: "after wake" }), + ).rejects.toMatchObject({ statusCode: 429, reason: "cap" }); + service.dispose(); + }, + ); + + it.each(ordinaryWakeCases)( + "releases an ordinary %s %s wake reservation when the runtime probe fails", + async (status, kind) => { + capAt(1); + trackRuntimes([]); + mockClaudeJsonlState("waiting"); + const due = "2026-03-18T10:00:00.000Z"; + const wakeOverrides: Partial = { status }; + if (status === "paused") wakeOverrides.stopReason = "manual_pause"; + if (kind === "interval") { + wakeOverrides.intervalWake = { + nextDueAt: due, + intervalMs: 60_000, + message: "interval wake", + stopCondition: "Stop condition", + }; + } else { + wakeOverrides.dailyWake = { + dailyAt: ["10:00"], + nextDueAt: due, + message: "daily wake", + stopCondition: "Stop condition", + }; + } + const sessions = createSessionStore(stoppedRecord("api-1", wakeOverrides)); + const service = await createDisposedSessionService(); + const held = holdWakeProbe(service, true); + await held.probeStarted; + const reservationCountAtProbe = held.reservationCount(); + held.releaseProbe(); + await held.pendingWake; + + expect(reservationCountAtProbe).toBe(1); + expect(held.internals.admissionReservations.size).toBe(0); + expect(held.internals.countLiveSessions().total).toBe(0); + expect(sessions.get("api-1")?.status).not.toBe("running"); + service.dispose(); + }, + ); + + describe("unexpected admission check failure", () => { + function failFirstCheck(service: unknown): void { + vi.spyOn( + service as { assertAdmissible(...args: unknown[]): unknown }, + "assertAdmissible", + ).mockImplementationOnce(() => { + throw new Error("host memory read blew up"); + }); + } + + function failureEvents(kind: "interval" | "daily") { + return [`session.wake.${kind}_deferred`, `session.wake.${kind}_failed`].flatMap((name) => + events(name), + ); + } + + it("defers only the failing interval record and still wakes the next one", async () => { + capAt(10); + trackRuntimes([]); + mockClaudeJsonlState("waiting"); + const due = "2026-03-18T10:04:59.000Z"; + const wake = (id: string) => ({ + nextDueAt: due, + intervalMs: 60_000, + message: `interval-${id}`, + stopCondition: "Stop condition", + }); + const sessions = createSessionStore( + stoppedRecord("api-1", { intervalWake: wake("api-1") }), + stoppedRecord("api-2", { intervalWake: wake("api-2") }), + ); + const service = await createDisposedSessionService(); + failFirstCheck(service); + + await tick(service); + + await waitForRealTime(() => expect(wakeSends("interval-api-2")).toBe(1)); + expect(wakeSends("interval-api-1")).toBe(0); + const logged = failureEvents("interval"); + expect(logged).toHaveLength(1); + expect(logged[0]).toMatchObject({ + event: "session.wake.interval_deferred", + sessionId: "api-1", + level: "error", + details: { reason: "admission_check_failed" }, + }); + expect(logged[0]?.message).toContain("host memory read blew up"); + expect(sessions.get("api-1")?.intervalWake?.nextDueAt).toBe("2026-03-18T10:05:59.000Z"); + service.dispose(); + }); + + it("defers only the failing daily record and still wakes the next one", async () => { + capAt(10); + trackRuntimes([]); + mockClaudeJsonlState("waiting"); + const at = (hours: number, minutes: number, seconds = 0) => + new Date(2026, 2, 19, hours, minutes, seconds).getTime(); + const wake = (id: string) => ({ + dailyAt: ["09:00", "09:40"], + nextDueAt: new Date(at(9, 0)).toISOString(), + message: `daily-${id}`, + stopCondition: "Stop condition", + }); + const sessions = createSessionStore( + stoppedRecord("api-1", { dailyWake: wake("api-1") }), + stoppedRecord("api-2", { dailyWake: wake("api-2") }), + ); + vi.setSystemTime(at(9, 0, 1)); + const service = await createDisposedSessionService(); + failFirstCheck(service); + + await tick(service); + + await waitForRealTime(() => expect(wakeSends("daily-api-2")).toBe(1)); + expect(wakeSends("daily-api-1")).toBe(0); + const logged = failureEvents("daily"); + expect(logged).toHaveLength(1); + expect(logged[0]).toMatchObject({ + event: "session.wake.daily_deferred", + sessionId: "api-1", + level: "error", + details: { reason: "admission_check_failed" }, + }); + expect(logged[0]?.message).toContain("host memory read blew up"); + expect(sessions.get("api-1")?.dailyWake?.admissionDeferredOccurrence).toBe( + new Date(at(9, 0)).toISOString(), + ); + service.dispose(); + }); + }); + + describe("interval", () => { + const DUE = "2026-03-18T10:04:59.000Z"; + const ids = ["api-1", "api-2", "api-3", "api-4"]; + + function intervalWake(id: string) { + return { + nextDueAt: DUE, + intervalMs: 60_000, + message: `interval-${id}`, + stopCondition: "Stop condition", + }; + } + + it("relaunches only up to the cap and consumes denied occurrences", async () => { + capAt(2); + trackRuntimes([]); + mockClaudeJsonlState("waiting"); + const sessions = createSessionStore( + ...ids.map((id) => stoppedRecord(id, { intervalWake: intervalWake(id) })), + ); + const service = await createDisposedSessionService(); + + await tick(service); + + expect(events("session.recover.completed")).toHaveLength(2); + await waitForRealTime(() => { + expect(wakeSends("interval-api-1")).toBe(1); + expect(wakeSends("interval-api-2")).toBe(1); + }); + expect(wakeSends("interval-api-3")).toBe(0); + expect(wakeSends("interval-api-4")).toBe(0); + const deferred = events("session.wake.interval_deferred"); + expect(deferred.map((entry) => entry.sessionId)).toEqual(["api-3", "api-4"]); + expect(deferred.map((entry) => entry.details?.["reason"])).toEqual([ + "admission", + "admission", + ]); + expect(events("session.wake.interval_failed")).toHaveLength(0); + expect(sessions.get("api-3")?.intervalWake?.nextDueAt).toBe("2026-03-18T10:05:59.000Z"); + expect(sessions.get("api-4")?.intervalWake?.nextDueAt).toBe("2026-03-18T10:05:59.000Z"); + + const settled = watchedCount(); + await idleTicks(service, 9); + expect(watchedCount()).toBe(settled); + + service.dispose(); + const restarted = await createDisposedSessionService(); + await idleTicks(restarted, 5); + expect(watchedCount()).toBe(settled); + + await tick(restarted, Date.parse("2026-03-18T10:06:00.000Z")); + expect(events("session.wake.interval_deferred", "api-3")).toHaveLength(2); + expect(events("session.wake.interval_deferred", "api-4")).toHaveLength(2); + await waitForRealTime(() => { + expect(wakeSends("interval-api-1")).toBe(2); + expect(wakeSends("interval-api-2")).toBe(2); + }); + expect(events("session.recover.completed")).toHaveLength(2); + restarted.dispose(); + }); + + it("keeps the parked rollback for a stale-parked record", async () => { + capAt(1); + trackRuntimes(["occupier"]); + mockClaudeJsonlState("waiting"); + const sessions = createSessionStore( + runningSession({ id: "occupier" }), + stoppedRecord("api-1", { + stopReason: "stale_timeout", + intervalWake: intervalWake("api-1"), + }), + ); + const service = await createDisposedSessionService(); + + await tick(service); + + expect(sessions.get("api-1")?.intervalWake?.nextDueAt).toBe(DUE); + const deferred = events("session.wake.interval_deferred", "api-1"); + expect(deferred).toHaveLength(1); + expect(deferred[0]?.details).not.toHaveProperty("reason"); + service.dispose(); + }); + + it("keeps the parked rollback for a memory-shed record", async () => { + capAt(1); + trackRuntimes(["occupier"]); + mockClaudeJsonlState("waiting"); + const sessions = createSessionStore( + runningSession({ id: "occupier" }), + stoppedRecord("api-1", { + stopReason: "memory_shed", + intervalWake: intervalWake("api-1"), + }), + ); + const service = await createDisposedSessionService(); + + await tick(service); + + expect(sessions.get("api-1")?.intervalWake?.nextDueAt).toBe(DUE); + const deferred = events("session.wake.interval_deferred", "api-1"); + expect(deferred).toHaveLength(1); + expect(deferred[0]?.details).not.toHaveProperty("reason"); + service.dispose(); + }); + + it("gates every stopped and paused wake, healed or not", async () => { + capAt(1); + trackRuntimes(["occupier"]); + mockClaudeJsonlState("waiting"); + const sessions = createSessionStore( + runningSession({ id: "occupier" }), + stoppedRecord("api-1", { intervalWake: intervalWake("api-1") }), + stoppedRecord("api-2", { status: "paused", intervalWake: intervalWake("api-2") }), + stoppedRecord("api-3", { + status: "budget_limited", + intervalWake: intervalWake("api-3"), + }), + ); + const service = await createDisposedSessionService(); + + await tick(service); + + for (const id of ["api-1", "api-2"]) { + expect(sessions.get(id)?.intervalWake?.nextDueAt).toBe("2026-03-18T10:05:59.000Z"); + expect( + events("session.wake.interval_deferred", id).map((e) => e.details?.["reason"]), + ).toEqual(["admission"]); + } + expect(events("session.admission.denied")).toHaveLength(2); + expect(createTmuxSessionMock).not.toHaveBeenCalled(); + expect(sendMessageToTmuxMock).not.toHaveBeenCalled(); + service.dispose(); + }); + }); + + describe("daily", () => { + // Local slots on the day after the fixture records, in any time zone. + const at = (hours: number, minutes: number, seconds = 0) => + new Date(2026, 2, 19, hours, minutes, seconds).getTime(); + const iso = (ms: number) => new Date(ms).toISOString(); + + function seedDaily(overrides: Partial = {}) { + capAt(1); + const panes = trackRuntimes(["occupier"]); + mockClaudeJsonlState("waiting"); + const sessions = createSessionStore( + runningSession({ id: "occupier" }), + stoppedRecord("api-1", { + dailyWake: { + dailyAt: ["09:00", "09:40"], + nextDueAt: iso(at(9, 0)), + message: "daily-api-1", + stopCondition: "Stop condition", + ...overrides, + }, + }), + ); + const freeSlot = () => { + const occupier = sessions.get("occupier"); + if (occupier) sessions.set("occupier", { ...occupier, status: "stopped" }); + panes.delete("occupier"); + }; + const takeSlot = () => { + const occupier = sessions.get("occupier"); + if (occupier) sessions.set("occupier", { ...occupier, status: "running" }); + panes.add("occupier"); + }; + return { sessions, panes, freeSlot, takeSlot }; + } + + it.each(["daily-api-1", "new-message"])( + "serializes a denied claim before supported rearm with %s", + async (message) => { + const { sessions } = seedDaily(); + vi.setSystemTime(at(9, 0, 1)); + const service = await createDisposedSessionService(); + const internals = service as unknown as { + assertAdmissible(...args: unknown[]): unknown; + }; + const original = internals.assertAdmissible.bind(service); + let rearm: Promise | undefined; + vi.spyOn(internals, "assertAdmissible").mockImplementationOnce((...args) => { + vi.setSystemTime(at(9, 0, 2)); + rearm = service.scheduleWake("api-1", { + dailyAt: ["09:00", "09:40"], + message, + stopCondition: "Stop condition", + }); + return original(...args); + }); + await tick(service); + expect(rearm).toBeDefined(); + await rearm; + expect(sessions.get("api-1")?.dailyWake).toEqual({ + dailyAt: ["09:00", "09:40"], + nextDueAt: iso(at(9, 40)), + message, + stopCondition: "Stop condition", + }); + expect(wakeSends(message)).toBe(0); + const writes = writeSessionMock.mock.calls + .map(([, record]) => record) + .filter((record) => record.id === "api-1"); + expect(writes.map((record) => record.dailyWake?.nextDueAt)).toEqual([ + iso(at(9, 40)), + iso(at(9, 15, 1)), + iso(at(9, 40)), + ]); + expect(events("session.wake.daily_deferred")).toHaveLength(1); + service.dispose(); + }, + ); + + it("retries a denied occurrence until admitted, then sends it once", async () => { + const { sessions, freeSlot } = seedDaily(); + vi.setSystemTime(at(9, 0, 1)); + const service = await createDisposedSessionService(); + + await tick(service); + + expect(events("session.admission.denied")).toHaveLength(1); + const deferred = events("session.wake.daily_deferred"); + expect(deferred).toHaveLength(1); + expect(deferred[0]?.details?.["reason"]).toBe("admission"); + expect(wakeSends("daily-api-1")).toBe(0); + expect(sessions.get("api-1")?.dailyWake).toMatchObject({ + nextDueAt: iso(at(9, 15, 1)), + admissionDeferredOccurrence: iso(at(9, 0)), + }); + const settled = watchedCount(); + await idleTicks(service, 10); + expect(watchedCount()).toBe(settled); + + freeSlot(); + await tick(service, at(9, 15, 2)); + await waitForRealTime(() => expect(wakeSends("daily-api-1")).toBe(1)); + expect(events("session.wake.daily_sent")).toHaveLength(1); + expect(events("session.wake.daily_sent")[0]?.details?.["nextDueAt"]).toBe(iso(at(9, 0))); + expect(sessions.get("api-1")?.dailyWake?.nextDueAt).toBe(iso(at(9, 40))); + expect(sessions.get("api-1")?.dailyWake).not.toHaveProperty( + "admissionDeferredOccurrence", + ); + await idleTicks(service, 10); + expect(wakeSends("daily-api-1")).toBe(1); + service.dispose(); + }); + + it("leaves a due occurrence armed while a lifecycle operation is pending", async () => { + const { sessions } = seedDaily(); + vi.setSystemTime(at(9, 0, 1)); + const service = await createDisposedSessionService(); + const lifecycle = (service as unknown as { lifecycle: SessionLifecycleRegister }) + .lifecycle; + const pending = lifecycle.begin("restore", ["api-1"]); + + await tick(service); + + expect(sessions.get("api-1")?.dailyWake).toEqual({ + dailyAt: ["09:00", "09:40"], + nextDueAt: iso(at(9, 0)), + message: "daily-api-1", + stopCondition: "Stop condition", + }); + expect(events("session.admission.denied")).toHaveLength(0); + expect(events("session.wake.daily_deferred")).toHaveLength(0); + + lifecycle.settle( + pending, + true, + () => "succeeded", + () => true, + ); + await tick(service, at(9, 0, 2)); + expect(sessions.get("api-1")?.dailyWake).toMatchObject({ + nextDueAt: iso(at(9, 15, 2)), + admissionDeferredOccurrence: iso(at(9, 0)), + }); + expect(events("session.wake.daily_deferred")).toHaveLength(1); + service.dispose(); + }); + + it("skips the occurrence once its retry window reaches the next slot", async () => { + const { sessions } = seedDaily(); + vi.setSystemTime(at(9, 0, 1)); + const service = await createDisposedSessionService(); + + await tick(service); + await idleTicks(service, 10); + await tick(service, at(9, 15, 2)); + await idleTicks(service, 10); + await tick(service, at(9, 30, 3)); + const settled = watchedCount(); + await idleTicks(service, 10); + expect(watchedCount()).toBe(settled); + + expect(wakeSends("daily-api-1")).toBe(0); + expect(events("session.admission.denied")).toHaveLength(3); + expect(events("session.wake.daily_deferred")).toHaveLength(1); + const exhausted = events("session.wake.daily_exhausted"); + expect(exhausted).toHaveLength(1); + expect(exhausted[0]?.details?.["occurrence"]).toBe(iso(at(9, 0))); + expect(sessions.get("api-1")?.dailyWake?.nextDueAt).toBe(iso(at(9, 40))); + expect(sessions.get("api-1")?.dailyWake).not.toHaveProperty( + "admissionDeferredOccurrence", + ); + service.dispose(); + }); + + it("keeps the retry occurrence and budget across a restart", async () => { + const { sessions, freeSlot } = seedDaily(); + vi.setSystemTime(at(9, 0, 1)); + const first = await createDisposedSessionService(); + await tick(first); + first.dispose(); + + const restarted = await createDisposedSessionService(); + const settled = watchedCount(); + await idleTicks(restarted, 5); + expect(watchedCount()).toBe(settled); + + await tick(restarted, at(9, 15, 2)); + expect(events("session.admission.denied")).toHaveLength(2); + expect(events("session.wake.daily_deferred")).toHaveLength(1); + expect(sessions.get("api-1")?.dailyWake?.admissionDeferredOccurrence).toBe(iso(at(9, 0))); + + freeSlot(); + await tick(restarted, at(9, 30, 3)); + await waitForRealTime(() => expect(wakeSends("daily-api-1")).toBe(1)); + await idleTicks(restarted, 5); + expect(wakeSends("daily-api-1")).toBe(1); + restarted.dispose(); + }); + + it("starts a fresh retry budget for the next occurrence", async () => { + const { sessions, panes, freeSlot, takeSlot } = seedDaily(); + vi.setSystemTime(at(9, 0, 1)); + const service = await createDisposedSessionService(); + await tick(service); + freeSlot(); + await tick(service, at(9, 15, 2)); + await waitForRealTime(() => expect(wakeSends("daily-api-1")).toBe(1)); + + const woken = sessions.get("api-1"); + if (woken) sessions.set("api-1", { ...woken, status: "stopped" }); + panes.delete("api-1"); + takeSlot(); + await tick(service, at(9, 40, 1)); + + const deferred = events("session.wake.daily_deferred"); + expect(deferred).toHaveLength(2); + expect(deferred[1]?.details?.["occurrence"]).toBe(iso(at(9, 40))); + service.dispose(); + }); + + it.each([ + { + name: "re-armed", + rearm: (record: SessionRecord): SessionRecord => ({ + ...record, + dailyWake: { + dailyAt: ["09:50"], + nextDueAt: iso(at(9, 50)), + message: "rearmed", + stopCondition: "Stop condition", + }, + }), + }, + { + name: "cancelled", + rearm: (record: SessionRecord): SessionRecord => { + const { dailyWake: _cancelled, ...rest } = record; + return rest; + }, + }, + ])("keeps a $name daily wake when the denial write loses the claim", async ({ rearm }) => { + const { sessions } = seedDaily(); + vi.setSystemTime(at(9, 0, 1)); + const service = await createDisposedSessionService(); + const internals = service as unknown as { + assertAdmissible(...args: unknown[]): unknown; + }; + const original = internals.assertAdmissible.bind(service); + vi.spyOn(internals, "assertAdmissible").mockImplementation((...args: unknown[]) => { + const current = sessions.get("api-1"); + if (current) sessions.set("api-1", rearm(current)); + return original(...args); + }); + + await tick(service); + + expect(events("session.admission.denied")).toHaveLength(1); + expect(events("session.wake.daily_deferred")).toHaveLength(0); + expect(sessions.get("api-1")?.dailyWake).toEqual(rearm(stoppedRecord("api-1")).dailyWake); + service.dispose(); + }); + + it("drops the retry marker when a malformed dailyAt skips the occurrence", async () => { + const { sessions } = seedDaily({ + dailyAt: ["99:99"], + nextDueAt: iso(at(9, 15, 1)), + admissionDeferredOccurrence: iso(at(9, 0)), + }); + vi.setSystemTime(at(9, 15, 2)); + const service = await createDisposedSessionService(); + + await tick(service); + + expect(events("session.wake.daily_failed")).toHaveLength(1); + expect(sessions.get("api-1")?.dailyWake).toMatchObject({ + nextDueAt: iso(at(9, 15, 2) + 24 * 60 * 60 * 1000), + }); + expect(sessions.get("api-1")?.dailyWake).not.toHaveProperty( + "admissionDeferredOccurrence", + ); + service.dispose(); + }); + + it("drops the retry marker on a manual daily dispatch", async () => { + const { sessions, freeSlot } = seedDaily({ + nextDueAt: iso(at(9, 15, 1)), + admissionDeferredOccurrence: iso(at(9, 0)), + }); + freeSlot(); + vi.setSystemTime(at(9, 5)); + const service = await createDisposedSessionService(); + + await service.dispatchWake("api-1", { target: "daily", dispatch: true }); + + expect(sessions.get("api-1")?.dailyWake).not.toHaveProperty( + "admissionDeferredOccurrence", + ); + service.dispose(); + }); + }); + }); + it("fires a far-past-due daily wake once and jumps to the next occurrence", async () => { const sessions = seedShepherdSession({ dailyWake: { @@ -63466,24 +66162,18 @@ describe("SessionService", () => { }); it("refuses to launch a replacement when the pane still reads alive but nothing was captured", async () => { - // capturePaneAgentProcessesMock stays at its ok+empty default: the - // matchers found nothing owned. Classification reads dead; the fresh - // probe after capture sees a process those ownership matchers missed. + // Only the liveness check after an empty ownership capture sees the + // foreign process; earlier classification remains dead. mockClaudeJsonlState("waiting"); findAgentSessionIdMock.mockResolvedValueOnce(null).mockResolvedValue("session-uuid"); createSessionStore(runningSession({ id: "api-1" })); mockExitedThenRestoredProcess(); agentLaunchUsesForeignBinaryMock.mockReturnValue(true); - // The first fresh lookup (restore's own) resolves unavailable; every - // later lookup, including the kill path's fresh one, resolves an ok pane - // without a pid. - let freshLookups = 0; - lookupTmuxPanePidMock.mockImplementation(async (_name, options) => { - if (options?.fresh) { - freshLookups += 1; - if (freshLookups === 1) return { status: "unavailable" }; - } - return { status: "ok", panePid: null, paneDead: false }; + lookupTmuxPanePidMock.mockResolvedValue({ status: "ok", panePid: null, paneDead: false }); + let emptyOwnershipCaptured = false; + capturePaneAgentProcessesMock.mockImplementation(async () => { + emptyOwnershipCaptured = true; + return { status: "ok", processes: [] }; }); isProcessRunningInTmuxMock.mockImplementation( async ( @@ -63491,11 +66181,7 @@ describe("SessionService", () => { _matchers: string[], opts?: { fresh?: boolean; paneChildFallback?: boolean }, ) => { - return ( - opts?.fresh === true && - opts?.paneChildFallback === true && - capturePaneAgentProcessesMock.mock.calls.length > 0 - ); + return opts?.fresh === true && opts?.paneChildFallback === true && emptyOwnershipCaptured; }, ); @@ -63597,14 +66283,17 @@ describe("SessionService", () => { createTmuxSessionMock.mockImplementation(async () => { restoredTmuxCreated = true; }); - // Classification stays dead until launch. An ownership-blind check - // after capture would read alive, but an unavailable PID must skip it. + // Bind liveness to capture and creation, not classification probe count. + // A forbidden fresh capture probe reads alive; readiness after creation + // also reads alive, while every pre-capture classification reads dead. + let captured = false; + capturePaneAgentProcessesMock.mockImplementation(async () => { + captured = true; + return { status: "ok", processes: [] }; + }); isProcessRunningInTmuxMock.mockImplementation( async (_tmuxSession: string, _matchers: string[], opts?: { fresh?: boolean }) => { - return ( - restoredTmuxCreated || - (opts?.fresh === true && capturePaneAgentProcessesMock.mock.calls.length > 0) - ); + return restoredTmuxCreated || (opts?.fresh === true && captured); }, ); @@ -71741,6 +74430,102 @@ describe("SessionService", () => { const holdAgents = ["claude", "codex", "cursor", "opencode"] as const; const afterHold = Date.parse(PENDING_LAUNCH_AT) + 30_000; + it.each(["needs_input", "rate_limited", "unconfirmed"] as const)( + "historical transport recovery: blocks queued input under current %s", + async (guard) => { + timerPromisesSleepMock.mockImplementation( + (ms) => new Promise((resolve) => setTimeout(resolve, ms)), + ); + const sessions = createSessionStore(); + seedSendHold(sessions, "codex", { + error: "Session status detection failed: ps failed: malformed process table output", + errorSource: "other", + queuedMessages: { messages: ["A", "B"], awaitingPrompt: true }, + }); + mockAgentActivity("codex", Date.now(), "working"); + mockHoldAckScan(guard !== "unconfirmed"); + if (guard === "needs_input") { + readAgentHookStateMock.mockReturnValue({ + state: "needs_input", + updatedAt: new Date().toISOString(), + }); + } else if (guard === "rate_limited") { + readCodexRolloutStateMock.mockResolvedValue({ + rollout: null, + rateLimit: { limited: true, reason: "codex out of credits" }, + }); + } + const service = await deferredService(); + try { + const result = await service.get("api-1"); + expect(sessions.get("api-1")?.error).toBeUndefined(); + expect(result.state).toBe(guard === "unconfirmed" ? "working" : guard); + await vi.advanceTimersByTimeAsync(30_000); + expect(tmuxTexts()).toEqual([]); + expect(sessions.get("api-1")?.queuedMessages?.messages).toEqual(["A", "B"]); + if (guard === "unconfirmed") { + expect(sessions.get("api-1")?.submitUnconfirmedAt).toBe(PENDING_LAUNCH_AT); + } + expect(sendInterruptKeysToTmuxMock).not.toHaveBeenCalled(); + } finally { + service.dispose(); + } + }, + ); + + it.each([ + "Session status detection failed: ps failed: malformed process table output", + "Session status detection failed: ps failed: Command failed: ps -eo pid,ppid,pgid,tpgid,tty,rss,args\n", + ])("historical transport recovery: own Codex ACK resumes FIFO once for %j", async (error) => { + timerPromisesSleepMock.mockImplementation( + (ms) => new Promise((resolve) => setTimeout(resolve, ms)), + ); + const sessions = createSessionStore(); + seedSendHold(sessions, "codex", { + error, + errorSource: "other", + queuedMessages: { messages: ["A", "B"], awaitingPrompt: true }, + }); + mockAgentActivity("codex", afterHold, "working"); + const scan = mockHoldAckScan(true); + sendMessageToTmuxMock.mockImplementation(async () => { + mockAgentActivity("codex", Date.now(), "working"); + }); + const service = await deferredService(); + try { + await service.get("api-1"); + expect(scan).toHaveBeenCalledWith("typed"); + expect(sessions.get("api-1")?.submitUnconfirmedAt).toBeUndefined(); + expect(sessions.get("api-1")?.queuedMessageTyped).toBeUndefined(); + expect(sessions.get("api-1")?.error).toBeUndefined(); + await vi.advanceTimersByTimeAsync(60_000); + expect(tmuxTexts()).toEqual([]); + + mockAgentActivity("codex", Date.now(), "waiting"); + await vi.advanceTimersByTimeAsync(10_000); + expect(tmuxTexts()).toEqual(["A"]); + await vi.advanceTimersByTimeAsync(60_000); + expect(tmuxTexts()).toEqual(["A"]); + expect(sessions.get("api-1")?.queuedMessages?.messages).toEqual(["B"]); + + mockAgentActivity("codex", Date.now(), "waiting"); + await vi.advanceTimersByTimeAsync(10_000); + expect(tmuxTexts()).toEqual(["A", "B"]); + await vi.advanceTimersByTimeAsync(30_000); + expect(tmuxTexts()).toEqual(["A", "B"]); + expect(sessions.get("api-1")?.agentSessionId).toBe("ses_1"); + expect(eventsNamed("session.message.requeued")).toHaveLength(0); + expect(sendInterruptKeysToTmuxMock).not.toHaveBeenCalled(); + expect(createTmuxSessionMock).not.toHaveBeenCalled(); + expect(eventsNamed("session.message.sent")).toEqual([ + expect.objectContaining({ details: expect.objectContaining({ interrupt: false }) }), + expect.objectContaining({ details: expect.objectContaining({ interrupt: false }) }), + ]); + } finally { + service.dispose(); + } + }); + async function historicalAckFixture(probe: string) { const sessions = createSessionStore(); const record = runningSession({ diff --git a/v2/test/fast/tmux-exit-net.fixture.test.ts b/v2/test/fast/tmux-exit-net.fixture.test.ts index 5b014b4b2..12cde0d55 100644 --- a/v2/test/fast/tmux-exit-net.fixture.test.ts +++ b/v2/test/fast/tmux-exit-net.fixture.test.ts @@ -53,6 +53,9 @@ describe("tmux-exit-net fixture", () => { const markerPath = process.env["TMUX_EXIT_NET_MARKER_FILE"]; if (markerPath) { writeFileSync(markerPath, JSON.stringify({ socketA, socketB })); + // Registered after the exit net: observing this proves its sweep finished + // in this worker, beyond the supervising Vitest CLI's exit. + process.once("exit", () => writeFileSync(`${markerPath}.exited`, "exited")); } // Deliberately no cleanup()/kill-server call and no afterAll — the exit diff --git a/v2/test/fast/tmux-exit-net.test.ts b/v2/test/fast/tmux-exit-net.test.ts index 662861b44..257ed7648 100644 --- a/v2/test/fast/tmux-exit-net.test.ts +++ b/v2/test/fast/tmux-exit-net.test.ts @@ -12,7 +12,7 @@ // `const socketName = activeTmuxSocketName` closure) leaves the SECOND // socket alive after this test's fixture exits. import { execFile } from "node:child_process"; -import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { promisify } from "node:util"; @@ -36,6 +36,7 @@ describe.skipIf(!tmuxOk)("tmux exit net (abnormal-exit safety)", () => { const vitestBin = join(process.cwd(), "node_modules", ".bin", "vitest"); const markerDir = mkdtempSync(join(tmpdir(), "spur-tmux-exit-net-marker-")); const markerPath = join(markerDir, "marker.json"); + let sockets: string[] = []; try { await execFileAsync( @@ -51,10 +52,15 @@ describe.skipIf(!tmuxOk)("tmux exit net (abnormal-exit safety)", () => { socketA: string; socketB: string; }; + sockets = [marker.socketA, marker.socketB]; + await expect.poll(() => existsSync(`${markerPath}.exited`)).toBe(true); expect(await tmuxHasSession(marker.socketA)).toBe(false); expect(await tmuxHasSession(marker.socketB)).toBe(false); } finally { + for (const socket of sockets) { + await execFileAsync("tmux", ["-L", socket, "kill-server"]).catch(() => undefined); + } rmSync(markerDir, { recursive: true, force: true }); } }, 60_000); diff --git a/v2/test/helpers/session-service-harness.ts b/v2/test/helpers/session-service-harness.ts index e86e1c4dc..ca227e53e 100644 --- a/v2/test/helpers/session-service-harness.ts +++ b/v2/test/helpers/session-service-harness.ts @@ -1298,6 +1298,7 @@ type SessionServiceInternals = { sessionBatch?: SessionRecord[], ): Promise<{ view: SessionView; classified: unknown }>; restoreWarmupUntil: Map; + restoreOwnsRuntimeRecovery(sessionId: string): boolean; withWorkspaceLifecycleLocks(sessionId: string, task: () => Promise): Promise; confirmAgentExited( session: Pick,