fix(responses): expose continuation spill write health (carry of #3525)

docs-site/src/content/docs/reference/management-api.md

@@ -244,7 +244,7 @@ whether to star the repository.
 
 | Method and path | Purpose | Notable errors |
 | --- | --- | --- |
-| `GET /api/system/memory` | Return scalar process, heap, stream, response-state, watchdog, and active-turn metrics | — |
+| `GET /api/system/memory` | Return scalar process, heap, stream, response-state, watchdog, and active-turn metrics. Response-state diagnostics include spill-write status, consecutive failures, fixed privacy-safe failure class, and last failure/success timestamps; raw errors and paths are never returned. | — |
 | `POST /api/system/restart` | Begin a drain-aware process restart without removing client injection | Returns 202; repeated calls report the existing drain |
 | `POST /api/stop` | Stop the service, restore native Codex, remove managed Grok injection, and drain the proxy | 409 service ownership conflict; 409 `respawnable_service` when a Windows Task Scheduler wrapper could respawn the proxy and the caller is not `ocx stop` (nothing is changed); 409 when the installed manager refuses to stop; 409 `service_state_unknown` when the Task Scheduler state cannot be read (nothing is changed; repair the query and retry) |
 | `GET /api/system/codex-app-server` | Report whether running Codex app-servers predate the current model catalog | — |

docs-site/src/content/docs/troubleshooting/windows-memory.md

@@ -50,15 +50,28 @@ runtime the leak itself remains an upstream problem:
   whereas a flat `responseState` under rising observed memory points away from
   that store. The values are scalar-only — no request bodies, tokens, paths, or
   account identifiers — and the read is side-effect free (it never prunes or
-  evicts). The dashboard's **Memory observability** card renders the
-  same fields and offers a confirm-gated **Drain & restart** action: it shows
+  evicts). Spill-write health is reported separately inside that block:
+  `spillWriteStatus` starts as `initial`, becomes `degraded` after a failed
+  publication, and returns to `healthy` after the next successful publication.
+  `spillWriteConsecutiveFailures`, the last failure time, and the last success
+  time show whether failures are accumulating or recovering in the same process.
+  The last failure is a fixed privacy-safe class such as `EACCES`, `ENOSPC`,
+  `ETIMEDOUT`, or `EACLRETRYEXHAUSTED`; raw error messages and filesystem paths
+  are never returned. These diagnostics stay on the authenticated management
+  endpoint and are intentionally absent from `/healthz`, which remains a liveness
+  signal. The dashboard's **Memory observability** card renders the memory and
+  continuation-size fields from this endpoint and offers a confirm-gated
+  **Drain & restart** action: it shows
   the current active-turn count, waits up to 60s for active turns (reusing
   the existing 503 + `Retry-After` drain), then aborts any remaining turns.
   The running proxy owns restart authorization and drain coordination, then
   exits; an installed service manager launches the replacement when applicable.
   The action reports success only after a different, identity-verified process
   is healthy on the same port, without tearing down Codex injection. That is a
   longer, informed recycle than the short drain on `POST /api/stop`.
+
+  For a scriptable snapshot of the complete authenticated payload, run
+  `ocx observe memory --json`; the CLI forwards the same response-state fields.
 - **A gated alternative stream path** — a bounded single-reader relay that
   removes the unbounded buffering shape entirely. On Windows it becomes the
   default automatically once a bundled Bun release verifiably carries the

src/responses/state.ts

@@ -170,6 +170,77 @@ async function snapshotOnDiskMatches(path: string, payload: string, payloadBytes
   }
 }
 const spillCounters = { writes: 0, writeFailures: 0, readFailures: 0 };
+
+export type ResponseSpillWriteFailureCode =
+  | "EACLRETRYEXHAUSTED"
+  | "ETIMEDOUT"
+  | "EACCES"
+  | "ENOSPC"
+  | "EFBIG"
+  | "EIO"
+  | "ECAPACITY"
+  | "ELOOP"
+  | "EUNKNOWN";
+
+export type ResponseSpillWriteStatus = "initial" | "healthy" | "degraded";
+
+interface ResponseSpillWriteHealth {
+  consecutiveFailures: number;
+  lastFailureCode: ResponseSpillWriteFailureCode | null;
+  lastFailureAt: number | null;
+  lastSuccessAt: number | null;
+}
+
+const spillWriteHealth: ResponseSpillWriteHealth = {
+  consecutiveFailures: 0,
+  lastFailureCode: null,
+  lastFailureAt: null,
+  lastSuccessAt: null,
+};
+
+/**
+ * Collapse filesystem/runtime errors into a fixed privacy-safe diagnostic union.
+ * Messages and paths are deliberately ignored: this projection is returned by the
+ * authenticated memory endpoint, and a nested `cause` can contain a username or
+ * workspace path even when the public wrapper does not.
+ */
+function classifySpillWriteFailure(error: unknown): ResponseSpillWriteFailureCode {
+  let cursor = error;
+  for (let depth = 0; depth < 4 && cursor && typeof cursor === "object"; depth += 1) {
+    const record = cursor as { code?: unknown; cause?: unknown };
+    const code = typeof record.code === "string" ? record.code.toUpperCase() : "";
+    switch (code) {
+      case "EACLRETRYEXHAUSTED": return "EACLRETRYEXHAUSTED";
+      case "ETIMEDOUT": return "ETIMEDOUT";
+      case "EACCES":
+      case "EPERM": return "EACCES";
+      case "ENOSPC":
+      case "EDQUOT": return "ENOSPC";
+      case "EFBIG": return "EFBIG";
+      case "EIO": return "EIO";
+      case "ECAPACITY": return "ECAPACITY";
+      case "ELOOP": return "ELOOP";
+    }
+    cursor = record.cause;
+  }
+  return "EUNKNOWN";
+}
+
+function noteSpillWriteSuccess(): void {
+  spillCounters.writes += 1;
+  spillWriteHealth.consecutiveFailures = 0;
+  spillWriteHealth.lastSuccessAt = now();
+}
+
+function noteSpillWriteFailure(
+  error: unknown,
+  override?: ResponseSpillWriteFailureCode,
+): void {
+  spillCounters.writeFailures += 1;
+  spillWriteHealth.consecutiveFailures += 1;
+  spillWriteHealth.lastFailureCode = override ?? classifySpillWriteFailure(error);
+  spillWriteHealth.lastFailureAt = now();
+}
 /**
  * Admission-boundary observability (test-visible). directSpills: oversized
  * candidates routed straight to durable spill without a resident stay or
@@ -346,6 +417,7 @@ async function runPendingResponseSpill(job: PendingResponseSpill): Promise<void>
   job.running = true;
   const candidate = job.candidate;
   let ref: ResponseSpillRef | null = null;
+  let exhaustedAclRetry = false;
   try {
     const state = spillPayloadForResident(candidate);
     try {
@@ -357,11 +429,16 @@ async function runPendingResponseSpill(job: PendingResponseSpill): Promise<void>
       if (!isAclTimeout(error)) throw error;
       // The ACL helper permits exactly one caller-owned recovery budget. The resident generation
       // remains replayable during both attempts, so a transient timeout never becomes a tombstone.
-      ref = await writeResponseSpillDurablyAsync(job.id, state, {
-        aclBudgetMs: responseSpillAsyncAclAttemptBudgetMs(),
-        retryTimedOutOnce: true,
-        publicationControl: job.publicationControl,
-      });
+      try {
+        ref = await writeResponseSpillDurablyAsync(job.id, state, {
+          aclBudgetMs: responseSpillAsyncAclAttemptBudgetMs(),
+          retryTimedOutOnce: true,
+          publicationControl: job.publicationControl,
+        });
+      } catch (retryError) {
+        exhaustedAclRetry = isAclTimeout(retryError);
+        throw retryError;
+      }
     }
     if (ref.payloadBytes > responseSpillPayloadCap()) {
       deleteResponseSpill(ref);
@@ -376,14 +453,14 @@ async function runPendingResponseSpill(job: PendingResponseSpill): Promise<void>
     }
     if (swapResidentForSpill(job.id, candidate, ref)) {
       ref = null;
-      spillCounters.writes += 1;
+      noteSpillWriteSuccess();
       if (job.directAdmission) admissionCounters.directSpills += 1;
       deferSupersededSpill(job.supersededSpill);
     }
-  } catch {
+  } catch (error) {
     if (ref) deleteResponseSpill(ref);
     if (states.get(job.id) === candidate && !job.cancelled) {
-      spillCounters.writeFailures += 1;
+      noteSpillWriteFailure(error, exhaustedAclRetry ? "EACLRETRYEXHAUSTED" : undefined);
       replaceWithSpillFailure(job.id, candidate);
       deferSupersededSpill(job.supersededSpill);
     }
@@ -406,7 +483,7 @@ function queuePendingResponseSpill(
 ): void {
   const inheritedSpill = cancelPendingResponseSpill(id) ?? options.supersededSpill;
   if (pendingResponseSpillBytes + candidate.sizeBytes > MAX_PENDING_RESPONSE_SPILL_BYTES) {
-    spillCounters.writeFailures += 1;
+    noteSpillWriteFailure(null, "ECAPACITY");
     replaceWithSpillFailure(id, candidate);
     deferSupersededSpill(inheritedSpill);
     return;
@@ -424,7 +501,7 @@ function queuePendingResponseSpill(
   if (accountedResponseSpillBytes() + footprint + inheritedBytes > spillByteCap()) {
     enforceSpilledResponseBudget();
     if (accountedResponseSpillBytes() + footprint + inheritedBytes > spillByteCap()) {
-      spillCounters.writeFailures += 1;
+      noteSpillWriteFailure(null, "ECAPACITY");
       replaceWithSpillFailure(id, candidate);
       deferSupersededSpill(inheritedSpill);
       return;
@@ -560,7 +637,7 @@ function installShutdownFallbackSpill(
       enforceSpilledResponseBudget();
       if (accountedResponseSpillBytes() + supersededBytes > spillByteCap()) {
         if (states.get(job.id) === candidate) {
-          spillCounters.writeFailures += 1;
+          noteSpillWriteFailure(null, "ECAPACITY");
           replaceWithSpillFailure(job.id, candidate);
           deferSupersededSpill(job.supersededSpill);
         }
@@ -581,14 +658,14 @@ function installShutdownFallbackSpill(
     }
     if (swapResidentForSpill(job.id, candidate, ref)) {
       ref = null;
-      spillCounters.writes += 1;
+      noteSpillWriteSuccess();
       if (job.directAdmission) admissionCounters.directSpills += 1;
       deferSupersededSpill(job.supersededSpill);
     }
   } catch (error) {
     if (ref) deleteResponseSpill(ref);
     if (states.get(job.id) === candidate) {
-      spillCounters.writeFailures += 1;
+      noteSpillWriteFailure(error);
       replaceWithSpillFailure(job.id, candidate);
       deferSupersededSpill(job.supersededSpill);
     }
@@ -601,9 +678,10 @@ function installShutdownFallbackSpill(
 function terminalizeShutdownFallbackCandidate(
   job: PendingResponseSpill,
   candidate: ResidentResponseState,
+  failureCode: ResponseSpillWriteFailureCode = "ETIMEDOUT",
 ): void {
   if (states.get(job.id) !== candidate) return;
-  spillCounters.writeFailures += 1;
+  noteSpillWriteFailure(null, failureCode);
   replaceWithSpillFailure(job.id, candidate);
   deferSupersededSpill(job.supersededSpill);
 }
@@ -652,11 +730,11 @@ function stopAtShutdownTerminalizationPassLimit(
   failures.push(Object.assign(new Error("Response spill shutdown terminalization pass limit exceeded"), { code: "ELOOP" }));
   supersedeShutdownFallbackBatch(pending, failures);
   for (const { job, candidate } of pending) {
-    terminalizeShutdownFallbackCandidate(job, candidate);
+    terminalizeShutdownFallbackCandidate(job, candidate, "ELOOP");
   }
   for (const [id, state] of [...states]) {
     if (state.kind !== "resident") continue;
-    spillCounters.writeFailures += 1;
+    noteSpillWriteFailure(null, "ELOOP");
     replaceWithSpillFailure(id, state);
   }
   recomputeOldestResident();
@@ -986,7 +1064,7 @@ function replaceSpillEntryAtomically(
       deleteResponseSpill(ref);
       return;
     }
-    spillCounters.writes += 1;
+    noteSpillWriteSuccess();
     noteStubSwapForTest();
     // The old generation is NOT unlinked here (review C1-1): the new stub is
     // only durable once the debounced snapshot flushes — a crash before that
@@ -996,8 +1074,8 @@ function replaceSpillEntryAtomically(
     while (pendingSpillUnlinks.length > PENDING_SPILL_UNLINKS_MAX) {
       deleteResponseSpill(pendingSpillUnlinks.shift()!);
     }
-  } catch {
-    spillCounters.writeFailures += 1;
+  } catch (error) {
+    noteSpillWriteFailure(error);
     // deferSpillUnlink: the durable snapshot may still reference the old
     // generation; deleting it now would strand the old stub after a crash.
     replaceWithSpillFailure(id, expected, { deferSpillUnlink: true });
@@ -1087,7 +1165,7 @@ function admitOversizedCandidate(
       deleteResponseSpill(ref);
       return;
     }
-    spillCounters.writes += 1;
+    noteSpillWriteSuccess();
     admissionCounters.directSpills += 1;
     noteStubSwapForTest();
     if (expected?.kind === "spill") {
@@ -1099,8 +1177,8 @@ function admitOversizedCandidate(
         deleteResponseSpill(pendingSpillUnlinks.shift()!);
       }
     }
-  } catch {
-    spillCounters.writeFailures += 1;
+  } catch (error) {
+    noteSpillWriteFailure(error);
     replaceWithSpillFailure(id, expected, { deferSpillUnlink: true });
   }
 }
@@ -1812,9 +1890,9 @@ function pruneResponses(at = now()): void {
         ...(entry.providerOutputStart !== undefined ? { providerOutputStart: entry.providerOutputStart } : {}),
         ...(entry.providers ? { providers: entry.providers } : {}),
       });
-      if (swapResidentForSpill(oldestId, entry, ref)) spillCounters.writes += 1;
-    } catch {
-      spillCounters.writeFailures += 1;
+      if (swapResidentForSpill(oldestId, entry, ref)) noteSpillWriteSuccess();
+    } catch (error) {
+      noteSpillWriteFailure(error);
       replaceWithSpillFailure(oldestId, entry);
     }
   }
@@ -1929,9 +2007,9 @@ export function evictOldestResponseContinuationForBudget(): number {
       ...(entry.providerOutputStart !== undefined ? { providerOutputStart: entry.providerOutputStart } : {}),
       ...(entry.providers ? { providers: entry.providers } : {}),
     });
-    if (swapResidentForSpill(id, entry, ref)) spillCounters.writes += 1;
-  } catch {
-    spillCounters.writeFailures += 1;
+    if (swapResidentForSpill(id, entry, ref)) noteSpillWriteSuccess();
+  } catch (error) {
+    noteSpillWriteFailure(error);
     replaceWithSpillFailure(id, entry);
   }
   schedulePersist();
@@ -2107,6 +2185,11 @@ export interface ResponseStateMetrics {
   oldestAgeMs: number;
   spillWrites: number;
   spillWriteFailures: number;
+  spillWriteStatus: ResponseSpillWriteStatus;
+  spillWriteConsecutiveFailures: number;
+  spillLastWriteFailureCode: ResponseSpillWriteFailureCode | null;
+  spillLastWriteFailureAt: number | null;
+  spillLastWriteSuccessAt: number | null;
   spillReadFailures: number;
   replayScopeMismatchDrops: number;
 }
@@ -2150,6 +2233,15 @@ export function responseStateMetrics(): ResponseStateMetrics {
     oldestAgeMs: states.size > 0 ? at - oldestCreatedAt : 0,
     spillWrites: spillCounters.writes,
     spillWriteFailures: spillCounters.writeFailures,
+    spillWriteStatus: spillWriteHealth.consecutiveFailures > 0
+      ? "degraded"
+      : spillWriteHealth.lastSuccessAt !== null
+        ? "healthy"
+        : "initial",
+    spillWriteConsecutiveFailures: spillWriteHealth.consecutiveFailures,
+    spillLastWriteFailureCode: spillWriteHealth.lastFailureCode,
+    spillLastWriteFailureAt: spillWriteHealth.lastFailureAt,
+    spillLastWriteSuccessAt: spillWriteHealth.lastSuccessAt,
     spillReadFailures: spillCounters.readFailures,
     replayScopeMismatchDrops,
   };
@@ -2268,6 +2360,10 @@ export function clearResponseStateMemoryForTests(): void {
   spillCounters.writes = 0;
   spillCounters.writeFailures = 0;
   spillCounters.readFailures = 0;
+  spillWriteHealth.consecutiveFailures = 0;
+  spillWriteHealth.lastFailureCode = null;
+  spillWriteHealth.lastFailureAt = null;
+  spillWriteHealth.lastSuccessAt = null;
   replayScopeMismatchDrops = 0;
   replayOverlapSkips = 0;
   persistAttemptHookForTests = null;

src/server/management/system-routes.ts

@@ -15,8 +15,9 @@
  * further: it is the proxy's previous_response_id continuation store, so a
  * growing responseState.totalBytes under rising observed memory points at
  * conversation retention rather than the runtime allocator. Spill counts,
- * payload-byte totals, tombstones, and failure counters remain finite scalars;
- * response ids, filenames, digests, paths, and payload content never leave the owner.
+ * payload-byte totals, tombstones, failure counters, fixed health/error enums,
+ * and event timestamps remain finite scalars; response ids, raw errors,
+ * filenames, digests, paths, and payload content never leave the owner.
  *
  * `activeTurnCount` / `isDraining` are scalar lifecycle counters for the
  * dashboard drain-and-restart confirm UX — never request bodies or IDs.