From 288506dc6883fa8433cf89014e72d01c1675317d Mon Sep 17 00:00:00 2001 From: Ingwannu Date: Fri, 4 Sep 2026 20:34:20 +0000 Subject: [PATCH 1/2] fix(responses): expose continuation spill write health --- .../content/docs/reference/management-api.md | 2 +- .../docs/troubleshooting/windows-memory.md | 17 +- src/responses/state.ts | 152 ++++++++++++++---- src/server/management/system-routes.ts | 5 +- structure/05_gui-and-management-api.md | 10 +- tests/memory-watchdog.test.ts | 32 +++- tests/responses/continuation-dedup.test.ts | 5 +- tests/responses/responses-state.test.ts | 125 +++++++++++++- 8 files changed, 305 insertions(+), 43 deletions(-) diff --git a/docs-site/src/content/docs/reference/management-api.md b/docs-site/src/content/docs/reference/management-api.md index 1a40b9f37e..5fb66378f3 100644 --- a/docs-site/src/content/docs/reference/management-api.md +++ b/docs-site/src/content/docs/reference/management-api.md @@ -244,7 +244,7 @@ whether to star the repository. | Method and path | Purpose | Notable errors | | --- | --- | --- | -| `GET /api/system/memory` | Return scalar process, heap, stream, response-state, watchdog, and active-turn metrics | — | +| `GET /api/system/memory` | Return scalar process, heap, stream, response-state, watchdog, and active-turn metrics. Response-state diagnostics include spill-write status, consecutive failures, fixed privacy-safe failure class, and last failure/success timestamps; raw errors and paths are never returned. | — | | `POST /api/system/restart` | Begin a drain-aware process restart without removing client injection | Returns 202; repeated calls report the existing drain | | `POST /api/stop` | Stop the service, restore native Codex, remove managed Grok injection, and drain the proxy | 409 service ownership conflict; 409 `respawnable_service` when a Windows Task Scheduler wrapper could respawn the proxy and the caller is not `ocx stop` (nothing is changed); 409 when the installed manager refuses to stop; 409 `service_state_unknown` when the Task Scheduler state cannot be read (nothing is changed; repair the query and retry) | | `GET /api/system/codex-app-server` | Report whether running Codex app-servers predate the current model catalog | — | diff --git a/docs-site/src/content/docs/troubleshooting/windows-memory.md b/docs-site/src/content/docs/troubleshooting/windows-memory.md index 8bbcd8214b..cea5b3c233 100644 --- a/docs-site/src/content/docs/troubleshooting/windows-memory.md +++ b/docs-site/src/content/docs/troubleshooting/windows-memory.md @@ -50,8 +50,18 @@ runtime the leak itself remains an upstream problem: whereas a flat `responseState` under rising observed memory points away from that store. The values are scalar-only — no request bodies, tokens, paths, or account identifiers — and the read is side-effect free (it never prunes or - evicts). The dashboard's **Memory observability** card renders the - same fields and offers a confirm-gated **Drain & restart** action: it shows + evicts). Spill-write health is reported separately inside that block: + `spillWriteStatus` starts as `initial`, becomes `degraded` after a failed + publication, and returns to `healthy` after the next successful publication. + `spillWriteConsecutiveFailures`, the last failure time, and the last success + time show whether failures are accumulating or recovering in the same process. + The last failure is a fixed privacy-safe class such as `EACCES`, `ENOSPC`, + `ETIMEDOUT`, or `EACLRETRYEXHAUSTED`; raw error messages and filesystem paths + are never returned. These diagnostics stay on the authenticated management + endpoint and are intentionally absent from `/healthz`, which remains a liveness + signal. The dashboard's **Memory observability** card renders the memory and + continuation-size fields from this endpoint and offers a confirm-gated + **Drain & restart** action: it shows the current active-turn count, waits up to 60s for active turns (reusing the existing 503 + `Retry-After` drain), then aborts any remaining turns. The running proxy owns restart authorization and drain coordination, then @@ -59,6 +69,9 @@ runtime the leak itself remains an upstream problem: The action reports success only after a different, identity-verified process is healthy on the same port, without tearing down Codex injection. That is a longer, informed recycle than the short drain on `POST /api/stop`. + + For a scriptable snapshot of the complete authenticated payload, run + `ocx observe memory --json`; the CLI forwards the same response-state fields. - **A gated alternative stream path** — a bounded single-reader relay that removes the unbounded buffering shape entirely. On Windows it becomes the default automatically once a bundled Bun release verifiably carries the diff --git a/src/responses/state.ts b/src/responses/state.ts index 6d8c6a3a96..e581653725 100644 --- a/src/responses/state.ts +++ b/src/responses/state.ts @@ -170,6 +170,77 @@ async function snapshotOnDiskMatches(path: string, payload: string, payloadBytes } } const spillCounters = { writes: 0, writeFailures: 0, readFailures: 0 }; + +export type ResponseSpillWriteFailureCode = + | "EACLRETRYEXHAUSTED" + | "ETIMEDOUT" + | "EACCES" + | "ENOSPC" + | "EFBIG" + | "EIO" + | "ECAPACITY" + | "ELOOP" + | "EUNKNOWN"; + +export type ResponseSpillWriteStatus = "initial" | "healthy" | "degraded"; + +interface ResponseSpillWriteHealth { + consecutiveFailures: number; + lastFailureCode: ResponseSpillWriteFailureCode | null; + lastFailureAt: number | null; + lastSuccessAt: number | null; +} + +const spillWriteHealth: ResponseSpillWriteHealth = { + consecutiveFailures: 0, + lastFailureCode: null, + lastFailureAt: null, + lastSuccessAt: null, +}; + +/** + * Collapse filesystem/runtime errors into a fixed privacy-safe diagnostic union. + * Messages and paths are deliberately ignored: this projection is returned by the + * authenticated memory endpoint, and a nested `cause` can contain a username or + * workspace path even when the public wrapper does not. + */ +function classifySpillWriteFailure(error: unknown): ResponseSpillWriteFailureCode { + let cursor = error; + for (let depth = 0; depth < 4 && cursor && typeof cursor === "object"; depth += 1) { + const record = cursor as { code?: unknown; cause?: unknown }; + const code = typeof record.code === "string" ? record.code.toUpperCase() : ""; + switch (code) { + case "EACLRETRYEXHAUSTED": return "EACLRETRYEXHAUSTED"; + case "ETIMEDOUT": return "ETIMEDOUT"; + case "EACCES": + case "EPERM": return "EACCES"; + case "ENOSPC": + case "EDQUOT": return "ENOSPC"; + case "EFBIG": return "EFBIG"; + case "EIO": return "EIO"; + case "ECAPACITY": return "ECAPACITY"; + case "ELOOP": return "ELOOP"; + } + cursor = record.cause; + } + return "EUNKNOWN"; +} + +function noteSpillWriteSuccess(): void { + spillCounters.writes += 1; + spillWriteHealth.consecutiveFailures = 0; + spillWriteHealth.lastSuccessAt = now(); +} + +function noteSpillWriteFailure( + error: unknown, + override?: ResponseSpillWriteFailureCode, +): void { + spillCounters.writeFailures += 1; + spillWriteHealth.consecutiveFailures += 1; + spillWriteHealth.lastFailureCode = override ?? classifySpillWriteFailure(error); + spillWriteHealth.lastFailureAt = now(); +} /** * Admission-boundary observability (test-visible). directSpills: oversized * candidates routed straight to durable spill without a resident stay or @@ -346,6 +417,7 @@ async function runPendingResponseSpill(job: PendingResponseSpill): Promise job.running = true; const candidate = job.candidate; let ref: ResponseSpillRef | null = null; + let exhaustedAclRetry = false; try { const state = spillPayloadForResident(candidate); try { @@ -357,11 +429,16 @@ async function runPendingResponseSpill(job: PendingResponseSpill): Promise if (!isAclTimeout(error)) throw error; // The ACL helper permits exactly one caller-owned recovery budget. The resident generation // remains replayable during both attempts, so a transient timeout never becomes a tombstone. - ref = await writeResponseSpillDurablyAsync(job.id, state, { - aclBudgetMs: responseSpillAsyncAclAttemptBudgetMs(), - retryTimedOutOnce: true, - publicationControl: job.publicationControl, - }); + try { + ref = await writeResponseSpillDurablyAsync(job.id, state, { + aclBudgetMs: responseSpillAsyncAclAttemptBudgetMs(), + retryTimedOutOnce: true, + publicationControl: job.publicationControl, + }); + } catch (retryError) { + exhaustedAclRetry = isAclTimeout(retryError); + throw retryError; + } } if (ref.payloadBytes > responseSpillPayloadCap()) { deleteResponseSpill(ref); @@ -376,14 +453,14 @@ async function runPendingResponseSpill(job: PendingResponseSpill): Promise } if (swapResidentForSpill(job.id, candidate, ref)) { ref = null; - spillCounters.writes += 1; + noteSpillWriteSuccess(); if (job.directAdmission) admissionCounters.directSpills += 1; deferSupersededSpill(job.supersededSpill); } - } catch { + } catch (error) { if (ref) deleteResponseSpill(ref); if (states.get(job.id) === candidate && !job.cancelled) { - spillCounters.writeFailures += 1; + noteSpillWriteFailure(error, exhaustedAclRetry ? "EACLRETRYEXHAUSTED" : undefined); replaceWithSpillFailure(job.id, candidate); deferSupersededSpill(job.supersededSpill); } @@ -406,7 +483,7 @@ function queuePendingResponseSpill( ): void { const inheritedSpill = cancelPendingResponseSpill(id) ?? options.supersededSpill; if (pendingResponseSpillBytes + candidate.sizeBytes > MAX_PENDING_RESPONSE_SPILL_BYTES) { - spillCounters.writeFailures += 1; + noteSpillWriteFailure(null, "ECAPACITY"); replaceWithSpillFailure(id, candidate); deferSupersededSpill(inheritedSpill); return; @@ -424,7 +501,7 @@ function queuePendingResponseSpill( if (accountedResponseSpillBytes() + footprint + inheritedBytes > spillByteCap()) { enforceSpilledResponseBudget(); if (accountedResponseSpillBytes() + footprint + inheritedBytes > spillByteCap()) { - spillCounters.writeFailures += 1; + noteSpillWriteFailure(null, "ECAPACITY"); replaceWithSpillFailure(id, candidate); deferSupersededSpill(inheritedSpill); return; @@ -560,7 +637,7 @@ function installShutdownFallbackSpill( enforceSpilledResponseBudget(); if (accountedResponseSpillBytes() + supersededBytes > spillByteCap()) { if (states.get(job.id) === candidate) { - spillCounters.writeFailures += 1; + noteSpillWriteFailure(null, "ECAPACITY"); replaceWithSpillFailure(job.id, candidate); deferSupersededSpill(job.supersededSpill); } @@ -581,14 +658,14 @@ function installShutdownFallbackSpill( } if (swapResidentForSpill(job.id, candidate, ref)) { ref = null; - spillCounters.writes += 1; + noteSpillWriteSuccess(); if (job.directAdmission) admissionCounters.directSpills += 1; deferSupersededSpill(job.supersededSpill); } } catch (error) { if (ref) deleteResponseSpill(ref); if (states.get(job.id) === candidate) { - spillCounters.writeFailures += 1; + noteSpillWriteFailure(error); replaceWithSpillFailure(job.id, candidate); deferSupersededSpill(job.supersededSpill); } @@ -601,9 +678,10 @@ function installShutdownFallbackSpill( function terminalizeShutdownFallbackCandidate( job: PendingResponseSpill, candidate: ResidentResponseState, + failureCode: ResponseSpillWriteFailureCode = "ETIMEDOUT", ): void { if (states.get(job.id) !== candidate) return; - spillCounters.writeFailures += 1; + noteSpillWriteFailure(null, failureCode); replaceWithSpillFailure(job.id, candidate); deferSupersededSpill(job.supersededSpill); } @@ -652,11 +730,11 @@ function stopAtShutdownTerminalizationPassLimit( failures.push(Object.assign(new Error("Response spill shutdown terminalization pass limit exceeded"), { code: "ELOOP" })); supersedeShutdownFallbackBatch(pending, failures); for (const { job, candidate } of pending) { - terminalizeShutdownFallbackCandidate(job, candidate); + terminalizeShutdownFallbackCandidate(job, candidate, "ELOOP"); } for (const [id, state] of [...states]) { if (state.kind !== "resident") continue; - spillCounters.writeFailures += 1; + noteSpillWriteFailure(null, "ELOOP"); replaceWithSpillFailure(id, state); } recomputeOldestResident(); @@ -986,7 +1064,7 @@ function replaceSpillEntryAtomically( deleteResponseSpill(ref); return; } - spillCounters.writes += 1; + noteSpillWriteSuccess(); noteStubSwapForTest(); // The old generation is NOT unlinked here (review C1-1): the new stub is // only durable once the debounced snapshot flushes — a crash before that @@ -996,8 +1074,8 @@ function replaceSpillEntryAtomically( while (pendingSpillUnlinks.length > PENDING_SPILL_UNLINKS_MAX) { deleteResponseSpill(pendingSpillUnlinks.shift()!); } - } catch { - spillCounters.writeFailures += 1; + } catch (error) { + noteSpillWriteFailure(error); // deferSpillUnlink: the durable snapshot may still reference the old // generation; deleting it now would strand the old stub after a crash. replaceWithSpillFailure(id, expected, { deferSpillUnlink: true }); @@ -1087,7 +1165,7 @@ function admitOversizedCandidate( deleteResponseSpill(ref); return; } - spillCounters.writes += 1; + noteSpillWriteSuccess(); admissionCounters.directSpills += 1; noteStubSwapForTest(); if (expected?.kind === "spill") { @@ -1099,8 +1177,8 @@ function admitOversizedCandidate( deleteResponseSpill(pendingSpillUnlinks.shift()!); } } - } catch { - spillCounters.writeFailures += 1; + } catch (error) { + noteSpillWriteFailure(error); replaceWithSpillFailure(id, expected, { deferSpillUnlink: true }); } } @@ -1812,9 +1890,9 @@ function pruneResponses(at = now()): void { ...(entry.providerOutputStart !== undefined ? { providerOutputStart: entry.providerOutputStart } : {}), ...(entry.providers ? { providers: entry.providers } : {}), }); - if (swapResidentForSpill(oldestId, entry, ref)) spillCounters.writes += 1; - } catch { - spillCounters.writeFailures += 1; + if (swapResidentForSpill(oldestId, entry, ref)) noteSpillWriteSuccess(); + } catch (error) { + noteSpillWriteFailure(error); replaceWithSpillFailure(oldestId, entry); } } @@ -1929,9 +2007,9 @@ export function evictOldestResponseContinuationForBudget(): number { ...(entry.providerOutputStart !== undefined ? { providerOutputStart: entry.providerOutputStart } : {}), ...(entry.providers ? { providers: entry.providers } : {}), }); - if (swapResidentForSpill(id, entry, ref)) spillCounters.writes += 1; - } catch { - spillCounters.writeFailures += 1; + if (swapResidentForSpill(id, entry, ref)) noteSpillWriteSuccess(); + } catch (error) { + noteSpillWriteFailure(error); replaceWithSpillFailure(id, entry); } schedulePersist(); @@ -2107,6 +2185,11 @@ export interface ResponseStateMetrics { oldestAgeMs: number; spillWrites: number; spillWriteFailures: number; + spillWriteStatus: ResponseSpillWriteStatus; + spillWriteConsecutiveFailures: number; + spillLastWriteFailureCode: ResponseSpillWriteFailureCode | null; + spillLastWriteFailureAt: number | null; + spillLastWriteSuccessAt: number | null; spillReadFailures: number; replayScopeMismatchDrops: number; } @@ -2150,6 +2233,15 @@ export function responseStateMetrics(): ResponseStateMetrics { oldestAgeMs: states.size > 0 ? at - oldestCreatedAt : 0, spillWrites: spillCounters.writes, spillWriteFailures: spillCounters.writeFailures, + spillWriteStatus: spillWriteHealth.consecutiveFailures > 0 + ? "degraded" + : spillWriteHealth.lastSuccessAt !== null + ? "healthy" + : "initial", + spillWriteConsecutiveFailures: spillWriteHealth.consecutiveFailures, + spillLastWriteFailureCode: spillWriteHealth.lastFailureCode, + spillLastWriteFailureAt: spillWriteHealth.lastFailureAt, + spillLastWriteSuccessAt: spillWriteHealth.lastSuccessAt, spillReadFailures: spillCounters.readFailures, replayScopeMismatchDrops, }; @@ -2268,6 +2360,10 @@ export function clearResponseStateMemoryForTests(): void { spillCounters.writes = 0; spillCounters.writeFailures = 0; spillCounters.readFailures = 0; + spillWriteHealth.consecutiveFailures = 0; + spillWriteHealth.lastFailureCode = null; + spillWriteHealth.lastFailureAt = null; + spillWriteHealth.lastSuccessAt = null; replayScopeMismatchDrops = 0; replayOverlapSkips = 0; persistAttemptHookForTests = null; diff --git a/src/server/management/system-routes.ts b/src/server/management/system-routes.ts index b163b8a8ad..3819a8e52d 100644 --- a/src/server/management/system-routes.ts +++ b/src/server/management/system-routes.ts @@ -15,8 +15,9 @@ * further: it is the proxy's previous_response_id continuation store, so a * growing responseState.totalBytes under rising observed memory points at * conversation retention rather than the runtime allocator. Spill counts, - * payload-byte totals, tombstones, and failure counters remain finite scalars; - * response ids, filenames, digests, paths, and payload content never leave the owner. + * payload-byte totals, tombstones, failure counters, fixed health/error enums, + * and event timestamps remain finite scalars; response ids, raw errors, + * filenames, digests, paths, and payload content never leave the owner. * * `activeTurnCount` / `isDraining` are scalar lifecycle counters for the * dashboard drain-and-restart confirm UX — never request bodies or IDs. diff --git a/structure/05_gui-and-management-api.md b/structure/05_gui-and-management-api.md index 725fbdcd5e..6bfa70aaa5 100644 --- a/structure/05_gui-and-management-api.md +++ b/structure/05_gui-and-management-api.md @@ -129,7 +129,7 @@ this document owns is which module holds which area and what invariant that area | V2 / Multi-agent mode | `GET/PUT /api/v2` — reports/sets the codex `multi_agent_v2` feature flag, the 3-state `multiAgentMode` override (`v1`/`default`/`v2`), the `keepNativeChatGptOnV1` hybrid pin, and the logical maximum thread count. Selecting `v2` normally enables the native flag; with the hybrid pin it disables that global override so native rows can resolve to v1 while routed rows resolve to v2. Selecting `v1` disables the flag; `default` leaves it unchanged. PUT rejects an explicit enabled flag that conflicts with the selected mode or hybrid pin. Every transition preserves the logical thread limit, is rollback-safe, and resyncs the catalog. | | Logs & Debug | One sidebar entry (`/#logs`) with two tabs. Logs tab: request/runtime logs for local diagnosis. Debug tab (`/#logs/debug`; legacy `/#debug` deep links redirect there): provider + usage toggles, refresh/follow log viewer. `GET/PUT /api/debug`; `GET /api/debug/logs` and `GET /api/debug/usage-logs` (monotonic `after` cursor, legacy `since` accepted). CLI: `ocx debug provider|usage …` (both streams via running proxy API). | | Usage | `GET /api/usage` aggregate read-only summary derived from the complete `~/.opencodex/usage.jsonl`; the ledger is streamed in fixed 1 MiB chunks, so the former read-byte and parsed-row caps cannot omit its prefix. The response includes measured / reported / unreported / unsupported / estimated counts, a daily zero-filled grid, and model and provider breakdowns. Never exposes prompts. | -| System | `POST /api/system/restart` restarts the proxy in place. Local CLI/tray callers first attest the exact runtime PID and port, then send a process-scoped HMAC capability bound to that method, path, PID, and port; the capability authorizes no other management route and is invalid after replacement. The caller observes one absolute deadline and accepts success only after a different runtime PID is healthy on the same port. `GET /api/system/health` is the authenticated scalar-only identity used by shared-plane Dashboard status and restart reconnect polling; it does not widen a Remote Hub management ingress to unauthenticated `/healthz`. `GET /api/system/memory` — service-process runtime/memory identity (pid, Bun version/revision, optional `bunRuntimeSource` provenance, platform, RSS/heap/external/ArrayBuffers scalars, observed memory = max(RSS, external, ArrayBuffers), `bun:jsc` heap context, streamMode + eager-relay gate decision, watchdog snapshot sliced to the last 60 samples) plus privacy-safe `appOwnedBytes` retained-store totals/counters under static store ids. Scalar-only payload; dashboard/admin callers use the standard management gate, while `ocx doctor` may use only the exact process-scoped local-read capability. It must never move to unauthenticated `/healthz`. | +| System | `POST /api/system/restart` restarts the proxy in place. Local CLI/tray callers first attest the exact runtime PID and port, then send a process-scoped HMAC capability bound to that method, path, PID, and port; the capability authorizes no other management route and is invalid after replacement. The caller observes one absolute deadline and accepts success only after a different runtime PID is healthy on the same port. `GET /api/system/health` is the authenticated scalar-only identity used by shared-plane Dashboard status and restart reconnect polling; it does not widen a Remote Hub management ingress to unauthenticated `/healthz`. `GET /api/system/memory` — service-process runtime/memory identity (pid, Bun version/revision, optional `bunRuntimeSource` provenance, platform, RSS/heap/external/ArrayBuffers scalars, observed memory = max(RSS, external, ArrayBuffers), `bun:jsc` heap context, streamMode + eager-relay gate decision, watchdog snapshot sliced to the last 60 samples) plus privacy-safe `appOwnedBytes` retained-store totals/counters under static store ids. Its response-state block also reports spill-write `initial`/`healthy`/`degraded` status, a consecutive-failure streak, fixed error class, and failure/success timestamps. A successful publication clears the streak in the same process; raw error text and paths never enter this surface. Scalar-only payload; dashboard/admin callers use the standard management gate, while `ocx doctor` may use only the exact process-scoped local-read capability. It must never move to unauthenticated `/healthz`. | | Stop | `POST /api/stop` — restore native Codex, stop any installed service, and exit the proxy. | | Diagnostics/sync | `src/server/management/config-routes.ts` — `GET /api/diagnostics/project-config` reports project-level Codex config that bypasses managed routing; `POST /api/sync` re-runs catalog/config sync. The diagnostic reports the bypass; it does not rewrite the project file. | | Sidecar/shadow-call settings | `src/server/management/config-routes.ts` — `GET/PUT /api/sidecar-settings` and `GET/PUT /api/shadow-call-settings`. PUT accepts model and backend (web-search union: openai/anthropic/xai/gemini/exa; xAI is live through stored Grok OAuth, while Gemini/Exa remain inert until their executors ship) plus validated `webSearch.xSearch`, optional `webSearch.exaApiKey` (write/clear only — never echoed by GET or the PUT response; redact.ts strips it from logs), `webSearch.reasoning`, `vision.reasoning`, `vision.enabled`, `vision.maxDescriptionsPerTurn`, and `vision.timeoutMs`; the read and PUT-response payload reports model, backend, reasoning, enabled, the vision per-turn limit, and timeout. `timeoutMs` is validated against the runtime integer bounds in `src/vision/timeout-bounds.ts`. Provider/OAuth credentials live in their stores; `exaApiKey` is the one sidecar-owned secret and follows the write-only contract above. Both shadow-call responses also report the resolved `sourceModels` — the prefixes the runtime actually intercepts (`src/lib/shadow-call.ts`, default `gpt-5.4-mini` + `gpt-5.6-luna`), so no client hard-codes a helper slug that a Codex release can invalidate. | @@ -143,6 +143,14 @@ this document owns is which module holds which area and what invariant that area | Sidebar | `src/server/management/sidebar-routes.ts` — `GET/POST /api/github/star` and `GET /api/update/badge`. Sidebar state is cosmetic; a failed fetch degrades silently. | | Logs | `src/server/management/logs-usage-routes.ts` — `GET /api/logs`, `GET /api/claude/inbound-debug`, and `GET /api/debug/injection-logs` join the debug streams described above. | +[Decision Log] +- 목적과 의도: Distinguish a healthy process from a continuation spill writer that is repeatedly failing, especially on Windows where the ACL publication lane is asynchronous. +- 기존 구현 및 제약 조건: `/healthz` intentionally reports liveness only, while `spillWriteFailures` was cumulative and discarded the failure class, event time, and recovery boundary. +- 검토한 주요 대안: Make `/healthz` fail on a spill error; publish raw error messages; expose a fixed classified health projection only on the authenticated memory route. +- 선택한 방식: Keep liveness unchanged and add a consecutive streak, fixed error class, and last failure/success timestamps to the existing authenticated response-state metrics. +- 다른 대안 대신 이 방식을 선택한 이유: One failed cache demotion must not restart or remove an otherwise serving proxy, and raw filesystem errors can disclose user paths while still failing to show whether the next write recovered. +- 장점, 단점 및 영향: Operators can identify accumulating failures and same-process recovery without sensitive text. The status is process-local and resets to `initial` on restart, so historical diagnosis still requires external metric collection. + Provider writes must not round-trip masked API keys as real secrets. Dashboard actions that change model visibility or subagent selection should trigger catalog/cache sync behavior through the server path that owns it. diff --git a/tests/memory-watchdog.test.ts b/tests/memory-watchdog.test.ts index edc9cc757a..f755f6d5c1 100644 --- a/tests/memory-watchdog.test.ts +++ b/tests/memory-watchdog.test.ts @@ -193,6 +193,12 @@ describe("GET /api/system/memory", () => { count: number; residentCount: number; spillStubCount: number; tombstoneCount: number; totalBytes: number; spillPayloadBytes: number; largestBytes: number; oldestAgeMs: number; spillWrites: number; spillWriteFailures: number; spillReadFailures: number; + spillWriteStatus: "initial" | "healthy" | "degraded"; + spillWriteConsecutiveFailures: number; + spillLastWriteFailureCode: string | null; + spillLastWriteFailureAt: number | null; + spillLastWriteSuccessAt: number | null; + replayScopeMismatchDrops: number; }; appOwnedBytes: ReturnType; inspectionCounters: { @@ -212,13 +218,27 @@ describe("GET /api/system/memory", () => { expect(body.observedBytes).toBeGreaterThan(0); expect(["rss", "external", "arrayBuffers"]).toContain(body.observedMetric); expect(body.jscHeap?.heapSize).toBeGreaterThan(0); - // responseState is a scalar-only continuation-store attribution block: every field is a - // finite number (no paths, tokens, or account identifiers), so it is safe on this surface. + // responseState is a scalar-only continuation-store attribution block: numbers plus fixed + // enum/null fields (no paths, messages, tokens, or account identifiers). // The exact count is pinned on purpose: a new field must be reviewed for privacy safety - // before it reaches this surface. 12 since #1597 added `replayScopeMismatchDrops`. - const responseStateValues = Object.values(body.responseState); - expect(responseStateValues).toHaveLength(12); - expect(responseStateValues.every(value => typeof value === "number" && Number.isFinite(value))).toBe(true); + // before it reaches this surface. 17 after #3522 added spill-write health diagnostics. + expect(Object.keys(body.responseState)).toHaveLength(17); + const { + spillWriteStatus, + spillLastWriteFailureCode, + spillLastWriteFailureAt, + spillLastWriteSuccessAt, + ...numericResponseState + } = body.responseState; + expect(Object.values(numericResponseState) + .every(value => typeof value === "number" && Number.isFinite(value))).toBe(true); + expect(["initial", "healthy", "degraded"]).toContain(spillWriteStatus); + expect(spillLastWriteFailureCode === null || [ + "EACLRETRYEXHAUSTED", "ETIMEDOUT", "EACCES", "ENOSPC", "EFBIG", + "EIO", "ECAPACITY", "ELOOP", "EUNKNOWN", + ].includes(spillLastWriteFailureCode)).toBe(true); + expect(spillLastWriteFailureAt === null || Number.isFinite(spillLastWriteFailureAt)).toBe(true); + expect(spillLastWriteSuccessAt === null || Number.isFinite(spillLastWriteSuccessAt)).toBe(true); expect(body.responseState.count).toBeGreaterThanOrEqual(0); expect(body.appOwnedBytes).toEqual({ budgetBytes: expect.any(Number), diff --git a/tests/responses/continuation-dedup.test.ts b/tests/responses/continuation-dedup.test.ts index fa6c3a0737..5e9efd2a74 100644 --- a/tests/responses/continuation-dedup.test.ts +++ b/tests/responses/continuation-dedup.test.ts @@ -310,8 +310,9 @@ describe("replay overlap: contracts held elsewhere", () => { }); test("the skip counter is not published on the memory surface", () => { - // /api/system/memory pins exactly 12 privacy-reviewed scalar fields. - expect(Object.keys(responseStateMetrics())).toHaveLength(12); + // /api/system/memory pins exactly 17 privacy-reviewed scalar fields. The five + // spill-health additions are enums, counters, or timestamps — never error text. + expect(Object.keys(responseStateMetrics())).toHaveLength(17); }); test("clearing state for tests resets the skip counter", () => { diff --git a/tests/responses/responses-state.test.ts b/tests/responses/responses-state.test.ts index 1217bbbe25..e4ae716a41 100644 --- a/tests/responses/responses-state.test.ts +++ b/tests/responses/responses-state.test.ts @@ -1076,9 +1076,61 @@ describe("Responses previous_response_id state", () => { tombstoneCount: 0, spillWrites: 1, spillWriteFailures: 0, + spillWriteStatus: "healthy", + spillWriteConsecutiveFailures: 0, }); }); + test("Windows spill reports exhausted ACL retry and recovers after a healthy runner", async () => { + forceWindowsAclLane(); + let clock = 0; + setNowForTests(() => clock); + setResponseSpillNowForTests(() => clock); + setResponseSpillAsyncAclAttemptBudgetForTests(100); + const spillDir = responseSpillDirectory(); + let failTemporaryPath = true; + setAsyncIcaclsRunnerForTests(async args => { + if (String(args[0]) === spillDir || !failTemporaryPath) { + return { success: true, exitCode: 0, timedOut: false, stdout: "" }; + } + clock += 100; + return { success: false, exitCode: null, timedOut: true, stdout: "private path omitted" }; + }); + setResponseStateByteCapForTests(1_024); + + rememberLarge("resp_async_acl_exhausted", "x".repeat(8_000)); + await flushPendingResponseSpillsForTests(); + + const metrics = responseStateMetrics(); + expect(metrics).toMatchObject({ + residentCount: 0, + spillStubCount: 0, + tombstoneCount: 1, + spillWriteFailures: 1, + spillWriteStatus: "degraded", + spillWriteConsecutiveFailures: 1, + spillLastWriteFailureCode: "EACLRETRYEXHAUSTED", + spillLastWriteSuccessAt: null, + }); + expect(metrics.spillLastWriteFailureAt).toBeGreaterThanOrEqual(0); + + failTemporaryPath = false; + rememberLarge("resp_async_acl_recovered", "r".repeat(8_000)); + await flushPendingResponseSpillsForTests(); + + const recovered = responseStateMetrics(); + expect(recovered).toMatchObject({ + spillStubCount: 1, + spillWrites: 1, + spillWriteFailures: 1, + spillWriteStatus: "healthy", + spillWriteConsecutiveFailures: 0, + spillLastWriteFailureCode: "EACLRETRYEXHAUSTED", + }); + expect(typeof recovered.spillLastWriteSuccessAt === "number" + && recovered.spillLastWriteSuccessAt >= (recovered.spillLastWriteFailureAt ?? 0)).toBe(true); + }); + test("Windows async spill attempts share one bounded ACL budget across every harden", async () => { forceWindowsAclLane(); let clock = 0; @@ -2422,7 +2474,19 @@ describe("Responses previous_response_id state", () => { setResponseStateByteCapForTests(1_024); rememberLarge("resp_private_metric_id", "secret-content".repeat(1_000)); const metrics = responseStateMetrics(); - expect(Object.values(metrics).every(value => typeof value === "number" && Number.isFinite(value))).toBe(true); + const { + spillWriteStatus, + spillLastWriteFailureCode, + spillLastWriteFailureAt, + spillLastWriteSuccessAt, + ...numericMetrics + } = metrics; + expect(Object.values(numericMetrics) + .every(value => typeof value === "number" && Number.isFinite(value))).toBe(true); + expect(spillWriteStatus).toBe("healthy"); + expect(spillLastWriteFailureCode).toBeNull(); + expect(spillLastWriteFailureAt).toBeNull(); + expect(typeof spillLastWriteSuccessAt === "number" && Number.isFinite(spillLastWriteSuccessAt)).toBe(true); const serialized = JSON.stringify(metrics); expect(serialized).not.toContain("resp_private_metric_id"); expect(serialized).not.toContain("secret-content"); @@ -3178,11 +3242,65 @@ describe("Responses previous_response_id state", () => { oldestAgeMs: 0, spillWrites: 0, spillWriteFailures: 0, + spillWriteStatus: "initial", + spillWriteConsecutiveFailures: 0, + spillLastWriteFailureCode: null, + spillLastWriteFailureAt: null, + spillLastWriteSuccessAt: null, spillReadFailures: 0, replayScopeMismatchDrops: 0, }); }); + test("a successful spill clears a repeated failure streak without erasing the last failure", () => { + const realNow = Date.now; + let clock = 1_000; + Date.now = () => clock; + try { + setResponseStateByteCapForTests(1_024); + setSpillIoForTest({ + write: () => { throw Object.assign(new Error("denied"), { code: "EACCES" }); }, + }); + rememberLarge("resp_health_failure", "f".repeat(8_000)); + + expect(responseStateMetrics()).toMatchObject({ + spillWriteFailures: 1, + spillWriteStatus: "degraded", + spillWriteConsecutiveFailures: 1, + spillLastWriteFailureCode: "EACCES", + spillLastWriteFailureAt: 1_000, + spillLastWriteSuccessAt: null, + }); + + clock = 1_500; + rememberLarge("resp_health_failure_again", "g".repeat(8_000)); + expect(responseStateMetrics()).toMatchObject({ + spillWriteFailures: 2, + spillWriteStatus: "degraded", + spillWriteConsecutiveFailures: 2, + spillLastWriteFailureCode: "EACCES", + spillLastWriteFailureAt: 1_500, + spillLastWriteSuccessAt: null, + }); + + clock = 2_000; + setSpillIoForTest(null); + rememberLarge("resp_health_recovered", "s".repeat(8_000)); + + expect(responseStateMetrics()).toMatchObject({ + spillWrites: 1, + spillWriteFailures: 2, + spillWriteStatus: "healthy", + spillWriteConsecutiveFailures: 0, + spillLastWriteFailureCode: "EACCES", + spillLastWriteFailureAt: 1_500, + spillLastWriteSuccessAt: 2_000, + }); + } finally { + Date.now = realNow; + } + }); + test("largest entry over 200KB is reflected, and total is >= largest", () => { const small = buildResponseJSON([{ type: "text_delta", text: "tiny" }, { type: "done" }], "gpt-5.5"); rememberResponseState({ model: "gpt-5.5", input: "small" }, small); @@ -3236,6 +3354,11 @@ describe("Responses previous_response_id state", () => { oldestAgeMs: 0, spillWrites: 0, spillWriteFailures: 0, + spillWriteStatus: "initial", + spillWriteConsecutiveFailures: 0, + spillLastWriteFailureCode: null, + spillLastWriteFailureAt: null, + spillLastWriteSuccessAt: null, spillReadFailures: 0, replayScopeMismatchDrops: 0, }); From 16c5df4a172df548523934360d461166eb33008c Mon Sep 17 00:00:00 2001 From: jun Date: Sat, 5 Sep 2026 07:39:36 +0900 Subject: [PATCH 2/2] chore: carry #3525 onto current dev Co-authored-by: Ingwannu <186453546+Ingwannu@users.noreply.github.com>