diff --git a/docs-site/src/content/docs/reference/management-api.md b/docs-site/src/content/docs/reference/management-api.md index 2c508f3857..a4ea3ad3d9 100644 --- a/docs-site/src/content/docs/reference/management-api.md +++ b/docs-site/src/content/docs/reference/management-api.md @@ -347,7 +347,7 @@ whether to star the repository. | Method and path | Purpose | Notable errors | | --- | --- | --- | -| `GET /api/system/memory` | Return scalar process, heap, stream, response-state, watchdog, and active-turn metrics. Response-state diagnostics include spill-write status, consecutive failures, fixed privacy-safe failure class, and last failure/success timestamps; raw errors and paths are never returned. | — | +| `GET /api/system/memory` | Return scalar process, heap, stream, response-state, watchdog, and active-turn metrics. Response-state diagnostics include spill-write status, consecutive failures, fixed privacy-safe failure class, and last failure/success timestamps. `spillLastWriteFailureOrigin` is `retry_returned_timeout`, `timeout_memo_refusal`, or null; cumulative `spillAclRetryReturnedTimeouts` and `spillAclTimeoutMemoRefusals` count terminal failed publications. See [Windows spill diagnostics](/troubleshooting/windows-memory/) for process-local semantics. Raw errors and paths are never returned. | — | | `POST /api/system/restart` | Begin a drain-aware process restart without removing client injection | Returns 202; repeated calls report the existing drain | | `POST /api/stop` | Stop the service, restore native Codex, remove managed Grok injection, and drain the proxy | 409 service ownership conflict; 409 `respawnable_service` when a Windows Task Scheduler wrapper could respawn the proxy and the caller is not `ocx stop` (nothing is changed); 409 when the installed manager refuses to stop; 409 `service_state_unknown` when the Task Scheduler state cannot be read (nothing is changed; repair the query and retry) | | `GET /api/system/codex-app-server` | Report whether running Codex app-servers predate the current model catalog | — | diff --git a/docs-site/src/content/docs/troubleshooting/windows-memory.md b/docs-site/src/content/docs/troubleshooting/windows-memory.md index cea5b3c233..10bc3799cc 100644 --- a/docs-site/src/content/docs/troubleshooting/windows-memory.md +++ b/docs-site/src/content/docs/troubleshooting/windows-memory.md @@ -57,7 +57,22 @@ runtime the leak itself remains an upstream problem: time show whether failures are accumulating or recovering in the same process. The last failure is a fixed privacy-safe class such as `EACCES`, `ENOSPC`, `ETIMEDOUT`, or `EACLRETRYEXHAUSTED`; raw error messages and filesystem paths - are never returned. These diagnostics stay on the authenticated management + are never returned. `spillLastWriteFailureOrigin` adds a fixed origin or null: + `retry_returned_timeout` means the existing second spill attempt returned a + timeout; `timeout_memo_refusal` means the ACL helper refused through its + remembered timeout state. Other failures use null. The cumulative + `spillAclRetryReturnedTimeouts` and `spillAclTimeoutMemoRefusals` count terminal + failed publications, not individual ACL commands or transient first attempts. + Success clears the failure streak but retains the last failure fields and + cumulative counts; a later unrelated failure sets the last origin to null. + These values are process-local, so compare snapshots from the same process. + Neither origin identifies an OS command: the attempt budget can expire before + a command starts, and an optional compliance inspection can run before a memo + refusal. A separate process succeeding does not prove that the live process's + memo recovered. These observations do not add retries, clear memos, weaken + required ACLs, or automatically restart the service. + + These diagnostics stay on the authenticated management endpoint and are intentionally absent from `/healthz`, which remains a liveness signal. The dashboard's **Memory observability** card renders the memory and continuation-size fields from this endpoint and offers a confirm-gated diff --git a/src/config.ts b/src/config.ts index 728b3969ea..d5ef05c33f 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,5 +1,5 @@ import { createHash } from "node:crypto"; -import { chmodSync, constants as fsConstants, copyFileSync, existsSync, linkSync, mkdirSync, readFileSync, truncateSync, unlinkSync, writeFileSync } from "node:fs"; +import { chmodSync, constants as fsConstants, copyFileSync, existsSync, linkSync, lstatSync, mkdirSync, readFileSync, truncateSync, unlinkSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { Database } from "bun:sqlite"; import * as z from "zod/v4"; @@ -123,6 +123,7 @@ export { type AtomicWriteIO, } from "./config/atomic-write"; import { getConfigDir, getConfigPath, hardenConfigDir } from "./config/paths"; +import { InitialConfigPublicationError, publishInitialConfigNoReplace, type InitialConfigPublicationIO } from "./config/initialize"; import { describeProxyForLog, readWindowsSystemProxy, @@ -2811,6 +2812,16 @@ export function readConfigDiagnostics(): ConfigDiagnostics { return readConfigFileSnapshot().diagnostics; } +/** Read-only init preflight. Occupied unsafe entries are never treated as absence. */ +export function observeInitialConfigState(): "missing" | "exists" | "invalid" { + try { + if (!lstatSync(getConfigPath()).isFile()) return "invalid"; + } catch (error) { + return isMissingPathError(error) ? "missing" : "invalid"; + } + return readConfigFileSnapshot().diagnostics.source === "file" ? "exists" : "invalid"; +} + /** * The persisted config, plus a digest of the EXACT bytes it was parsed from. * @@ -3123,6 +3134,44 @@ function persistConfigUnlocked(config: OcxConfig): boolean { return true; } +export type PersistedConfigInitializationOutcome = "created" | "exists" | "invalid"; + +/** Initialize only a missing config; ordinary explicit updates still use saveConfig. */ +export function initializePersistedConfigIfMissing( + config: OcxConfig, + io?: Partial, +): PersistedConfigInitializationOutcome { + assertNotRealHomeUnderTest(getConfigDir()); + const before = observeInitialConfigState(); + if (before !== "missing") return before; + let published = false; + try { + const persisted = withConfigMutationLockSync((): OcxConfig | "exists" | "invalid" => { + const current = observeInitialConfigState(); + if (current !== "missing") return current; + const projected = projectCustomModelCatalogMigration(undefined, projectConfigRebaseProvenance(config)); + if (!validateConfigCandidate(projected).ok) throw new Error("Initial configuration is invalid."); + if (!publishInitialConfigNoReplace(getConfigPath(), JSON.stringify(projected, null, 2) + "\n", io)) { + return observeInitialConfigState() === "exists" ? "exists" : "invalid"; + } + published = true; + recordOwnedConfigPath(getConfigDir(), getConfigPath()); + bumpGenerationForCooperatingConfigWrite(); + return projected; + }); + if (typeof persisted === "string") return persisted; + adoptCustomModelCatalogMigration(config, persisted); + if (persisted.configRebaseProvenance === undefined) delete config.configRebaseProvenance; + else config.configRebaseProvenance = structuredClone(persisted.configRebaseProvenance); + clearPendingConfigTopLevelDeletions(config); + refreshUserCostOverlays(persisted); + return "created"; + } catch (cause) { + if (published) throw new InitialConfigPublicationError("published", false, false, { cause }); + throw cause; + } +} + /** Persist `config` to config.json under the config-mutation lock. */ export function saveConfig(config: OcxConfig): void { // Keep the real-home assertion ahead of even lock-directory preparation. diff --git a/src/config/initialize.ts b/src/config/initialize.ts new file mode 100644 index 0000000000..864b09b036 --- /dev/null +++ b/src/config/initialize.ts @@ -0,0 +1,132 @@ +import { + closeSync, constants, fchmodSync, fstatSync, linkSync, lstatSync, + openSync, unlinkSync, writeFileSync, +} from "node:fs"; +import { dirname } from "node:path"; +import { assertNotRealHomeUnderTest } from "../lib/test-home-guard"; +import { forgetEphemeralSecretPath, hardenSecretPath } from "../lib/windows-secret-acl"; +import { isMissingPathError, nextAtomicTempSequence } from "./atomic-write"; + +type PublicationState = "not-published" | "published" | "uncertain"; + +/** Messages contain no candidate bytes or raw filesystem error text. */ +export class InitialConfigPublicationError extends Error { + constructor( + readonly publication: PublicationState, + readonly residualTemp: boolean, + readonly hardLinkUnavailable: boolean, + options?: ErrorOptions, + ) { + super(hardLinkUnavailable + ? "Initial config requires hard-link publication; the filesystem or its permissions denied it." + : "Initial config publication did not finish.", options); + this.name = "InitialConfigPublicationError"; + } +} + +/** Narrow fault boundary; publication must be a single link operation. */ +export interface InitialConfigPublicationIO { + harden(fd: number, temp: string, target: string): void; + write(fd: number, bytes: string): void; + link(temp: string, target: string): void; + unlink(temp: string): void; + close(fd: number): void; +} + +function hardenInitialConfig(fd: number, temp: string, target: string): void { + if (process.platform === "win32") { + hardenSecretPath(temp, { required: true, timeoutMemoKey: target }); + } else { + fchmodSync(fd, 0o600); + } +} + +function identifiesDescriptor(fd: number, path: string): boolean { + const opened = fstatSync(fd); + const entry = lstatSync(path); + return opened.isFile() && entry.isFile() + && opened.dev === entry.dev && opened.ino === entry.ino; +} + +function verifyPrivateTemp(fd: number, temp: string): void { + if (!identifiesDescriptor(fd, temp) + || (process.platform !== "win32" && (fstatSync(fd).mode & 0o777) !== 0o600)) { + throw new Error("Initial config temporary file identity or permissions changed."); + } +} + +function removeOwnedTemp(fd: number, temp: string, unlink: (path: string) => void): boolean { + for (let attempt = 0; attempt < 2; attempt++) { + try { + if (!identifiesDescriptor(fd, temp)) return false; + unlink(temp); + forgetEphemeralSecretPath(temp); + return true; + } catch (error) { + if (isMissingPathError(error)) { + forgetEphemeralSecretPath(temp); + return true; + } + } + } + return false; +} + +/** + * Publish complete bytes without replacing any entry at target. Never truncate: + * even an error from link can mean a remote filesystem already published the inode. + * Cleanup removes only our temporary name, never the target or another inode. + */ +export function publishInitialConfigNoReplace( + target: string, + bytes: string, + io: Partial = {}, +): boolean { + assertNotRealHomeUnderTest(dirname(target)); + const temp = `${target}.ocx.${process.pid}.${nextAtomicTempSequence()}.tmp`; + let fd: number | undefined; + let publication: PublicationState = "not-published"; + let collided = false; + let failure: unknown; + let failed = false; + let hardLinkUnavailable = false; + let residualTemp = false; + try { + fd = openSync(temp, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL, 0o600); + (io.harden ?? hardenInitialConfig)(fd, temp, target); + verifyPrivateTemp(fd, temp); + (io.write ?? ((descriptor: number, value: string) => writeFileSync(descriptor, value, { encoding: "utf8" })))(fd, bytes); + verifyPrivateTemp(fd, temp); + try { + publication = "uncertain"; + (io.link ?? linkSync)(temp, target); + publication = "published"; + } catch (error) { + const code = (error as NodeJS.ErrnoException | undefined)?.code; + // EEXIST normally means a competitor won. A shared target means our + // publication may nevertheless have happened (e.g. a remote FS retry). + if (code === "EEXIST" && !identifiesDescriptor(fd, target)) collided = true; + else { + hardLinkUnavailable = ["EOPNOTSUPP", "ENOTSUP", "ENOSYS", "EXDEV", "EPERM"].includes(code ?? ""); + throw error; + } + } + if (!collided && !identifiesDescriptor(fd, target)) { + throw new Error("Initial config published target identity changed."); + } + } catch (error) { + failed = true; + failure = error; + } finally { + if (fd !== undefined) { + // Unlink-only cleanup preserves all bytes if another name shares this inode. + residualTemp = !removeOwnedTemp(fd, temp, io.unlink ?? unlinkSync); + try { (io.close ?? closeSync)(fd); } + catch (error) { if (!failed) failure = error; failed = true; } + } + } + if (failed || residualTemp) { + throw new InitialConfigPublicationError(publication, residualTemp, hardLinkUnavailable, { cause: failure }); + } + return !collided; +} diff --git a/src/lib/windows-secret-acl.ts b/src/lib/windows-secret-acl.ts index dbf1f06b79..dc8bb5749f 100644 --- a/src/lib/windows-secret-acl.ts +++ b/src/lib/windows-secret-acl.ts @@ -715,18 +715,22 @@ function sanitizedAclError(diagnostics: string, cause: unknown): NodeJS.ErrnoExc return error; } -function previousTimeoutError(retryConsumed: boolean): NodeJS.ErrnoException { +type TimeoutMemoRefusalError = NodeJS.ErrnoException & { + aclFailureOrigin: "timeout_memo_refusal"; +}; + +function previousTimeoutError(retryConsumed: boolean): TimeoutMemoRefusalError { if (retryConsumed) { const error = new Error( "ACL hardening skipped — the previous timeout recovery was already consumed", ) as NodeJS.ErrnoException; error.code = "EACLRETRYEXHAUSTED"; - return error; + return Object.assign(error, { aclFailureOrigin: "timeout_memo_refusal" as const }); } - return sanitizedAclError( + return Object.assign(sanitizedAclError( "ACL hardening skipped — previous attempt timed out", Object.assign(new Error("timeout"), { code: "ETIMEDOUT" }), - ); + ), { aclFailureOrigin: "timeout_memo_refusal" as const }); } /** Consume, but never reset, the single explicit recovery attempt for this key. */ diff --git a/src/responses/state.ts b/src/responses/state.ts index e581653725..f9195196a2 100644 --- a/src/responses/state.ts +++ b/src/responses/state.ts @@ -169,7 +169,10 @@ async function snapshotOnDiskMatches(path: string, payload: string, payloadBytes return false; } } -const spillCounters = { writes: 0, writeFailures: 0, readFailures: 0 }; +const spillCounters = { + writes: 0, writeFailures: 0, readFailures: 0, + aclRetryReturnedTimeouts: 0, aclTimeoutMemoRefusals: 0, +}; export type ResponseSpillWriteFailureCode = | "EACLRETRYEXHAUSTED" @@ -184,9 +187,14 @@ export type ResponseSpillWriteFailureCode = export type ResponseSpillWriteStatus = "initial" | "healthy" | "degraded"; +export type ResponseSpillWriteFailureOrigin = + | "retry_returned_timeout" + | "timeout_memo_refusal"; + interface ResponseSpillWriteHealth { consecutiveFailures: number; lastFailureCode: ResponseSpillWriteFailureCode | null; + lastFailureOrigin: ResponseSpillWriteFailureOrigin | null; lastFailureAt: number | null; lastSuccessAt: number | null; } @@ -194,6 +202,7 @@ interface ResponseSpillWriteHealth { const spillWriteHealth: ResponseSpillWriteHealth = { consecutiveFailures: 0, lastFailureCode: null, + lastFailureOrigin: null, lastFailureAt: null, lastSuccessAt: null, }; @@ -226,6 +235,20 @@ function classifySpillWriteFailure(error: unknown): ResponseSpillWriteFailureCod return "EUNKNOWN"; } +/** The spill writer preserves ACL errors in cause; only a fixed memo marker is diagnostic. */ +function spillAclMemoRefusalOrigin(error: unknown): "timeout_memo_refusal" | null { + let cursor = error; + for (let depth = 0; depth < 4 && cursor && typeof cursor === "object"; depth += 1) { + const record = cursor as { code?: unknown; aclFailureOrigin?: unknown; cause?: unknown }; + if ((record.code === "ETIMEDOUT" || record.code === "EACLRETRYEXHAUSTED") + && record.aclFailureOrigin === "timeout_memo_refusal") { + return "timeout_memo_refusal"; + } + cursor = record.cause; + } + return null; +} + function noteSpillWriteSuccess(): void { spillCounters.writes += 1; spillWriteHealth.consecutiveFailures = 0; @@ -235,11 +258,20 @@ function noteSpillWriteSuccess(): void { function noteSpillWriteFailure( error: unknown, override?: ResponseSpillWriteFailureCode, + retryOrigin: ResponseSpillWriteFailureOrigin | null = null, ): void { + const code = override ?? classifySpillWriteFailure(error); + const origin = code === "ETIMEDOUT" || code === "EACLRETRYEXHAUSTED" + ? spillAclMemoRefusalOrigin(error) ?? retryOrigin + : null; spillCounters.writeFailures += 1; spillWriteHealth.consecutiveFailures += 1; - spillWriteHealth.lastFailureCode = override ?? classifySpillWriteFailure(error); + spillWriteHealth.lastFailureCode = code; + spillWriteHealth.lastFailureOrigin = origin; spillWriteHealth.lastFailureAt = now(); + // Count terminal publications, not ACL calls or a transient first attempt. + if (origin === "retry_returned_timeout") spillCounters.aclRetryReturnedTimeouts += 1; + else if (origin === "timeout_memo_refusal") spillCounters.aclTimeoutMemoRefusals += 1; } /** * Admission-boundary observability (test-visible). directSpills: oversized @@ -418,6 +450,7 @@ async function runPendingResponseSpill(job: PendingResponseSpill): Promise const candidate = job.candidate; let ref: ResponseSpillRef | null = null; let exhaustedAclRetry = false; + let aclRetryFailureOrigin: ResponseSpillWriteFailureOrigin | null = null; try { const state = spillPayloadForResident(candidate); try { @@ -437,6 +470,9 @@ async function runPendingResponseSpill(job: PendingResponseSpill): Promise }); } catch (retryError) { exhaustedAclRetry = isAclTimeout(retryError); + // A returned timeout can also mean an exhausted budget before the next OS command. + aclRetryFailureOrigin = spillAclMemoRefusalOrigin(retryError) + ?? (exhaustedAclRetry ? "retry_returned_timeout" : null); throw retryError; } } @@ -460,7 +496,7 @@ async function runPendingResponseSpill(job: PendingResponseSpill): Promise } catch (error) { if (ref) deleteResponseSpill(ref); if (states.get(job.id) === candidate && !job.cancelled) { - noteSpillWriteFailure(error, exhaustedAclRetry ? "EACLRETRYEXHAUSTED" : undefined); + noteSpillWriteFailure(error, exhaustedAclRetry ? "EACLRETRYEXHAUSTED" : undefined, aclRetryFailureOrigin); replaceWithSpillFailure(job.id, candidate); deferSupersededSpill(job.supersededSpill); } @@ -2188,6 +2224,9 @@ export interface ResponseStateMetrics { spillWriteStatus: ResponseSpillWriteStatus; spillWriteConsecutiveFailures: number; spillLastWriteFailureCode: ResponseSpillWriteFailureCode | null; + spillLastWriteFailureOrigin: ResponseSpillWriteFailureOrigin | null; + spillAclRetryReturnedTimeouts: number; + spillAclTimeoutMemoRefusals: number; spillLastWriteFailureAt: number | null; spillLastWriteSuccessAt: number | null; spillReadFailures: number; @@ -2240,6 +2279,9 @@ export function responseStateMetrics(): ResponseStateMetrics { : "initial", spillWriteConsecutiveFailures: spillWriteHealth.consecutiveFailures, spillLastWriteFailureCode: spillWriteHealth.lastFailureCode, + spillLastWriteFailureOrigin: spillWriteHealth.lastFailureOrigin, + spillAclRetryReturnedTimeouts: spillCounters.aclRetryReturnedTimeouts, + spillAclTimeoutMemoRefusals: spillCounters.aclTimeoutMemoRefusals, spillLastWriteFailureAt: spillWriteHealth.lastFailureAt, spillLastWriteSuccessAt: spillWriteHealth.lastSuccessAt, spillReadFailures: spillCounters.readFailures, @@ -2360,8 +2402,11 @@ export function clearResponseStateMemoryForTests(): void { spillCounters.writes = 0; spillCounters.writeFailures = 0; spillCounters.readFailures = 0; + spillCounters.aclRetryReturnedTimeouts = 0; + spillCounters.aclTimeoutMemoRefusals = 0; spillWriteHealth.consecutiveFailures = 0; spillWriteHealth.lastFailureCode = null; + spillWriteHealth.lastFailureOrigin = null; spillWriteHealth.lastFailureAt = null; spillWriteHealth.lastSuccessAt = null; replayScopeMismatchDrops = 0; diff --git a/tests/config/config-mutation-lock.test.ts b/tests/config/config-mutation-lock.test.ts index c0b37c28b4..06a18dd99d 100644 --- a/tests/config/config-mutation-lock.test.ts +++ b/tests/config/config-mutation-lock.test.ts @@ -1,8 +1,10 @@ import { afterEach, beforeEach, expect, test } from "bun:test"; -import { existsSync, mkdtempSync, writeFileSync } from "node:fs"; +import { closeSync, existsSync, linkSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, renameSync, symlinkSync, unlinkSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { pathToFileURL } from "node:url"; -import { ConfigMutationLockError, loadConfig, saveConfig, withConfigMutationLockSync } from "../../src/config"; +import { ConfigMutationLockError, deleteConfigTopLevelKey, getConfigPath, initializePersistedConfigIfMissing, loadConfig, observeInitialConfigState, readConfigGeneration, saveConfig, withConfigMutationLockSync } from "../../src/config"; +import { InitialConfigPublicationError, publishInitialConfigNoReplace } from "../../src/config/initialize"; +import { nextAtomicTempSequence } from "../../src/config/atomic-write"; import { CodexCredentialRefreshLockTimeoutError, getCodexAccountCredential, saveCodexAccountCredential } from "../../src/codex/account-store"; import type { OcxConfig } from "../../src/types"; import { ManagementRequest, managementHeaders } from "../helpers/management-auth"; @@ -13,7 +15,13 @@ let testRoot = ""; let previousOpencodexHome: string | undefined; function config(port = 10100): OcxConfig { - return { port, providers: {}, defaultProvider: "openai" }; + // Initial publication validates the candidate before reaching the filesystem; + // unlike a replacing save, it cannot accept a dangling default provider. + return { + port, + providers: { openai: { adapter: "openai-chat", baseUrl: "https://example.test/v1" } }, + defaultProvider: "openai", + }; } async function waitForPath(path: string): Promise { @@ -79,6 +87,11 @@ test("a live cross-process holder is not stolen and runtime writers fail immedia } const startedAt = performance.now(); expect(() => saveConfig(config(20200))).toThrow(ConfigMutationLockError); + // A busy initializer must not steal the holder even when its target is absent. + unlinkSync(getConfigPath()); + expect(() => initializePersistedConfigIfMissing(config(20200))).toThrow(ConfigMutationLockError); + expect(existsSync(getConfigPath())).toBe(false); + writeFileSync(getConfigPath(), JSON.stringify(config())); expect(() => saveCodexAccountCredential("busy-account", { accessToken: "busy-access", refreshToken: "busy-refresh", @@ -139,6 +152,194 @@ test("a throwing mutation releases the lock and leaves writers available", () => expect(loadConfig().port).toBe(50500); }); +const initTemps = () => readdirSync(testRoot).filter(name => name.includes(".ocx.") && name.endsWith(".tmp")); + +test("initial publication rejects a missing default provider before writing candidate bytes", () => { + let wrote = false; + expect(() => initializePersistedConfigIfMissing({ ...config(), providers: {} }, { + write() { wrote = true; }, + })).toThrow("Initial configuration is invalid."); + expect(wrote).toBe(false); + expect(existsSync(getConfigPath())).toBe(false); +}); + +test("initial creation keeps candidate values and existing bytes; the explicit saver still updates", () => { + const candidate = { ...config(21001), operatorNote: "keep unknown fields" }; + expect(initializePersistedConfigIfMissing(candidate)).toBe("created"); + expect(JSON.parse(readFileSync(getConfigPath(), "utf8"))).toEqual(candidate); + if (process.platform !== "win32") expect(lstatSync(getConfigPath()).mode & 0o777).toBe(0o600); + expect(readConfigGeneration()).toMatchObject({ generation: { value: 1 } }); + const bytes = readFileSync(getConfigPath(), "utf8"); + expect(initializePersistedConfigIfMissing(config(21002))).toBe("exists"); + expect(readFileSync(getConfigPath(), "utf8")).toBe(bytes); + saveConfig(config(21003)); + expect(loadConfig().port).toBe(21003); + expect(initTemps()).toEqual([]); +}); + +test.each(["", "not-json\n", '{"port":"broken"}', '\uFEFF{ "port":21002, "providers":{}, "defaultProvider":"openai", "unknown":42 }\n'])( + "init preserves occupied bytes without lock or backup creation: %j", bytes => { + writeFileSync(getConfigPath(), bytes); + const candidate = config(21001); + const original = structuredClone(candidate); + expect(initializePersistedConfigIfMissing(candidate)).toBe(bytes.startsWith("\uFEFF") ? "exists" : "invalid"); + expect(readFileSync(getConfigPath(), "utf8")).toBe(bytes); + expect(candidate).toEqual(original); + expect(readdirSync(testRoot)).toEqual(["config.json"]); + }, +); + +test("init refuses a directory and a dangling symlink without following either", () => { + mkdirSync(getConfigPath()); + expect(observeInitialConfigState()).toBe("invalid"); + expect(initializePersistedConfigIfMissing(config())).toBe("invalid"); + removeTreeWithRetry(getConfigPath()); + const absent = join(testRoot, "absent"); + symlinkSync(absent, getConfigPath(), "file"); + expect(initializePersistedConfigIfMissing(config())).toBe("invalid"); + expect(lstatSync(getConfigPath()).isSymbolicLink()).toBe(true); + expect(existsSync(absent)).toBe(false); +}); + +test("real link collision preserves the winner and does not advance generation or mutate the candidate", () => { + withConfigMutationLockSync(() => {}); + const generation = readConfigGeneration(); + const winner = JSON.stringify(config(21002)) + "\n"; + const candidate = config(21001); + const original = structuredClone(candidate); + expect(initializePersistedConfigIfMissing(candidate, { + link(temp, target) { + writeFileSync(target, winner, { flag: "wx" }); + linkSync(temp, target); + }, + })).toBe("exists"); + expect(readFileSync(getConfigPath(), "utf8")).toBe(winner); + expect(readConfigGeneration()).toEqual(generation); + expect(candidate).toEqual(original); + expect(initTemps()).toEqual([]); +}); + +test("exclusive temp collision does not remove or modify somebody else's file", () => { + const sequence = nextAtomicTempSequence() + 1; + const occupied = `${getConfigPath()}.ocx.${process.pid}.${sequence}.tmp`; + writeFileSync(occupied, "other staged bytes", { flag: "wx" }); + expect(() => publishInitialConfigNoReplace(getConfigPath(), "candidate bytes")).toThrow(InitialConfigPublicationError); + expect(readFileSync(occupied, "utf8")).toBe("other staged bytes"); + expect(existsSync(getConfigPath())).toBe(false); +}); + +test("failed hardening occurs before candidate bytes are written", () => { + let wrote = false; + expect(() => initializePersistedConfigIfMissing(config(), { + harden(_fd, temp) { + expect(readFileSync(temp, "utf8")).toBe(""); + throw new Error("ACL denied"); + }, + write() { wrote = true; }, + })).toThrow(InitialConfigPublicationError); + expect(wrote).toBe(false); + expect(existsSync(getConfigPath())).toBe(false); + expect(initTemps()).toEqual([]); +}); + +test("partial write failure removes only the unpublished temporary name", () => { + expect(() => initializePersistedConfigIfMissing(config(), { + write(fd, bytes) { writeFileSync(fd, bytes.slice(0, 10)); throw new Error("disk full"); }, + })).toThrow(InitialConfigPublicationError); + expect(existsSync(getConfigPath())).toBe(false); + expect(initTemps()).toEqual([]); +}); + +test.each(["EOPNOTSUPP", "ENOTSUP", "ENOSYS", "EXDEV", "EPERM"])("unsupported/denied link %s never falls back to replacement", code => { + try { + initializePersistedConfigIfMissing(config(), { + link() { throw Object.assign(new Error("do not print raw error"), { code }); }, + }); + throw new Error("expected link refusal"); + } catch (error) { + expect(error).toBeInstanceOf(InitialConfigPublicationError); + expect((error as InitialConfigPublicationError).hardLinkUnavailable).toBe(true); + } + expect(existsSync(getConfigPath())).toBe(false); + expect(initTemps()).toEqual([]); +}); + +test("a syscall error after a real link leaves the entire published candidate intact", () => { + const bytes = 'complete candidate bytes\n'; + expect(() => publishInitialConfigNoReplace(getConfigPath(), bytes, { + link(temp, target) { linkSync(temp, target); throw Object.assign(new Error("uncertain completion"), { code: "EIO" }); }, + })).toThrow(InitialConfigPublicationError); + expect(readFileSync(getConfigPath(), "utf8")).toBe(bytes); + expect(initTemps()).toEqual([]); +}); + +test("post-link identity failure cannot remove a concurrent replacement", () => { + expect(() => initializePersistedConfigIfMissing(config(21001), { + link(temp, target) { + linkSync(temp, target); + const replacement = join(testRoot, "replacement"); + writeFileSync(replacement, "concurrent-winner\n"); + renameSync(replacement, target); + }, + })).toThrow(InitialConfigPublicationError); + expect(readFileSync(getConfigPath(), "utf8")).toBe("concurrent-winner\n"); +}); + +test("cleanup failure retains full published/shared bytes and closes the descriptor", () => { + let closed = false; + let failure: unknown; + try { + publishInitialConfigNoReplace(getConfigPath(), "complete bytes", { + unlink() { throw new Error("sharing violation"); }, + close(fd) { closed = true; closeSync(fd); }, + }); + } catch (error) { failure = error; } + expect(failure).toMatchObject({ publication: "published", residualTemp: true }); + expect(closed).toBe(true); + expect(readFileSync(getConfigPath(), "utf8")).toBe("complete bytes"); + const temps = initTemps(); + expect(temps).toHaveLength(1); + expect(readFileSync(join(testRoot, temps[0]!), "utf8")).toBe("complete bytes"); +}); + +test("a shared unpublished inode is never scrubbed", () => { + const otherName = join(testRoot, "shared-candidate"); + expect(() => publishInitialConfigNoReplace(getConfigPath(), "candidate bytes", { + link(temp) { linkSync(temp, otherName); throw new Error("publication failed"); }, + })).toThrow(InitialConfigPublicationError); + expect(readFileSync(otherName, "utf8")).toBe("candidate bytes"); + expect(existsSync(getConfigPath())).toBe(false); + expect(initTemps()).toEqual([]); +}); + +test("a swapped temporary symlink is neither written through nor removed as our inode", () => { + const victim = join(testRoot, "victim"); + writeFileSync(victim, "untouched"); + expect(() => publishInitialConfigNoReplace(getConfigPath(), "candidate bytes", { + harden(_fd, temp) { unlinkSync(temp); symlinkSync(victim, temp, "file"); }, + })).toThrow(InitialConfigPublicationError); + expect(readFileSync(victim, "utf8")).toBe("untouched"); + expect(existsSync(getConfigPath())).toBe(false); + expect(lstatSync(join(testRoot, initTemps()[0]!)).isSymbolicLink()).toBe(true); +}); + +test("descriptor close failure cannot scrub an already published config", () => { + expect(() => publishInitialConfigNoReplace(getConfigPath(), "complete bytes", { + close(fd) { closeSync(fd); throw new Error("close failed"); }, + })).toThrow(InitialConfigPublicationError); + expect(readFileSync(getConfigPath(), "utf8")).toBe("complete bytes"); +}); + +test("successful init adopts deletion provenance before a subsequent explicit save", () => { + const candidate = config(); + deleteConfigTopLevelKey(candidate, "hostname"); + expect(initializePersistedConfigIfMissing(candidate)).toBe("created"); + expect(candidate.configRebaseProvenance).toEqual({ version: 1, deletedTopLevelKeys: ["hostname"] }); + candidate.hostname = "127.0.0.1"; + saveConfig(candidate); + expect(JSON.parse(readFileSync(getConfigPath(), "utf8")).hostname).toBe("127.0.0.1"); +}); + test("management API maps config mutation lock contention to retryable 503", async () => { saveConfig(config()); const readyPath = join(testRoot, "mgmt-holder-ready"); diff --git a/tests/responses/continuation-dedup.test.ts b/tests/responses/continuation-dedup.test.ts index 5e9efd2a74..c5b55b10fd 100644 --- a/tests/responses/continuation-dedup.test.ts +++ b/tests/responses/continuation-dedup.test.ts @@ -310,9 +310,17 @@ describe("replay overlap: contracts held elsewhere", () => { }); test("the skip counter is not published on the memory surface", () => { - // /api/system/memory pins exactly 17 privacy-reviewed scalar fields. The five - // spill-health additions are enums, counters, or timestamps — never error text. - expect(Object.keys(responseStateMetrics())).toHaveLength(17); + // Pin the reviewed public fields, not just their count: no replay-skip + // counter or arbitrary diagnostic may replace a permitted field unnoticed. + expect(Object.keys(responseStateMetrics()).sort()).toEqual([ + "count", "residentCount", "spillStubCount", "tombstoneCount", + "totalBytes", "spillPayloadBytes", "largestBytes", "oldestAgeMs", + "spillWrites", "spillWriteFailures", "spillReadFailures", + "spillWriteStatus", "spillWriteConsecutiveFailures", + "spillLastWriteFailureCode", "spillLastWriteFailureOrigin", + "spillAclRetryReturnedTimeouts", "spillAclTimeoutMemoRefusals", + "spillLastWriteFailureAt", "spillLastWriteSuccessAt", "replayScopeMismatchDrops", + ].sort()); }); test("clearing state for tests resets the skip counter", () => { diff --git a/tests/responses/responses-state.test.ts b/tests/responses/responses-state.test.ts index 8ba5da86b2..464642cda7 100644 --- a/tests/responses/responses-state.test.ts +++ b/tests/responses/responses-state.test.ts @@ -1079,6 +1079,9 @@ describe("Responses previous_response_id state", () => { spillWriteFailures: 0, spillWriteStatus: "healthy", spillWriteConsecutiveFailures: 0, + spillLastWriteFailureOrigin: null, + spillAclRetryReturnedTimeouts: 0, + spillAclTimeoutMemoRefusals: 0, }); }); @@ -1112,6 +1115,9 @@ describe("Responses previous_response_id state", () => { spillWriteConsecutiveFailures: 1, spillLastWriteFailureCode: "EACLRETRYEXHAUSTED", spillLastWriteSuccessAt: null, + spillLastWriteFailureOrigin: "retry_returned_timeout", + spillAclRetryReturnedTimeouts: 1, + spillAclTimeoutMemoRefusals: 0, }); expect(metrics.spillLastWriteFailureAt).toBeGreaterThanOrEqual(0); @@ -1127,11 +1133,65 @@ describe("Responses previous_response_id state", () => { spillWriteStatus: "healthy", spillWriteConsecutiveFailures: 0, spillLastWriteFailureCode: "EACLRETRYEXHAUSTED", + spillLastWriteFailureOrigin: "retry_returned_timeout", + spillAclRetryReturnedTimeouts: 1, + spillAclTimeoutMemoRefusals: 0, }); expect(typeof recovered.spillLastWriteSuccessAt === "number" && recovered.spillLastWriteSuccessAt >= (recovered.spillLastWriteFailureAt ?? 0)).toBe(true); }); + test("Windows stable-directory memo refusals stay distinct after the runner becomes healthy", async () => { + forceWindowsAclLane(); + const previousVerify = process.env.OPENCODEX_ACL_VERIFY_EXISTING; + delete process.env.OPENCODEX_ACL_VERIFY_EXISTING; + let clock = 0; + let grantCalls = 0; + setNowForTests(() => clock); + setResponseSpillNowForTests(() => clock); + setResponseSpillAsyncAclAttemptBudgetForTests(100); + setResponseStateByteCapForTests(1_024); + const spillDir = responseSpillDirectory(); + let healthy = false; + setAsyncIcaclsRunnerForTests(async args => { + if (args[0] !== spillDir) return ICACLS_OK; + if (args.includes("/grant:r")) grantCalls += 1; + if (healthy) return ICACLS_OK; + clock += 100; + return { success: false, exitCode: null, timedOut: true, stdout: "private-acl-output" }; + }); + try { + rememberLarge("resp_stable_timeout", "x".repeat(8_000)); + await flushPendingResponseSpillsForTests(); + expect(responseStateMetrics()).toMatchObject({ + spillWrites: 0, spillWriteFailures: 1, + spillLastWriteFailureCode: "EACLRETRYEXHAUSTED", + spillLastWriteFailureOrigin: "retry_returned_timeout", + spillAclRetryReturnedTimeouts: 1, spillAclTimeoutMemoRefusals: 0, + }); + expect(grantCalls).toBe(2); + healthy = true; // Same stable directory and process; no memo reset between jobs. + for (let refusal = 1; refusal <= 2; refusal += 1) { + rememberLarge(`resp_stable_refusal_${refusal}`, "y".repeat(8_000)); + await flushPendingResponseSpillsForTests(); + expect(responseStateMetrics()).toMatchObject({ + spillWrites: 0, spillWriteFailures: 1 + refusal, + spillWriteStatus: "degraded", spillWriteConsecutiveFailures: 1 + refusal, + spillLastWriteFailureCode: "EACLRETRYEXHAUSTED", + spillLastWriteFailureOrigin: "timeout_memo_refusal", + spillAclRetryReturnedTimeouts: 1, spillAclTimeoutMemoRefusals: refusal, + spillLastWriteSuccessAt: null, spillStubCount: 0, + }); + expect(grantCalls).toBe(2); + expect(spillFileNames(home)).toHaveLength(0); + expect(spillTempNames(home)).toHaveLength(0); + } + } finally { + if (previousVerify === undefined) delete process.env.OPENCODEX_ACL_VERIFY_EXISTING; + else process.env.OPENCODEX_ACL_VERIFY_EXISTING = previousVerify; + } + }); + test("Windows async spill attempts share one bounded ACL budget across every harden", async () => { forceWindowsAclLane(); let clock = 0; @@ -2591,6 +2651,7 @@ describe("Responses previous_response_id state", () => { const { spillWriteStatus, spillLastWriteFailureCode, + spillLastWriteFailureOrigin, spillLastWriteFailureAt, spillLastWriteSuccessAt, ...numericMetrics @@ -2599,6 +2660,7 @@ describe("Responses previous_response_id state", () => { .every(value => typeof value === "number" && Number.isFinite(value))).toBe(true); expect(spillWriteStatus).toBe("healthy"); expect(spillLastWriteFailureCode).toBeNull(); + expect(spillLastWriteFailureOrigin).toBeNull(); expect(spillLastWriteFailureAt).toBeNull(); expect(typeof spillLastWriteSuccessAt === "number" && Number.isFinite(spillLastWriteSuccessAt)).toBe(true); const serialized = JSON.stringify(metrics); @@ -3359,6 +3421,9 @@ describe("Responses previous_response_id state", () => { spillWriteStatus: "initial", spillWriteConsecutiveFailures: 0, spillLastWriteFailureCode: null, + spillLastWriteFailureOrigin: null, + spillAclRetryReturnedTimeouts: 0, + spillAclTimeoutMemoRefusals: 0, spillLastWriteFailureAt: null, spillLastWriteSuccessAt: null, spillReadFailures: 0, @@ -3366,6 +3431,53 @@ describe("Responses previous_response_id state", () => { }); }); + test("spill failure origin decoding stays bounded, closed and paired with the effective code", () => { + setResponseStateByteCapForTests(1_024); + const memoError = Object.assign(new Error("private-path-and-payload"), { + code: "ETIMEDOUT", aclFailureOrigin: "timeout_memo_refusal", + }); + const cycle: { code: string; cause?: unknown; aclFailureOrigin: string } = { + code: "ETIMEDOUT", aclFailureOrigin: "private-origin", + }; + cycle.cause = cycle; + const cases = [ + { error: new Error("wrapper", { cause: memoError }), code: "ETIMEDOUT", origin: "timeout_memo_refusal" }, + { error: Object.assign(new Error("denied", { cause: memoError }), { code: "EACCES" }), code: "EACCES", origin: null }, + { error: { code: "EACLRETRYEXHAUSTED" }, code: "EACLRETRYEXHAUSTED", origin: null }, + { error: { code: "ETIMEDOUT", aclFailureOrigin: "private-origin" }, code: "ETIMEDOUT", origin: null }, + { error: { code: "ETIMEDOUT", aclFailureOrigin: ["timeout_memo_refusal"] }, code: "ETIMEDOUT", origin: null }, + { error: cycle, code: "ETIMEDOUT", origin: null }, + // Including the writer's wrapper, the marker is beyond the four-object scan. + { error: { code: "ETIMEDOUT", cause: { cause: { cause: memoError } } }, code: "ETIMEDOUT", origin: null }, + ]; + cases.forEach(({ error, code, origin }, index) => { + setSpillIoForTest({ write: () => { throw error; } }); + rememberLarge(`resp_private_origin_${index}`, "private-content".repeat(1_000)); + const metrics = responseStateMetrics(); + expect(metrics).toMatchObject({ + spillWriteFailures: index + 1, + spillLastWriteFailureCode: code, + spillLastWriteFailureOrigin: origin, + spillAclRetryReturnedTimeouts: 0, spillAclTimeoutMemoRefusals: 1, + }); + const serialized = JSON.stringify(metrics); + for (const privateValue of ["private-path-and-payload", "private-origin", "private-content", "resp_private_origin", home]) { + expect(serialized).not.toContain(privateValue); + } + }); + setSpillIoForTest(null); + rememberLarge("resp_after_origin_failures", "healthy".repeat(1_500)); + expect(responseStateMetrics()).toMatchObject({ + spillWriteStatus: "healthy", spillWriteConsecutiveFailures: 0, + spillLastWriteFailureCode: "ETIMEDOUT", spillLastWriteFailureOrigin: null, + spillAclRetryReturnedTimeouts: 0, spillAclTimeoutMemoRefusals: 1, + }); + clearResponseStateMemoryForTests(); + expect(responseStateMetrics()).toMatchObject({ + spillLastWriteFailureOrigin: null, spillAclRetryReturnedTimeouts: 0, spillAclTimeoutMemoRefusals: 0, + }); + }); + test("a successful spill clears a repeated failure streak without erasing the last failure", () => { const realNow = Date.now; let clock = 1_000; @@ -3471,6 +3583,9 @@ describe("Responses previous_response_id state", () => { spillWriteStatus: "initial", spillWriteConsecutiveFailures: 0, spillLastWriteFailureCode: null, + spillLastWriteFailureOrigin: null, + spillAclRetryReturnedTimeouts: 0, + spillAclTimeoutMemoRefusals: 0, spillLastWriteFailureAt: null, spillLastWriteSuccessAt: null, spillReadFailures: 0, diff --git a/tests/server/memory-watchdog.test.ts b/tests/server/memory-watchdog.test.ts index 80c38dd020..918503456c 100644 --- a/tests/server/memory-watchdog.test.ts +++ b/tests/server/memory-watchdog.test.ts @@ -196,6 +196,9 @@ describe("GET /api/system/memory", () => { spillWriteStatus: "initial" | "healthy" | "degraded"; spillWriteConsecutiveFailures: number; spillLastWriteFailureCode: string | null; + spillLastWriteFailureOrigin: string | null; + spillAclRetryReturnedTimeouts: number; + spillAclTimeoutMemoRefusals: number; spillLastWriteFailureAt: number | null; spillLastWriteSuccessAt: number | null; replayScopeMismatchDrops: number; @@ -221,11 +224,12 @@ describe("GET /api/system/memory", () => { // responseState is a scalar-only continuation-store attribution block: numbers plus fixed // enum/null fields (no paths, messages, tokens, or account identifiers). // The exact count is pinned on purpose: a new field must be reviewed for privacy safety - // before it reaches this surface. 17 after #3522 added spill-write health diagnostics. - expect(Object.keys(body.responseState)).toHaveLength(17); + // before it reaches this surface. 20 after #3522 added failure origins and counters. + expect(Object.keys(body.responseState)).toHaveLength(20); const { spillWriteStatus, spillLastWriteFailureCode, + spillLastWriteFailureOrigin, spillLastWriteFailureAt, spillLastWriteSuccessAt, ...numericResponseState @@ -233,6 +237,9 @@ describe("GET /api/system/memory", () => { expect(Object.values(numericResponseState) .every(value => typeof value === "number" && Number.isFinite(value))).toBe(true); expect(["initial", "healthy", "degraded"]).toContain(spillWriteStatus); + expect(spillLastWriteFailureOrigin === null || [ + "retry_returned_timeout", "timeout_memo_refusal", + ].includes(spillLastWriteFailureOrigin)).toBe(true); expect(spillLastWriteFailureCode === null || [ "EACLRETRYEXHAUSTED", "ETIMEDOUT", "EACCES", "ENOSPC", "EFBIG", "EIO", "ECAPACITY", "ELOOP", "EUNKNOWN", diff --git a/tests/windows/windows-secret-acl.test.ts b/tests/windows/windows-secret-acl.test.ts index dddbd38084..aa011bb516 100644 --- a/tests/windows/windows-secret-acl.test.ts +++ b/tests/windows/windows-secret-acl.test.ts @@ -376,6 +376,42 @@ describe("opt-in existing ACL proof", () => { expect(calls).toEqual([[target]]); }); + test("async compliance inspection can precede a memo refusal or an existing compliant success", async () => { + const target = join(testDir, "memo-compliance.json"); + writeFileSync(target, "secret"); + let clock = 0; + setNowForTests(() => clock); + setAsyncWindowsPrincipalRunnerForTests(async () => success(`${ownerSid}\n${ownerName}\n`)); + seedIdentity(); + delete process.env.OPENCODEX_ACL_VERIFY_EXISTING; + setAsyncIcaclsRunnerForTests(async () => { + clock += 100; + return { success: false, exitCode: null, timedOut: true, stdout: "" }; + }); + try { + await expect(hardenSecretPathAsync(target, { required: true, deadlineMs: 100 })) + .rejects.toMatchObject({ code: "ETIMEDOUT" }); + await expect(hardenSecretPathAsync(target, { required: true, deadlineMs: 100, retryTimedOutOnce: true })) + .rejects.toMatchObject({ code: "ETIMEDOUT" }); + process.env.OPENCODEX_ACL_VERIFY_EXISTING = "1"; + const calls: string[][] = []; + let compliant = false; + setAsyncIcaclsRunnerForTests(async args => { + calls.push(args); + return success(compliant ? `${target} ${ownerName}:(F)\r\n` : "unverified"); + }); + await expect(hardenSecretPathAsync(target, { required: true, deadlineMs: 100 })) + .rejects.toMatchObject({ code: "EACLRETRYEXHAUSTED", aclFailureOrigin: "timeout_memo_refusal" }); + expect(calls).toEqual([[target]]); // Inspection ran, but no grant was launched. + compliant = true; + await expect(hardenSecretPathAsync(target, { required: true, deadlineMs: 100 })).resolves.toEqual({ ok: true }); + expect(calls).toEqual([[target], [target]]); + expect(timedOutSecretPathCountForTests()).toBe(1); // Existing proof did not clear the memo. + } finally { + setNowForTests(null); + } + }); + test("an inherited owner ACE falls through to the mutation sequence", () => { const target = join(testDir, "inherited.json"); writeFileSync(target, "secret"); @@ -1014,7 +1050,7 @@ describe("async hardenSecretPath (issue #612)", () => { expect(timedOutSecretPathCountForTests()).toBe(0); }); - test("the explicit timeout recovery cannot be consumed more than once", async () => { + test.each(["sync", "async"] as const)("%s timeout origin distinguishes memo refusal without another recovery", async lane => { // Pinned: this asserts recovery CARDINALITY. At the 30s default the first call would // succeed on its internal retry and the cardinality claim would never be exercised. process.env.OPENCODEX_ACL_TIMEOUT_MS = "5000"; @@ -1022,27 +1058,43 @@ describe("async hardenSecretPath (issue #612)", () => { let now = 0; let grantCalls = 0; setNowForTests(() => now); - setAsyncIcaclsRunnerForTests(async args => { + const runner = (args: string[]): IcaclsResult => { if (args.includes("/grant:r")) grantCalls += 1; now += 5_000; return timeout; - }); + }; + setIcaclsRunnerForTests(runner); + setAsyncIcaclsRunnerForTests(async args => runner(args)); + const identity = { ...ok, stdout: "S-1-5-21-1-2-3-1001\nocx-test\n" }; + setWindowsPrincipalRunnerForTests(() => identity); + setAsyncWindowsPrincipalRunnerForTests(async () => identity); + const harden = async (retryTimedOutOnce = false) => lane === "sync" + ? hardenSecretPath(target, { required: true, retryTimedOutOnce }) + : hardenSecretPathAsync(target, { required: true, retryTimedOutOnce }); - await expect(hardenSecretPathAsync(target, { required: true })).rejects.toMatchObject({ - code: "ETIMEDOUT", - }); - await expect(hardenSecretPathAsync(target, { - required: true, - retryTimedOutOnce: true, - })).rejects.toMatchObject({ code: "ETIMEDOUT" }); - const callsAfterRecovery = grantCalls; - await expect(hardenSecretPathAsync(target, { - required: true, - retryTimedOutOnce: true, - })).rejects.toMatchObject({ code: "EACLRETRYEXHAUSTED" }); - expect(grantCalls).toBe(callsAfterRecovery); - expect(grantCalls).toBe(2); - expect(timedOutSecretPathCountForTests()).toBe(1); + try { + const first = await harden().catch(error => error); + expect(first).toMatchObject({ code: "ETIMEDOUT" }); + expect(first).not.toHaveProperty("aclFailureOrigin"); + await expect(harden()).rejects.toMatchObject({ + code: "ETIMEDOUT", aclFailureOrigin: "timeout_memo_refusal", + }); + expect(grantCalls).toBe(1); + const recovery = await harden(true).catch(error => error); + expect(recovery).toMatchObject({ code: "ETIMEDOUT" }); + expect(recovery).not.toHaveProperty("aclFailureOrigin"); + const callsAfterRecovery = grantCalls; + await expect(harden(true)).rejects.toMatchObject({ + code: "EACLRETRYEXHAUSTED", aclFailureOrigin: "timeout_memo_refusal", + }); + expect(grantCalls).toBe(callsAfterRecovery); + expect(grantCalls).toBe(2); + expect(timedOutSecretPathCountForTests()).toBe(1); + } finally { + setWindowsPrincipalRunnerForTests(null); + setAsyncWindowsPrincipalRunnerForTests(null); + resetWindowsPrincipalForTests(); + } }); test("optional timeout memo does not poison a later required harden of the same path", () => {