Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs-site/src/content/docs/reference/management-api.md
Original file line number Diff line number Diff line change
Expand Up @@ -347,7 +347,7 @@ whether to star the repository.

| Method and path | Purpose | Notable errors |
| --- | --- | --- |
| `GET /api/system/memory` | Return scalar process, heap, stream, response-state, watchdog, and active-turn metrics. Response-state diagnostics include spill-write status, consecutive failures, fixed privacy-safe failure class, and last failure/success timestamps; raw errors and paths are never returned. | — |
| `GET /api/system/memory` | Return scalar process, heap, stream, response-state, watchdog, and active-turn metrics. Response-state diagnostics include spill-write status, consecutive failures, fixed privacy-safe failure class, and last failure/success timestamps. `spillLastWriteFailureOrigin` is `retry_returned_timeout`, `timeout_memo_refusal`, or null; cumulative `spillAclRetryReturnedTimeouts` and `spillAclTimeoutMemoRefusals` count terminal failed publications. See [Windows spill diagnostics](/troubleshooting/windows-memory/) for process-local semantics. Raw errors and paths are never returned. | — |
| `POST /api/system/restart` | Begin a drain-aware process restart without removing client injection | Returns 202; repeated calls report the existing drain |
| `POST /api/stop` | Stop the service, restore native Codex, remove managed Grok injection, and drain the proxy | 409 service ownership conflict; 409 `respawnable_service` when a Windows Task Scheduler wrapper could respawn the proxy and the caller is not `ocx stop` (nothing is changed); 409 when the installed manager refuses to stop; 409 `service_state_unknown` when the Task Scheduler state cannot be read (nothing is changed; repair the query and retry) |
| `GET /api/system/codex-app-server` | Report whether running Codex app-servers predate the current model catalog | — |
Expand Down
17 changes: 16 additions & 1 deletion docs-site/src/content/docs/troubleshooting/windows-memory.md
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,22 @@ runtime the leak itself remains an upstream problem:
time show whether failures are accumulating or recovering in the same process.
The last failure is a fixed privacy-safe class such as `EACCES`, `ENOSPC`,
`ETIMEDOUT`, or `EACLRETRYEXHAUSTED`; raw error messages and filesystem paths
are never returned. These diagnostics stay on the authenticated management
are never returned. `spillLastWriteFailureOrigin` adds a fixed origin or null:
`retry_returned_timeout` means the existing second spill attempt returned a
timeout; `timeout_memo_refusal` means the ACL helper refused through its
remembered timeout state. Other failures use null. The cumulative
`spillAclRetryReturnedTimeouts` and `spillAclTimeoutMemoRefusals` count terminal
failed publications, not individual ACL commands or transient first attempts.
Success clears the failure streak but retains the last failure fields and
cumulative counts; a later unrelated failure sets the last origin to null.
These values are process-local, so compare snapshots from the same process.
Neither origin identifies an OS command: the attempt budget can expire before
a command starts, and an optional compliance inspection can run before a memo
refusal. A separate process succeeding does not prove that the live process's
memo recovered. These observations do not add retries, clear memos, weaken
required ACLs, or automatically restart the service.

These diagnostics stay on the authenticated management
endpoint and are intentionally absent from `/healthz`, which remains a liveness
signal. The dashboard's **Memory observability** card renders the memory and
continuation-size fields from this endpoint and offers a confirm-gated
Expand Down
51 changes: 50 additions & 1 deletion src/config.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { createHash } from "node:crypto";
import { chmodSync, constants as fsConstants, copyFileSync, existsSync, linkSync, mkdirSync, readFileSync, truncateSync, unlinkSync, writeFileSync } from "node:fs";
import { chmodSync, constants as fsConstants, copyFileSync, existsSync, linkSync, lstatSync, mkdirSync, readFileSync, truncateSync, unlinkSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { Database } from "bun:sqlite";
import * as z from "zod/v4";
Expand Down Expand Up @@ -123,6 +123,7 @@ export {
type AtomicWriteIO,
} from "./config/atomic-write";
import { getConfigDir, getConfigPath, hardenConfigDir } from "./config/paths";
import { InitialConfigPublicationError, publishInitialConfigNoReplace, type InitialConfigPublicationIO } from "./config/initialize";
import {
describeProxyForLog,
readWindowsSystemProxy,
Expand Down Expand Up @@ -2811,6 +2812,16 @@ export function readConfigDiagnostics(): ConfigDiagnostics {
return readConfigFileSnapshot().diagnostics;
}

/** Read-only init preflight. Occupied unsafe entries are never treated as absence. */
export function observeInitialConfigState(): "missing" | "exists" | "invalid" {
try {
if (!lstatSync(getConfigPath()).isFile()) return "invalid";
} catch (error) {
return isMissingPathError(error) ? "missing" : "invalid";
}
return readConfigFileSnapshot().diagnostics.source === "file" ? "exists" : "invalid";
}

/**
* The persisted config, plus a digest of the EXACT bytes it was parsed from.
*
Expand Down Expand Up @@ -3123,6 +3134,44 @@ function persistConfigUnlocked(config: OcxConfig): boolean {
return true;
}

export type PersistedConfigInitializationOutcome = "created" | "exists" | "invalid";

/** Initialize only a missing config; ordinary explicit updates still use saveConfig. */
export function initializePersistedConfigIfMissing(
config: OcxConfig,
io?: Partial<InitialConfigPublicationIO>,
): PersistedConfigInitializationOutcome {
assertNotRealHomeUnderTest(getConfigDir());
const before = observeInitialConfigState();
if (before !== "missing") return before;
let published = false;
try {
const persisted = withConfigMutationLockSync((): OcxConfig | "exists" | "invalid" => {
const current = observeInitialConfigState();
if (current !== "missing") return current;
const projected = projectCustomModelCatalogMigration(undefined, projectConfigRebaseProvenance(config));
if (!validateConfigCandidate(projected).ok) throw new Error("Initial configuration is invalid.");
if (!publishInitialConfigNoReplace(getConfigPath(), JSON.stringify(projected, null, 2) + "\n", io)) {
return observeInitialConfigState() === "exists" ? "exists" : "invalid";
}
published = true;
recordOwnedConfigPath(getConfigDir(), getConfigPath());
bumpGenerationForCooperatingConfigWrite();
return projected;
});
if (typeof persisted === "string") return persisted;
adoptCustomModelCatalogMigration(config, persisted);
if (persisted.configRebaseProvenance === undefined) delete config.configRebaseProvenance;
else config.configRebaseProvenance = structuredClone(persisted.configRebaseProvenance);
clearPendingConfigTopLevelDeletions(config);
refreshUserCostOverlays(persisted);
return "created";
} catch (cause) {
if (published) throw new InitialConfigPublicationError("published", false, false, { cause });
throw cause;
}
}

/** Persist `config` to config.json under the config-mutation lock. */
export function saveConfig(config: OcxConfig): void {
// Keep the real-home assertion ahead of even lock-directory preparation.
Expand Down
132 changes: 132 additions & 0 deletions src/config/initialize.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
import {
closeSync, constants, fchmodSync, fstatSync, linkSync, lstatSync,
openSync, unlinkSync, writeFileSync,
} from "node:fs";
import { dirname } from "node:path";
import { assertNotRealHomeUnderTest } from "../lib/test-home-guard";
import { forgetEphemeralSecretPath, hardenSecretPath } from "../lib/windows-secret-acl";
import { isMissingPathError, nextAtomicTempSequence } from "./atomic-write";

type PublicationState = "not-published" | "published" | "uncertain";

/** Messages contain no candidate bytes or raw filesystem error text. */
export class InitialConfigPublicationError extends Error {
constructor(
readonly publication: PublicationState,
readonly residualTemp: boolean,
readonly hardLinkUnavailable: boolean,
options?: ErrorOptions,
) {
super(hardLinkUnavailable
? "Initial config requires hard-link publication; the filesystem or its permissions denied it."
: "Initial config publication did not finish.", options);
this.name = "InitialConfigPublicationError";
}
}

/** Narrow fault boundary; publication must be a single link operation. */
export interface InitialConfigPublicationIO {
harden(fd: number, temp: string, target: string): void;
write(fd: number, bytes: string): void;
link(temp: string, target: string): void;
unlink(temp: string): void;
close(fd: number): void;
}

function hardenInitialConfig(fd: number, temp: string, target: string): void {
if (process.platform === "win32") {
hardenSecretPath(temp, { required: true, timeoutMemoKey: target });
} else {
fchmodSync(fd, 0o600);
}
}

function identifiesDescriptor(fd: number, path: string): boolean {
const opened = fstatSync(fd);
const entry = lstatSync(path);
return opened.isFile() && entry.isFile()
&& opened.dev === entry.dev && opened.ino === entry.ino;
}

function verifyPrivateTemp(fd: number, temp: string): void {
if (!identifiesDescriptor(fd, temp)
|| (process.platform !== "win32" && (fstatSync(fd).mode & 0o777) !== 0o600)) {
throw new Error("Initial config temporary file identity or permissions changed.");
}
}

function removeOwnedTemp(fd: number, temp: string, unlink: (path: string) => void): boolean {
for (let attempt = 0; attempt < 2; attempt++) {
try {
if (!identifiesDescriptor(fd, temp)) return false;
unlink(temp);
forgetEphemeralSecretPath(temp);
return true;
} catch (error) {
if (isMissingPathError(error)) {
forgetEphemeralSecretPath(temp);
return true;
}
}
}
return false;
}

/**
* Publish complete bytes without replacing any entry at target. Never truncate:
* even an error from link can mean a remote filesystem already published the inode.
* Cleanup removes only our temporary name, never the target or another inode.
*/
export function publishInitialConfigNoReplace(
target: string,
bytes: string,
io: Partial<InitialConfigPublicationIO> = {},
): boolean {
assertNotRealHomeUnderTest(dirname(target));
const temp = `${target}.ocx.${process.pid}.${nextAtomicTempSequence()}.tmp`;
let fd: number | undefined;
let publication: PublicationState = "not-published";
let collided = false;
let failure: unknown;
let failed = false;
let hardLinkUnavailable = false;
let residualTemp = false;
try {
fd = openSync(temp, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL, 0o600);
(io.harden ?? hardenInitialConfig)(fd, temp, target);
verifyPrivateTemp(fd, temp);
(io.write ?? ((descriptor: number, value: string) => writeFileSync(descriptor, value, { encoding: "utf8" })))(fd, bytes);
verifyPrivateTemp(fd, temp);
try {
publication = "uncertain";
(io.link ?? linkSync)(temp, target);
publication = "published";
} catch (error) {
const code = (error as NodeJS.ErrnoException | undefined)?.code;
// EEXIST normally means a competitor won. A shared target means our
// publication may nevertheless have happened (e.g. a remote FS retry).
if (code === "EEXIST" && !identifiesDescriptor(fd, target)) collided = true;
else {
hardLinkUnavailable = ["EOPNOTSUPP", "ENOTSUP", "ENOSYS", "EXDEV", "EPERM"].includes(code ?? "");
throw error;
}
}
if (!collided && !identifiesDescriptor(fd, target)) {
throw new Error("Initial config published target identity changed.");
}
} catch (error) {
failed = true;
failure = error;
} finally {
if (fd !== undefined) {
// Unlink-only cleanup preserves all bytes if another name shares this inode.
residualTemp = !removeOwnedTemp(fd, temp, io.unlink ?? unlinkSync);
try { (io.close ?? closeSync)(fd); }
catch (error) { if (!failed) failure = error; failed = true; }
}
}
if (failed || residualTemp) {
throw new InitialConfigPublicationError(publication, residualTemp, hardLinkUnavailable, { cause: failure });
}
return !collided;
}
12 changes: 8 additions & 4 deletions src/lib/windows-secret-acl.ts
Original file line number Diff line number Diff line change
Expand Up @@ -715,18 +715,22 @@ function sanitizedAclError(diagnostics: string, cause: unknown): NodeJS.ErrnoExc
return error;
}

function previousTimeoutError(retryConsumed: boolean): NodeJS.ErrnoException {
type TimeoutMemoRefusalError = NodeJS.ErrnoException & {
aclFailureOrigin: "timeout_memo_refusal";
};

function previousTimeoutError(retryConsumed: boolean): TimeoutMemoRefusalError {
if (retryConsumed) {
const error = new Error(
"ACL hardening skipped — the previous timeout recovery was already consumed",
) as NodeJS.ErrnoException;
error.code = "EACLRETRYEXHAUSTED";
return error;
return Object.assign(error, { aclFailureOrigin: "timeout_memo_refusal" as const });
}
return sanitizedAclError(
return Object.assign(sanitizedAclError(
"ACL hardening skipped — previous attempt timed out",
Object.assign(new Error("timeout"), { code: "ETIMEDOUT" }),
);
), { aclFailureOrigin: "timeout_memo_refusal" as const });
}

/** Consume, but never reset, the single explicit recovery attempt for this key. */
Expand Down
Loading
Loading