From 80e07d0ff8f0ed7d28402243d25e4112cc022368 Mon Sep 17 00:00:00 2001 From: JUN Date: Sun, 13 Sep 2026 15:56:36 +0900 Subject: [PATCH 1/2] fix(cli): bind hub status credentials to connection snapshots [skip ci] `ocx status` on a connected client read the connection snapshot and the current `service-api-token` independently, so a reconnect or a key rotation between the two reads could send the new token to the snapshot's hub, or the snapshot's hub the new connection's token. The window is real: `collectRemoteHubStatus` awaits a dynamic import before it reads credentials. It now rereads the persisted connection and passes a token only when that connection still matches the snapshot's `serverUrl`, `apiKeyId` and `connectedAt` AND the token fingerprint matches that connection's `tokenFingerprint`. Otherwise it skips the live request and falls back to the snapshot owner's cache, or reports `unavailable`. A withheld token now carries its own cause. `resolveHubState` reported every null token as "this client has no usable data-plane token", which is false for a client that reconnected and holds a perfectly good token for a different hub - the operator would go re-enroll a credential that is not the problem. The caller supplies the reason through the new `withheldTokenReason`, so a changed connection, a missing token file and a fingerprint mismatch are named separately. That folds the maintainer review finding on #4382; it is the same misdiagnosis class as #4169 in the stop path. Verification: bun test tests/cli/cli-status-hub-state.test.ts (20 pass), tests/clients/client-hub-state.test.ts + tests/server/v1-hub-state.test.ts (35 pass), bun run typecheck, bun run structure:check, bun run privacy:scan, and the docs-site build required by docs-site/AGENTS.md (441 pages) - the one actionable CodeRabbit finding on the source pull request. Carried from #4382 by @luvs01. Co-authored-by: luvs01 <27862058+luvs01@users.noreply.github.com> --- .../src/content/docs/guides/remote-hub.md | 4 + .../src/content/docs/ko/guides/remote-hub.md | 7 ++ src/cli/status.ts | 46 ++++++-- src/client/hub-state.ts | 10 +- structure/clients/claude-desktop.md | 3 + structure/config.md | 3 + structure/ops/docs-and-release.md | 4 + structure/runtime.md | 11 ++ tests/cli/cli-status-hub-state.test.ts | 101 +++++++++++++++++- 9 files changed, 180 insertions(+), 9 deletions(-) diff --git a/docs-site/src/content/docs/guides/remote-hub.md b/docs-site/src/content/docs/guides/remote-hub.md index 1f2db4486c..dce98d2a86 100644 --- a/docs-site/src/content/docs/guides/remote-hub.md +++ b/docs-site/src/content/docs/guides/remote-hub.md @@ -92,6 +92,10 @@ local login state, and `ocx config show` on a client prints a `_remoteHub` note credentials and model availability live on the hub. The hub read uses the per-client data key only; no admin token and no provider secret ever reaches a client. +`ocx status` makes a live hub-state request only when the saved connection still matches the +status snapshot and the data-token file matches that connection. If either check fails, it skips +the request and shows matching cached hub state, or `unavailable` if no matching cache exists. + ## Linux systemd or macOS launchd Bind the data listener to the hub's Tailscale address, enable the loopback companion so the hub's diff --git a/docs-site/src/content/docs/ko/guides/remote-hub.md b/docs-site/src/content/docs/ko/guides/remote-hub.md index 72e96ae263..f47ae658fe 100644 --- a/docs-site/src/content/docs/ko/guides/remote-hub.md +++ b/docs-site/src/content/docs/ko/guides/remote-hub.md @@ -36,6 +36,13 @@ ocx sync 허브가 발급한 클라이언트별 키는 권한이 제한된 `service-api-token` 파일에 저장됩니다. `config.json`에는 저장되지 않습니다. 연결 중 사용량은 허브 기록에서 해당 `apiKeyId`만 조회하고, 연결을 끊은 뒤에는 로컬 기록을 봅니다. 두 기록은 서로 복제되지 않습니다. +### 연결된 클라이언트의 상태 표시 + +연결된 클라이언트의 `ocx status`는 허브 상태를 표시합니다. 상태 수집 중 연결 정보가 +바뀌거나 데이터 토큰 파일이 현재 연결과 일치하지 않으면 실시간 허브 조회를 건너뜁니다. +이때 상태 조회 대상 연결의 캐시를 표시하고, 일치하는 캐시가 없으면 `unavailable`로 보고합니다. +`ocx status --json`의 `remoteHub.stateSource`는 `hub`, `cache`, `unavailable` 중 하나입니다. + ## systemd 또는 launchd 데이터 리스너는 허브의 Tailscale 주소에 바인드하고, 허브 자신의 프로세스가 같은 포트를 자격 증명 없이 쓸 수 있도록 루프백 companion을 켜고, 관리 평면은 따로 공개합니다. 아래 값은 예시입니다. diff --git a/src/cli/status.ts b/src/cli/status.ts index cc1f4506bf..e120b5de70 100644 --- a/src/cli/status.ts +++ b/src/cli/status.ts @@ -18,6 +18,7 @@ import { effectiveLoopbackListenerPort } from "../codex/loopback-target"; import { claudeDesktopIntegrationEnabled } from "../codex/desired-state"; import { claudeDesktopPolicyHealth, probeClaudeDesktopPolicy, type ClaudeDesktopPolicyHealth } from "../claude/desktop-policy"; import { collectClientConnectionStatus, type ClientConnectionStatus } from "./connect"; +import { readClientConnectionState, sameClientConnectionOwner } from "../client/state"; import type { HubStateOAuthEntry, HubStateProvider } from "../remote/hub-state"; import type { HubStateSource } from "../client/hub-state"; import { readServiceApiTokenState, serviceApiTokenFilePath } from "../lib/service-secrets"; @@ -324,6 +325,35 @@ export function disconnectedRemoteHubStatus(): CliRemoteHubStatus { }; } +/** + * The data key this status snapshot may spend, and the cause when it may not. + * + * A reconnect or a rotation can replace both files between the snapshot and this read, so a + * matching cache owner alone does not authorize sending the current token. Withholding is only + * half the job: reporting every withheld case as "no usable data-plane token" is false for a + * client that reconnected and holds a perfectly good token for a different hub, and a cause the + * operator acts on has to be the real one (#4169 is the same defect in the stop path). + */ +function boundHubStateToken( + current: ReturnType, + token: ReturnType, + owner: { serverUrl: string; apiKeyId: string; connectedAt: string }, +): { token: string | null; withheldReason?: string } { + if (current.kind !== "connected") { + return { token: null, withheldReason: "this client is no longer connected" }; + } + if (!sameClientConnectionOwner(current.value, owner)) { + return { token: null, withheldReason: "the saved connection no longer matches the one this status reports" }; + } + if (token.kind !== "present") { + return { token: null, withheldReason: "this client has no usable data-plane token" }; + } + if (token.fingerprint !== current.value.tokenFingerprint) { + return { token: null, withheldReason: "the data-plane token no longer belongs to the saved connection" }; + } + return { token: token.token }; +} + /** * Ask the hub what it can serve, with a bounded read and a cache fallback. * @@ -340,14 +370,16 @@ export async function collectRemoteHubStatus( return disconnectedRemoteHubStatus(); } const { resolveHubState } = await import("../client/hub-state"); - const token = readServiceApiTokenState(); + const owner = { + serverUrl: connection.serverUrl, + apiKeyId: connection.apiKeyId, + connectedAt: connection.connectedAt, + }; + const bound = boundHubStateToken(readClientConnectionState(), readServiceApiTokenState(), owner); const resolved = await resolveHubState({ - owner: { - serverUrl: connection.serverUrl, - apiKeyId: connection.apiKeyId, - connectedAt: connection.connectedAt, - }, - token: token.kind === "present" ? token.token : null, + owner, + token: bound.token, + ...(bound.withheldReason ? { withheldTokenReason: bound.withheldReason } : {}), ...(options.fetchImpl ? { fetchImpl: options.fetchImpl } : {}), ...(options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs }), ...(options.now === undefined ? {} : { now: options.now }), diff --git a/src/client/hub-state.ts b/src/client/hub-state.ts index 655bcbdd9c..8b0916b514 100644 --- a/src/client/hub-state.ts +++ b/src/client/hub-state.ts @@ -161,6 +161,12 @@ export interface ResolveHubStateOptions { owner: HubStateOwner; /** The per-client data key. Null when the token file is missing or unsafe. */ token: string | null; + /** + * Why the caller withheld the token, when it withheld one it holds. A null token reads the + * same here whether the file is missing or the caller declined to send a perfectly good + * token for a different connection, and the operator acts on that difference. + */ + withheldTokenReason?: string; timeoutMs?: number; fetchImpl?: typeof fetch; now?: number; @@ -195,7 +201,9 @@ export async function resolveHubState(options: ResolveHubStateOptions): Promise< ? withAge("cache", cached.state, cached.fetchedAt, now, reason) : withAge("unavailable", null, undefined, now, reason); }; - if (!options.token) return fromCache("this client has no usable data-plane token"); + if (!options.token) { + return fromCache(options.withheldTokenReason ?? "this client has no usable data-plane token"); + } if (options.allowNetwork === false) return fromCache("a live hub read was not attempted"); let state: HubStateDTO; try { diff --git a/structure/clients/claude-desktop.md b/structure/clients/claude-desktop.md index 1412a498d1..c557b4ab2e 100644 --- a/structure/clients/claude-desktop.md +++ b/structure/clients/claude-desktop.md @@ -69,6 +69,9 @@ or profile-upload API. Thinking replay and prompt caching remain separate in #37 The shared Responses path follows the [bounded multipart recovery contract](../subagents.md#multipart-encrypted-task-recovery); credential admission and retry policy remain unchanged. +Connected `ocx status` diagnostics follow the shared +[status credential binding](../runtime.md#remote-hub-status-credential-binding). + ## Claude Desktop config-library resolution The Desktop profile writer and the management status probe share diff --git a/structure/config.md b/structure/config.md index 96f63e2a2e..1e30795cbe 100644 --- a/structure/config.md +++ b/structure/config.md @@ -209,6 +209,9 @@ the residual directory for manual review; there is no recursive-delete fallback. ## Remote client key files +The connection's `tokenFingerprint` participates in +[`ocx status` credential binding](runtime.md#remote-hub-status-credential-binding). + Client connection metadata stores a stable `apiKeyId` and a non-secret rotation `pendingOperation`. The current data secret remains only in `service-api-token`; a bounded rotation temporarily keeps the old secret in owner-only `service-api-token.prev`. Commit or recovery clears the marker before orphan cleanup. `ocx disconnect` is local-only and leaves remote revocation to the hub's **Integrations → API Keys** page. Hub and local usage stores are not mirrored. Codex display-cache expiry, retained main-policy evidence, and reset history follow the diff --git a/structure/ops/docs-and-release.md b/structure/ops/docs-and-release.md index 0aa5928944..c5f0aad889 100644 --- a/structure/ops/docs-and-release.md +++ b/structure/ops/docs-and-release.md @@ -308,6 +308,10 @@ The shared Responses path follows the [bounded multipart recovery contract](../s ## Remote Hub locale and release gate +The Remote Hub guide describes the +[status credential binding](../runtime.md#remote-hub-status-credential-binding) +and its matching-cache or `unavailable` result. + The Remote Hub guide and affected CLI, server-config, management-API, and dashboard references have eight sources: root English plus `fr`, `ko`, `zh-cn`, `zh-tw`, `ru`, `ja`, and `tr`. English is canonical; commands, defaults, endpoint auth, and warnings remain exact in translations. A release requires the remote-only focused/full gates, privacy scan, GUI/docs builds, protocol compatibility receipts, and the MAINTAINERS security review for the exact head. Codex display-cache expiry, retained main-policy evidence, and reset history follow the diff --git a/structure/runtime.md b/structure/runtime.md index 35b1e1f31a..48c3968db4 100644 --- a/structure/runtime.md +++ b/structure/runtime.md @@ -214,6 +214,17 @@ The shared Responses path follows the [bounded multipart recovery contract](suba `src/remote/protocol.ts` owns pure interval/feature negotiation. `src/remote/hub-state.ts` owns the `GET|HEAD /v1/hub-state` contract, its caps, and the parser both sides share. `src/client/hub-client.ts` owns bounded, schema-validated remote catalog consumption, hub-state reads, and key-id probes; `src/client/hub-state.ts` owns the resolution and the owner-stamped 0600 cache, and a failed read reports "unavailable" rather than degrading to the client's own local provider and login state. `src/client/hub-relay.ts` is a fixed-authority management relay with URL, header, body, redirect, and stream bounds. The public data listener remains the direct client→hub path; the loopback management ingress never serves data-plane routes. +### Remote Hub status credential binding + +`src/cli/status.ts` rereads the persisted client connection and `service-api-token` state before +requesting hub state. It passes a usable token only when the current connection matches the +status snapshot's `serverUrl`, `apiKeyId`, and `connectedAt`, and the token fingerprint matches +the current connection's `tokenFingerprint`. Otherwise it skips the live request and uses the +snapshot owner's matching cached hub state, or reports `unavailable`. +A withheld token carries its own cause into the reported `reason` through +`resolveHubState`'s `withheldTokenReason`, so a changed connection, a missing token file, and a +fingerprint mismatch are named separately rather than all reported as a missing data key. + Codex display-cache expiry, retained main-policy evidence, and reset history follow the [quota cache contract](providers/openai-tiers.md#quota-cache-and-short-window-history). diff --git a/tests/cli/cli-status-hub-state.test.ts b/tests/cli/cli-status-hub-state.test.ts index e396f023e1..d7f808b23b 100644 --- a/tests/cli/cli-status-hub-state.test.ts +++ b/tests/cli/cli-status-hub-state.test.ts @@ -27,6 +27,7 @@ import { type CliRemoteHubStatus, } from "../../src/cli/status"; import type { HubStateDTO } from "../../src/remote/hub-state"; +import { writeCachedHubState } from "../../src/client/hub-state"; import { removeTreeWithRetry } from "../helpers/remove-tree"; import { SPAWN_BUDGET_MS } from "../helpers/test-budget"; @@ -88,6 +89,27 @@ function jsonFetch(body: unknown): typeof fetch { })) as unknown as typeof fetch; } +function observedHubFetch() { + const requests: { url: string; token: string | null }[] = []; + const fetchImpl = (async (input: RequestInfo | URL, init?: RequestInit) => { + requests.push({ + url: String(input), + token: new Headers(init?.headers).get("x-opencodex-api-key"), + }); + return Response.json(hubState()); + }) as typeof fetch; + return { requests, fetchImpl }; +} + +function connectionSnapshot() { + return { + state: "connected" as const, + serverUrl: "https://hub.example.test:8443", + apiKeyId: "status-hub-state", + connectedAt: "2026-09-06T00:00:00.000Z", + }; +} + beforeEach(() => { testHome = mkdtempSync(join(tmpdir(), "ocx-status-hub-")); process.env.OPENCODEX_HOME = testHome; @@ -103,10 +125,15 @@ afterEach(() => { describe("collectRemoteHubStatus", () => { test("a connected client with a live hub reports the hub's facts", async () => { writeConnectedHome(testHome, "https://hub.example.test:8443"); + const probe = observedHubFetch(); const remoteHub = await collectRemoteHubStatus( { state: "connected", serverUrl: "https://hub.example.test:8443", apiKeyId: "status-hub-state", connectedAt: "2026-09-06T00:00:00.000Z" }, - { fetchImpl: jsonFetch(hubState()) }, + { fetchImpl: probe.fetchImpl }, ); + expect(probe.requests).toEqual([{ + url: "https://hub.example.test:8443/v1/hub-state", + token: FIXTURE_TOKEN, + }]); expect(remoteHub.connected).toBe(true); expect(remoteHub.stateSource).toBe("hub"); expect(remoteHub.hubVersion).toBe("2.51.0"); @@ -115,6 +142,78 @@ describe("collectRemoteHubStatus", () => { expect(remoteHub.claudeCodeEnabled).toBe(true); }); + test.each([ + ["origin", { serverUrl: "https://other-hub.example.test" }], + ["key", { apiKeyId: "another-client-key" }], + ["enrollment", { connectedAt: "2026-09-07T00:00:00.000Z" }], + ])("a changed %s never sends a credential for the old snapshot", async (_label, changed) => { + const snapshot = connectionSnapshot(); + const config = connectedConfig(snapshot.serverUrl); + Object.assign(config.client, changed); + writeFileSync(join(testHome, "config.json"), JSON.stringify(config)); + writeFileSync(join(testHome, "service-api-token"), FIXTURE_TOKEN, { mode: 0o600 }); + const probe = observedHubFetch(); + + const remoteHub = await collectRemoteHubStatus(snapshot, { fetchImpl: probe.fetchImpl }); + + expect(probe.requests).toEqual([]); + expect(remoteHub.stateSource).toBe("unavailable"); + expect(remoteHub.providers).toEqual([]); + // The token file is intact and usable — it simply belongs to a different connection now. + // Reporting "no usable data-plane token" here would send the operator to reconnect a + // credential that is not the problem. + expect(remoteHub.reason).toContain("no longer matches"); + expect(remoteHub.reason).not.toContain("no usable data-plane token"); + }); + + test("a persistent token fingerprint mismatch never sends the changed token", async () => { + const snapshot = connectionSnapshot(); + writeConnectedHome(testHome, snapshot.serverUrl); + writeFileSync(join(testHome, "service-api-token"), "unowned-status-token", { mode: 0o600 }); + const probe = observedHubFetch(); + + const remoteHub = await collectRemoteHubStatus(snapshot, { fetchImpl: probe.fetchImpl }); + + expect(probe.requests).toEqual([]); + expect(remoteHub.stateSource).toBe("unavailable"); + expect(remoteHub.reason).toContain("data-plane token"); + }); + + test("a token rotation during the asynchronous boundary cannot reach the old hub", async () => { + const snapshot = connectionSnapshot(); + writeConnectedHome(testHome, snapshot.serverUrl); + const probe = observedHubFetch(); + const pending = collectRemoteHubStatus(snapshot, { fetchImpl: probe.fetchImpl }); + // The collector has yielded before reading credentials. Rotate the actual files before + // its continuation runs, including a fingerprint that legitimately owns the NEW token. + const config = connectedConfig("https://other-hub.example.test"); + const rotatedToken = "rotated-status-token"; + config.client.apiKeyId = "rotated-client-key"; + config.client.tokenFingerprint = createHash("sha256").update(rotatedToken).digest("hex"); + writeFileSync(join(testHome, "config.json"), JSON.stringify(config)); + writeFileSync(join(testHome, "service-api-token"), rotatedToken, { mode: 0o600 }); + + const remoteHub = await pending; + + expect(probe.requests).toEqual([]); + expect(remoteHub.stateSource).toBe("unavailable"); + }); + + test.each([true, false])("a mismatched connection uses only the snapshot's cache (matching=%s)", async matching => { + const snapshot = connectionSnapshot(); + const current = connectedConfig("https://other-hub.example.test"); + writeFileSync(join(testHome, "config.json"), JSON.stringify(current)); + writeFileSync(join(testHome, "service-api-token"), FIXTURE_TOKEN, { mode: 0o600 }); + expect(writeCachedHubState(matching ? snapshot : current.client, hubState(), "2026-09-06T00:00:00.000Z")).toBe(true); + const probe = observedHubFetch(); + + const remoteHub = await collectRemoteHubStatus(snapshot, { fetchImpl: probe.fetchImpl }); + + expect(probe.requests).toEqual([]); + expect(remoteHub.stateSource).toBe(matching ? "cache" : "unavailable"); + expect(remoteHub.providers).toEqual(matching ? hubState().providers : []); + }); + test("a client whose token file is missing is unavailable, not locally sourced", async () => { writeFileSync(join(testHome, "config.json"), JSON.stringify(connectedConfig("https://hub.example.test:8443"))); const remoteHub = await collectRemoteHubStatus( From 394b96dee1079a5573f1ca17f0ac876d03ead1c0 Mon Sep 17 00:00:00 2001 From: JUN Date: Sun, 13 Sep 2026 16:03:03 +0900 Subject: [PATCH 2/2] feat(codex): pull an authenticated remote catalog into local Codex state Adds `ocx catalog pull [--auth-env ] [--json] [--restart-codex]`, which installs a complete catalog served by another OpenCodex instance's `/v1/catalog` endpoint and then synchronizes `models_cache.json`. `ocx sync` builds a catalog from locally configured providers and `ocx sync-cache` rebuilds the cache from the catalog already on disk; neither consumes a finished catalog from another server, which is the gap #3729 has held open. Acquisition is fail-closed before any local write: HTTPS is required except on loopback, URL-embedded credentials, queries, fragments and redirects are refused, the body is bounded by size and inactivity, and slugs and input modalities are validated. The token is read only by environment-variable name and never from argv. Catalog and cache writes share the Codex catalog write lock and the atomic writer. Three review findings on the source pull request are folded in. A cache rebuild that failed AFTER the catalog write left a new catalog paired with a stale `models_cache.json` while reporting `catalogWritten: false`. The permit rolls back SQLite; `replaceActiveCodexCatalog` is an atomic file write that nothing else undid. The pull now restores the previous catalog bytes, or removes the file when the home had none, before it throws. Both cases have a regression test, and both fail without the restore. `--restart-codex` reported success when only some app-servers stopped, so `ok: true` and exit 0 could be returned while a stale app-server still served the previous catalog from memory. A restart now counts only when nothing failed, nothing survived, and every listed process stopped. An incomplete restart returns `code: "restart_incomplete"` and exit 1 while keeping `catalogWritten` and `cacheSynced` true, because the writes did land. The unused `statSync` import is removed. Documentation now covers the full `--json` envelope, every failure code and its exit status, the host-root `/v1/catalog` path contract, and the two deliberate Phase 1 omissions: no `ETag`/`If-None-Match` conditional request, and no Windows `--restart-desktop-app`. Identical bytes stay a complete no-op, so a home whose cache alone is broken is repaired by `ocx sync-cache` rather than by this command. The seven localized lifecycle pages carry the command too. Closes #3729 Verification: bun test on catalog-remote-pull, skill-ocx, both test-layout guards, and the CLI help/dispatch/registry suites (137 pass); bun run typecheck, structure:check, privacy:scan, skill:surface:check; docs-site build (441 pages). Carried from #4413 by @rrmlima. Co-authored-by: rrmlima <137737127+rrmlima@users.noreply.github.com> --- .../docs/fr/reference/cli/lifecycle.md | 16 ++ .../docs/ja/reference/cli/lifecycle.md | 14 + .../docs/ko/reference/cli/lifecycle.md | 12 + .../content/docs/reference/cli/lifecycle.md | 54 ++++ .../docs/ru/reference/cli/lifecycle.md | 14 + .../docs/tr/reference/cli/lifecycle.md | 15 ++ .../docs/zh-cn/reference/cli/lifecycle.md | 6 + .../docs/zh-tw/reference/cli/lifecycle.md | 6 + scripts/test-layout/layout.json | 1 + src/cli/catalog.ts | 109 ++++++++ src/cli/dispatch.ts | 4 + src/cli/help.ts | 1 + src/cli/registry.ts | 10 + src/codex/catalog/remote.ts | 233 +++++++++++++++++ .../catalog-remote-pull.test.ts | 242 ++++++++++++++++++ tests/fixtures/test-layout-expected.json | 1 + 16 files changed, 738 insertions(+) create mode 100644 src/cli/catalog.ts create mode 100644 src/codex/catalog/remote.ts create mode 100644 tests/codex-integration/catalog-remote-pull.test.ts diff --git a/docs-site/src/content/docs/fr/reference/cli/lifecycle.md b/docs-site/src/content/docs/fr/reference/cli/lifecycle.md index 2624d6388e..aa2254733e 100644 --- a/docs-site/src/content/docs/fr/reference/cli/lifecycle.md +++ b/docs-site/src/content/docs/fr/reference/cli/lifecycle.md @@ -155,6 +155,22 @@ Si des processus Codex `app-server` de longue durée sont encore actifs, `ocx sy Invalide le cache local du sélecteur de modèles de Codex afin qu’il soit reconstruit à partir du catalogue opencodex actif. Le même avertissement concernant un `app-server` obsolète et le même comportement facultatif `--restart-codex` que pour `ocx sync` s’appliquent. +### `ocx catalog pull [--auth-env ] [--json] [--restart-codex]` + +Installe un catalogue complet servi par le point de terminaison `/v1/catalog` d'une autre instance +OpenCodex, puis synchronise `models_cache.json`. L'URL doit être en HTTPS ; le HTTP est accepté +uniquement en loopback. Les identifiants intégrés à l'URL, les requêtes, les fragments, les +redirections, les réponses trop volumineuses et les catalogues invalides sont refusés avant toute +écriture locale. L'authentification est facultative et lue uniquement par référence à une variable +d'environnement (`--auth-env`), jamais depuis argv. + +Le catalogue et le cache sont écrits sous le verrou de catalogue Codex partagé ; un échec préserve +les derniers fichiers valides connus. Des octets identiques constituent une non-opération qui +préserve les mtimes. `--restart-codex` ne s'applique qu'après une écriture réelle. Les requêtes +conditionnelles `ETag` et le redémarrage de l'application Desktop ne font pas partie de cette +commande. Voir la [référence anglaise](/reference/cli/lifecycle/) pour l'enveloppe `--json` +complète et les codes de sortie. + ## Service d’arrière-plan ### `ocx service [install|repair|restart|start|stop|status|uninstall|remove]` diff --git a/docs-site/src/content/docs/ja/reference/cli/lifecycle.md b/docs-site/src/content/docs/ja/reference/cli/lifecycle.md index 39cec05483..6fd2dc6333 100644 --- a/docs-site/src/content/docs/ja/reference/cli/lifecycle.md +++ b/docs-site/src/content/docs/ja/reference/cli/lifecycle.md @@ -158,6 +158,20 @@ ocx status --json Codex のローカル モデル ピッカー キャッシュを無効にし、アクティブな opencodex カタログから再構築されるようにします。 `ocx sync` と同じ、古い `app-server` 警告とオプションの `--restart-codex` 動作が適用されます。 +### `ocx catalog pull [--auth-env ] [--json] [--restart-codex]` + +別の OpenCodex インスタンスの `/v1/catalog` エンドポイントが提供する完全なカタログをインストール +し、続いて `models_cache.json` を同期します。URL は HTTPS が必須で、HTTP はループバックのみ許可 +されます。URL 埋め込み資格情報、クエリ、フラグメント、リダイレクト、サイズ超過の応答、不正な +カタログは、ローカル書き込みの前に拒否されます。認証は任意で、環境変数参照 (`--auth-env`) から +のみ読み取られ、argv からは読み取られません。 + +カタログとキャッシュは共有の Codex カタログロックの下で書き込まれ、失敗時は last-known-good の +ファイルが保持されます。バイトが同一の場合は mtime を保持する no-op です。`--restart-codex` は +実際の書き込みの後にのみ適用されます。`ETag` 条件付きリクエストと Desktop アプリの再起動は、この +コマンドには含まれません。`--json` エンベロープと終了コードの詳細は +[英語版リファレンス](/reference/cli/lifecycle/)を参照してください。 + ## バックグラウンドサービス ### `ocx service [install|repair|restart|start|stop|status|uninstall|remove]` diff --git a/docs-site/src/content/docs/ko/reference/cli/lifecycle.md b/docs-site/src/content/docs/ko/reference/cli/lifecycle.md index f8b5dc6579..e5b3e8a69b 100644 --- a/docs-site/src/content/docs/ko/reference/cli/lifecycle.md +++ b/docs-site/src/content/docs/ko/reference/cli/lifecycle.md @@ -236,6 +236,18 @@ single-flight/lock 파일을 만들 수 있는지, 건강하지 않은 OAuth 또 Codex의 로컬 모델 선택기 캐시를 무효화하여, 활성 opencodex 카탈로그에서 다시 빌드되게 합니다. `ocx sync`와 같은 오래된 `app-server` 경고와 선택적 `--restart-codex` 동작이 적용됩니다. +### `ocx catalog pull [--auth-env ] [--json] [--restart-codex]` + +다른 OpenCodex 인스턴스의 `/v1/catalog` 엔드포인트가 제공하는 완성된 카탈로그를 설치한 뒤 +`models_cache.json`을 맞춥니다. URL은 HTTPS여야 하고 HTTP는 루프백만 허용합니다. URL에 박힌 +자격증명, 쿼리, 프래그먼트, 리다이렉트, 크기를 넘는 응답, 잘못된 카탈로그는 로컬에 쓰기 전에 +거절합니다. 인증은 선택이며 환경변수 이름(`--auth-env`)으로만 읽고 argv로는 받지 않습니다. + +카탈로그와 캐시는 공유 Codex 카탈로그 잠금 아래에서 쓰고, 실패하면 직전까지 정상이던 파일을 +그대로 둡니다. 바이트가 같으면 mtime까지 건드리지 않는 no-op입니다. `--restart-codex`는 실제로 +쓴 뒤에만 적용됩니다. `ETag` 조건부 요청과 Desktop 앱 재시작은 이 명령에 없습니다. `--json` +envelope 필드와 종료 코드는 [영문 레퍼런스](/reference/cli/lifecycle/)를 보세요. + ## 백그라운드 서비스 ### `ocx service [install|repair|restart|start|stop|status|uninstall|remove]` diff --git a/docs-site/src/content/docs/reference/cli/lifecycle.md b/docs-site/src/content/docs/reference/cli/lifecycle.md index ea1f0295e9..e49dc08a08 100644 --- a/docs-site/src/content/docs/reference/cli/lifecycle.md +++ b/docs-site/src/content/docs/reference/cli/lifecycle.md @@ -282,6 +282,60 @@ were updated. Pass `--restart-codex` to send `SIGTERM` only to matching `codex Invalidate Codex's local model picker cache so it is rebuilt from the active opencodex catalog. The same stale-`app-server` warning and optional `--restart-codex` behavior as `ocx sync` apply. +### `ocx catalog pull [--auth-env ] [--json] [--restart-codex]` + +Install a complete catalog served by another OpenCodex instance's `/v1/catalog` endpoint, then +synchronize `models_cache.json`. Unlike `ocx sync`, this command does not discover configured +providers or inject Codex configuration. Unlike `ocx sync-cache`, it replaces the active catalog +before rebuilding the cache. It works even when the local Codex integration desired state is off. + +The URL must be HTTPS; loopback HTTP is accepted for local testing. Embedded URL credentials, +queries, fragments, redirects, oversized responses, malformed JSON, duplicate or unsafe slugs, and +unknown `input_modalities` are refused before any local write. Authentication is optional and is +read only by environment-variable reference: + +```bash +export OPENCODEX_CATALOG_AUTH_TOKEN='...' +ocx catalog pull https://proxy.example.com/v1/catalog \ + --auth-env OPENCODEX_CATALOG_AUTH_TOKEN +``` + +The value is sent as a Bearer token but is never accepted as an argv value. Redirects are refused, +so authorization cannot cross origins. Catalog and cache writes use the shared Codex catalog lock +and atomic writer. A failed fetch, validation, lock acquisition, catalog write, or cache rebuild +preserves the last-known-good files. Identical catalog bytes are a no-op that preserves mtimes and +never touches processes. `--restart-codex` applies only after a real write and remains explicit; +Desktop restart is not part of this command. + +The URL must name `/v1/catalog` at the host root. A reverse proxy that serves the endpoint under a +path prefix is not supported by this command. + +Two behaviors are deliberately out of scope in this first cut. The command downloads the full +catalog and compares bytes locally instead of issuing an `ETag` / `If-None-Match` conditional +request, and it has no Windows `--restart-desktop-app`. Identical bytes are treated as a complete +no-op, so a home whose catalog is correct but whose `models_cache.json` is missing or stale is not +repaired by this command; use `ocx sync-cache` for that. + +`--json` emits one stable envelope on stdout. `schemaVersion`, `ok`, `status`, `catalogWritten`, +`cacheSynced`, and `codexRestarted` are always present. `status` is `updated`, `unchanged`, or +`failed`. A successful pull adds `modelCount`; a failure adds `code`, which is the field a script +branches on: + +| `code` | Meaning | Exit | +| --- | --- | --- | +| `usage` | The arguments were not a valid `catalog pull` invocation | 2 | +| `auth_env_missing` | `--auth-env` named a variable that is not set | 1 | +| `url_invalid`, `insecure_http_refused` | The URL was refused before any request | 1 | +| `request_failed`, `redirect_refused`, `http_error` | The request did not produce a usable response | 1 | +| `body_too_large`, `body_invalid`, `catalog_invalid` | The response was refused before any local write | 1 | +| `write_failed`, `lock_database`, `unsafe_path` | The coordinated write did not complete; files are unchanged | 1 | +| `lock_busy` | Another writer holds the Codex catalog lock | 3 | +| `restart_incomplete` | The catalog and cache landed, but a Codex app-server survived `--restart-codex` | 1 | + +`restart_incomplete` is the one failure that reports real writes: `catalogWritten` and +`cacheSynced` stay true and `ok` is false, because a surviving app-server still serves the +previous catalog from memory. + ## Background service ### `ocx service [install|repair|restart|start|stop|status|uninstall|remove]` diff --git a/docs-site/src/content/docs/ru/reference/cli/lifecycle.md b/docs-site/src/content/docs/ru/reference/cli/lifecycle.md index cbe330bf71..374e16202d 100644 --- a/docs-site/src/content/docs/ru/reference/cli/lifecycle.md +++ b/docs-site/src/content/docs/ru/reference/cli/lifecycle.md @@ -232,6 +232,20 @@ credential'ы и не выполняет repair. opencodex. Предупреждение о stale-`app-server` и optional `--restart-codex` работают так же, как и у `ocx sync`. +### `ocx catalog pull [--auth-env ] [--json] [--restart-codex]` + +Устанавливает полный каталог, который отдаёт эндпоинт `/v1/catalog` другого экземпляра OpenCodex, +и затем синхронизирует `models_cache.json`. URL должен быть HTTPS; HTTP допускается только на +loopback. Учётные данные в URL, query, фрагменты, редиректы, слишком большие ответы и невалидные +каталоги отклоняются до любой локальной записи. Аутентификация необязательна и читается только по +имени переменной окружения (`--auth-env`), но не из argv. + +Каталог и кэш пишутся под общей блокировкой каталога Codex; при сбое сохраняются last-known-good +файлы. Идентичные байты — это no-op, сохраняющий mtime. `--restart-codex` применяется только после +реальной записи. Условные запросы `ETag` и перезапуск Desktop-приложения в эту команду не входят. +Полная `--json`-обёртка и коды выхода описаны в +[английской справке](/reference/cli/lifecycle/). + ## Фоновая служба ### `ocx service [install|repair|restart|start|stop|status|uninstall|remove]` diff --git a/docs-site/src/content/docs/tr/reference/cli/lifecycle.md b/docs-site/src/content/docs/tr/reference/cli/lifecycle.md index 33d13c543b..dd6568b5ea 100644 --- a/docs-site/src/content/docs/tr/reference/cli/lifecycle.md +++ b/docs-site/src/content/docs/tr/reference/cli/lifecycle.md @@ -242,6 +242,21 @@ Codex'in yerel model seçici önbelleğini geçersiz kılın, böylece aktif ope kataloğundan yeniden oluşturulur. `ocx sync` ile aynı eski `app-server` uyarısı ve isteğe bağlı `--restart-codex` davranışı geçerlidir. +### `ocx catalog pull [--auth-env ] [--json] [--restart-codex]` + +Başka bir OpenCodex örneğinin `/v1/catalog` uç noktasının sunduğu eksiksiz kataloğu kurar ve +ardından `models_cache.json` dosyasını eşitler. URL HTTPS olmalıdır; HTTP yalnızca loopback için +kabul edilir. URL içine gömülü kimlik bilgileri, sorgular, parçalar, yönlendirmeler, boyutu aşan +yanıtlar ve geçersiz kataloglar, herhangi bir yerel yazma işleminden önce reddedilir. Kimlik +doğrulama isteğe bağlıdır ve yalnızca ortam değişkeni adıyla (`--auth-env`) okunur, argv'den +alınmaz. + +Katalog ve önbellek, paylaşılan Codex katalog kilidi altında yazılır; bir hata durumunda +last-known-good dosyalar korunur. Aynı baytlar, mtime değerlerini koruyan bir no-op'tur. +`--restart-codex` yalnızca gerçek bir yazmadan sonra uygulanır. `ETag` koşullu istekleri ve Desktop +uygulamasının yeniden başlatılması bu komutun kapsamında değildir. Tam `--json` zarfı ve çıkış +kodları için [İngilizce referansa](/reference/cli/lifecycle/) bakın. + ## Arka plan servisi ### `ocx service [install|repair|restart|start|stop|status|uninstall|remove]` diff --git a/docs-site/src/content/docs/zh-cn/reference/cli/lifecycle.md b/docs-site/src/content/docs/zh-cn/reference/cli/lifecycle.md index 1b1d98e178..98ff929156 100644 --- a/docs-site/src/content/docs/zh-cn/reference/cli/lifecycle.md +++ b/docs-site/src/content/docs/zh-cn/reference/cli/lifecycle.md @@ -154,6 +154,12 @@ ocx status --json 使 Codex 的本地模型选择器缓存失效，让它根据当前激活的 opencodex 目录重新生成。与 `ocx sync` 相同的陈旧 `app-server` 警告和可选 `--restart-codex` 行为同样适用。 +### `ocx catalog pull [--auth-env ] [--json] [--restart-codex]` + +安装由另一个 OpenCodex 实例的 `/v1/catalog` 端点提供的完整目录，然后同步 `models_cache.json`。URL 必须是 HTTPS；仅回环地址允许 HTTP。URL 内嵌凭据、查询、片段、重定向、超出大小的响应以及无效目录，都会在任何本地写入之前被拒绝。认证是可选的，并且只通过环境变量名（`--auth-env`）读取，不接受 argv 传入。 + +目录和缓存在共享的 Codex 目录锁下写入；失败时保留 last-known-good 文件。字节完全相同时是保留 mtime 的空操作。`--restart-codex` 仅在发生真实写入之后生效。`ETag` 条件请求和 Desktop 应用重启不属于此命令。完整的 `--json` 信封与退出码请参见[英文参考](/reference/cli/lifecycle/)。 + ## 后台服务 ### `ocx service [install|repair|restart|start|stop|status|uninstall|remove]` diff --git a/docs-site/src/content/docs/zh-tw/reference/cli/lifecycle.md b/docs-site/src/content/docs/zh-tw/reference/cli/lifecycle.md index 727de9bb54..015590cd3c 100644 --- a/docs-site/src/content/docs/zh-tw/reference/cli/lifecycle.md +++ b/docs-site/src/content/docs/zh-tw/reference/cli/lifecycle.md @@ -148,6 +148,12 @@ ocx status --json 使 Codex 的本機模型選擇器快取失效，使其從現用的 opencodex 目錄重建。與 `ocx sync` 相同的過時 `app-server` 警告與可選的 `--restart-codex` 行為適用。 +### `ocx catalog pull [--auth-env ] [--json] [--restart-codex]` + +安裝由另一個 OpenCodex 執行個體的 `/v1/catalog` 端點提供的完整目錄，接著同步 `models_cache.json`。URL 必須是 HTTPS；僅回送位址允許 HTTP。URL 內嵌憑證、查詢、片段、重新導向、超出大小的回應以及無效目錄，都會在任何本機寫入之前遭拒。驗證為選用，且只透過環境變數名稱（`--auth-env`）讀取，不接受 argv 傳入。 + +目錄與快取在共用的 Codex 目錄鎖之下寫入；失敗時保留 last-known-good 檔案。位元組完全相同時是保留 mtime 的無操作。`--restart-codex` 僅在實際寫入之後生效。`ETag` 條件式請求與 Desktop 應用程式重新啟動不屬於此命令。完整的 `--json` 信封與結束碼請參見[英文參考](/reference/cli/lifecycle/)。 + ## 背景服務 ### `ocx service [install|repair|restart|start|stop|status|uninstall|remove]` diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index ed73c48686..3b2bd12533 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -282,6 +282,7 @@ "catalog-input-modality-enum.test.ts": "codex-integration", "catalog-llamacpp-capabilities.test.ts": "codex-integration", "catalog-oauth-observation.test.ts": "codex-integration", + "catalog-remote-pull.test.ts": "codex-integration", "catalog-retain-models.test.ts": "codex-integration", "catalog-verbosity-default.test.ts": "codex-integration", "catalog-vision-sidecar-modalities.test.ts": "codex-integration", diff --git a/src/cli/catalog.ts b/src/cli/catalog.ts new file mode 100644 index 0000000000..e3d9aa3883 --- /dev/null +++ b/src/cli/catalog.ts @@ -0,0 +1,109 @@ +import { afterCatalogWriteHandleAppServers } from "../codex/app-server-processes"; +import { pullRemoteCatalog, RemoteCatalogError } from "../codex/catalog/remote"; +import { hasHelpFlag, printSubcommandUsage } from "./help"; + +export interface CatalogPullEnvelope { + schemaVersion: 1; + ok: boolean; + status: "updated" | "unchanged" | "failed"; + catalogWritten: boolean; + cacheSynced: boolean; + codexRestarted: boolean; + modelCount?: number; + code?: string; +} + +function optionValue(args: string[], name: string): string | undefined { + const index = args.indexOf(name); + return index >= 0 ? args[index + 1] : undefined; +} + +export async function handleCatalogCommand(args: string[]): Promise { + if (hasHelpFlag(args)) { printSubcommandUsage("catalog"); return 0; } + const json = args.includes("--json"); + const restartCodex = args.includes("--restart-codex"); + const authEnv = optionValue(args, "--auth-env"); + const positionals = args.filter((arg, index) => { + if (arg === "--auth-env") return false; + if (index > 0 && args[index - 1] === "--auth-env") return false; + return !arg.startsWith("-"); + }); + const knownFlags = new Set(["--json", "--restart-codex", "--auth-env"]); + const unknown = args.find((arg, index) => arg.startsWith("-") && !knownFlags.has(arg) && args[index - 1] !== "--auth-env"); + const validEnvName = authEnv === undefined || /^[A-Za-z_][A-Za-z0-9_]*$/.test(authEnv); + if (positionals[0] !== "pull" || positionals.length !== 2 || unknown || !validEnvName + || args.includes("--auth-env") !== (authEnv !== undefined)) { + const envelope: CatalogPullEnvelope = { + schemaVersion: 1, ok: false, status: "failed", catalogWritten: false, + cacheSynced: false, codexRestarted: false, code: "usage", + }; + if (json) console.log(JSON.stringify(envelope)); + else console.error("Usage: ocx catalog pull [--auth-env ] [--json] [--restart-codex]"); + return 2; + } + let token: string | undefined; + if (authEnv !== undefined) { + token = process.env[authEnv]; + if (token === undefined) { + const envelope: CatalogPullEnvelope = { + schemaVersion: 1, ok: false, status: "failed", catalogWritten: false, + cacheSynced: false, codexRestarted: false, code: "auth_env_missing", + }; + if (json) console.log(JSON.stringify(envelope)); + else console.error(`Catalog authentication environment variable ${authEnv} is not set.`); + return 1; + } + } + try { + const result = await pullRemoteCatalog(positionals[1]!, { token }); + let codexRestarted = false; + let restartIncomplete = false; + if (result.catalogWritten) { + const processLog = json + ? { log: (...values: unknown[]) => console.error(...values), error: (...values: unknown[]) => console.error(...values) } + : console; + const processResult = afterCatalogWriteHandleAppServers({ restart: restartCodex, log: processLog }); + const restart = processResult.restart; + if (restart) { + // A partial stop is not a restart. `restartCodexAppServers` reports failures and + // survivors without throwing, so counting `stopped` alone reported success while a + // stale app-server was still serving the previous catalog from memory. + codexRestarted = restart.failed.length === 0 + && restart.surviving.length === 0 + && restart.stopped.length === processResult.processes.length; + restartIncomplete = !codexRestarted; + } + } + if (restartIncomplete) { + // The catalog and cache landed; only the restart did not finish. Saying the pull failed + // and wrote nothing would be a second false report, so the envelope keeps the real + // write state and `ok` carries the failure. + const envelope: CatalogPullEnvelope = { + schemaVersion: 1, ok: false, status: result.status, + catalogWritten: result.catalogWritten, cacheSynced: result.cacheSynced, + codexRestarted: false, modelCount: result.modelCount, code: "restart_incomplete", + }; + if (json) console.log(JSON.stringify(envelope)); + else console.error("Remote Codex catalog installed, but a Codex app-server is still running the previous catalog."); + return 1; + } + const envelope: CatalogPullEnvelope = { + schemaVersion: 1, ok: true, status: result.status, + catalogWritten: result.catalogWritten, cacheSynced: result.cacheSynced, + codexRestarted, modelCount: result.modelCount, + }; + if (json) console.log(JSON.stringify(envelope)); + else if (result.status === "unchanged") console.log("Remote Codex catalog is unchanged; no files or processes were touched."); + else console.log(`Remote Codex catalog installed (${result.modelCount} models) and models_cache.json synchronized.`); + return 0; + } catch (error) { + const code = error instanceof RemoteCatalogError ? error.code : "write_failed"; + const envelope: CatalogPullEnvelope = { + schemaVersion: 1, ok: false, status: "failed", catalogWritten: false, + cacheSynced: false, codexRestarted: false, code, + }; + if (json) console.log(JSON.stringify(envelope)); + else console.error(error instanceof RemoteCatalogError ? error.message : "Remote catalog installation failed"); + return code === "lock_busy" ? 3 : 1; + } +} diff --git a/src/cli/dispatch.ts b/src/cli/dispatch.ts index 203d2d63d5..7ec21f9499 100644 --- a/src/cli/dispatch.ts +++ b/src/cli/dispatch.ts @@ -490,6 +490,10 @@ const commandRunners: Record = { const { handleDisconnectCommand } = await import("./connect"); return await handleDisconnectCommand(deps.args.slice(1)); }, + catalog: async deps => { + const { handleCatalogCommand } = await import("./catalog"); + return await handleCatalogCommand(deps.args.slice(1)); + }, "sync-cache": async deps => { const cacheArgs = deps.args.slice(1); const restartCodex = cacheArgs.includes("--restart-codex"); diff --git a/src/cli/help.ts b/src/cli/help.ts index 15f6ed711a..6765d27fcf 100644 --- a/src/cli/help.ts +++ b/src/cli/help.ts @@ -46,6 +46,7 @@ Usage: ocx sync [--restart-codex] Fetch models from providers and inject into Codex config ocx sync-cache [--restart-codex] Refresh Codex's model cache from the active catalog + ocx catalog pull Install a validated remote catalog and refresh the Codex cache ocx status Check proxy server status (on a hub: one block with its ports and token source) ocx doctor Diagnose environment/network issues (WSL, proxy, ChatGPT reachability) ocx doctor --reclaim-response-temps diff --git a/src/cli/registry.ts b/src/cli/registry.ts index 1883bb27f2..6a8a98478d 100644 --- a/src/cli/registry.ts +++ b/src/cli/registry.ts @@ -143,6 +143,16 @@ export const CLI_COMMANDS: CliCommandEntry[] = [ "--restart-desktop-app (Windows only, opt-in) fully restarts the Codex desktop app so its model picker re-reads the catalog. Never implied by --restart-codex: it ends live conversations.", ], }, + { + name: "catalog", + usage: "ocx catalog pull [--auth-env ] [--json] [--restart-codex]", + summary: "Install a validated remote /v1/catalog snapshot into Codex.", + details: [ + "Authentication is read only from the named environment variable and sent as a Bearer token.", + "HTTPS is required except for loopback HTTP; redirects are refused.", + "The catalog and models_cache.json are coordinated under the Codex catalog write lock.", + ], + }, { name: "status", usage: "ocx status", summary: "Check proxy server status." }, { name: "doctor", diff --git a/src/codex/catalog/remote.ts b/src/codex/catalog/remote.ts new file mode 100644 index 0000000000..46576b18e4 --- /dev/null +++ b/src/codex/catalog/remote.ts @@ -0,0 +1,233 @@ +import { existsSync, readFileSync, rmSync } from "node:fs"; + +import { MAX_REMOTE_CATALOG_BYTES } from "../../server/catalog-download"; +import { readBoundedResponseBytes } from "../../lib/bounded-body"; +import { withCatalogWriteSerialization, type CatalogSerializationOutcome, type CatalogWritePermit } from "../catalog-write-serialization"; +import { replaceActiveCodexCatalog } from "../internal/catalog-writer"; +import { resetCodexAppServerCatalogStateCache } from "../app-server-processes"; +import { getCodexHome } from "../paths"; +import { readCodexCatalogPathForHome } from "./parsing"; +import { invalidateCodexModelsCacheWithPermit } from "./sync"; + +const DEFAULT_TIMEOUT_MS = 15_000; +const MAX_MODELS = 2_000; +const MAX_SLUG_BYTES = 512; +const ALLOWED_MODALITIES = new Set(["text", "image", "audio"]); + +export type RemoteCatalogFailureCode = + | "url_invalid" | "insecure_http_refused" | "credential_invalid" | "request_failed" + | "redirect_refused" | "http_error" | "body_too_large" | "body_invalid" + | "catalog_invalid" | "write_failed" | "lock_busy" | "lock_database" | "unsafe_path"; + +export class RemoteCatalogError extends Error { + constructor(readonly code: RemoteCatalogFailureCode, message: string, readonly status?: number) { + super(message); + this.name = "RemoteCatalogError"; + } +} + +export interface RemoteCatalogDocument extends Record { + models: Record[]; +} + +export interface PullRemoteCatalogOptions { + token?: string; + timeoutMs?: number; + maxBytes?: number; + fetchImpl?: typeof fetch; + codexHome?: string; +} + +export interface PullRemoteCatalogResult { + status: "updated" | "unchanged"; + catalogWritten: boolean; + cacheSynced: boolean; + codexHome: string; + catalogPath: string; + modelCount: number; +} + +function isLoopback(hostname: string): boolean { + const host = hostname.toLowerCase(); + return host === "localhost" || host === "127.0.0.1" || host === "[::1]" || host === "::1"; +} + +export function validateRemoteCatalogUrl(input: string): URL { + let url: URL; + try { url = new URL(input); } catch { throw new RemoteCatalogError("url_invalid", "Catalog URL must be an absolute HTTPS URL"); } + if (url.username || url.password) throw new RemoteCatalogError("url_invalid", "Catalog URL must not contain credentials"); + if (url.protocol !== "https:" && !(url.protocol === "http:" && isLoopback(url.hostname))) { + throw new RemoteCatalogError("insecure_http_refused", "Catalog URL requires HTTPS (HTTP is allowed only on loopback)"); + } + if (url.pathname !== "/v1/catalog" || url.search || url.hash) { + throw new RemoteCatalogError("url_invalid", "Catalog URL must identify /v1/catalog without query or fragment"); + } + return url; +} + +function validateToken(token: string | undefined): string | undefined { + if (token === undefined) return undefined; + if (!token || token.length > 4096 || /[\r\n\0]/.test(token)) { + throw new RemoteCatalogError("credential_invalid", "Catalog authentication environment variable is invalid"); + } + return token; +} + +export function validateRemoteCatalogDocument(value: unknown): RemoteCatalogDocument { + const invalid = (message: string): never => { throw new RemoteCatalogError("catalog_invalid", message); }; + if (!value || typeof value !== "object" || Array.isArray(value)) invalid("Remote catalog must be a JSON object"); + const document = value as Record; + const rawModels = document.models; + if (!Array.isArray(rawModels) || rawModels.length === 0 || rawModels.length > MAX_MODELS) { + invalid("Remote catalog models must be a non-empty bounded array"); + } + const models = rawModels as unknown[]; + const slugs = new Set(); + for (const row of models) { + if (!row || typeof row !== "object" || Array.isArray(row) || Object.getPrototypeOf(row) !== Object.prototype) { + invalid("Remote catalog model rows must be plain objects"); + } + const model = row as Record; + const rawSlug = model.slug; + if (typeof rawSlug !== "string") invalid("Remote catalog contains an invalid model slug"); + const slug = rawSlug as string; + if (slug !== slug.trim() || !slug + || new TextEncoder().encode(slug).byteLength > MAX_SLUG_BYTES || /[\x00-\x1f\x7f]/.test(slug)) { + invalid("Remote catalog contains an invalid model slug"); + } + if (slugs.has(slug)) invalid("Remote catalog contains duplicate model slugs"); + slugs.add(slug); + if (Object.hasOwn(model, "input_modalities")) { + const modalities = model.input_modalities; + if (!Array.isArray(modalities) || modalities.length === 0 + || modalities.some(item => typeof item !== "string" || !ALLOWED_MODALITIES.has(item))) { + invalid("Remote catalog contains unsupported input modalities"); + } + } + } + return document as RemoteCatalogDocument; +} + +function safeTimeout(value: number | undefined): number { + return typeof value === "number" && Number.isFinite(value) && value > 0 + ? Math.min(Math.floor(value), 120_000) : DEFAULT_TIMEOUT_MS; +} + +export async function fetchRemoteCatalog( + input: string, + options: Pick = {}, +): Promise<{ document: RemoteCatalogDocument; content: string }> { + const url = validateRemoteCatalogUrl(input); + const token = validateToken(options.token); + const headers = new Headers({ Accept: "application/json" }); + if (token !== undefined) headers.set("Authorization", `Bearer ${token}`); + let response: Response; + try { + response = await (options.fetchImpl ?? fetch)(url, { + method: "GET", headers, redirect: "manual", signal: AbortSignal.timeout(safeTimeout(options.timeoutMs)), + }); + } catch { + throw new RemoteCatalogError("request_failed", "Remote catalog request did not complete"); + } + if (response.status >= 300 && response.status < 400) { + try { await response.body?.cancel(); } catch { /* best effort */ } + throw new RemoteCatalogError("redirect_refused", "Remote catalog redirect was refused", response.status); + } + if (!response.ok) { + try { await response.body?.cancel(); } catch { /* best effort */ } + throw new RemoteCatalogError("http_error", `Remote catalog request failed with HTTP ${response.status}`, response.status); + } + const contentType = response.headers.get("content-type")?.split(";", 1)[0]?.trim().toLowerCase(); + if (contentType !== "application/json" && contentType?.endsWith("+json") !== true) { + try { await response.body?.cancel(); } catch { /* best effort */ } + throw new RemoteCatalogError("body_invalid", "Remote catalog response was not JSON"); + } + const maxBytes = options.maxBytes ?? MAX_REMOTE_CATALOG_BYTES; + const declaredRaw = response.headers.get("content-length"); + if (declaredRaw !== null) { + const declared = Number(declaredRaw); + if (!Number.isSafeInteger(declared) || declared < 0 || declared > maxBytes) { + try { await response.body?.cancel(); } catch { /* best effort */ } + throw new RemoteCatalogError("body_too_large", "Remote catalog exceeded the allowed size"); + } + } + let bytes: Uint8Array; + try { + const bounded = await readBoundedResponseBytes(response, { maxBytes, inactivityTimeoutMs: safeTimeout(options.timeoutMs) }); + if (bounded.oversized) throw new RemoteCatalogError("body_too_large", "Remote catalog exceeded the allowed size"); + bytes = bounded.bytes; + } catch (error) { + if (error instanceof RemoteCatalogError) throw error; + throw new RemoteCatalogError("request_failed", "Remote catalog download did not complete"); + } + let text: string; + try { text = new TextDecoder("utf-8", { fatal: true }).decode(bytes); } + catch { throw new RemoteCatalogError("body_invalid", "Remote catalog was not valid UTF-8"); } + let parsed: unknown; + try { parsed = JSON.parse(text); } + catch { throw new RemoteCatalogError("body_invalid", "Remote catalog was not valid JSON"); } + const document = validateRemoteCatalogDocument(parsed); + return { document, content: `${JSON.stringify(document, null, 2)}\n` }; +} + +function mapSerializationFailure(outcome: CatalogSerializationOutcome): never { + if (outcome.kind === "completed") throw new RemoteCatalogError("write_failed", "Remote catalog installation failed"); + const code = outcome.reason === "busy" ? "lock_busy" : outcome.reason === "database" ? "lock_database" : "unsafe_path"; + throw new RemoteCatalogError(code, `Remote catalog installation unavailable (${outcome.reason})`); +} + +/** + * Put the catalog file back the way this pull found it. + * + * `replaceActiveCodexCatalog` is an atomic FILE write; the serialization permit rolls back the + * SQLite transaction and nothing on disk. Without this, a cache rebuild that fails after the + * catalog was replaced leaves a new catalog paired with a stale `models_cache.json` — the exact + * split the last-known-good guarantee exists to prevent — while the caller is told the pull + * failed and wrote nothing. + */ +function restorePreviousCatalog( + permit: CatalogWritePermit, + codexHome: string, + catalogPath: string, + previous: Buffer | null, +): void { + if (previous) { + replaceActiveCodexCatalog(permit, codexHome, { path: catalogPath, content: previous.toString("utf8") }); + return; + } + // There was no catalog before this pull, so last-known-good is its absence. + rmSync(catalogPath, { force: true }); + resetCodexAppServerCatalogStateCache(); +} + +export async function pullRemoteCatalog(input: string, options: PullRemoteCatalogOptions = {}): Promise { + // Network acquisition and fail-closed validation intentionally happen before K. + const fetched = await fetchRemoteCatalog(input, options); + const codexHome = options.codexHome ?? getCodexHome(); + const catalogPath = readCodexCatalogPathForHome(codexHome); + const current = existsSync(catalogPath) ? readFileSync(catalogPath) : null; + const candidate = Buffer.from(fetched.content, "utf8"); + if (current?.equals(candidate)) { + return { status: "unchanged", catalogWritten: false, cacheSynced: false, codexHome, catalogPath, modelCount: fetched.document.models.length }; + } + const outcome = withCatalogWriteSerialization(codexHome, permit => { + // Re-check under K: another writer may have installed these bytes while the request was in flight. + const lockedCurrent = existsSync(catalogPath) ? readFileSync(catalogPath) : null; + if (lockedCurrent?.equals(candidate)) return { catalogWritten: false, cacheSynced: false }; + replaceActiveCodexCatalog(permit, codexHome, { path: catalogPath, content: fetched.content }); + const cacheSynced = invalidateCodexModelsCacheWithPermit(permit, codexHome, { allowWhenDesiredDisabled: true }); + if (!cacheSynced) { + restorePreviousCatalog(permit, codexHome, catalogPath, lockedCurrent); + throw new RemoteCatalogError("write_failed", "Remote catalog cache synchronization failed"); + } + return { catalogWritten: true, cacheSynced: true }; + }); + if (outcome.kind !== "completed") return mapSerializationFailure(outcome); + return { + status: outcome.value.catalogWritten ? "updated" : "unchanged", + ...outcome.value, + codexHome, + catalogPath, + modelCount: fetched.document.models.length, + }; +} diff --git a/tests/codex-integration/catalog-remote-pull.test.ts b/tests/codex-integration/catalog-remote-pull.test.ts new file mode 100644 index 0000000000..02a6768f22 --- /dev/null +++ b/tests/codex-integration/catalog-remote-pull.test.ts @@ -0,0 +1,242 @@ +import { afterEach, describe, expect, mock, test } from "bun:test"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Database } from "bun:sqlite"; + +import { + fetchRemoteCatalog, + pullRemoteCatalog, + RemoteCatalogError, + validateRemoteCatalogDocument, + validateRemoteCatalogUrl, +} from "../../src/codex/catalog/remote"; +import { resolveCodexCatalogSerializationDatabasePath, resolveEffectiveUserIdentity } from "../../src/codex/user-identity"; +import { withCatalogWriteSerialization } from "../../src/codex/catalog-write-serialization"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; + +const homes: string[] = []; +afterEach(() => { while (homes.length) removeTreeWithRetry(homes.pop()!); }); + +function home(): string { + const value = mkdtempSync(join(tmpdir(), "ocx-catalog-pull-")); + homes.push(value); + return value; +} + +const catalog = { version: 1, models: [{ slug: "provider/model", input_modalities: ["text", "image"], extension: { safe: true } }] }; +const response = (value: unknown, init: ResponseInit = {}) => new Response(JSON.stringify(value), { + headers: { "Content-Type": "application/json", ...init.headers }, status: init.status, +}); + +describe("remote catalog acquisition", () => { + test("accepts HTTPS and loopback HTTP but rejects credentials and insecure remote HTTP", () => { + expect(validateRemoteCatalogUrl("https://hub.example.com/v1/catalog").href).toBe("https://hub.example.com/v1/catalog"); + expect(validateRemoteCatalogUrl("http://127.0.0.1:10100/v1/catalog").protocol).toBe("http:"); + expect(() => validateRemoteCatalogUrl("http://hub.example.com/v1/catalog")).toThrow(RemoteCatalogError); + expect(() => validateRemoteCatalogUrl("https://user:secret@example.com/v1/catalog")).toThrow(RemoteCatalogError); + expect(() => validateRemoteCatalogUrl("https://hub.example.com/v1/catalog?q=secret")).toThrow(RemoteCatalogError); + }); + + test("sends optional bearer authentication from the caller without following redirects", async () => { + const fetchImpl = mock(async (_input: RequestInfo | URL, init?: RequestInit) => { + expect(new Headers(init?.headers).get("authorization")).toBe("Bearer env-token"); + expect(init?.redirect).toBe("manual"); + return response(catalog); + }) as typeof fetch; + const fetched = await fetchRemoteCatalog("https://hub.example/v1/catalog", { token: "env-token", fetchImpl }); + expect(fetched.document).toEqual(catalog); + expect(fetchImpl).toHaveBeenCalledTimes(1); + + await expect(fetchRemoteCatalog("https://hub.example/v1/catalog", { + token: "secret-marker", fetchImpl: async () => new Response("body-marker", { status: 302, headers: { Location: "https://other.example/secret" } }), + })).rejects.toMatchObject({ code: "redirect_refused", message: "Remote catalog redirect was refused" }); + }); + + test("never reflects credentials, remote bodies, URLs, or transport causes", async () => { + for (const fetchImpl of [ + async () => new Response("remote-body-marker", { status: 401 }), + async () => { throw new Error("secret-marker https://private.example/path"); }, + ]) { + let caught: unknown; + try { await fetchRemoteCatalog("https://hub.example/v1/catalog", { token: "secret-marker", fetchImpl: fetchImpl as typeof fetch }); } + catch (error) { caught = error; } + expect(String(caught)).not.toContain("secret-marker"); + expect(String(caught)).not.toContain("remote-body-marker"); + expect(String(caught)).not.toContain("private.example"); + expect((caught as Error).cause).toBeUndefined(); + } + }); + + test.each([401, 403, 404, 500])("rejects HTTP %s", async status => { + await expect(fetchRemoteCatalog("https://hub.example/v1/catalog", { + fetchImpl: async () => new Response(null, { status }), + })).rejects.toMatchObject({ code: "http_error", status }); + }); + + test("enforces declared and streamed byte limits", async () => { + await expect(fetchRemoteCatalog("https://hub.example/v1/catalog", { + maxBytes: 10, fetchImpl: async () => new Response("{}", { headers: { "Content-Type": "application/json", "Content-Length": "11" } }), + })).rejects.toMatchObject({ code: "body_too_large" }); + const stream = new ReadableStream({ start(controller) { + controller.enqueue(new TextEncoder().encode('{"models":[')); + controller.enqueue(new Uint8Array(128)); controller.close(); + } }); + await expect(fetchRemoteCatalog("https://hub.example/v1/catalog", { + maxBytes: 32, fetchImpl: async () => new Response(stream, { headers: { "Content-Type": "application/json", "Content-Length": "1" } }), + })).rejects.toMatchObject({ code: "body_too_large" }); + }); + + test("bounds headers and stalled streams", async () => { + await expect(fetchRemoteCatalog("https://hub.example/v1/catalog", { + timeoutMs: 10, + fetchImpl: async (_input, init) => await new Promise((_resolve, reject) => { + init?.signal?.addEventListener("abort", () => reject(new Error("secret timeout cause")), { once: true }); + }), + })).rejects.toMatchObject({ code: "request_failed" }); + }); +}); + +describe("remote catalog validation", () => { + test.each([ + [null], [[]], [{}], [{ models: [] }], [{ models: [null] }], [{ models: [[]] }], + [{ models: [{}] }], [{ models: [{ slug: "" }] }], [{ models: [{ slug: " padded " }] }], + [{ models: [{ slug: "bad\u0000slug" }] }], [{ models: [{ slug: "a" }, { slug: "a" }] }], + [{ models: [{ slug: "a", input_modalities: [] }] }], + [{ models: [{ slug: "a", input_modalities: ["video"] }] }], + ])("rejects invalid document %#", value => { + expect(() => validateRemoteCatalogDocument(value)).toThrow(RemoteCatalogError); + }); + + test("preserves safe additive fields", () => { + expect(validateRemoteCatalogDocument(catalog)).toEqual(catalog); + }); +}); + +describe("remote catalog coordinated installation", () => { + test("updates catalog and cache under the shared writer even when desired integration is disabled", async () => { + const codexHome = home(); + const opencodexHome = home(); + const previous = process.env.OPENCODEX_HOME; + process.env.OPENCODEX_HOME = opencodexHome; + writeFileSync(join(opencodexHome, "config.json"), JSON.stringify({ providers: {}, defaultProvider: "openai", desiredIntegrations: { codex: false } })); + try { + const result = await pullRemoteCatalog("https://hub.example/v1/catalog", { + codexHome, fetchImpl: async () => response(catalog), + }); + expect(result).toMatchObject({ status: "updated", catalogWritten: true, cacheSynced: true, modelCount: 1 }); + expect(JSON.parse(readFileSync(result.catalogPath, "utf8"))).toEqual(catalog); + expect(JSON.parse(readFileSync(join(codexHome, "models_cache.json"), "utf8")).models).toEqual(catalog.models); + } finally { + if (previous === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previous; + } + }); + + test("an identical pull preserves catalog and cache mtimes", async () => { + const codexHome = home(); + const fetchImpl = async () => response(catalog); + const first = await pullRemoteCatalog("https://hub.example/v1/catalog", { codexHome, fetchImpl }); + const cachePath = join(codexHome, "models_cache.json"); + const before = [statSync(first.catalogPath).mtimeMs, statSync(cachePath).mtimeMs]; + await Bun.sleep(20); + const second = await pullRemoteCatalog("https://hub.example/v1/catalog", { codexHome, fetchImpl }); + expect(second).toMatchObject({ status: "unchanged", catalogWritten: false, cacheSynced: false }); + expect([statSync(first.catalogPath).mtimeMs, statSync(cachePath).mtimeMs]).toEqual(before); + }); + + test("lock contention is typed and preserves last-known-good files", async () => { + const codexHome = home(); + // Materialize K, then hold BEGIN IMMEDIATE from a separate connection while pull attempts it. + expect(withCatalogWriteSerialization(codexHome, () => null).kind).toBe("completed"); + const lockPath = resolveCodexCatalogSerializationDatabasePath(resolveEffectiveUserIdentity(), codexHome); + const holder = new Database(lockPath); + holder.exec("PRAGMA busy_timeout=0; BEGIN IMMEDIATE"); + try { + await expect(pullRemoteCatalog("https://hub.example/v1/catalog", { + codexHome, fetchImpl: async () => response(catalog), + })).rejects.toMatchObject({ code: "lock_busy" }); + expect(existsSync(join(codexHome, "opencodex-catalog.json"))).toBe(false); + expect(existsSync(join(codexHome, "models_cache.json"))).toBe(false); + } finally { + holder.exec("ROLLBACK"); holder.close(); + } + }); + + test("fetch and validation failures preserve last-known-good files", async () => { + const codexHome = home(); + const catalogPath = join(codexHome, "opencodex-catalog.json"); + const cachePath = join(codexHome, "models_cache.json"); + writeFileSync(catalogPath, "catalog-before"); writeFileSync(cachePath, "cache-before"); + for (const fetchImpl of [async () => new Response(null, { status: 500 }), async () => response({ models: [] })]) { + await expect(pullRemoteCatalog("https://hub.example/v1/catalog", { codexHome, fetchImpl: fetchImpl as typeof fetch })).rejects.toBeInstanceOf(RemoteCatalogError); + expect(readFileSync(catalogPath, "utf8")).toBe("catalog-before"); + expect(readFileSync(cachePath, "utf8")).toBe("cache-before"); + } + expect(existsSync(join(codexHome, "models_cache.json"))).toBe(true); + }); + + test("a cache rebuild that fails after the catalog write puts the previous catalog back", async () => { + const codexHome = home(); + const first = await pullRemoteCatalog("https://hub.example/v1/catalog", { + codexHome, fetchImpl: async () => response(catalog), + }); + const before = readFileSync(first.catalogPath, "utf8"); + // Make the cache write fail AFTER the catalog has already been replaced. The permit rolls + // back SQLite; the catalog is an atomic file write that nothing else undoes. + const cachePath = join(codexHome, "models_cache.json"); + rmSync(cachePath, { force: true }); + mkdirSync(cachePath); + const next = { version: 1, models: [{ slug: "provider/second-model" }] }; + + await expect(pullRemoteCatalog("https://hub.example/v1/catalog", { + codexHome, fetchImpl: async () => response(next), + })).rejects.toMatchObject({ code: "write_failed" }); + + // Without the restore this reads the SECOND catalog while the cache is stale, and the + // caller was told the pull wrote nothing. + expect(readFileSync(first.catalogPath, "utf8")).toBe(before); + }); + + test("a cache rebuild that fails on a first pull leaves no catalog behind", async () => { + const codexHome = home(); + const catalogPath = join(codexHome, "opencodex-catalog.json"); + const cachePath = join(codexHome, "models_cache.json"); + mkdirSync(cachePath); + + await expect(pullRemoteCatalog("https://hub.example/v1/catalog", { + codexHome, fetchImpl: async () => response(catalog), + })).rejects.toMatchObject({ code: "write_failed" }); + + // Last-known-good for a home that had no catalog is its absence, not a catalog whose + // cache was never built. + expect(existsSync(catalogPath)).toBe(false); + }); +}); + +describe("catalog pull CLI envelope", () => { + test("emits a stable JSON failure without reflecting the secret environment value", async () => { + const { handleCatalogCommand } = await import("../../src/cli/catalog"); + const old = process.env.OCX_CATALOG_TEST_TOKEN; + process.env.OCX_CATALOG_TEST_TOKEN = "secret-cli-marker"; + const output: string[] = []; + const errors: string[] = []; + const log = console.log; + const error = console.error; + console.log = (...values) => { output.push(values.map(String).join(" ")); }; + console.error = (...values) => { errors.push(values.map(String).join(" ")); }; + try { + expect(await handleCatalogCommand([ + "pull", "http://remote.example/v1/catalog", "--auth-env", "OCX_CATALOG_TEST_TOKEN", "--json", + ])).toBe(1); + expect(output).toHaveLength(1); + expect(JSON.parse(output[0]!)).toEqual({ + schemaVersion: 1, ok: false, status: "failed", catalogWritten: false, + cacheSynced: false, codexRestarted: false, code: "insecure_http_refused", + }); + expect(output.join("\n") + errors.join("\n")).not.toContain("secret-cli-marker"); + } finally { + console.log = log; console.error = error; + if (old === undefined) delete process.env.OCX_CATALOG_TEST_TOKEN; else process.env.OCX_CATALOG_TEST_TOKEN = old; + } + }); +}); diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 21bf29eb78..0dbf1402a2 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -115,6 +115,7 @@ "catalog-input-modality-enum.test.ts": "codex-integration", "catalog-llamacpp-capabilities.test.ts": "codex-integration", "catalog-oauth-observation.test.ts": "codex-integration", + "catalog-remote-pull.test.ts": "codex-integration", "catalog-retain-models.test.ts": "codex-integration", "catalog-verbosity-default.test.ts": "codex-integration", "catalog-vision-sidecar-modalities.test.ts": "codex-integration",