From 2aa821dff41ceb629164cfb63cf1fe97aa92896f Mon Sep 17 00:00:00 2001 From: Rafael Moreira Date: Sat, 12 Sep 2026 10:25:50 -0300 Subject: [PATCH] feat(codex): pull an authenticated remote catalog into local Codex state - Implement Phase 1 of `ocx catalog pull ` to safely install and synchronize an external `/v1/catalog` snapshot into local `CODEX_HOME` state. - Validate remote catalogs fail-closed before any local mutation: require HTTPS (except loopback HTTP), refuse redirects/queries/credentials, bound sizes, and enforce safe slugs and input modalities. - Coordinate catalog and `models_cache.json` updates through the existing shared write lock and atomic serialization paths. - Preserve last-known-good files on failure, and treat identical bytes as an unchanged no-op that preserves mtimes and avoids touching processes. - Read optional bearer credentials only via `--auth-env `, never argv. - Add unit tests in `tests/codex-integration/catalog-remote-pull.test.ts` (28 pass). Closes #3729 Co-authored-by: CommandCodeBot --- .../content/docs/reference/cli/lifecycle.md | 28 +++ scripts/test-layout/layout.json | 1 + src/cli/catalog.ts | 86 ++++++++ src/cli/dispatch.ts | 4 + src/cli/help.ts | 1 + src/cli/registry.ts | 10 + src/codex/catalog/remote.ts | 205 ++++++++++++++++++ .../catalog-remote-pull.test.ts | 205 ++++++++++++++++++ tests/fixtures/test-layout-expected.json | 1 + 9 files changed, 541 insertions(+) create mode 100644 src/cli/catalog.ts create mode 100644 src/codex/catalog/remote.ts create mode 100644 tests/codex-integration/catalog-remote-pull.test.ts diff --git a/docs-site/src/content/docs/reference/cli/lifecycle.md b/docs-site/src/content/docs/reference/cli/lifecycle.md index bc0c03c57f..dca658bf32 100644 --- a/docs-site/src/content/docs/reference/cli/lifecycle.md +++ b/docs-site/src/content/docs/reference/cli/lifecycle.md @@ -278,6 +278,34 @@ were updated. Pass `--restart-codex` to send `SIGTERM` only to matching `codex Invalidate Codex's local model picker cache so it is rebuilt from the active opencodex catalog. The same stale-`app-server` warning and optional `--restart-codex` behavior as `ocx sync` apply. +### `ocx catalog pull [--auth-env ] [--json] [--restart-codex]` + +Install a complete catalog served by another OpenCodex instance's `/v1/catalog` endpoint, then +synchronize `models_cache.json`. Unlike `ocx sync`, this command does not discover configured +providers or inject Codex configuration. Unlike `ocx sync-cache`, it replaces the active catalog +before rebuilding the cache. It works even when the local Codex integration desired state is off. + +The URL must be HTTPS; loopback HTTP is accepted for local testing. Embedded URL credentials, +queries, fragments, redirects, oversized responses, malformed JSON, duplicate or unsafe slugs, and +unknown `input_modalities` are refused before any local write. Authentication is optional and is +read only by environment-variable reference: + +```bash +export OPENCODEX_CATALOG_AUTH_TOKEN='...' +ocx catalog pull https://proxy.example.com/v1/catalog \ + --auth-env OPENCODEX_CATALOG_AUTH_TOKEN +``` + +The value is sent as a Bearer token but is never accepted as an argv value. Redirects are refused, +so authorization cannot cross origins. Catalog and cache writes use the shared Codex catalog lock +and atomic writer. A failed fetch, validation, lock acquisition, catalog write, or cache rebuild +preserves the last-known-good files. Identical catalog bytes are a no-op that preserves mtimes and +never touches processes. `--restart-codex` applies only after a real write and remains explicit; +Desktop restart is not part of this command. + +`--json` emits one stable envelope on stdout. The `status` field is `updated`, `unchanged`, or +`failed`; `catalogWritten`, `cacheSynced`, and `codexRestarted` are always present. + ## Background service ### `ocx service [install|repair|restart|start|stop|status|uninstall|remove]` diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index b2eabcbcc9..b0909e58db 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -276,6 +276,7 @@ "catalog-input-modality-enum.test.ts": "codex-integration", "catalog-llamacpp-capabilities.test.ts": "codex-integration", "catalog-oauth-observation.test.ts": "codex-integration", + "catalog-remote-pull.test.ts": "codex-integration", "catalog-retain-models.test.ts": "codex-integration", "catalog-verbosity-default.test.ts": "codex-integration", "catalog-vision-sidecar-modalities.test.ts": "codex-integration", diff --git a/src/cli/catalog.ts b/src/cli/catalog.ts new file mode 100644 index 0000000000..8b0df46031 --- /dev/null +++ b/src/cli/catalog.ts @@ -0,0 +1,86 @@ +import { afterCatalogWriteHandleAppServers } from "../codex/app-server-processes"; +import { pullRemoteCatalog, RemoteCatalogError } from "../codex/catalog/remote"; +import { hasHelpFlag, printSubcommandUsage } from "./help"; + +export interface CatalogPullEnvelope { + schemaVersion: 1; + ok: boolean; + status: "updated" | "unchanged" | "failed"; + catalogWritten: boolean; + cacheSynced: boolean; + codexRestarted: boolean; + modelCount?: number; + code?: string; +} + +function optionValue(args: string[], name: string): string | undefined { + const index = args.indexOf(name); + return index >= 0 ? args[index + 1] : undefined; +} + +export async function handleCatalogCommand(args: string[]): Promise { + if (hasHelpFlag(args)) { printSubcommandUsage("catalog"); return 0; } + const json = args.includes("--json"); + const restartCodex = args.includes("--restart-codex"); + const authEnv = optionValue(args, "--auth-env"); + const positionals = args.filter((arg, index) => { + if (arg === "--auth-env") return false; + if (index > 0 && args[index - 1] === "--auth-env") return false; + return !arg.startsWith("-"); + }); + const knownFlags = new Set(["--json", "--restart-codex", "--auth-env"]); + const unknown = args.find((arg, index) => arg.startsWith("-") && !knownFlags.has(arg) && args[index - 1] !== "--auth-env"); + const validEnvName = authEnv === undefined || /^[A-Za-z_][A-Za-z0-9_]*$/.test(authEnv); + if (positionals[0] !== "pull" || positionals.length !== 2 || unknown || !validEnvName + || args.includes("--auth-env") !== (authEnv !== undefined)) { + const envelope: CatalogPullEnvelope = { + schemaVersion: 1, ok: false, status: "failed", catalogWritten: false, + cacheSynced: false, codexRestarted: false, code: "usage", + }; + if (json) console.log(JSON.stringify(envelope)); + else console.error("Usage: ocx catalog pull [--auth-env ] [--json] [--restart-codex]"); + return 2; + } + let token: string | undefined; + if (authEnv !== undefined) { + token = process.env[authEnv]; + if (token === undefined) { + const envelope: CatalogPullEnvelope = { + schemaVersion: 1, ok: false, status: "failed", catalogWritten: false, + cacheSynced: false, codexRestarted: false, code: "auth_env_missing", + }; + if (json) console.log(JSON.stringify(envelope)); + else console.error(`Catalog authentication environment variable ${authEnv} is not set.`); + return 1; + } + } + try { + const result = await pullRemoteCatalog(positionals[1]!, { token }); + let codexRestarted = false; + if (result.catalogWritten) { + const processLog = json + ? { log: (...values: unknown[]) => console.error(...values), error: (...values: unknown[]) => console.error(...values) } + : console; + const processResult = afterCatalogWriteHandleAppServers({ restart: restartCodex, log: processLog }); + codexRestarted = (processResult.restart?.stopped.length ?? 0) > 0; + } + const envelope: CatalogPullEnvelope = { + schemaVersion: 1, ok: true, status: result.status, + catalogWritten: result.catalogWritten, cacheSynced: result.cacheSynced, + codexRestarted, modelCount: result.modelCount, + }; + if (json) console.log(JSON.stringify(envelope)); + else if (result.status === "unchanged") console.log("Remote Codex catalog is unchanged; no files or processes were touched."); + else console.log(`Remote Codex catalog installed (${result.modelCount} models) and models_cache.json synchronized.`); + return 0; + } catch (error) { + const code = error instanceof RemoteCatalogError ? error.code : "write_failed"; + const envelope: CatalogPullEnvelope = { + schemaVersion: 1, ok: false, status: "failed", catalogWritten: false, + cacheSynced: false, codexRestarted: false, code, + }; + if (json) console.log(JSON.stringify(envelope)); + else console.error(error instanceof RemoteCatalogError ? error.message : "Remote catalog installation failed"); + return code === "lock_busy" ? 3 : 1; + } +} diff --git a/src/cli/dispatch.ts b/src/cli/dispatch.ts index bf124af303..039edb4701 100644 --- a/src/cli/dispatch.ts +++ b/src/cli/dispatch.ts @@ -485,6 +485,10 @@ const commandRunners: Record = { const { handleDisconnectCommand } = await import("./connect"); return await handleDisconnectCommand(deps.args.slice(1)); }, + catalog: async deps => { + const { handleCatalogCommand } = await import("./catalog"); + return await handleCatalogCommand(deps.args.slice(1)); + }, "sync-cache": async deps => { const cacheArgs = deps.args.slice(1); const restartCodex = cacheArgs.includes("--restart-codex"); diff --git a/src/cli/help.ts b/src/cli/help.ts index 8608e5789d..fcb68d1d88 100644 --- a/src/cli/help.ts +++ b/src/cli/help.ts @@ -45,6 +45,7 @@ Usage: ocx sync [--restart-codex] Fetch models from providers and inject into Codex config ocx sync-cache [--restart-codex] Refresh Codex's model cache from the active catalog + ocx catalog pull Install a validated remote catalog and refresh the Codex cache ocx status Check proxy server status (on a hub: one block with its ports and token source) ocx doctor Diagnose environment/network issues (WSL, proxy, ChatGPT reachability) ocx doctor --reclaim-response-temps diff --git a/src/cli/registry.ts b/src/cli/registry.ts index 91eab8723b..d6381e8f55 100644 --- a/src/cli/registry.ts +++ b/src/cli/registry.ts @@ -132,6 +132,16 @@ export const CLI_COMMANDS: CliCommandEntry[] = [ "--restart-desktop-app (Windows only, opt-in) fully restarts the Codex desktop app so its model picker re-reads the catalog. Never implied by --restart-codex: it ends live conversations.", ], }, + { + name: "catalog", + usage: "ocx catalog pull [--auth-env ] [--json] [--restart-codex]", + summary: "Install a validated remote /v1/catalog snapshot into Codex.", + details: [ + "Authentication is read only from the named environment variable and sent as a Bearer token.", + "HTTPS is required except for loopback HTTP; redirects are refused.", + "The catalog and models_cache.json are coordinated under the Codex catalog write lock.", + ], + }, { name: "status", usage: "ocx status", summary: "Check proxy server status." }, { name: "doctor", diff --git a/src/codex/catalog/remote.ts b/src/codex/catalog/remote.ts new file mode 100644 index 0000000000..24e2e9b4f7 --- /dev/null +++ b/src/codex/catalog/remote.ts @@ -0,0 +1,205 @@ +import { existsSync, readFileSync, statSync } from "node:fs"; + +import { MAX_REMOTE_CATALOG_BYTES } from "../../server/catalog-download"; +import { readBoundedResponseBytes } from "../../lib/bounded-body"; +import { withCatalogWriteSerialization, type CatalogSerializationOutcome } from "../catalog-write-serialization"; +import { replaceActiveCodexCatalog } from "../internal/catalog-writer"; +import { getCodexHome } from "../paths"; +import { readCodexCatalogPathForHome } from "./parsing"; +import { invalidateCodexModelsCacheWithPermit } from "./sync"; + +const DEFAULT_TIMEOUT_MS = 15_000; +const MAX_MODELS = 2_000; +const MAX_SLUG_BYTES = 512; +const ALLOWED_MODALITIES = new Set(["text", "image", "audio"]); + +export type RemoteCatalogFailureCode = + | "url_invalid" | "insecure_http_refused" | "credential_invalid" | "request_failed" + | "redirect_refused" | "http_error" | "body_too_large" | "body_invalid" + | "catalog_invalid" | "write_failed" | "lock_busy" | "lock_database" | "unsafe_path"; + +export class RemoteCatalogError extends Error { + constructor(readonly code: RemoteCatalogFailureCode, message: string, readonly status?: number) { + super(message); + this.name = "RemoteCatalogError"; + } +} + +export interface RemoteCatalogDocument extends Record { + models: Record[]; +} + +export interface PullRemoteCatalogOptions { + token?: string; + timeoutMs?: number; + maxBytes?: number; + fetchImpl?: typeof fetch; + codexHome?: string; +} + +export interface PullRemoteCatalogResult { + status: "updated" | "unchanged"; + catalogWritten: boolean; + cacheSynced: boolean; + codexHome: string; + catalogPath: string; + modelCount: number; +} + +function isLoopback(hostname: string): boolean { + const host = hostname.toLowerCase(); + return host === "localhost" || host === "127.0.0.1" || host === "[::1]" || host === "::1"; +} + +export function validateRemoteCatalogUrl(input: string): URL { + let url: URL; + try { url = new URL(input); } catch { throw new RemoteCatalogError("url_invalid", "Catalog URL must be an absolute HTTPS URL"); } + if (url.username || url.password) throw new RemoteCatalogError("url_invalid", "Catalog URL must not contain credentials"); + if (url.protocol !== "https:" && !(url.protocol === "http:" && isLoopback(url.hostname))) { + throw new RemoteCatalogError("insecure_http_refused", "Catalog URL requires HTTPS (HTTP is allowed only on loopback)"); + } + if (url.pathname !== "/v1/catalog" || url.search || url.hash) { + throw new RemoteCatalogError("url_invalid", "Catalog URL must identify /v1/catalog without query or fragment"); + } + return url; +} + +function validateToken(token: string | undefined): string | undefined { + if (token === undefined) return undefined; + if (!token || token.length > 4096 || /[\r\n\0]/.test(token)) { + throw new RemoteCatalogError("credential_invalid", "Catalog authentication environment variable is invalid"); + } + return token; +} + +export function validateRemoteCatalogDocument(value: unknown): RemoteCatalogDocument { + const invalid = (message: string): never => { throw new RemoteCatalogError("catalog_invalid", message); }; + if (!value || typeof value !== "object" || Array.isArray(value)) invalid("Remote catalog must be a JSON object"); + const document = value as Record; + const rawModels = document.models; + if (!Array.isArray(rawModels) || rawModels.length === 0 || rawModels.length > MAX_MODELS) { + invalid("Remote catalog models must be a non-empty bounded array"); + } + const models = rawModels as unknown[]; + const slugs = new Set(); + for (const row of models) { + if (!row || typeof row !== "object" || Array.isArray(row) || Object.getPrototypeOf(row) !== Object.prototype) { + invalid("Remote catalog model rows must be plain objects"); + } + const model = row as Record; + const rawSlug = model.slug; + if (typeof rawSlug !== "string") invalid("Remote catalog contains an invalid model slug"); + const slug = rawSlug as string; + if (slug !== slug.trim() || !slug + || new TextEncoder().encode(slug).byteLength > MAX_SLUG_BYTES || /[\x00-\x1f\x7f]/.test(slug)) { + invalid("Remote catalog contains an invalid model slug"); + } + if (slugs.has(slug)) invalid("Remote catalog contains duplicate model slugs"); + slugs.add(slug); + if (Object.hasOwn(model, "input_modalities")) { + const modalities = model.input_modalities; + if (!Array.isArray(modalities) || modalities.length === 0 + || modalities.some(item => typeof item !== "string" || !ALLOWED_MODALITIES.has(item))) { + invalid("Remote catalog contains unsupported input modalities"); + } + } + } + return document as RemoteCatalogDocument; +} + +function safeTimeout(value: number | undefined): number { + return typeof value === "number" && Number.isFinite(value) && value > 0 + ? Math.min(Math.floor(value), 120_000) : DEFAULT_TIMEOUT_MS; +} + +export async function fetchRemoteCatalog( + input: string, + options: Pick = {}, +): Promise<{ document: RemoteCatalogDocument; content: string }> { + const url = validateRemoteCatalogUrl(input); + const token = validateToken(options.token); + const headers = new Headers({ Accept: "application/json" }); + if (token !== undefined) headers.set("Authorization", `Bearer ${token}`); + let response: Response; + try { + response = await (options.fetchImpl ?? fetch)(url, { + method: "GET", headers, redirect: "manual", signal: AbortSignal.timeout(safeTimeout(options.timeoutMs)), + }); + } catch { + throw new RemoteCatalogError("request_failed", "Remote catalog request did not complete"); + } + if (response.status >= 300 && response.status < 400) { + try { await response.body?.cancel(); } catch { /* best effort */ } + throw new RemoteCatalogError("redirect_refused", "Remote catalog redirect was refused", response.status); + } + if (!response.ok) { + try { await response.body?.cancel(); } catch { /* best effort */ } + throw new RemoteCatalogError("http_error", `Remote catalog request failed with HTTP ${response.status}`, response.status); + } + const contentType = response.headers.get("content-type")?.split(";", 1)[0]?.trim().toLowerCase(); + if (contentType !== "application/json" && contentType?.endsWith("+json") !== true) { + try { await response.body?.cancel(); } catch { /* best effort */ } + throw new RemoteCatalogError("body_invalid", "Remote catalog response was not JSON"); + } + const maxBytes = options.maxBytes ?? MAX_REMOTE_CATALOG_BYTES; + const declaredRaw = response.headers.get("content-length"); + if (declaredRaw !== null) { + const declared = Number(declaredRaw); + if (!Number.isSafeInteger(declared) || declared < 0 || declared > maxBytes) { + try { await response.body?.cancel(); } catch { /* best effort */ } + throw new RemoteCatalogError("body_too_large", "Remote catalog exceeded the allowed size"); + } + } + let bytes: Uint8Array; + try { + const bounded = await readBoundedResponseBytes(response, { maxBytes, inactivityTimeoutMs: safeTimeout(options.timeoutMs) }); + if (bounded.oversized) throw new RemoteCatalogError("body_too_large", "Remote catalog exceeded the allowed size"); + bytes = bounded.bytes; + } catch (error) { + if (error instanceof RemoteCatalogError) throw error; + throw new RemoteCatalogError("request_failed", "Remote catalog download did not complete"); + } + let text: string; + try { text = new TextDecoder("utf-8", { fatal: true }).decode(bytes); } + catch { throw new RemoteCatalogError("body_invalid", "Remote catalog was not valid UTF-8"); } + let parsed: unknown; + try { parsed = JSON.parse(text); } + catch { throw new RemoteCatalogError("body_invalid", "Remote catalog was not valid JSON"); } + const document = validateRemoteCatalogDocument(parsed); + return { document, content: `${JSON.stringify(document, null, 2)}\n` }; +} + +function mapSerializationFailure(outcome: CatalogSerializationOutcome): never { + if (outcome.kind === "completed") throw new RemoteCatalogError("write_failed", "Remote catalog installation failed"); + const code = outcome.reason === "busy" ? "lock_busy" : outcome.reason === "database" ? "lock_database" : "unsafe_path"; + throw new RemoteCatalogError(code, `Remote catalog installation unavailable (${outcome.reason})`); +} + +export async function pullRemoteCatalog(input: string, options: PullRemoteCatalogOptions = {}): Promise { + // Network acquisition and fail-closed validation intentionally happen before K. + const fetched = await fetchRemoteCatalog(input, options); + const codexHome = options.codexHome ?? getCodexHome(); + const catalogPath = readCodexCatalogPathForHome(codexHome); + const current = existsSync(catalogPath) ? readFileSync(catalogPath) : null; + const candidate = Buffer.from(fetched.content, "utf8"); + if (current?.equals(candidate)) { + return { status: "unchanged", catalogWritten: false, cacheSynced: false, codexHome, catalogPath, modelCount: fetched.document.models.length }; + } + const outcome = withCatalogWriteSerialization(codexHome, permit => { + // Re-check under K: another writer may have installed these bytes while the request was in flight. + const lockedCurrent = existsSync(catalogPath) ? readFileSync(catalogPath) : null; + if (lockedCurrent?.equals(candidate)) return { catalogWritten: false, cacheSynced: false }; + replaceActiveCodexCatalog(permit, codexHome, { path: catalogPath, content: fetched.content }); + const cacheSynced = invalidateCodexModelsCacheWithPermit(permit, codexHome, { allowWhenDesiredDisabled: true }); + if (!cacheSynced) throw new RemoteCatalogError("write_failed", "Remote catalog cache synchronization failed"); + return { catalogWritten: true, cacheSynced: true }; + }); + if (outcome.kind !== "completed") return mapSerializationFailure(outcome); + return { + status: outcome.value.catalogWritten ? "updated" : "unchanged", + ...outcome.value, + codexHome, + catalogPath, + modelCount: fetched.document.models.length, + }; +} diff --git a/tests/codex-integration/catalog-remote-pull.test.ts b/tests/codex-integration/catalog-remote-pull.test.ts new file mode 100644 index 0000000000..e82b3321e7 --- /dev/null +++ b/tests/codex-integration/catalog-remote-pull.test.ts @@ -0,0 +1,205 @@ +import { afterEach, describe, expect, mock, test } from "bun:test"; +import { existsSync, mkdtempSync, readFileSync, statSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Database } from "bun:sqlite"; + +import { + fetchRemoteCatalog, + pullRemoteCatalog, + RemoteCatalogError, + validateRemoteCatalogDocument, + validateRemoteCatalogUrl, +} from "../../src/codex/catalog/remote"; +import { resolveCodexCatalogSerializationDatabasePath, resolveEffectiveUserIdentity } from "../../src/codex/user-identity"; +import { withCatalogWriteSerialization } from "../../src/codex/catalog-write-serialization"; +import { removeTreeWithRetry } from "../helpers/remove-tree"; + +const homes: string[] = []; +afterEach(() => { while (homes.length) removeTreeWithRetry(homes.pop()!); }); + +function home(): string { + const value = mkdtempSync(join(tmpdir(), "ocx-catalog-pull-")); + homes.push(value); + return value; +} + +const catalog = { version: 1, models: [{ slug: "provider/model", input_modalities: ["text", "image"], extension: { safe: true } }] }; +const response = (value: unknown, init: ResponseInit = {}) => new Response(JSON.stringify(value), { + headers: { "Content-Type": "application/json", ...init.headers }, status: init.status, +}); + +describe("remote catalog acquisition", () => { + test("accepts HTTPS and loopback HTTP but rejects credentials and insecure remote HTTP", () => { + expect(validateRemoteCatalogUrl("https://hub.example.com/v1/catalog").href).toBe("https://hub.example.com/v1/catalog"); + expect(validateRemoteCatalogUrl("http://127.0.0.1:10100/v1/catalog").protocol).toBe("http:"); + expect(() => validateRemoteCatalogUrl("http://hub.example.com/v1/catalog")).toThrow(RemoteCatalogError); + expect(() => validateRemoteCatalogUrl("https://user:secret@example.com/v1/catalog")).toThrow(RemoteCatalogError); + expect(() => validateRemoteCatalogUrl("https://hub.example.com/v1/catalog?q=secret")).toThrow(RemoteCatalogError); + }); + + test("sends optional bearer authentication from the caller without following redirects", async () => { + const fetchImpl = mock(async (_input: RequestInfo | URL, init?: RequestInit) => { + expect(new Headers(init?.headers).get("authorization")).toBe("Bearer env-token"); + expect(init?.redirect).toBe("manual"); + return response(catalog); + }) as typeof fetch; + const fetched = await fetchRemoteCatalog("https://hub.example/v1/catalog", { token: "env-token", fetchImpl }); + expect(fetched.document).toEqual(catalog); + expect(fetchImpl).toHaveBeenCalledTimes(1); + + await expect(fetchRemoteCatalog("https://hub.example/v1/catalog", { + token: "secret-marker", fetchImpl: async () => new Response("body-marker", { status: 302, headers: { Location: "https://other.example/secret" } }), + })).rejects.toMatchObject({ code: "redirect_refused", message: "Remote catalog redirect was refused" }); + }); + + test("never reflects credentials, remote bodies, URLs, or transport causes", async () => { + for (const fetchImpl of [ + async () => new Response("remote-body-marker", { status: 401 }), + async () => { throw new Error("secret-marker https://private.example/path"); }, + ]) { + let caught: unknown; + try { await fetchRemoteCatalog("https://hub.example/v1/catalog", { token: "secret-marker", fetchImpl: fetchImpl as typeof fetch }); } + catch (error) { caught = error; } + expect(String(caught)).not.toContain("secret-marker"); + expect(String(caught)).not.toContain("remote-body-marker"); + expect(String(caught)).not.toContain("private.example"); + expect((caught as Error).cause).toBeUndefined(); + } + }); + + test.each([401, 403, 404, 500])("rejects HTTP %s", async status => { + await expect(fetchRemoteCatalog("https://hub.example/v1/catalog", { + fetchImpl: async () => new Response(null, { status }), + })).rejects.toMatchObject({ code: "http_error", status }); + }); + + test("enforces declared and streamed byte limits", async () => { + await expect(fetchRemoteCatalog("https://hub.example/v1/catalog", { + maxBytes: 10, fetchImpl: async () => new Response("{}", { headers: { "Content-Type": "application/json", "Content-Length": "11" } }), + })).rejects.toMatchObject({ code: "body_too_large" }); + const stream = new ReadableStream({ start(controller) { + controller.enqueue(new TextEncoder().encode('{"models":[')); + controller.enqueue(new Uint8Array(128)); controller.close(); + } }); + await expect(fetchRemoteCatalog("https://hub.example/v1/catalog", { + maxBytes: 32, fetchImpl: async () => new Response(stream, { headers: { "Content-Type": "application/json", "Content-Length": "1" } }), + })).rejects.toMatchObject({ code: "body_too_large" }); + }); + + test("bounds headers and stalled streams", async () => { + await expect(fetchRemoteCatalog("https://hub.example/v1/catalog", { + timeoutMs: 10, + fetchImpl: async (_input, init) => await new Promise((_resolve, reject) => { + init?.signal?.addEventListener("abort", () => reject(new Error("secret timeout cause")), { once: true }); + }), + })).rejects.toMatchObject({ code: "request_failed" }); + }); +}); + +describe("remote catalog validation", () => { + test.each([ + [null], [[]], [{}], [{ models: [] }], [{ models: [null] }], [{ models: [[]] }], + [{ models: [{}] }], [{ models: [{ slug: "" }] }], [{ models: [{ slug: " padded " }] }], + [{ models: [{ slug: "bad\u0000slug" }] }], [{ models: [{ slug: "a" }, { slug: "a" }] }], + [{ models: [{ slug: "a", input_modalities: [] }] }], + [{ models: [{ slug: "a", input_modalities: ["video"] }] }], + ])("rejects invalid document %#", value => { + expect(() => validateRemoteCatalogDocument(value)).toThrow(RemoteCatalogError); + }); + + test("preserves safe additive fields", () => { + expect(validateRemoteCatalogDocument(catalog)).toEqual(catalog); + }); +}); + +describe("remote catalog coordinated installation", () => { + test("updates catalog and cache under the shared writer even when desired integration is disabled", async () => { + const codexHome = home(); + const opencodexHome = home(); + const previous = process.env.OPENCODEX_HOME; + process.env.OPENCODEX_HOME = opencodexHome; + writeFileSync(join(opencodexHome, "config.json"), JSON.stringify({ providers: {}, defaultProvider: "openai", desiredIntegrations: { codex: false } })); + try { + const result = await pullRemoteCatalog("https://hub.example/v1/catalog", { + codexHome, fetchImpl: async () => response(catalog), + }); + expect(result).toMatchObject({ status: "updated", catalogWritten: true, cacheSynced: true, modelCount: 1 }); + expect(JSON.parse(readFileSync(result.catalogPath, "utf8"))).toEqual(catalog); + expect(JSON.parse(readFileSync(join(codexHome, "models_cache.json"), "utf8")).models).toEqual(catalog.models); + } finally { + if (previous === undefined) delete process.env.OPENCODEX_HOME; else process.env.OPENCODEX_HOME = previous; + } + }); + + test("an identical pull preserves catalog and cache mtimes", async () => { + const codexHome = home(); + const fetchImpl = async () => response(catalog); + const first = await pullRemoteCatalog("https://hub.example/v1/catalog", { codexHome, fetchImpl }); + const cachePath = join(codexHome, "models_cache.json"); + const before = [statSync(first.catalogPath).mtimeMs, statSync(cachePath).mtimeMs]; + await Bun.sleep(20); + const second = await pullRemoteCatalog("https://hub.example/v1/catalog", { codexHome, fetchImpl }); + expect(second).toMatchObject({ status: "unchanged", catalogWritten: false, cacheSynced: false }); + expect([statSync(first.catalogPath).mtimeMs, statSync(cachePath).mtimeMs]).toEqual(before); + }); + + test("lock contention is typed and preserves last-known-good files", async () => { + const codexHome = home(); + // Materialize K, then hold BEGIN IMMEDIATE from a separate connection while pull attempts it. + expect(withCatalogWriteSerialization(codexHome, () => null).kind).toBe("completed"); + const lockPath = resolveCodexCatalogSerializationDatabasePath(resolveEffectiveUserIdentity(), codexHome); + const holder = new Database(lockPath); + holder.exec("PRAGMA busy_timeout=0; BEGIN IMMEDIATE"); + try { + await expect(pullRemoteCatalog("https://hub.example/v1/catalog", { + codexHome, fetchImpl: async () => response(catalog), + })).rejects.toMatchObject({ code: "lock_busy" }); + expect(existsSync(join(codexHome, "opencodex-catalog.json"))).toBe(false); + expect(existsSync(join(codexHome, "models_cache.json"))).toBe(false); + } finally { + holder.exec("ROLLBACK"); holder.close(); + } + }); + + test("fetch and validation failures preserve last-known-good files", async () => { + const codexHome = home(); + const catalogPath = join(codexHome, "opencodex-catalog.json"); + const cachePath = join(codexHome, "models_cache.json"); + writeFileSync(catalogPath, "catalog-before"); writeFileSync(cachePath, "cache-before"); + for (const fetchImpl of [async () => new Response(null, { status: 500 }), async () => response({ models: [] })]) { + await expect(pullRemoteCatalog("https://hub.example/v1/catalog", { codexHome, fetchImpl: fetchImpl as typeof fetch })).rejects.toBeInstanceOf(RemoteCatalogError); + expect(readFileSync(catalogPath, "utf8")).toBe("catalog-before"); + expect(readFileSync(cachePath, "utf8")).toBe("cache-before"); + } + expect(existsSync(join(codexHome, "models_cache.json"))).toBe(true); + }); +}); + +describe("catalog pull CLI envelope", () => { + test("emits a stable JSON failure without reflecting the secret environment value", async () => { + const { handleCatalogCommand } = await import("../../src/cli/catalog"); + const old = process.env.OCX_CATALOG_TEST_TOKEN; + process.env.OCX_CATALOG_TEST_TOKEN = "secret-cli-marker"; + const output: string[] = []; + const errors: string[] = []; + const log = console.log; + const error = console.error; + console.log = (...values) => { output.push(values.map(String).join(" ")); }; + console.error = (...values) => { errors.push(values.map(String).join(" ")); }; + try { + expect(await handleCatalogCommand([ + "pull", "http://remote.example/v1/catalog", "--auth-env", "OCX_CATALOG_TEST_TOKEN", "--json", + ])).toBe(1); + expect(output).toHaveLength(1); + expect(JSON.parse(output[0]!)).toEqual({ + schemaVersion: 1, ok: false, status: "failed", catalogWritten: false, + cacheSynced: false, codexRestarted: false, code: "insecure_http_refused", + }); + expect(output.join("\n") + errors.join("\n")).not.toContain("secret-cli-marker"); + } finally { + console.log = log; console.error = error; + if (old === undefined) delete process.env.OCX_CATALOG_TEST_TOKEN; else process.env.OCX_CATALOG_TEST_TOKEN = old; + } + }); +}); diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index b56ab03905..49aeb6c2cd 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -108,6 +108,7 @@ "catalog-input-modality-enum.test.ts": "codex-integration", "catalog-llamacpp-capabilities.test.ts": "codex-integration", "catalog-oauth-observation.test.ts": "codex-integration", + "catalog-remote-pull.test.ts": "codex-integration", "catalog-retain-models.test.ts": "codex-integration", "catalog-verbosity-default.test.ts": "codex-integration", "catalog-vision-sidecar-modalities.test.ts": "codex-integration",