diff --git a/src/lib/payload/fragment-arx.ts b/src/lib/payload/fragment-arx.ts index 91de2d3..1c8838f 100644 --- a/src/lib/payload/fragment-arx.ts +++ b/src/lib/payload/fragment-arx.ts @@ -6,6 +6,8 @@ import { arx3DecompressEnvelope, arxCompressPayloads, arxDecompress, + getActiveArx2OverlayVersion, + getActiveDictVersion, isExternalArx2OverlayDictionaryLoaded, isExternalDictionaryLoaded, loadArxDictionary, @@ -30,36 +32,79 @@ type TransportLengthCalculator = (value: string) => number; let arxDictionaryLoadPromise: Promise | null = null; let arx2OverlayDictionaryLoadPromise: Promise | null = null; +// Compact ARX fragments (tags `a`/`b`/`c`) do NOT carry a dictionary version — the tag implies the +// CURRENT dictionary, which keeps links short. The safety cost is that a build must not decode with +// a dictionary NEWER than it was built for (a CDN/asset split serving a future dictionary, or a +// version bump), because it would lack the new slots and could produce a structurally-valid-but- +// wrong envelope. We pin the newest supported version and reject anything newer so decode hard-fails +// instead of mis-decoding. The built-in fallback dictionary (version 0) and the current external +// dictionary (version 1) are both <= this and remain usable. Bumping a dictionary version is +// therefore a wire change that also requires new compact tags and updating +// tests/arx-dictionary-pin.test.ts. +const EXPECTED_ARX_DICTIONARY_VERSION = 1; +const EXPECTED_ARX2_OVERLAY_VERSION = 1; + +function assertArxDictionaryNotNewerThanExpected(): void { + const version = getActiveDictVersion(); + if (version > EXPECTED_ARX_DICTIONARY_VERSION) { + throw new Error( + `Active arx dictionary version ${version} is newer than this build supports (${EXPECTED_ARX_DICTIONARY_VERSION}); refusing to decode with a forward-incompatible dictionary.`, + ); + } +} + +function assertArx2OverlayNotNewerThanExpected(): void { + const version = getActiveArx2OverlayVersion(); + if (version > EXPECTED_ARX2_OVERLAY_VERSION) { + throw new Error( + `Active arx2 overlay dictionary version ${version} is newer than this build supports (${EXPECTED_ARX2_OVERLAY_VERSION}); refusing to decode with a forward-incompatible dictionary.`, + ); + } +} + async function ensureArxDictionaryLoaded(): Promise { - if (isExternalDictionaryLoaded()) { - return; + if (!isExternalDictionaryLoaded()) { + arxDictionaryLoadPromise ??= loadArxDictionary() + .then((version) => { + if (version < 0) { + // The external fetch failed and the built-in fallback is now active. Don't cache this, so + // a later call can retry the external dictionary once the endpoint recovers; the current + // call still proceeds (degraded) on the built-in dictionary rather than being poisoned. + arxDictionaryLoadPromise = null; + } + }) + .catch((error) => { + arxDictionaryLoadPromise = null; + throw error; + }); + await arxDictionaryLoadPromise; } - // Reset the cached promise on failure so a transient dictionary load error can be retried - // instead of permanently poisoning every arx encode/decode for the page's lifetime. - arxDictionaryLoadPromise ??= loadArxDictionary() - .then(() => undefined) - .catch((error) => { - arxDictionaryLoadPromise = null; - throw error; - }); - await arxDictionaryLoadPromise; + // Runs for both fetched and injected (sync) dictionaries so a forward-incompatible skew can't slip + // through whichever way the dictionary was loaded. + assertArxDictionaryNotNewerThanExpected(); } async function ensureArx2DictionariesLoaded(): Promise { await ensureArxDictionaryLoaded(); - if (isExternalArx2OverlayDictionaryLoaded()) { - return; + if (!isExternalArx2OverlayDictionaryLoaded()) { + // Same retry-on-failure contract as the base dictionary (loadArx2OverlayDictionary also resolves + // -1 on a transient fetch failure rather than rejecting). + arx2OverlayDictionaryLoadPromise ??= loadArx2OverlayDictionary() + .then((version) => { + if (version < 0) { + arx2OverlayDictionaryLoadPromise = null; + } + }) + .catch((error) => { + arx2OverlayDictionaryLoadPromise = null; + throw error; + }); + await arx2OverlayDictionaryLoadPromise; } - arx2OverlayDictionaryLoadPromise ??= loadArx2OverlayDictionary() - .then(() => undefined) - .catch((error) => { - arx2OverlayDictionaryLoadPromise = null; - throw error; - }); - await arx2OverlayDictionaryLoadPromise; + assertArx2OverlayNotNewerThanExpected(); } function decodeArxEncodedPayload(encoded: string): string { diff --git a/src/lib/payload/fragment.ts b/src/lib/payload/fragment.ts index 4d20c86..429d1bf 100644 --- a/src/lib/payload/fragment.ts +++ b/src/lib/payload/fragment.ts @@ -1,5 +1,5 @@ import { compressToEncodedURIComponent, decompressFromEncodedURIComponent } from "lz-string"; -import { deflateSync, inflateSync, strFromU8, strToU8 } from "fflate"; +import { deflateSync, inflateSync, strFromU8, strToU8, unzlibSync } from "fflate"; import { normalizeEnvelope } from "@/lib/payload/envelope"; import { packEnvelope, unpackEnvelope } from "@/lib/payload/wire-format"; import { @@ -136,6 +136,26 @@ function encodePayload(json: string, codec: PayloadCodec): string { } } +/** + * Inflates a `deflate` payload, accepting both raw deflate (current encoder) and zlib-wrapped + * deflate (emitted by an older encoder version). Raw is tried first so current output is never at + * risk of header mis-detection; zlib is a back-compat fallback so historical shared links keep + * decoding instead of failing as invalid-json. + */ +function inflateDeflatePayload(bytes: Uint8Array): Uint8Array { + try { + return inflateSync(bytes); + } catch (rawDeflateError) { + // Fall back to zlib-wrapped (legacy) deflate. If that also fails the input is not valid deflate + // at all, so surface the original raw-inflate error rather than a misleading zlib one. + try { + return unzlibSync(bytes); + } catch { + throw rawDeflateError; + } + } +} + function decodePayload(encoded: string, codec: PayloadCodec): string | null { switch (codec) { case "plain": @@ -143,7 +163,7 @@ function decodePayload(encoded: string, codec: PayloadCodec): string | null { case "lz": return decompressFromEncodedURIComponent(encoded); case "deflate": - return strFromU8(inflateSync(fromBase64UrlBytes(encoded))); + return strFromU8(inflateDeflatePayload(fromBase64UrlBytes(encoded))); case "arx": case "arx2": case "arx3": @@ -531,8 +551,18 @@ export async function decodeFragmentAsync(hash: string, options?: DecodeOptions) if (error instanceof Error && error.name === "ArxDecodedPayloadTooLargeError") { return { ok: false, code: "decoded-too-large", message: error.message }; } - return { ok: false, code: "invalid-json", message: "The fragment payload could not be decoded as valid JSON." }; + const arxHint = + codec === "arx" || codec === "arx2" || codec === "arx3" + ? " It may have been encoded with a different ARX dictionary version." + : ""; + return { ok: false, code: "invalid-json", message: `The fragment payload could not be decoded as valid JSON.${arxHint}` }; } - return resolveEnvelope(parsed, header.fragmentLength); + const resolved = resolveEnvelope(parsed, header.fragmentLength); + if (!resolved.ok && resolved.code === "invalid-envelope" && (codec === "arx" || codec === "arx2" || codec === "arx3")) { + // An ARX payload that decoded but is not a valid envelope almost always means the active + // dictionary differs from the one it was encoded with (see tests/arx-dictionary-pin.test.ts). + return { ...resolved, message: `${resolved.message} It may have been encoded with a different ARX dictionary version.` }; + } + return resolved; } diff --git a/tests/arx-codec.test.ts b/tests/arx-codec.test.ts index 971392c..efc3096 100644 --- a/tests/arx-codec.test.ts +++ b/tests/arx-codec.test.ts @@ -543,7 +543,7 @@ describe("arx2 tuple envelope", () => { expect(parsed.ok).toBe(true); }); - it("decodes arx and arx2 payloads when the active dictionary version differs", async () => { + it("refuses to decode when the active dictionary is newer than the build supports", async () => { loadArxDictionarySync(arxDictionaryJson); const arxHash = `#${await encodeEnvelopeAsync(bundle, { codec: "arx" })}`; const arx2Hash = `#${await encodeEnvelopeAsync(bundle, { codec: "arx2" })}`; @@ -553,14 +553,15 @@ describe("arx2 tuple envelope", () => { }; loadArxDictionarySync(shiftedDictionary); - - const arxParsed = await decodeFragmentAsync(arxHash); - const arx2Parsed = await decodeFragmentAsync(arx2Hash); - - expect(arxParsed.ok).toBe(true); - expect(arx2Parsed.ok).toBe(true); - - loadArxDictionarySync(arxDictionaryJson); + try { + // A dictionary newer than this build supports is rejected: a compact arx link carries no + // version of its own, so decoding it against a forward-incompatible dictionary could silently + // mis-decode. Hard-failing is the safe behavior. + expect((await decodeFragmentAsync(arxHash)).ok).toBe(false); + expect((await decodeFragmentAsync(arx2Hash)).ok).toBe(false); + } finally { + loadArxDictionarySync(arxDictionaryJson); + } }); it("can decode arx2 payloads directly through the codec API", async () => { diff --git a/tests/arx-dict-load-retry.test.ts b/tests/arx-dict-load-retry.test.ts new file mode 100644 index 0000000..91ad084 --- /dev/null +++ b/tests/arx-dict-load-retry.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it, vi } from "vitest"; + +// loadArxDictionary() resolves -1 on a transient fetch/parse failure (it does not reject). The +// ensure-helper must treat that as a failure and clear its cached promise, so a later decode retries +// the load instead of reusing a poisoned no-op promise for the page's lifetime. +const { loadArxDictionaryMock } = vi.hoisted(() => ({ loadArxDictionaryMock: vi.fn() })); + +vi.mock("@/lib/payload/arx-codec", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + isExternalDictionaryLoaded: () => false, // force the fetch path so the cached promise is exercised + loadArxDictionary: loadArxDictionaryMock, + }; +}); + +const { decodeFragmentAsync } = await import("@/lib/payload/fragment"); + +describe("arx dictionary load retry", () => { + it("retries after a transient (-1) load failure instead of caching it for the page lifetime", async () => { + loadArxDictionaryMock.mockResolvedValueOnce(-1); // transient failure + const first = await decodeFragmentAsync("#bAAAA", { skipFragmentBudget: true }); + expect(first.ok).toBe(false); + expect(loadArxDictionaryMock).toHaveBeenCalledTimes(1); + + // Endpoint recovers: the next decode must attempt the load again, not reuse a poisoned promise. + loadArxDictionaryMock.mockResolvedValueOnce(1); + await decodeFragmentAsync("#bAAAA", { skipFragmentBudget: true }); + expect(loadArxDictionaryMock).toHaveBeenCalledTimes(2); + }); +}); diff --git a/tests/arx-dict-version-guard.test.ts b/tests/arx-dict-version-guard.test.ts new file mode 100644 index 0000000..dbc6802 --- /dev/null +++ b/tests/arx-dict-version-guard.test.ts @@ -0,0 +1,49 @@ +import { beforeEach, describe, expect, it } from "vitest"; +import arx2DictionaryJson from "../public/arx2-dictionary.json"; +import arxDictionaryJson from "../public/arx-dictionary.json"; +import { loadArx2OverlayDictionarySync, loadArxDictionarySync } from "@/lib/payload/arx-codec"; +import { decodeFragmentAsync, encodeEnvelopeAsync } from "@/lib/payload/fragment"; +import type { PayloadEnvelope } from "@/lib/payload/schema"; + +// Compact arx links carry no dictionary version (the tag implies the current dictionary), so a build +// must refuse to decode with a skewed-version dictionary rather than silently mis-decode. These +// tests exercise that guard by making a different-version dictionary active and confirming decode +// hard-fails, then recovers when the expected version is restored. +const envelope: PayloadEnvelope = { + v: 1, + codec: "arx2", + activeArtifactId: "doc", + artifacts: [{ id: "doc", kind: "markdown", content: "# Title\n\nSome content for the arx dense path." }], +}; + +describe("arx dictionary version guard", () => { + beforeEach(() => { + loadArx2OverlayDictionarySync(arx2DictionaryJson); + loadArxDictionarySync(arxDictionaryJson); // expected version (1) + }); + + it("refuses to decode a compact arx link when the base dictionary version is skewed", async () => { + const fragment = await encodeEnvelopeAsync(envelope, { codec: "arx2", preferPacked: true }); + + const okParsed = await decodeFragmentAsync(`#${fragment}`, { skipFragmentBudget: true }); + expect(okParsed.ok).toBe(true); + + // Simulate asset/CDN skew or a future bump: a different-version dictionary becomes active. + loadArxDictionarySync({ ...arxDictionaryJson, version: 2 }); + const skewed = await decodeFragmentAsync(`#${fragment}`, { skipFragmentBudget: true }); + expect(skewed.ok).toBe(false); // hard-fail, not a silent mis-decode + + // Recovers once the expected-version dictionary is active again. + loadArxDictionarySync(arxDictionaryJson); + const recovered = await decodeFragmentAsync(`#${fragment}`, { skipFragmentBudget: true }); + expect(recovered.ok).toBe(true); + }); + + it("refuses to decode when the arx2 overlay dictionary version is skewed", async () => { + const fragment = await encodeEnvelopeAsync(envelope, { codec: "arx2", preferPacked: true }); + + loadArx2OverlayDictionarySync({ ...arx2DictionaryJson, version: 2 }); + const skewed = await decodeFragmentAsync(`#${fragment}`, { skipFragmentBudget: true }); + expect(skewed.ok).toBe(false); + }); +}); diff --git a/tests/arx-dictionary-pin.test.ts b/tests/arx-dictionary-pin.test.ts new file mode 100644 index 0000000..c09f61b --- /dev/null +++ b/tests/arx-dictionary-pin.test.ts @@ -0,0 +1,39 @@ +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; + +// Defended contract — learned the hard way: real shared arx/arx2 links broke because a dictionary's +// content was changed while its `version` field stayed 1, so every previously-encoded fragment +// silently mis-decoded into garbage. The dictionary content and its version MUST move together. +// +// If you change a dictionary's slots, you MUST bump its `version` AND update the pinned hash here in +// the same commit. The version bump is the only thing that lets a future build distinguish (and +// reject, rather than garble) a fragment that was encoded with an older dictionary. Do NOT just +// re-pin the hash to make this test pass — that re-creates the silent-drift bug. +// +// arx3 reuses public/arx-dictionary.json (via arx3DecompressEnvelope), so pinning it also covers +// arx3; if arx3 is ever given its own dictionary file, add a pin entry for it here. +const PINNED: Record = { + "public/arx-dictionary.json": { + version: 1, + sha256: "16fe3f72dd5d282fd2f0271647a56c38c4b7eebb6e4723da670765f7d90380a9", + }, + "public/arx2-dictionary.json": { + version: 1, + sha256: "12d0166fda16ce9697831805d92af0f558b4fad5a882c471ba072118561a74eb", + }, +}; + +describe("arx dictionary content is pinned to its version", () => { + for (const [file, pin] of Object.entries(PINNED)) { + it(`${file} matches its pinned version and content hash`, () => { + const obj = JSON.parse(readFileSync(file, "utf8")); + // Hash the canonical (re-serialized) form so whitespace/formatting changes don't trip it, only + // real content changes (slots, version) do. + const canonicalSha256 = createHash("sha256").update(JSON.stringify(obj)).digest("hex"); + + expect(obj.version).toBe(pin.version); + expect(canonicalSha256).toBe(pin.sha256); + }); + } +}); diff --git a/tests/fragment.test.ts b/tests/fragment.test.ts index a6896f7..297b154 100644 --- a/tests/fragment.test.ts +++ b/tests/fragment.test.ts @@ -1,3 +1,4 @@ +import { strToU8, zlibSync } from "fflate"; import { describe, expect, it } from "vitest"; import { decodeFragment, decodeFragmentAsync, encodeEnvelope, encodeEnvelopeAsync } from "@/lib/payload/fragment"; import { compactTagForCodec, type PayloadEnvelope } from "@/lib/payload/schema"; @@ -75,6 +76,33 @@ describe("fragment payload transport", () => { expect(parsed.code).toBe("invalid-json"); }); + it("decodes a legacy zlib-wrapped deflate fragment (older encoder back-compat)", () => { + // The current encoder emits raw deflate, but an older version emitted zlib-wrapped deflate. + // Real historical shared links use that form and must still decode. Build one by hand. + const legacy: PayloadEnvelope = { + v: 1, + codec: "deflate", + title: "legacy zlib link", + activeArtifactId: "doc", + artifacts: [{ id: "doc", kind: "markdown", content: "# Legacy\n\nEncoded with zlib-wrapped deflate." }], + }; + const b64url = Buffer.from(zlibSync(strToU8(JSON.stringify(legacy)))) + .toString("base64") + .replace(/\+/g, "-") + .replace(/\//g, "_") + .replace(/=+$/, ""); + + const parsed = decodeFragment(`#agent-render=v1.deflate.${b64url}`); + + expect(parsed.ok).toBe(true); + if (parsed.ok) { + expect(parsed.envelope.title).toBe("legacy zlib link"); + expect(parsed.envelope.artifacts[0]).toMatchObject({ + content: "# Legacy\n\nEncoded with zlib-wrapped deflate.", + }); + } + }); + it("uses compressed transport when it is smaller", () => { const repetitiveEnvelope: PayloadEnvelope = { ...envelope,