From 5d65f2868d917f52e9665dd48538160dc47d0595 Mon Sep 17 00:00:00 2001 From: Aanish Bhirud Date: Fri, 19 Jun 2026 18:27:24 +0530 Subject: [PATCH 1/4] Decode legacy zlib-wrapped deflate fragments An older encoder version emitted zlib-wrapped deflate, but the current decoder uses fflate's raw `inflateSync`, so those historical shared links fail with `invalid-json`. Verified against a real sha256-checked link (`#agent-render=v1.deflate.eNqN...`, a "cline/kanban analysis" envelope) that the current decoder rejects but is a complete, valid zlib stream. Decode now tries raw inflate first (so current output is never at risk of header mis-detection) and falls back to `unzlibSync` for the zlib-wrapped legacy form. Encoding is unchanged (still raw deflate); this is decode-only and wire-safe. Co-Authored-By: Claude Opus 4.8 --- src/lib/payload/fragment.ts | 18 ++++++++++++++++-- tests/fragment.test.ts | 28 ++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/src/lib/payload/fragment.ts b/src/lib/payload/fragment.ts index 4d20c86..e23fc45 100644 --- a/src/lib/payload/fragment.ts +++ b/src/lib/payload/fragment.ts @@ -1,5 +1,5 @@ import { compressToEncodedURIComponent, decompressFromEncodedURIComponent } from "lz-string"; -import { deflateSync, inflateSync, strFromU8, strToU8 } from "fflate"; +import { deflateSync, inflateSync, strFromU8, strToU8, unzlibSync } from "fflate"; import { normalizeEnvelope } from "@/lib/payload/envelope"; import { packEnvelope, unpackEnvelope } from "@/lib/payload/wire-format"; import { @@ -136,6 +136,20 @@ function encodePayload(json: string, codec: PayloadCodec): string { } } +/** + * Inflates a `deflate` payload, accepting both raw deflate (current encoder) and zlib-wrapped + * deflate (emitted by an older encoder version). Raw is tried first so current output is never at + * risk of header mis-detection; zlib is a back-compat fallback so historical shared links keep + * decoding instead of failing as invalid-json. + */ +function inflateDeflatePayload(bytes: Uint8Array): Uint8Array { + try { + return inflateSync(bytes); + } catch { + return unzlibSync(bytes); + } +} + function decodePayload(encoded: string, codec: PayloadCodec): string | null { switch (codec) { case "plain": @@ -143,7 +157,7 @@ function decodePayload(encoded: string, codec: PayloadCodec): string | null { case "lz": return decompressFromEncodedURIComponent(encoded); case "deflate": - return strFromU8(inflateSync(fromBase64UrlBytes(encoded))); + return strFromU8(inflateDeflatePayload(fromBase64UrlBytes(encoded))); case "arx": case "arx2": case "arx3": diff --git a/tests/fragment.test.ts b/tests/fragment.test.ts index a6896f7..297b154 100644 --- a/tests/fragment.test.ts +++ b/tests/fragment.test.ts @@ -1,3 +1,4 @@ +import { strToU8, zlibSync } from "fflate"; import { describe, expect, it } from "vitest"; import { decodeFragment, decodeFragmentAsync, encodeEnvelope, encodeEnvelopeAsync } from "@/lib/payload/fragment"; import { compactTagForCodec, type PayloadEnvelope } from "@/lib/payload/schema"; @@ -75,6 +76,33 @@ describe("fragment payload transport", () => { expect(parsed.code).toBe("invalid-json"); }); + it("decodes a legacy zlib-wrapped deflate fragment (older encoder back-compat)", () => { + // The current encoder emits raw deflate, but an older version emitted zlib-wrapped deflate. + // Real historical shared links use that form and must still decode. Build one by hand. + const legacy: PayloadEnvelope = { + v: 1, + codec: "deflate", + title: "legacy zlib link", + activeArtifactId: "doc", + artifacts: [{ id: "doc", kind: "markdown", content: "# Legacy\n\nEncoded with zlib-wrapped deflate." }], + }; + const b64url = Buffer.from(zlibSync(strToU8(JSON.stringify(legacy)))) + .toString("base64") + .replace(/\+/g, "-") + .replace(/\//g, "_") + .replace(/=+$/, ""); + + const parsed = decodeFragment(`#agent-render=v1.deflate.${b64url}`); + + expect(parsed.ok).toBe(true); + if (parsed.ok) { + expect(parsed.envelope.title).toBe("legacy zlib link"); + expect(parsed.envelope.artifacts[0]).toMatchObject({ + content: "# Legacy\n\nEncoded with zlib-wrapped deflate.", + }); + } + }); + it("uses compressed transport when it is smaller", () => { const repetitiveEnvelope: PayloadEnvelope = { ...envelope, From 866fe5cb0d821ca3b8d3e49be237ca7296da6f87 Mon Sep 17 00:00:00 2001 From: Aanish Bhirud Date: Fri, 19 Jun 2026 18:27:45 +0530 Subject: [PATCH 2/4] Pin arx dictionary content to its version; flag dict mismatch on decode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Real sha256-verified arx/arx2 links (dictVer=1) decode to garbage: brotli and the dictionary substitution run, but produce a non-envelope object. The `public/arx-dictionary.json` content drifted from what those fragments were encoded with while its `version` field stayed 1, so old links silently mis-decode — a back-compat hazard with no signal. - tests/arx-dictionary-pin.test.ts pins each dictionary's `version` to its canonical content hash. Changing slots now fails CI unless the version is bumped in the same commit, so content and version can never drift apart silently again. - decodeFragmentAsync now appends a dictionary-version hint to the error when an arx payload fails to decode or isn't a valid envelope, instead of a generic invalid-json message. Co-Authored-By: Claude Opus 4.8 --- src/lib/payload/fragment.ts | 14 +++++++++++-- tests/arx-dictionary-pin.test.ts | 36 ++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 tests/arx-dictionary-pin.test.ts diff --git a/src/lib/payload/fragment.ts b/src/lib/payload/fragment.ts index e23fc45..9cca242 100644 --- a/src/lib/payload/fragment.ts +++ b/src/lib/payload/fragment.ts @@ -545,8 +545,18 @@ export async function decodeFragmentAsync(hash: string, options?: DecodeOptions) if (error instanceof Error && error.name === "ArxDecodedPayloadTooLargeError") { return { ok: false, code: "decoded-too-large", message: error.message }; } - return { ok: false, code: "invalid-json", message: "The fragment payload could not be decoded as valid JSON." }; + const arxHint = + codec === "arx" || codec === "arx2" || codec === "arx3" + ? " It may have been encoded with a different ARX dictionary version." + : ""; + return { ok: false, code: "invalid-json", message: `The fragment payload could not be decoded as valid JSON.${arxHint}` }; } - return resolveEnvelope(parsed, header.fragmentLength); + const resolved = resolveEnvelope(parsed, header.fragmentLength); + if (!resolved.ok && resolved.code === "invalid-envelope" && (codec === "arx" || codec === "arx2" || codec === "arx3")) { + // An ARX payload that decoded but is not a valid envelope almost always means the active + // dictionary differs from the one it was encoded with (see tests/arx-dictionary-pin.test.ts). + return { ...resolved, message: `${resolved.message} It may have been encoded with a different ARX dictionary version.` }; + } + return resolved; } diff --git a/tests/arx-dictionary-pin.test.ts b/tests/arx-dictionary-pin.test.ts new file mode 100644 index 0000000..d941f0a --- /dev/null +++ b/tests/arx-dictionary-pin.test.ts @@ -0,0 +1,36 @@ +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; + +// Defended contract — learned the hard way: real shared arx/arx2 links broke because a dictionary's +// content was changed while its `version` field stayed 1, so every previously-encoded fragment +// silently mis-decoded into garbage. The dictionary content and its version MUST move together. +// +// If you change a dictionary's slots, you MUST bump its `version` AND update the pinned hash here in +// the same commit. The version bump is the only thing that lets a future build distinguish (and +// reject, rather than garble) a fragment that was encoded with an older dictionary. Do NOT just +// re-pin the hash to make this test pass — that re-creates the silent-drift bug. +const PINNED: Record = { + "public/arx-dictionary.json": { + version: 1, + sha256: "16fe3f72dd5d282fd2f0271647a56c38c4b7eebb6e4723da670765f7d90380a9", + }, + "public/arx2-dictionary.json": { + version: 1, + sha256: "12d0166fda16ce9697831805d92af0f558b4fad5a882c471ba072118561a74eb", + }, +}; + +describe("arx dictionary content is pinned to its version", () => { + for (const [file, pin] of Object.entries(PINNED)) { + it(`${file} matches its pinned version and content hash`, () => { + const obj = JSON.parse(readFileSync(file, "utf8")); + // Hash the canonical (re-serialized) form so whitespace/formatting changes don't trip it, only + // real content changes (slots, version) do. + const canonicalSha256 = createHash("sha256").update(JSON.stringify(obj)).digest("hex"); + + expect(obj.version).toBe(pin.version); + expect(canonicalSha256).toBe(pin.sha256); + }); + } +}); From 7e589e5b528396e27d069dd6d3c19c7ddf0a72f5 Mon Sep 17 00:00:00 2001 From: Aanish Bhirud Date: Fri, 19 Jun 2026 21:26:21 +0530 Subject: [PATCH 3/4] Address review nits: preserve original deflate error; note arx3 dict reuse - inflateDeflatePayload: when the zlib-wrapped fallback also fails, re-throw the original raw-inflate error instead of a misleading zlib error, so genuinely corrupt deflate surfaces an accurate failure (greptile). - arx-dictionary-pin test: document that arx3 reuses arx-dictionary.json, so the pin also covers arx3 and a future arx3-specific dictionary needs its own entry. Co-Authored-By: Claude Opus 4.8 --- src/lib/payload/fragment.ts | 10 ++++++++-- tests/arx-dictionary-pin.test.ts | 3 +++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/lib/payload/fragment.ts b/src/lib/payload/fragment.ts index 9cca242..429d1bf 100644 --- a/src/lib/payload/fragment.ts +++ b/src/lib/payload/fragment.ts @@ -145,8 +145,14 @@ function encodePayload(json: string, codec: PayloadCodec): string { function inflateDeflatePayload(bytes: Uint8Array): Uint8Array { try { return inflateSync(bytes); - } catch { - return unzlibSync(bytes); + } catch (rawDeflateError) { + // Fall back to zlib-wrapped (legacy) deflate. If that also fails the input is not valid deflate + // at all, so surface the original raw-inflate error rather than a misleading zlib one. + try { + return unzlibSync(bytes); + } catch { + throw rawDeflateError; + } } } diff --git a/tests/arx-dictionary-pin.test.ts b/tests/arx-dictionary-pin.test.ts index d941f0a..c09f61b 100644 --- a/tests/arx-dictionary-pin.test.ts +++ b/tests/arx-dictionary-pin.test.ts @@ -10,6 +10,9 @@ import { describe, expect, it } from "vitest"; // the same commit. The version bump is the only thing that lets a future build distinguish (and // reject, rather than garble) a fragment that was encoded with an older dictionary. Do NOT just // re-pin the hash to make this test pass — that re-creates the silent-drift bug. +// +// arx3 reuses public/arx-dictionary.json (via arx3DecompressEnvelope), so pinning it also covers +// arx3; if arx3 is ever given its own dictionary file, add a pin entry for it here. const PINNED: Record = { "public/arx-dictionary.json": { version: 1, From 969748337307bd6e5a6757ad18a11987e030db0b Mon Sep 17 00:00:00 2001 From: Aanish Bhirud Date: Fri, 19 Jun 2026 22:15:18 +0530 Subject: [PATCH 4/4] Harden arx dictionary loading (adversarial review) Addresses two findings from an adversarial review of the stack: - Transient dictionary-load failures were cached for the page lifetime. loadArxDictionary() resolves -1 (it falls back to the built-in dictionary instead of rejecting), so the ensure helpers treated a failed external fetch as "loaded" and never retried. Now a -1 result clears the cached promise: the current call still proceeds on the built-in fallback, but a later call retries the external dictionary once the endpoint recovers. - Compact arx links carry no dictionary version (the tag implies the current dictionary), so decoding against a skewed dictionary (CDN/asset split or a future version bump) could silently produce a structurally-valid-but-wrong envelope. Decode now rejects any active dictionary NEWER than the build supports; the built-in (v0) and current (v1) dictionaries remain usable. A dictionary version bump is documented as a wire change that also needs new compact tags. Tests: load-retry (transient -1 then recovery), version-skew guard for base and overlay dictionaries, and the prior version-difference test updated to assert the new hard-fail behavior. Co-Authored-By: Claude Opus 4.8 --- src/lib/payload/fragment-arx.ts | 85 +++++++++++++++++++++------- tests/arx-codec.test.ts | 19 ++++--- tests/arx-dict-load-retry.test.ts | 31 ++++++++++ tests/arx-dict-version-guard.test.ts | 49 ++++++++++++++++ 4 files changed, 155 insertions(+), 29 deletions(-) create mode 100644 tests/arx-dict-load-retry.test.ts create mode 100644 tests/arx-dict-version-guard.test.ts diff --git a/src/lib/payload/fragment-arx.ts b/src/lib/payload/fragment-arx.ts index 91de2d3..1c8838f 100644 --- a/src/lib/payload/fragment-arx.ts +++ b/src/lib/payload/fragment-arx.ts @@ -6,6 +6,8 @@ import { arx3DecompressEnvelope, arxCompressPayloads, arxDecompress, + getActiveArx2OverlayVersion, + getActiveDictVersion, isExternalArx2OverlayDictionaryLoaded, isExternalDictionaryLoaded, loadArxDictionary, @@ -30,36 +32,79 @@ type TransportLengthCalculator = (value: string) => number; let arxDictionaryLoadPromise: Promise | null = null; let arx2OverlayDictionaryLoadPromise: Promise | null = null; +// Compact ARX fragments (tags `a`/`b`/`c`) do NOT carry a dictionary version — the tag implies the +// CURRENT dictionary, which keeps links short. The safety cost is that a build must not decode with +// a dictionary NEWER than it was built for (a CDN/asset split serving a future dictionary, or a +// version bump), because it would lack the new slots and could produce a structurally-valid-but- +// wrong envelope. We pin the newest supported version and reject anything newer so decode hard-fails +// instead of mis-decoding. The built-in fallback dictionary (version 0) and the current external +// dictionary (version 1) are both <= this and remain usable. Bumping a dictionary version is +// therefore a wire change that also requires new compact tags and updating +// tests/arx-dictionary-pin.test.ts. +const EXPECTED_ARX_DICTIONARY_VERSION = 1; +const EXPECTED_ARX2_OVERLAY_VERSION = 1; + +function assertArxDictionaryNotNewerThanExpected(): void { + const version = getActiveDictVersion(); + if (version > EXPECTED_ARX_DICTIONARY_VERSION) { + throw new Error( + `Active arx dictionary version ${version} is newer than this build supports (${EXPECTED_ARX_DICTIONARY_VERSION}); refusing to decode with a forward-incompatible dictionary.`, + ); + } +} + +function assertArx2OverlayNotNewerThanExpected(): void { + const version = getActiveArx2OverlayVersion(); + if (version > EXPECTED_ARX2_OVERLAY_VERSION) { + throw new Error( + `Active arx2 overlay dictionary version ${version} is newer than this build supports (${EXPECTED_ARX2_OVERLAY_VERSION}); refusing to decode with a forward-incompatible dictionary.`, + ); + } +} + async function ensureArxDictionaryLoaded(): Promise { - if (isExternalDictionaryLoaded()) { - return; + if (!isExternalDictionaryLoaded()) { + arxDictionaryLoadPromise ??= loadArxDictionary() + .then((version) => { + if (version < 0) { + // The external fetch failed and the built-in fallback is now active. Don't cache this, so + // a later call can retry the external dictionary once the endpoint recovers; the current + // call still proceeds (degraded) on the built-in dictionary rather than being poisoned. + arxDictionaryLoadPromise = null; + } + }) + .catch((error) => { + arxDictionaryLoadPromise = null; + throw error; + }); + await arxDictionaryLoadPromise; } - // Reset the cached promise on failure so a transient dictionary load error can be retried - // instead of permanently poisoning every arx encode/decode for the page's lifetime. - arxDictionaryLoadPromise ??= loadArxDictionary() - .then(() => undefined) - .catch((error) => { - arxDictionaryLoadPromise = null; - throw error; - }); - await arxDictionaryLoadPromise; + // Runs for both fetched and injected (sync) dictionaries so a forward-incompatible skew can't slip + // through whichever way the dictionary was loaded. + assertArxDictionaryNotNewerThanExpected(); } async function ensureArx2DictionariesLoaded(): Promise { await ensureArxDictionaryLoaded(); - if (isExternalArx2OverlayDictionaryLoaded()) { - return; + if (!isExternalArx2OverlayDictionaryLoaded()) { + // Same retry-on-failure contract as the base dictionary (loadArx2OverlayDictionary also resolves + // -1 on a transient fetch failure rather than rejecting). + arx2OverlayDictionaryLoadPromise ??= loadArx2OverlayDictionary() + .then((version) => { + if (version < 0) { + arx2OverlayDictionaryLoadPromise = null; + } + }) + .catch((error) => { + arx2OverlayDictionaryLoadPromise = null; + throw error; + }); + await arx2OverlayDictionaryLoadPromise; } - arx2OverlayDictionaryLoadPromise ??= loadArx2OverlayDictionary() - .then(() => undefined) - .catch((error) => { - arx2OverlayDictionaryLoadPromise = null; - throw error; - }); - await arx2OverlayDictionaryLoadPromise; + assertArx2OverlayNotNewerThanExpected(); } function decodeArxEncodedPayload(encoded: string): string { diff --git a/tests/arx-codec.test.ts b/tests/arx-codec.test.ts index 971392c..efc3096 100644 --- a/tests/arx-codec.test.ts +++ b/tests/arx-codec.test.ts @@ -543,7 +543,7 @@ describe("arx2 tuple envelope", () => { expect(parsed.ok).toBe(true); }); - it("decodes arx and arx2 payloads when the active dictionary version differs", async () => { + it("refuses to decode when the active dictionary is newer than the build supports", async () => { loadArxDictionarySync(arxDictionaryJson); const arxHash = `#${await encodeEnvelopeAsync(bundle, { codec: "arx" })}`; const arx2Hash = `#${await encodeEnvelopeAsync(bundle, { codec: "arx2" })}`; @@ -553,14 +553,15 @@ describe("arx2 tuple envelope", () => { }; loadArxDictionarySync(shiftedDictionary); - - const arxParsed = await decodeFragmentAsync(arxHash); - const arx2Parsed = await decodeFragmentAsync(arx2Hash); - - expect(arxParsed.ok).toBe(true); - expect(arx2Parsed.ok).toBe(true); - - loadArxDictionarySync(arxDictionaryJson); + try { + // A dictionary newer than this build supports is rejected: a compact arx link carries no + // version of its own, so decoding it against a forward-incompatible dictionary could silently + // mis-decode. Hard-failing is the safe behavior. + expect((await decodeFragmentAsync(arxHash)).ok).toBe(false); + expect((await decodeFragmentAsync(arx2Hash)).ok).toBe(false); + } finally { + loadArxDictionarySync(arxDictionaryJson); + } }); it("can decode arx2 payloads directly through the codec API", async () => { diff --git a/tests/arx-dict-load-retry.test.ts b/tests/arx-dict-load-retry.test.ts new file mode 100644 index 0000000..91ad084 --- /dev/null +++ b/tests/arx-dict-load-retry.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it, vi } from "vitest"; + +// loadArxDictionary() resolves -1 on a transient fetch/parse failure (it does not reject). The +// ensure-helper must treat that as a failure and clear its cached promise, so a later decode retries +// the load instead of reusing a poisoned no-op promise for the page's lifetime. +const { loadArxDictionaryMock } = vi.hoisted(() => ({ loadArxDictionaryMock: vi.fn() })); + +vi.mock("@/lib/payload/arx-codec", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + isExternalDictionaryLoaded: () => false, // force the fetch path so the cached promise is exercised + loadArxDictionary: loadArxDictionaryMock, + }; +}); + +const { decodeFragmentAsync } = await import("@/lib/payload/fragment"); + +describe("arx dictionary load retry", () => { + it("retries after a transient (-1) load failure instead of caching it for the page lifetime", async () => { + loadArxDictionaryMock.mockResolvedValueOnce(-1); // transient failure + const first = await decodeFragmentAsync("#bAAAA", { skipFragmentBudget: true }); + expect(first.ok).toBe(false); + expect(loadArxDictionaryMock).toHaveBeenCalledTimes(1); + + // Endpoint recovers: the next decode must attempt the load again, not reuse a poisoned promise. + loadArxDictionaryMock.mockResolvedValueOnce(1); + await decodeFragmentAsync("#bAAAA", { skipFragmentBudget: true }); + expect(loadArxDictionaryMock).toHaveBeenCalledTimes(2); + }); +}); diff --git a/tests/arx-dict-version-guard.test.ts b/tests/arx-dict-version-guard.test.ts new file mode 100644 index 0000000..dbc6802 --- /dev/null +++ b/tests/arx-dict-version-guard.test.ts @@ -0,0 +1,49 @@ +import { beforeEach, describe, expect, it } from "vitest"; +import arx2DictionaryJson from "../public/arx2-dictionary.json"; +import arxDictionaryJson from "../public/arx-dictionary.json"; +import { loadArx2OverlayDictionarySync, loadArxDictionarySync } from "@/lib/payload/arx-codec"; +import { decodeFragmentAsync, encodeEnvelopeAsync } from "@/lib/payload/fragment"; +import type { PayloadEnvelope } from "@/lib/payload/schema"; + +// Compact arx links carry no dictionary version (the tag implies the current dictionary), so a build +// must refuse to decode with a skewed-version dictionary rather than silently mis-decode. These +// tests exercise that guard by making a different-version dictionary active and confirming decode +// hard-fails, then recovers when the expected version is restored. +const envelope: PayloadEnvelope = { + v: 1, + codec: "arx2", + activeArtifactId: "doc", + artifacts: [{ id: "doc", kind: "markdown", content: "# Title\n\nSome content for the arx dense path." }], +}; + +describe("arx dictionary version guard", () => { + beforeEach(() => { + loadArx2OverlayDictionarySync(arx2DictionaryJson); + loadArxDictionarySync(arxDictionaryJson); // expected version (1) + }); + + it("refuses to decode a compact arx link when the base dictionary version is skewed", async () => { + const fragment = await encodeEnvelopeAsync(envelope, { codec: "arx2", preferPacked: true }); + + const okParsed = await decodeFragmentAsync(`#${fragment}`, { skipFragmentBudget: true }); + expect(okParsed.ok).toBe(true); + + // Simulate asset/CDN skew or a future bump: a different-version dictionary becomes active. + loadArxDictionarySync({ ...arxDictionaryJson, version: 2 }); + const skewed = await decodeFragmentAsync(`#${fragment}`, { skipFragmentBudget: true }); + expect(skewed.ok).toBe(false); // hard-fail, not a silent mis-decode + + // Recovers once the expected-version dictionary is active again. + loadArxDictionarySync(arxDictionaryJson); + const recovered = await decodeFragmentAsync(`#${fragment}`, { skipFragmentBudget: true }); + expect(recovered.ok).toBe(true); + }); + + it("refuses to decode when the arx2 overlay dictionary version is skewed", async () => { + const fragment = await encodeEnvelopeAsync(envelope, { codec: "arx2", preferPacked: true }); + + loadArx2OverlayDictionarySync({ ...arx2DictionaryJson, version: 2 }); + const skewed = await decodeFragmentAsync(`#${fragment}`, { skipFragmentBudget: true }); + expect(skewed.ok).toBe(false); + }); +});