Harden legacy fragment decoding (zlib-wrapped deflate + arx dictionary drift)

src/lib/payload/fragment-arx.ts

@@ -6,6 +6,8 @@ import {
   arx3DecompressEnvelope,
   arxCompressPayloads,
   arxDecompress,
+  getActiveArx2OverlayVersion,
+  getActiveDictVersion,
   isExternalArx2OverlayDictionaryLoaded,
   isExternalDictionaryLoaded,
   loadArxDictionary,
@@ -30,36 +32,79 @@ type TransportLengthCalculator = (value: string) => number;
 let arxDictionaryLoadPromise: Promise<void> | null = null;
 let arx2OverlayDictionaryLoadPromise: Promise<void> | null = null;
 
+// Compact ARX fragments (tags `a`/`b`/`c`) do NOT carry a dictionary version — the tag implies the
+// CURRENT dictionary, which keeps links short. The safety cost is that a build must not decode with
+// a dictionary NEWER than it was built for (a CDN/asset split serving a future dictionary, or a
+// version bump), because it would lack the new slots and could produce a structurally-valid-but-
+// wrong envelope. We pin the newest supported version and reject anything newer so decode hard-fails
+// instead of mis-decoding. The built-in fallback dictionary (version 0) and the current external
+// dictionary (version 1) are both <= this and remain usable. Bumping a dictionary version is
+// therefore a wire change that also requires new compact tags and updating
+// tests/arx-dictionary-pin.test.ts.
+const EXPECTED_ARX_DICTIONARY_VERSION = 1;
+const EXPECTED_ARX2_OVERLAY_VERSION = 1;
+
+function assertArxDictionaryNotNewerThanExpected(): void {
+  const version = getActiveDictVersion();
+  if (version > EXPECTED_ARX_DICTIONARY_VERSION) {
+    throw new Error(
+      `Active arx dictionary version ${version} is newer than this build supports (${EXPECTED_ARX_DICTIONARY_VERSION}); refusing to decode with a forward-incompatible dictionary.`,
+    );
+  }
+}
+
+function assertArx2OverlayNotNewerThanExpected(): void {
+  const version = getActiveArx2OverlayVersion();
+  if (version > EXPECTED_ARX2_OVERLAY_VERSION) {
+    throw new Error(
+      `Active arx2 overlay dictionary version ${version} is newer than this build supports (${EXPECTED_ARX2_OVERLAY_VERSION}); refusing to decode with a forward-incompatible dictionary.`,
+    );
+  }
+}
+
 async function ensureArxDictionaryLoaded(): Promise<void> {
-  if (isExternalDictionaryLoaded()) {
-    return;
+  if (!isExternalDictionaryLoaded()) {
+    arxDictionaryLoadPromise ??= loadArxDictionary()
+      .then((version) => {
+        if (version < 0) {
+          // The external fetch failed and the built-in fallback is now active. Don't cache this, so
+          // a later call can retry the external dictionary once the endpoint recovers; the current
+          // call still proceeds (degraded) on the built-in dictionary rather than being poisoned.
+          arxDictionaryLoadPromise = null;
+        }
+      })
+      .catch((error) => {
+        arxDictionaryLoadPromise = null;
+        throw error;
+      });
+    await arxDictionaryLoadPromise;
   }
 
-  // Reset the cached promise on failure so a transient dictionary load error can be retried
-  // instead of permanently poisoning every arx encode/decode for the page's lifetime.
-  arxDictionaryLoadPromise ??= loadArxDictionary()
-    .then(() => undefined)
-    .catch((error) => {
-      arxDictionaryLoadPromise = null;
-      throw error;
-    });
-  await arxDictionaryLoadPromise;
+  // Runs for both fetched and injected (sync) dictionaries so a forward-incompatible skew can't slip
+  // through whichever way the dictionary was loaded.
+  assertArxDictionaryNotNewerThanExpected();
 }
 
 async function ensureArx2DictionariesLoaded(): Promise<void> {
   await ensureArxDictionaryLoaded();
 
-  if (isExternalArx2OverlayDictionaryLoaded()) {
-    return;
+  if (!isExternalArx2OverlayDictionaryLoaded()) {
+    // Same retry-on-failure contract as the base dictionary (loadArx2OverlayDictionary also resolves
+    // -1 on a transient fetch failure rather than rejecting).
+    arx2OverlayDictionaryLoadPromise ??= loadArx2OverlayDictionary()
+      .then((version) => {
+        if (version < 0) {
+          arx2OverlayDictionaryLoadPromise = null;
+        }
+      })
+      .catch((error) => {
+        arx2OverlayDictionaryLoadPromise = null;
+        throw error;
+      });
+    await arx2OverlayDictionaryLoadPromise;
   }
 
-  arx2OverlayDictionaryLoadPromise ??= loadArx2OverlayDictionary()
-    .then(() => undefined)
-    .catch((error) => {
-      arx2OverlayDictionaryLoadPromise = null;
-      throw error;
-    });
-  await arx2OverlayDictionaryLoadPromise;
+  assertArx2OverlayNotNewerThanExpected();
 }
 
 function decodeArxEncodedPayload(encoded: string): string {

src/lib/payload/fragment.ts

@@ -1,5 +1,5 @@
 import { compressToEncodedURIComponent, decompressFromEncodedURIComponent } from "lz-string";
-import { deflateSync, inflateSync, strFromU8, strToU8 } from "fflate";
+import { deflateSync, inflateSync, strFromU8, strToU8, unzlibSync } from "fflate";
 import { normalizeEnvelope } from "@/lib/payload/envelope";
 import { packEnvelope, unpackEnvelope } from "@/lib/payload/wire-format";
 import {
@@ -136,14 +136,34 @@ function encodePayload(json: string, codec: PayloadCodec): string {
   }
 }
 
+/**
+ * Inflates a `deflate` payload, accepting both raw deflate (current encoder) and zlib-wrapped
+ * deflate (emitted by an older encoder version). Raw is tried first so current output is never at
+ * risk of header mis-detection; zlib is a back-compat fallback so historical shared links keep
+ * decoding instead of failing as invalid-json.
+ */
+function inflateDeflatePayload(bytes: Uint8Array): Uint8Array {
+  try {
+    return inflateSync(bytes);
+  } catch (rawDeflateError) {
+    // Fall back to zlib-wrapped (legacy) deflate. If that also fails the input is not valid deflate
+    // at all, so surface the original raw-inflate error rather than a misleading zlib one.
+    try {
+      return unzlibSync(bytes);
+    } catch {
+      throw rawDeflateError;
+    }
+  }
+}
+
 function decodePayload(encoded: string, codec: PayloadCodec): string | null {
   switch (codec) {
     case "plain":
       return fromBase64Url(encoded);
     case "lz":
       return decompressFromEncodedURIComponent(encoded);
     case "deflate":
-      return strFromU8(inflateSync(fromBase64UrlBytes(encoded)));
+      return strFromU8(inflateDeflatePayload(fromBase64UrlBytes(encoded)));
     case "arx":
     case "arx2":
     case "arx3":
@@ -531,8 +551,18 @@ export async function decodeFragmentAsync(hash: string, options?: DecodeOptions)
     if (error instanceof Error && error.name === "ArxDecodedPayloadTooLargeError") {
       return { ok: false, code: "decoded-too-large", message: error.message };
     }
-    return { ok: false, code: "invalid-json", message: "The fragment payload could not be decoded as valid JSON." };
+    const arxHint =
+      codec === "arx" || codec === "arx2" || codec === "arx3"
+        ? " It may have been encoded with a different ARX dictionary version."
+        : "";
+    return { ok: false, code: "invalid-json", message: `The fragment payload could not be decoded as valid JSON.${arxHint}` };
   }
 
-  return resolveEnvelope(parsed, header.fragmentLength);
+  const resolved = resolveEnvelope(parsed, header.fragmentLength);
+  if (!resolved.ok && resolved.code === "invalid-envelope" && (codec === "arx" || codec === "arx2" || codec === "arx3")) {
+    // An ARX payload that decoded but is not a valid envelope almost always means the active
+    // dictionary differs from the one it was encoded with (see tests/arx-dictionary-pin.test.ts).
+    return { ...resolved, message: `${resolved.message} It may have been encoded with a different ARX dictionary version.` };
+  }
+  return resolved;
 }

tests/arx-codec.test.ts

@@ -543,7 +543,7 @@ describe("arx2 tuple envelope", () => {
     expect(parsed.ok).toBe(true);
   });
 
-  it("decodes arx and arx2 payloads when the active dictionary version differs", async () => {
+  it("refuses to decode when the active dictionary is newer than the build supports", async () => {
     loadArxDictionarySync(arxDictionaryJson);
     const arxHash = `#${await encodeEnvelopeAsync(bundle, { codec: "arx" })}`;
     const arx2Hash = `#${await encodeEnvelopeAsync(bundle, { codec: "arx2" })}`;
@@ -553,14 +553,15 @@ describe("arx2 tuple envelope", () => {
     };
 
     loadArxDictionarySync(shiftedDictionary);
-
-    const arxParsed = await decodeFragmentAsync(arxHash);
-    const arx2Parsed = await decodeFragmentAsync(arx2Hash);
-
-    expect(arxParsed.ok).toBe(true);
-    expect(arx2Parsed.ok).toBe(true);
-
-    loadArxDictionarySync(arxDictionaryJson);
+    try {
+      // A dictionary newer than this build supports is rejected: a compact arx link carries no
+      // version of its own, so decoding it against a forward-incompatible dictionary could silently
+      // mis-decode. Hard-failing is the safe behavior.
+      expect((await decodeFragmentAsync(arxHash)).ok).toBe(false);
+      expect((await decodeFragmentAsync(arx2Hash)).ok).toBe(false);
+    } finally {
+      loadArxDictionarySync(arxDictionaryJson);
+    }
   });
 
   it("can decode arx2 payloads directly through the codec API", async () => {

tests/arx-dict-load-retry.test.ts

@@ -0,0 +1,31 @@
+import { describe, expect, it, vi } from "vitest";
+
+// loadArxDictionary() resolves -1 on a transient fetch/parse failure (it does not reject). The
+// ensure-helper must treat that as a failure and clear its cached promise, so a later decode retries
+// the load instead of reusing a poisoned no-op promise for the page's lifetime.
+const { loadArxDictionaryMock } = vi.hoisted(() => ({ loadArxDictionaryMock: vi.fn() }));
+
+vi.mock("@/lib/payload/arx-codec", async (importOriginal) => {
+  const actual = await importOriginal<typeof import("@/lib/payload/arx-codec")>();
+  return {
+    ...actual,
+    isExternalDictionaryLoaded: () => false, // force the fetch path so the cached promise is exercised
+    loadArxDictionary: loadArxDictionaryMock,
+  };
+});
+
+const { decodeFragmentAsync } = await import("@/lib/payload/fragment");
+
+describe("arx dictionary load retry", () => {
+  it("retries after a transient (-1) load failure instead of caching it for the page lifetime", async () => {
+    loadArxDictionaryMock.mockResolvedValueOnce(-1); // transient failure
+    const first = await decodeFragmentAsync("#bAAAA", { skipFragmentBudget: true });
+    expect(first.ok).toBe(false);
+    expect(loadArxDictionaryMock).toHaveBeenCalledTimes(1);
+
+    // Endpoint recovers: the next decode must attempt the load again, not reuse a poisoned promise.
+    loadArxDictionaryMock.mockResolvedValueOnce(1);
+    await decodeFragmentAsync("#bAAAA", { skipFragmentBudget: true });
+    expect(loadArxDictionaryMock).toHaveBeenCalledTimes(2);
+  });
+});

tests/arx-dict-version-guard.test.ts

@@ -0,0 +1,49 @@
+import { beforeEach, describe, expect, it } from "vitest";
+import arx2DictionaryJson from "../public/arx2-dictionary.json";
+import arxDictionaryJson from "../public/arx-dictionary.json";
+import { loadArx2OverlayDictionarySync, loadArxDictionarySync } from "@/lib/payload/arx-codec";
+import { decodeFragmentAsync, encodeEnvelopeAsync } from "@/lib/payload/fragment";
+import type { PayloadEnvelope } from "@/lib/payload/schema";
+
+// Compact arx links carry no dictionary version (the tag implies the current dictionary), so a build
+// must refuse to decode with a skewed-version dictionary rather than silently mis-decode. These
+// tests exercise that guard by making a different-version dictionary active and confirming decode
+// hard-fails, then recovers when the expected version is restored.
+const envelope: PayloadEnvelope = {
+  v: 1,
+  codec: "arx2",
+  activeArtifactId: "doc",
+  artifacts: [{ id: "doc", kind: "markdown", content: "# Title\n\nSome content for the arx dense path." }],
+};
+
+describe("arx dictionary version guard", () => {
+  beforeEach(() => {
+    loadArx2OverlayDictionarySync(arx2DictionaryJson);
+    loadArxDictionarySync(arxDictionaryJson); // expected version (1)
+  });
+
+  it("refuses to decode a compact arx link when the base dictionary version is skewed", async () => {
+    const fragment = await encodeEnvelopeAsync(envelope, { codec: "arx2", preferPacked: true });
+
+    const okParsed = await decodeFragmentAsync(`#${fragment}`, { skipFragmentBudget: true });
+    expect(okParsed.ok).toBe(true);
+
+    // Simulate asset/CDN skew or a future bump: a different-version dictionary becomes active.
+    loadArxDictionarySync({ ...arxDictionaryJson, version: 2 });
+    const skewed = await decodeFragmentAsync(`#${fragment}`, { skipFragmentBudget: true });
+    expect(skewed.ok).toBe(false); // hard-fail, not a silent mis-decode
+
+    // Recovers once the expected-version dictionary is active again.
+    loadArxDictionarySync(arxDictionaryJson);
+    const recovered = await decodeFragmentAsync(`#${fragment}`, { skipFragmentBudget: true });
+    expect(recovered.ok).toBe(true);
+  });
+
+  it("refuses to decode when the arx2 overlay dictionary version is skewed", async () => {
+    const fragment = await encodeEnvelopeAsync(envelope, { codec: "arx2", preferPacked: true });
+
+    loadArx2OverlayDictionarySync({ ...arx2DictionaryJson, version: 2 });
+    const skewed = await decodeFragmentAsync(`#${fragment}`, { skipFragmentBudget: true });
+    expect(skewed.ok).toBe(false);
+  });
+});

tests/arx-dictionary-pin.test.ts

@@ -0,0 +1,39 @@
+import { createHash } from "node:crypto";
+import { readFileSync } from "node:fs";
+import { describe, expect, it } from "vitest";
+
+// Defended contract — learned the hard way: real shared arx/arx2 links broke because a dictionary's
+// content was changed while its `version` field stayed 1, so every previously-encoded fragment
+// silently mis-decoded into garbage. The dictionary content and its version MUST move together.
+//
+// If you change a dictionary's slots, you MUST bump its `version` AND update the pinned hash here in
+// the same commit. The version bump is the only thing that lets a future build distinguish (and
+// reject, rather than garble) a fragment that was encoded with an older dictionary. Do NOT just
+// re-pin the hash to make this test pass — that re-creates the silent-drift bug.
+//
+// arx3 reuses public/arx-dictionary.json (via arx3DecompressEnvelope), so pinning it also covers
+// arx3; if arx3 is ever given its own dictionary file, add a pin entry for it here.
+const PINNED: Record<string, { version: number; sha256: string }> = {
+  "public/arx-dictionary.json": {
+    version: 1,
+    sha256: "16fe3f72dd5d282fd2f0271647a56c38c4b7eebb6e4723da670765f7d90380a9",
+  },
+  "public/arx2-dictionary.json": {
+    version: 1,
+    sha256: "12d0166fda16ce9697831805d92af0f558b4fad5a882c471ba072118561a74eb",
+  },
+};
+
+describe("arx dictionary content is pinned to its version", () => {
+  for (const [file, pin] of Object.entries(PINNED)) {
+    it(`${file} matches its pinned version and content hash`, () => {
+      const obj = JSON.parse(readFileSync(file, "utf8"));
+      // Hash the canonical (re-serialized) form so whitespace/formatting changes don't trip it, only
+      // real content changes (slots, version) do.
+      const canonicalSha256 = createHash("sha256").update(JSON.stringify(obj)).digest("hex");
+
+      expect(obj.version).toBe(pin.version);
+      expect(canonicalSha256).toBe(pin.sha256);
+    });
+  }
+});

tests/fragment.test.ts

@@ -1,3 +1,4 @@
+import { strToU8, zlibSync } from "fflate";
 import { describe, expect, it } from "vitest";
 import { decodeFragment, decodeFragmentAsync, encodeEnvelope, encodeEnvelopeAsync } from "@/lib/payload/fragment";
 import { compactTagForCodec, type PayloadEnvelope } from "@/lib/payload/schema";
@@ -75,6 +76,33 @@ describe("fragment payload transport", () => {
     expect(parsed.code).toBe("invalid-json");
   });
 
+  it("decodes a legacy zlib-wrapped deflate fragment (older encoder back-compat)", () => {
+    // The current encoder emits raw deflate, but an older version emitted zlib-wrapped deflate.
+    // Real historical shared links use that form and must still decode. Build one by hand.
+    const legacy: PayloadEnvelope = {
+      v: 1,
+      codec: "deflate",
+      title: "legacy zlib link",
+      activeArtifactId: "doc",
+      artifacts: [{ id: "doc", kind: "markdown", content: "# Legacy\n\nEncoded with zlib-wrapped deflate." }],
+    };
+    const b64url = Buffer.from(zlibSync(strToU8(JSON.stringify(legacy))))
+      .toString("base64")
+      .replace(/\+/g, "-")
+      .replace(/\//g, "_")
+      .replace(/=+$/, "");
+
+    const parsed = decodeFragment(`#agent-render=v1.deflate.${b64url}`);
+
+    expect(parsed.ok).toBe(true);
+    if (parsed.ok) {
+      expect(parsed.envelope.title).toBe("legacy zlib link");
+      expect(parsed.envelope.artifacts[0]).toMatchObject({
+        content: "# Legacy\n\nEncoded with zlib-wrapped deflate.",
+      });
+    }
+  });
+
   it("uses compressed transport when it is smaller", () => {
     const repetitiveEnvelope: PayloadEnvelope = {
       ...envelope,