normalize display math fences attached to content / nested in lists

lib/markdown.test.mjs

@@ -0,0 +1,83 @@
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+
+import { normalizeDisplayMath } from "./markdown.ts";
+
+describe("normalizeDisplayMath", () => {
+  describe("single-line $$…$$", () => {
+    it("splits a single-line block into three lines", () => {
+      assert.equal(normalizeDisplayMath("$$a + b$$"), "$$\na + b\n$$");
+    });
+
+    it("preserves the indent of the surrounding list item", () => {
+      assert.equal(
+        normalizeDisplayMath("- item:\n  $$a + b$$\n- next"),
+        "- item:\n  $$\n  a + b\n  $$\n- next",
+      );
+    });
+  });
+
+  describe("multi-line blocks with glued delimiters", () => {
+    it("moves a glued opening delimiter to its own line", () => {
+      const input = "$$\n\\frac{a}{b} = c\n<d$$\n\nafter";
+      assert.equal(normalizeDisplayMath(input), "$$\n\\frac{a}{b} = c\n<d\n$$\n\nafter");
+    });
+
+    it("moves a glued closing delimiter to its own line", () => {
+      const input = "$$\nx = y\nz = w$$\n\nafter";
+      assert.equal(normalizeDisplayMath(input), "$$\nx = y\nz = w\n$$\n\nafter");
+    });
+  });
+
+  describe("blocks nested in GFM list items", () => {
+    it("re-indents lazy content lines of an indented bare-fence block", () => {
+      const input = "- item:\n  $$\nx = y\n  $$\n- next";
+      assert.equal(normalizeDisplayMath(input), "- item:\n  $$\n  x = y\n  $$\n- next");
+    });
+  });
+
+  describe("blocks that must stay untouched", () => {
+    it("leaves a top-level block with detached delimiters untouched", () => {
+      const input = "$$\n\\frac{a}{b}\n$$\n\nend";
+      assert.equal(normalizeDisplayMath(input), input);
+    });
+
+    it("leaves content inside fenced code blocks untouched", () => {
+      const input = "```\n$$ not math $$\n$$\n```\n\nreal $$x = 1$$ end";
+      const normalized = normalizeDisplayMath(input);
+      assert.ok(normalized.includes("```\n$$ not math $$\n$$\n```"));
+      // every `$$` is preserved: 3 inside the fence + 2 in inline math
+      assert.equal(normalized.match(/\$\$/g)?.length, 5);
+    });
+
+    it("leaves inline math and plain prose untouched", () => {
+      const input = "text $x = 1$ and $$a + b$$ more";
+      assert.equal(normalizeDisplayMath(input), input);
+    });
+
+    it("does not treat a glued opener with mid-line $$ as a block", () => {
+      const input = "$$x$$ and text";
+      assert.equal(normalizeDisplayMath(input), input);
+    });
+  });
+
+  describe("\\[ … \\] blocks", () => {
+    it("normalizes single-line brackets", () => {
+      assert.equal(normalizeDisplayMath("\\[a + b\\]"), "$$\na + b\n$$");
+    });
+
+    it("keeps content indented when nested in a list item", () => {
+      assert.equal(
+        normalizeDisplayMath("- item:\n  \\[a + b\\]\n- next"),
+        "- item:\n  $$\n  a + b\n  $$\n- next",
+      );
+    });
+
+    it("normalizes multi-line brackets without double-indenting", () => {
+      assert.equal(
+        normalizeDisplayMath("- item:\n  \\[\n  x = y\n  \\]\n- next"),
+        "- item:\n  $$\n  x = y\n  $$\n- next",
+      );
+    });
+  });
+});

lib/markdown.ts

@@ -73,7 +73,16 @@ export function normalizeDisplayMath(markdown: string): string {
     if (bracketDisplayOneLine) {
       const math = bracketDisplayOneLine[2].trim();
       if (math) {
-        normalized.push(`${bracketDisplayOneLine[1]}$$`, math, `${bracketDisplayOneLine[1]}$$`);
+        // Keep the content line indented together with the `$$` fence. When the
+        // formula is nested inside a GFM list item (indented `$$`), a content line
+        // at column 0 becomes a "lazy continuation" line, which makes remark-math
+        // mis-parse the fence pair: the opening `$$` turns into an empty math node
+        // and the closing one swallows the rest of the document as math content.
+        normalized.push(
+          `${bracketDisplayOneLine[1]}$$`,
+          `${bracketDisplayOneLine[1]}${math}`,
+          `${bracketDisplayOneLine[1]}$$`,
+        );
         continue;
       }
     }
@@ -82,9 +91,13 @@ export function normalizeDisplayMath(markdown: string): string {
     if (bracketDisplayStart) {
       const closingIndex = findBracketDisplayClose(lines, index + 1);
       if (closingIndex !== -1) {
+        // Same lazy-continuation guard as above: indent content lines that sit at
+        // column 0 so the block stays parseable when nested inside a list item.
         normalized.push(
           `${bracketDisplayStart[1]}$$`,
-          ...lines.slice(index + 1, closingIndex),
+          ...lines.slice(index + 1, closingIndex).map((mathLine) =>
+            /^\s/.test(mathLine) ? mathLine : `${bracketDisplayStart[1]}${mathLine}`,
+          ),
           `${bracketDisplayStart[1]}$$`,
         );
         index = closingIndex;
@@ -96,7 +109,101 @@ export function normalizeDisplayMath(markdown: string): string {
     if (displayMathMatch) {
       const math = displayMathMatch[2].trim();
       if (math) {
-        normalized.push(`${displayMathMatch[1]}$$`, math, `${displayMathMatch[1]}$$`);
+        // See the comment on bracketDisplayOneLine: without matching indentation,
+        // a formula nested in a GFM list item is mis-parsed by remark-math and the
+        // text after the formula renders as a garbled KaTeX error block.
+        normalized.push(
+          `${displayMathMatch[1]}$$`,
+          `${displayMathMatch[1]}${math}`,
+          `${displayMathMatch[1]}$$`,
+        );
+        continue;
+      }
+    }
+
+    // remark-math requires both `$$` delimiters to sit on their own lines, but
+    // models also emit display math as a multi-line block where the opening `$$`
+    // is glued to the first formula line and/or the closing `$$` is glued to the
+    // end of the last one (`$$x = 1` + `y = 2$$`). Without normalization such a
+    // block swallows the following text as math content and renders as garbage.
+    const displayMathMultiLine = line.match(/^([ \t]{0,3})\$\$(.+)$/);
+    if (displayMathMultiLine) {
+      const indent = displayMathMultiLine[1];
+      const firstLine = displayMathMultiLine[2].trimEnd();
+      // Only treat this as a block opener if no other `$$` is embedded mid-line
+      // (e.g. `$$x$$ and text` stays untouched and is rendered as inline math).
+      if (firstLine && !firstLine.includes("$$")) {
+        let closingIndex = -1;
+        let closingRest = "";
+        for (let j = index + 1; j < lines.length; j++) {
+          const candidate = lines[j];
+          if (/^ {0,3}\$\$\s*$/.test(candidate)) {
+            closingIndex = j;
+            break;
+          }
+          const endMatch = candidate.match(/^(.*)\$\$\s*$/);
+          if (endMatch) {
+            closingIndex = j;
+            closingRest = endMatch[1].trimEnd();
+            break;
+          }
+          // Do not pair delimiters across another Markdown block boundary.
+          if (/^ {0,3}(`{3,}|~{3,})/.test(candidate)) break;
+        }
+        if (closingIndex !== -1) {
+          normalized.push(`${indent}$$`, `${indent}${firstLine}`);
+          for (let j = index + 1; j < closingIndex; j++) {
+            const middle = lines[j];
+            normalized.push(/^\s/.test(middle) ? middle : `${indent}${middle}`);
+          }
+          if (closingRest) normalized.push(`${indent}${closingRest}`);
+          normalized.push(`${indent}$$`);
+          index = closingIndex;
+          continue;
+        }
+      }
+    }
+
+    // Bare `$$` opener (possibly indented inside a GFM list item). Two problems
+    // need fixing: (1) when the closing `$$` is glued to the last content line
+    // (e.g. `z = w$$`) remark-math never finds a valid closing fence and swallows
+    // the rest of the document; (2) inside a list item, content lines at column 0
+    // are lazy continuations that break the math flow. Both are fixed by moving
+    // the closing `$$` to its own line and re-indenting lazy content lines.
+    // A column-0 block with a properly detached closing `$$` is left untouched
+    // (remark-math already parses it correctly).
+    const displayMathBareOpen = line.match(/^([ \t]{0,3})\$\$\s*$/);
+    if (displayMathBareOpen) {
+      const indent = displayMathBareOpen[1];
+      let closingIndex = -1;
+      let closingGlued = false;
+      for (let j = index + 1; j < lines.length; j++) {
+        const candidate = lines[j];
+        if (/^ {0,3}\$\$\s*$/.test(candidate)) {
+          closingIndex = j;
+          break;
+        }
+        const endMatch = candidate.match(/^(.*)\$\$\s*$/);
+        if (endMatch) {
+          closingIndex = j;
+          closingGlued = true;
+          break;
+        }
+        // Do not pair delimiters across another Markdown block boundary.
+        if (/^ {0,3}(`{3,}|~{3,})/.test(candidate)) break;
+      }
+      if (closingIndex !== -1 && (closingGlued || indent !== "")) {
+        normalized.push(`${indent}$$`);
+        for (let j = index + 1; j < closingIndex; j++) {
+          const middle = lines[j];
+          normalized.push(/^\s/.test(middle) ? middle : `${indent}${middle}`);
+        }
+        if (closingGlued) {
+          const rest = lines[closingIndex].match(/^(.*)\$\$\s*$/)?.[1]?.trimEnd();
+          if (rest) normalized.push(/^\s/.test(rest) ? rest : `${indent}${rest}`);
+        }
+        normalized.push(`${indent}$$`);
+        index = closingIndex;
         continue;
       }
     }

package.json

@@ -34,6 +34,7 @@
     "start": "next start -H 127.0.0.1 -p 30141",
     "start:lan": "next start -H 0.0.0.0 -p 30141",
     "lint": "eslint .",
+    "test": "node --experimental-strip-types --test lib/markdown.test.mjs",
     "release": "npm version patch --no-git-tag-version && npm run build && npm publish --access public"
   },
   "dependencies": {