From a9ba44c9fd83893456cbb1ea12fc0e99cc532415 Mon Sep 17 00:00:00 2001 From: Kaylee <65376239+KayleeWilliams@users.noreply.github.com> Date: Sun, 23 Aug 2026 22:05:16 +0100 Subject: [PATCH 01/29] fix(llm): preserve valid Setext heading anchors --- .changeset/setext-heading-scanner.md | 5 ++ .../leadtype/src/internal/docs-heading.ts | 28 +++++++-- packages/leadtype/src/llm/llm.test.ts | 29 +++++++++ packages/leadtype/src/search/search.test.ts | 60 +++++++++++++++++++ 4 files changed, 116 insertions(+), 6 deletions(-) create mode 100644 .changeset/setext-heading-scanner.md diff --git a/.changeset/setext-heading-scanner.md b/.changeset/setext-heading-scanner.md new file mode 100644 index 00000000..b9be81a6 --- /dev/null +++ b/.changeset/setext-heading-scanner.md @@ -0,0 +1,5 @@ +--- +"leadtype": patch +--- + +Keep table-of-contents and search anchors aligned for Setext headings with inline markup, and ignore underlines indented as code. diff --git a/packages/leadtype/src/internal/docs-heading.ts b/packages/leadtype/src/internal/docs-heading.ts index e5788d45..cff9efdd 100644 --- a/packages/leadtype/src/internal/docs-heading.ts +++ b/packages/leadtype/src/internal/docs-heading.ts @@ -1,13 +1,20 @@ const DIACRITIC_PATTERN = /[\u0300-\u036f]/g; const FRONTMATTER_PATTERN = /^---\s*\n[\s\S]*?\n---\s*\n?/; const HEADING_PATTERN = /^(#{1,6})(?:\s+(.*))?$/; -const SETEXT_H1_PATTERN = /^=+\s*$/; -const SETEXT_H2_PATTERN = /^-+\s*$/; +const SETEXT_H1_PATTERN = /^ {0,3}=+[ \t]*$/; +const SETEXT_H2_PATTERN = /^ {0,3}-+[ \t]*$/; const FENCE_PATTERN = /^(`{3,}|~{3,})/; const INDENTED_CODE_PATTERN = /^(?: {4}|\t)/; const BLOCKQUOTE_PATTERN = /^ {0,3}>/; const LIST_ITEM_PATTERN = /^ {0,3}(?:[*+-]|\d{1,9}[.)])(?:[ \t]+|$)/; -const HTML_OR_MDX_BLOCK_PATTERN = /^ {0,3}[<{]/; +const HTML_BLOCK_START_PATTERN = + /^ {0,3}(?:<(?:pre|script|style|textarea)(?:[ \t>]|$)|")) { + output.push(" "); + tagBuffer = ""; + } + continue; + } if (quote !== null) { if (character === quote) { quote = null; @@ -132,8 +139,7 @@ function stripHtmlTags(input: string): string { } function cleanHeadingText(input: string): string { - return stripHtmlTags(input) - .replace(HEADING_CLOSING_SEQUENCE_PATTERN, "") + return stripHtmlTags(input.replace(HEADING_CLOSING_SEQUENCE_PATTERN, "")) .replace(MARKDOWN_LINK_PATTERN, "$1") .replace(HEADING_INLINE_PATTERN, " ") .replace(WHITESPACE_PATTERN, " ") diff --git a/packages/leadtype/src/llm/llm.test.ts b/packages/leadtype/src/llm/llm.test.ts index a85aad68..8d49f0b2 100644 --- a/packages/leadtype/src/llm/llm.test.ts +++ b/packages/leadtype/src/llm/llm.test.ts @@ -4338,6 +4338,7 @@ describe("extractDocsTableOfContents", () => { heading: 'Install', title: "Install", }, + { heading: "Install ", title: "Install" }, { heading: "
Note
", title: "Note" }, ]; @@ -4358,6 +4359,17 @@ describe("extractDocsTableOfContents", () => { } }); + it("keeps a trailing marker wrapped in inline HTML in the heading title", () => { + const toc = extractDocsTableOfContents("## Anchors and #", { + urlPath: "/docs/example", + absoluteUrl: "https://leadtype.dev/docs/example", + }); + + expect(toc.map((item) => ({ id: item.id, title: item.title }))).toEqual([ + { id: "anchors-and", title: "Anchors and #" }, + ]); + }); + it("does not treat a thematic break after a blank line as a Setext heading", () => { const toc = extractDocsTableOfContents( ["A paragraph.", "", "---", "## After"].join("\n"), diff --git a/packages/leadtype/src/search/search.test.ts b/packages/leadtype/src/search/search.test.ts index f2db741a..3397c6e8 100644 --- a/packages/leadtype/src/search/search.test.ts +++ b/packages/leadtype/src/search/search.test.ts @@ -648,6 +648,13 @@ describe("createDocsSearchIndex and searchDocs", () => { "## Note", "The ATX section covers sprockets.", ].join("\n"), + [ + "Install ", + "---", + "Inline comment section covers widgets.", + "## Install", + "The ATX section covers sprockets.", + ].join("\n"), ]; for (const content of fixtures) { @@ -683,6 +690,29 @@ describe("createDocsSearchIndex and searchDocs", () => { } }); + it("keeps inline HTML text in search heading labels", () => { + const index = createDocsSearchIndex( + [ + { + id: "anchors", + title: "Anchors", + urlPath: "/docs/anchors", + absoluteUrl: "https://leadtype.dev/docs/anchors", + relativePath: "anchors.mdx", + content: [ + "## Anchors and #", + "Literal marker covers widgets.", + ].join("\n"), + }, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + + const result = searchDocs(index, "widgets")[0]; + expect(result?.headingPath.at(-1)).toBe("Anchors and #"); + expect(result?.urlWithHash).toBe("/docs/anchors#anchors-and"); + }); + it("slugifies headings for hash links", () => { expect(slugifyDocsHeading("Café API: Quick Start!")).toBe( "cafe-api-quick-start" From 64d2da5f0f41bbf6d2784f007b04827933949b14 Mon Sep 17 00:00:00 2001 From: Kaylee <65376239+KayleeWilliams@users.noreply.github.com> Date: Sun, 23 Aug 2026 23:01:25 +0100 Subject: [PATCH 06/29] fix(llm): classify quoted standalone tags --- .../leadtype/src/internal/docs-heading.ts | 2 +- packages/leadtype/src/llm/llm.test.ts | 12 ++++++ packages/leadtype/src/search/search.test.ts | 37 +++++++++++++++++++ 3 files changed, 50 insertions(+), 1 deletion(-) diff --git a/packages/leadtype/src/internal/docs-heading.ts b/packages/leadtype/src/internal/docs-heading.ts index b48809e2..6c23ed40 100644 --- a/packages/leadtype/src/internal/docs-heading.ts +++ b/packages/leadtype/src/internal/docs-heading.ts @@ -78,7 +78,7 @@ const HTML_BLOCK_TAG_PATTERN = new RegExp( "i" ); const STANDALONE_HTML_TAG_PATTERN = - /^ {0,3}<\/?[A-Za-z][A-Za-z0-9-]*(?:[ \t]+[^<>]*)?\/?>[ \t]*$/; + /^ {0,3}<\/?[A-Za-z][A-Za-z0-9-]*(?:[ \t]+(?:[^<>"']|"[^"<]*"|'[^'<]*')*)?\/?>[ \t]*$/; const MDX_BLOCK_START_PATTERN = /^ {0,3}(?:\{|<\/?[A-Z][A-Za-z0-9_.:-]*(?:[ \t/>]|$))/; const LINK_DEFINITION_PATTERN = /^ {0,3}\[[^\]]+\]:/; diff --git a/packages/leadtype/src/llm/llm.test.ts b/packages/leadtype/src/llm/llm.test.ts index 8d49f0b2..e540e7e1 100644 --- a/packages/leadtype/src/llm/llm.test.ts +++ b/packages/leadtype/src/llm/llm.test.ts @@ -4409,6 +4409,18 @@ describe("extractDocsTableOfContents", () => { } }); + it("recognizes standalone HTML tags with quoted delimiters", () => { + const toc = extractDocsTableOfContents( + ['', "---", "", "## !!!", "## !!!"].join("\n"), + { + urlPath: "/docs/example", + absoluteUrl: "https://leadtype.dev/docs/example", + } + ); + + expect(toc.map((item) => item.id)).toEqual(["", "-1"]); + }); + it("respects custom heading level ranges", () => { const toc = extractDocsTableOfContents( ["# Page", "## Section", "### Child", "#### Detail"].join("\n"), diff --git a/packages/leadtype/src/search/search.test.ts b/packages/leadtype/src/search/search.test.ts index 3397c6e8..7636859b 100644 --- a/packages/leadtype/src/search/search.test.ts +++ b/packages/leadtype/src/search/search.test.ts @@ -713,6 +713,43 @@ describe("createDocsSearchIndex and searchDocs", () => { expect(result?.urlWithHash).toBe("/docs/anchors#anchors-and"); }); + it("keeps standalone HTML tags from shifting empty heading anchors", () => { + const content = [ + '', + "---", + "", + "## !!!", + "First punctuation section covers widgets.", + "## !!!", + "Second punctuation section covers sprockets.", + ].join("\n"); + const index = createDocsSearchIndex( + [ + { + id: "fixture", + title: "Fixture", + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + relativePath: "fixture.mdx", + content, + }, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + const tocIds = flattenTocIds( + extractDocsTableOfContents(content, { + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + }) + ); + const searchAnchors = index.chunks.map( + (chunk) => chunk[CHUNK_ANCHOR_INDEX] + ); + + expect(tocIds).toEqual(["", "-1"]); + expect(searchAnchors).toEqual(["", ...tocIds]); + }); + it("slugifies headings for hash links", () => { expect(slugifyDocsHeading("Café API: Quick Start!")).toBe( "cafe-api-quick-start" From c5b7257e073aa885160b70603a21195c36982d26 Mon Sep 17 00:00:00 2001 From: Kaylee <65376239+KayleeWilliams@users.noreply.github.com> Date: Sun, 23 Aug 2026 23:12:02 +0100 Subject: [PATCH 07/29] fix(llm): allow quoted tag delimiters --- packages/leadtype/src/internal/docs-heading.ts | 2 +- packages/leadtype/src/llm/llm.test.ts | 4 +++- packages/leadtype/src/search/search.test.ts | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/leadtype/src/internal/docs-heading.ts b/packages/leadtype/src/internal/docs-heading.ts index 6c23ed40..3e88f6ce 100644 --- a/packages/leadtype/src/internal/docs-heading.ts +++ b/packages/leadtype/src/internal/docs-heading.ts @@ -78,7 +78,7 @@ const HTML_BLOCK_TAG_PATTERN = new RegExp( "i" ); const STANDALONE_HTML_TAG_PATTERN = - /^ {0,3}<\/?[A-Za-z][A-Za-z0-9-]*(?:[ \t]+(?:[^<>"']|"[^"<]*"|'[^'<]*')*)?\/?>[ \t]*$/; + /^ {0,3}<\/?[A-Za-z][A-Za-z0-9-]*(?:[ \t]+(?:[^<>"']|"[^"]*"|'[^']*')*)?\/?>[ \t]*$/; const MDX_BLOCK_START_PATTERN = /^ {0,3}(?:\{|<\/?[A-Z][A-Za-z0-9_.:-]*(?:[ \t/>]|$))/; const LINK_DEFINITION_PATTERN = /^ {0,3}\[[^\]]+\]:/; diff --git a/packages/leadtype/src/llm/llm.test.ts b/packages/leadtype/src/llm/llm.test.ts index e540e7e1..e5b91897 100644 --- a/packages/leadtype/src/llm/llm.test.ts +++ b/packages/leadtype/src/llm/llm.test.ts @@ -4411,7 +4411,9 @@ describe("extractDocsTableOfContents", () => { it("recognizes standalone HTML tags with quoted delimiters", () => { const toc = extractDocsTableOfContents( - ['', "---", "", "## !!!", "## !!!"].join("\n"), + ['', "---", "", "## !!!", "## !!!"].join( + "\n" + ), { urlPath: "/docs/example", absoluteUrl: "https://leadtype.dev/docs/example", diff --git a/packages/leadtype/src/search/search.test.ts b/packages/leadtype/src/search/search.test.ts index 7636859b..d6f6b75c 100644 --- a/packages/leadtype/src/search/search.test.ts +++ b/packages/leadtype/src/search/search.test.ts @@ -715,7 +715,7 @@ describe("createDocsSearchIndex and searchDocs", () => { it("keeps standalone HTML tags from shifting empty heading anchors", () => { const content = [ - '', + '', "---", "", "## !!!", From 4a26db491d82bb0570e881220c5f370b5e0b6790 Mon Sep 17 00:00:00 2001 From: Kaylee <65376239+KayleeWilliams@users.noreply.github.com> Date: Sun, 23 Aug 2026 23:28:33 +0100 Subject: [PATCH 08/29] fix(llm): parse heading HTML constructs safely --- .../leadtype/src/internal/docs-heading.ts | 95 +++++++++++++------ packages/leadtype/src/llm/llm.test.ts | 34 ++++++- packages/leadtype/src/search/search.test.ts | 67 +++++++++++-- 3 files changed, 154 insertions(+), 42 deletions(-) diff --git a/packages/leadtype/src/internal/docs-heading.ts b/packages/leadtype/src/internal/docs-heading.ts index 3e88f6ce..1c22fdb4 100644 --- a/packages/leadtype/src/internal/docs-heading.ts +++ b/packages/leadtype/src/internal/docs-heading.ts @@ -9,6 +9,8 @@ const BLOCKQUOTE_PATTERN = /^ {0,3}>/; const LIST_ITEM_PATTERN = /^ {0,3}(?:[*+-]|\d{1,9}[.)])(?:[ \t]+|$)/; const HTML_BLOCK_START_PATTERN = /^ {0,3}(?:<(?:pre|script|style|textarea)(?:[ \t>]|$)|" | "?>" | "]]>"; tracksQuotes: false } + | { closingSequence: ">"; tracksQuotes: boolean }; - for (const character of input) { - if (!tagBuffer) { - if (character === "<") { - tagBuffer = character; - } else { - output.push(character); - } - continue; - } +function getHtmlConstruct(input: string): HtmlConstruct | null { + if (input.startsWith("", tracksQuotes: false }; + } + if (input.startsWith("", tracksQuotes: false }; + } + if (input.startsWith("", tracksQuotes: false }; + } + if (HTML_DECLARATION_START_PATTERN.test(input)) { + return { closingSequence: ">", tracksQuotes: false }; + } + if (HTML_TAG_START_PATTERN.test(input)) { + return { closingSequence: ">", tracksQuotes: true }; + } + return null; +} - tagBuffer += character; - if (tagBuffer.startsWith("")) { - output.push(" "); - tagBuffer = ""; - } - continue; - } +function findHtmlConstructEnd( + input: string, + start: number, + construct: HtmlConstruct +): number { + if (!construct.tracksQuotes) { + const closingIndex = input.indexOf(construct.closingSequence, start); + return closingIndex < 0 + ? -1 + : closingIndex + construct.closingSequence.length; + } + + let quote: '"' | "'" | null = null; + for (let index = start; index < input.length; index += 1) { + const character = input[index]; if (quote !== null) { if (character === quote) { quote = null; @@ -127,14 +144,38 @@ function stripHtmlTags(input: string): string { continue; } if (character === ">") { - output.push(" "); - tagBuffer = ""; + return index + 1; } } + return -1; +} - if (tagBuffer) { - output.push(tagBuffer); +function stripHtmlTags(input: string): string { + const output: string[] = []; + let cursor = 0; + + while (cursor < input.length) { + const tagStart = input.indexOf("<", cursor); + if (tagStart < 0) { + output.push(input.slice(cursor)); + break; + } + output.push(input.slice(cursor, tagStart)); + const construct = getHtmlConstruct(input.slice(tagStart)); + if (!construct) { + output.push("<"); + cursor = tagStart + 1; + continue; + } + const constructEnd = findHtmlConstructEnd(input, tagStart, construct); + if (constructEnd < 0) { + output.push(input.slice(tagStart)); + break; + } + output.push(" "); + cursor = constructEnd; } + return output.join(""); } diff --git a/packages/leadtype/src/llm/llm.test.ts b/packages/leadtype/src/llm/llm.test.ts index e5b91897..a6c67c90 100644 --- a/packages/leadtype/src/llm/llm.test.ts +++ b/packages/leadtype/src/llm/llm.test.ts @@ -4409,18 +4409,42 @@ describe("extractDocsTableOfContents", () => { } }); - it("recognizes standalone HTML tags with quoted delimiters", () => { + it("recognizes standalone HTML tags with quoted delimiters for LF and CRLF", () => { + for (const lineEnding of ["\n", "\r\n"]) { + const toc = extractDocsTableOfContents( + ['', "---", "", "## !!!", "## !!!"].join( + lineEnding + ), + { + urlPath: "/docs/example", + absoluteUrl: "https://leadtype.dev/docs/example", + } + ); + + expect(toc.map((item) => item.id)).toEqual(["", "-1"]); + } + }); + + it("preserves comparisons and strips declarations from heading text", () => { const toc = extractDocsTableOfContents( - ['', "---", "", "## !!!", "## !!!"].join( - "\n" - ), + [ + "## 1 < 2 > 0", + "## Install ", + "## Install ", + "## Install", + ].join("\n"), { urlPath: "/docs/example", absoluteUrl: "https://leadtype.dev/docs/example", } ); - expect(toc.map((item) => item.id)).toEqual(["", "-1"]); + expect(toc.map((item) => ({ id: item.id, title: item.title }))).toEqual([ + { id: "1-2-0", title: "1 < 2 > 0" }, + { id: "install", title: "Install" }, + { id: "install-1", title: "Install" }, + { id: "install-2", title: "Install" }, + ]); }); it("respects custom heading level ranges", () => { diff --git a/packages/leadtype/src/search/search.test.ts b/packages/leadtype/src/search/search.test.ts index d6f6b75c..dc83d6c4 100644 --- a/packages/leadtype/src/search/search.test.ts +++ b/packages/leadtype/src/search/search.test.ts @@ -713,15 +713,53 @@ describe("createDocsSearchIndex and searchDocs", () => { expect(result?.urlWithHash).toBe("/docs/anchors#anchors-and"); }); - it("keeps standalone HTML tags from shifting empty heading anchors", () => { + it("keeps standalone HTML tags from shifting LF or CRLF heading anchors", () => { + for (const lineEnding of ["\n", "\r\n"]) { + const content = [ + '', + "---", + "", + "## !!!", + "First punctuation section covers widgets.", + "## !!!", + "Second punctuation section covers sprockets.", + ].join(lineEnding); + const index = createDocsSearchIndex( + [ + { + id: "fixture", + title: "Fixture", + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + relativePath: "fixture.mdx", + content, + }, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + const tocIds = flattenTocIds( + extractDocsTableOfContents(content, { + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + }) + ); + const searchAnchors = index.chunks.map( + (chunk) => chunk[CHUNK_ANCHOR_INDEX] + ); + + expect(tocIds).toEqual(["", "-1"]); + expect(searchAnchors).toEqual(["", ...tocIds]); + } + }); + + it("keeps comparison and declaration heading labels aligned with anchors", () => { const content = [ - '', - "---", - "", - "## !!!", - "First punctuation section covers widgets.", - "## !!!", - "Second punctuation section covers sprockets.", + "## 1 < 2 > 0", + "Comparison section covers widgets.", + "## Install ", + "Processing section covers sprockets.", + "## Install ", + "Declaration section covers gadgets.", ].join("\n"); const index = createDocsSearchIndex( [ @@ -736,6 +774,10 @@ describe("createDocsSearchIndex and searchDocs", () => { ], { generatedAt: "2026-01-01T00:00:00.000Z" } ); + + const comparison = searchDocs(index, "widgets")[0]; + const processing = searchDocs(index, "sprockets")[0]; + const declaration = searchDocs(index, "gadgets")[0]; const tocIds = flattenTocIds( extractDocsTableOfContents(content, { urlPath: "/docs/fixture", @@ -746,8 +788,13 @@ describe("createDocsSearchIndex and searchDocs", () => { (chunk) => chunk[CHUNK_ANCHOR_INDEX] ); - expect(tocIds).toEqual(["", "-1"]); - expect(searchAnchors).toEqual(["", ...tocIds]); + expect(searchAnchors).toEqual(tocIds); + expect(comparison?.headingPath.at(-1)).toBe("1 < 2 > 0"); + expect(comparison?.urlWithHash).toBe("/docs/fixture#1-2-0"); + expect(processing?.headingPath.at(-1)).toBe("Install"); + expect(processing?.urlWithHash).toBe("/docs/fixture#install"); + expect(declaration?.headingPath.at(-1)).toBe("Install"); + expect(declaration?.urlWithHash).toBe("/docs/fixture#install-1"); }); it("slugifies headings for hash links", () => { From 308956c203e0dd28e61e62671943938d1de29518 Mon Sep 17 00:00:00 2001 From: Kaylee <65376239+KayleeWilliams@users.noreply.github.com> Date: Mon, 24 Aug 2026 05:11:43 +0100 Subject: [PATCH 09/29] fix(llm): align MDX heading constructs --- .changeset/setext-heading-scanner.md | 2 +- .../leadtype/src/internal/docs-heading.ts | 14 +-- packages/leadtype/src/llm/llm.test.ts | 63 ++++++++++++++ packages/leadtype/src/search/search.test.ts | 86 +++++++++++++++++++ 4 files changed, 157 insertions(+), 8 deletions(-) diff --git a/.changeset/setext-heading-scanner.md b/.changeset/setext-heading-scanner.md index b9be81a6..2ff1d9e3 100644 --- a/.changeset/setext-heading-scanner.md +++ b/.changeset/setext-heading-scanner.md @@ -2,4 +2,4 @@ "leadtype": patch --- -Keep table-of-contents and search anchors aligned for Setext headings with inline markup, and ignore underlines indented as code. +Keep table-of-contents and search anchors aligned with rendered Setext and ATX headings containing inline HTML or MDX markup, including indentation and line-ending edge cases. diff --git a/packages/leadtype/src/internal/docs-heading.ts b/packages/leadtype/src/internal/docs-heading.ts index 1c22fdb4..c720ea12 100644 --- a/packages/leadtype/src/internal/docs-heading.ts +++ b/packages/leadtype/src/internal/docs-heading.ts @@ -8,9 +8,10 @@ const INDENTED_CODE_PATTERN = /^(?: {4}|\t)/; const BLOCKQUOTE_PATTERN = /^ {0,3}>/; const LIST_ITEM_PATTERN = /^ {0,3}(?:[*+-]|\d{1,9}[.)])(?:[ \t]+|$)/; const HTML_BLOCK_START_PATTERN = - /^ {0,3}(?:<(?:pre|script|style|textarea)(?:[ \t>]|$)|" | "?>" | "]]>"; tracksQuotes: false } diff --git a/packages/leadtype/src/llm/llm.test.ts b/packages/leadtype/src/llm/llm.test.ts index a6c67c90..06ee0287 100644 --- a/packages/leadtype/src/llm/llm.test.ts +++ b/packages/leadtype/src/llm/llm.test.ts @@ -4370,6 +4370,21 @@ describe("extractDocsTableOfContents", () => { ]); }); + it("strips MDX member-expression tags from ATX heading text", () => { + const toc = extractDocsTableOfContents( + ["## Install", "## Install"].join("\n"), + { + urlPath: "/docs/example", + absoluteUrl: "https://leadtype.dev/docs/example", + } + ); + + expect(toc.map((item) => ({ id: item.id, title: item.title }))).toEqual([ + { id: "install", title: "Install" }, + { id: "install-1", title: "Install" }, + ]); + }); + it("does not treat a thematic break after a blank line as a Setext heading", () => { const toc = extractDocsTableOfContents( ["A paragraph.", "", "---", "## After"].join("\n"), @@ -4409,6 +4424,39 @@ describe("extractDocsTableOfContents", () => { } }); + it("recognizes lowercase MDX member-expression flow tags", () => { + const toc = extractDocsTableOfContents( + [ + "", + "---", + "", + "## Components Note", + ].join("\n"), + { + urlPath: "/docs/example", + absoluteUrl: "https://leadtype.dev/docs/example", + } + ); + + expect(toc.map((item) => item.id)).toEqual(["components-note"]); + }); + + it("recognizes bare block starters for LF and CRLF", () => { + for (const blockStart of [" item.id)).toEqual(["after"]); + } + } + }); + it("recognizes standalone HTML tags with quoted delimiters for LF and CRLF", () => { for (const lineEnding of ["\n", "\r\n"]) { const toc = extractDocsTableOfContents( @@ -4447,6 +4495,21 @@ describe("extractDocsTableOfContents", () => { ]); }); + it("strips CDATA and preserves unterminated HTML constructs", () => { + const toc = extractDocsTableOfContents( + ["## Install ", '## Install ({ id: item.id, title: item.title }))).toEqual([ + { id: "install", title: "Install" }, + { id: "install-em-title-x", title: 'Install { const toc = extractDocsTableOfContents( ["# Page", "## Section", "### Child", "#### Detail"].join("\n"), diff --git a/packages/leadtype/src/search/search.test.ts b/packages/leadtype/src/search/search.test.ts index dc83d6c4..3e40c5dd 100644 --- a/packages/leadtype/src/search/search.test.ts +++ b/packages/leadtype/src/search/search.test.ts @@ -713,6 +713,92 @@ describe("createDocsSearchIndex and searchDocs", () => { expect(result?.urlWithHash).toBe("/docs/anchors#anchors-and"); }); + it("keeps MDX member-expression heading labels aligned with anchors", () => { + const content = [ + "## Install", + "Icon section covers widgets.", + "## Install", + "Plain section covers sprockets.", + "", + "---", + "", + "## Components Note", + "Member flow section covers gadgets.", + ].join("\n"); + const index = createDocsSearchIndex( + [ + { + id: "fixture", + title: "Fixture", + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + relativePath: "fixture.mdx", + content, + }, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + const tocIds = flattenTocIds( + extractDocsTableOfContents(content, { + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + }) + ); + const searchAnchors = index.chunks.map( + (chunk) => chunk[CHUNK_ANCHOR_INDEX] + ); + + expect(searchAnchors).toEqual(tocIds); + expect(searchDocs(index, "widgets")[0]?.urlWithHash).toBe( + "/docs/fixture#install" + ); + expect(searchDocs(index, "sprockets")[0]?.urlWithHash).toBe( + "/docs/fixture#install-1" + ); + expect(searchDocs(index, "gadgets")[0]?.urlWithHash).toBe( + "/docs/fixture#components-note" + ); + }); + + it("keeps bare block starters aligned for LF and CRLF", () => { + for (const blockStart of [" chunk[CHUNK_ANCHOR_INDEX] + ); + + expect(tocIds).toEqual(["after"]); + expect(searchAnchors).toEqual(["", ...tocIds]); + } + } + }); + it("keeps standalone HTML tags from shifting LF or CRLF heading anchors", () => { for (const lineEnding of ["\n", "\r\n"]) { const content = [ From a653e8e7ff0922d3fedabfeeda1d5b135becdc2e Mon Sep 17 00:00:00 2001 From: Kaylee <65376239+KayleeWilliams@users.noreply.github.com> Date: Mon, 24 Aug 2026 08:29:05 +0100 Subject: [PATCH 10/29] fix(llm): recognize MDX heading fragments --- .../leadtype/src/internal/docs-heading.ts | 13 ++++-- packages/leadtype/src/llm/llm.test.ts | 17 +++++++ packages/leadtype/src/search/search.test.ts | 45 +++++++++++++++++++ 3 files changed, 72 insertions(+), 3 deletions(-) diff --git a/packages/leadtype/src/internal/docs-heading.ts b/packages/leadtype/src/internal/docs-heading.ts index c720ea12..ef5ca4a3 100644 --- a/packages/leadtype/src/internal/docs-heading.ts +++ b/packages/leadtype/src/internal/docs-heading.ts @@ -10,7 +10,10 @@ const LIST_ITEM_PATTERN = /^ {0,3}(?:[*+-]|\d{1,9}[.)])(?:[ \t]+|$)/; const HTML_BLOCK_START_PATTERN = /^ {0,3}(?:<(?:pre|script|style|textarea)(?:[ \t\r>]|$)|" | "?>" | "]]>"; tracksQuotes: false } | { closingSequence: ">"; tracksQuotes: boolean }; +type JavaScriptBraceContext = { + allowsRegexAfterClose: boolean; + statementBody: boolean; +}; + +type JavaScriptParenthesisKind = "control" | "function" | "other"; + function getHtmlConstruct(input: string): HtmlConstruct | null { if (input.startsWith("", tracksQuotes: false }; @@ -149,72 +156,6 @@ function getHtmlConstruct(input: string): HtmlConstruct | null { return null; } -const closesJavaScriptControlBlock = (prefix: string): boolean => { - if (!prefix.endsWith(")")) { - return false; - } - - let parenthesisDepth = 0; - for (let index = prefix.length - 1; index >= 0; index -= 1) { - const character = prefix[index]; - if (character === ")") { - parenthesisDepth += 1; - continue; - } - if (character !== "(") { - continue; - } - parenthesisDepth -= 1; - if (parenthesisDepth !== 0) { - continue; - } - - const beforeParenthesis = prefix.slice(0, index).trimEnd(); - let identifierStart = beforeParenthesis.length; - while ( - identifierStart > 0 && - JAVASCRIPT_IDENTIFIER_PART_PATTERN.test( - beforeParenthesis[identifierStart - 1] ?? "" - ) - ) { - identifierStart -= 1; - } - const identifier = beforeParenthesis.slice(identifierStart); - const beforeIdentifier = beforeParenthesis - .slice(0, identifierStart) - .trimEnd(); - return ( - !beforeIdentifier.endsWith(".") && - JAVASCRIPT_CONTROL_KEYWORDS.has(identifier) - ); - } - return false; -}; - -const closesJavaScriptStatementBlock = (prefix: string): boolean => { - let depth = 0; - for (let index = prefix.length - 1; index >= 0; index -= 1) { - const character = prefix[index]; - if (character === "}") { - depth += 1; - continue; - } - if (character !== "{") { - continue; - } - depth -= 1; - if (depth !== 0) { - continue; - } - const beforeBlock = prefix.slice(0, index).trimEnd(); - return ( - closesJavaScriptControlBlock(beforeBlock) || - /(?:^|[^A-Za-z0-9_$])(?:do|else|finally|try)$/.test(beforeBlock) - ); - } - return false; -}; - function findHtmlConstructEnd( input: string, start: number, @@ -232,8 +173,19 @@ function findHtmlConstructEnd( let javascriptComment: "block" | "line" | null = null; let javascriptRegex = false; let javascriptRegexAllowed = true; - let nextParenthesisIsControl = false; - const parenthesisKinds: boolean[] = []; + let javascriptStatementStart = false; + let nextIdentifierIsProperty = false; + let nextBraceContext: JavaScriptBraceContext | null = null; + let pendingAsyncDeclaration: boolean | null = null; + let pendingClass: { + allowsRegexAfterClose: boolean; + parenthesisDepth: number; + } | null = null; + let pendingControlParenthesis: "for" | "other" | null = null; + let pendingFunctionDeclaration: boolean | null = null; + let awaitingFunctionParameters = false; + const braceContexts: JavaScriptBraceContext[] = []; + const parenthesisKinds: JavaScriptParenthesisKind[] = []; let quote: '"' | "'" | null = null; let regexCharacterClass = false; const templateInterpolationDepths: Array = []; @@ -273,6 +225,8 @@ function findHtmlConstructEnd( if (character === "/" && !regexCharacterClass) { javascriptRegex = false; javascriptRegexAllowed = false; + javascriptStatementStart = false; + nextIdentifierIsProperty = false; } continue; } @@ -292,13 +246,22 @@ function findHtmlConstructEnd( if (character === "`") { templateInterpolationDepths.pop(); javascriptRegexAllowed = false; + javascriptStatementStart = false; + nextIdentifierIsProperty = false; continue; } if (character === "$" && nextCharacter === "{") { braceDepth += 1; + braceContexts.push({ + allowsRegexAfterClose: false, + statementBody: false, + }); templateInterpolationDepths[templateInterpolationDepths.length - 1] = braceDepth; javascriptRegexAllowed = true; + javascriptStatementStart = false; + nextIdentifierIsProperty = false; + nextBraceContext = null; index += 1; } continue; @@ -316,6 +279,8 @@ function findHtmlConstructEnd( quote = null; if (braceDepth > 0) { javascriptRegexAllowed = false; + javascriptStatementStart = false; + nextIdentifierIsProperty = false; } } continue; @@ -334,17 +299,29 @@ function findHtmlConstructEnd( if (javascriptRegexAllowed) { javascriptRegex = true; regexCharacterClass = false; + javascriptStatementStart = false; + nextIdentifierIsProperty = false; + nextBraceContext = null; continue; } javascriptRegexAllowed = true; + javascriptStatementStart = false; + nextIdentifierIsProperty = false; + nextBraceContext = null; continue; } if (character === '"' || character === "'") { quote = character; + javascriptStatementStart = false; + nextIdentifierIsProperty = false; + nextBraceContext = null; continue; } if (braceDepth > 0 && character === "`") { templateInterpolationDepths.push(null); + javascriptStatementStart = false; + nextIdentifierIsProperty = false; + nextBraceContext = null; continue; } if ( @@ -360,8 +337,55 @@ function findHtmlConstructEnd( identifierEnd += 1; } const identifier = input.slice(index, identifierEnd); - nextParenthesisIsControl = JAVASCRIPT_CONTROL_KEYWORDS.has(identifier); - javascriptRegexAllowed = JAVASCRIPT_REGEX_PREFIX_KEYWORDS.has(identifier); + const wasStatementStart: boolean = javascriptStatementStart; + const isKeywordPosition = !nextIdentifierIsProperty; + const isControlKeyword = + isKeywordPosition && JAVASCRIPT_CONTROL_KEYWORDS.has(identifier); + const preservesForAwait = + identifier === "await" && pendingControlParenthesis === "for"; + if (isControlKeyword) { + pendingControlParenthesis = identifier === "for" ? "for" : "other"; + } else if (!preservesForAwait) { + pendingControlParenthesis = null; + } + + if (isKeywordPosition && identifier === "async") { + pendingAsyncDeclaration = wasStatementStart; + } else if (isKeywordPosition && identifier === "function") { + pendingFunctionDeclaration = + wasStatementStart || pendingAsyncDeclaration; + awaitingFunctionParameters = true; + pendingAsyncDeclaration = null; + } else { + pendingAsyncDeclaration = null; + } + + if (isKeywordPosition && identifier === "class") { + pendingClass = { + allowsRegexAfterClose: wasStatementStart, + parenthesisDepth: parenthesisKinds.length, + }; + } + + if ( + isKeywordPosition && + (identifier === "do" || + identifier === "else" || + identifier === "finally" || + identifier === "try") + ) { + nextBraceContext = { + allowsRegexAfterClose: true, + statementBody: true, + }; + javascriptStatementStart = true; + } else { + nextBraceContext = null; + javascriptStatementStart = false; + } + javascriptRegexAllowed = + isKeywordPosition && JAVASCRIPT_REGEX_PREFIX_KEYWORDS.has(identifier); + nextIdentifierIsProperty = false; index = identifierEnd - 1; continue; } @@ -374,54 +398,126 @@ function findHtmlConstructEnd( numberEnd += 1; } javascriptRegexAllowed = false; - nextParenthesisIsControl = false; + javascriptStatementStart = false; + nextIdentifierIsProperty = false; + nextBraceContext = null; + pendingControlParenthesis = null; index = numberEnd - 1; continue; } if (braceDepth > 0 && character === "(") { - parenthesisKinds.push(nextParenthesisIsControl); - nextParenthesisIsControl = false; + let parenthesisKind: JavaScriptParenthesisKind = "other"; + if (awaitingFunctionParameters) { + parenthesisKind = "function"; + } else if (pendingControlParenthesis) { + parenthesisKind = "control"; + } + parenthesisKinds.push(parenthesisKind); + awaitingFunctionParameters = false; + pendingControlParenthesis = null; javascriptRegexAllowed = true; + javascriptStatementStart = false; + nextIdentifierIsProperty = false; + nextBraceContext = null; continue; } if (braceDepth > 0 && character === ")") { - javascriptRegexAllowed = parenthesisKinds.pop() ?? false; - nextParenthesisIsControl = false; + const parenthesisKind = parenthesisKinds.pop() ?? "other"; + if (parenthesisKind === "control") { + nextBraceContext = { + allowsRegexAfterClose: true, + statementBody: true, + }; + javascriptRegexAllowed = true; + javascriptStatementStart = true; + } else if (parenthesisKind === "function") { + nextBraceContext = { + allowsRegexAfterClose: pendingFunctionDeclaration ?? false, + statementBody: true, + }; + pendingFunctionDeclaration = null; + javascriptRegexAllowed = false; + javascriptStatementStart = false; + } else { + nextBraceContext = { + allowsRegexAfterClose: false, + statementBody: true, + }; + javascriptRegexAllowed = false; + javascriptStatementStart = false; + } + pendingControlParenthesis = null; + nextIdentifierIsProperty = false; continue; } if (character === "{") { + let braceContext: JavaScriptBraceContext | null = nextBraceContext; + const startsClassBody = + pendingClass !== null && + pendingClass.parenthesisDepth === parenthesisKinds.length; + if (startsClassBody && pendingClass) { + braceContext = { + allowsRegexAfterClose: pendingClass.allowsRegexAfterClose, + statementBody: false, + }; + pendingClass = null; + } + braceContext ??= javascriptStatementStart + ? { allowsRegexAfterClose: true, statementBody: true } + : { allowsRegexAfterClose: false, statementBody: false }; braceDepth += 1; + braceContexts.push(braceContext); javascriptRegexAllowed = true; - nextParenthesisIsControl = false; + javascriptStatementStart = braceContext.statementBody; + nextIdentifierIsProperty = false; + nextBraceContext = null; + pendingControlParenthesis = null; continue; } if (character === "}" && braceDepth > 0) { if (templateInterpolationDepth === braceDepth) { braceDepth -= 1; + braceContexts.pop(); templateInterpolationDepths[templateInterpolationDepths.length - 1] = null; + javascriptStatementStart = false; + nextIdentifierIsProperty = false; continue; } braceDepth -= 1; - javascriptRegexAllowed = closesJavaScriptStatementBlock( - input.slice(0, index + 1) - ); - nextParenthesisIsControl = false; + const braceContext = braceContexts.pop(); + javascriptRegexAllowed = braceContext?.allowsRegexAfterClose ?? false; + javascriptStatementStart = braceContext?.allowsRegexAfterClose ?? false; + nextIdentifierIsProperty = false; + nextBraceContext = null; + pendingControlParenthesis = null; continue; } if (braceDepth > 0 && character === "[") { javascriptRegexAllowed = true; - nextParenthesisIsControl = false; + javascriptStatementStart = false; + nextIdentifierIsProperty = false; + nextBraceContext = null; + pendingControlParenthesis = null; continue; } if (braceDepth > 0 && (character === "]" || character === ".")) { javascriptRegexAllowed = false; - nextParenthesisIsControl = false; + javascriptStatementStart = false; + nextIdentifierIsProperty = character === "."; + nextBraceContext = null; + pendingControlParenthesis = null; continue; } if (braceDepth > 0 && character === "=" && nextCharacter === ">") { javascriptRegexAllowed = true; - nextParenthesisIsControl = false; + javascriptStatementStart = false; + nextIdentifierIsProperty = false; + nextBraceContext = { + allowsRegexAfterClose: false, + statementBody: true, + }; + pendingControlParenthesis = null; index += 1; continue; } @@ -431,7 +527,10 @@ function findHtmlConstructEnd( ",:;?=.!&|+-*%^~<>".includes(character) ) { javascriptRegexAllowed = true; - nextParenthesisIsControl = false; + javascriptStatementStart = character === ";"; + nextIdentifierIsProperty = false; + nextBraceContext = null; + pendingControlParenthesis = null; continue; } if (character === ">" && braceDepth === 0) { diff --git a/packages/leadtype/src/llm/llm.test.ts b/packages/leadtype/src/llm/llm.test.ts index b0796e14..4a1be15f 100644 --- a/packages/leadtype/src/llm/llm.test.ts +++ b/packages/leadtype/src/llm/llm.test.ts @@ -4483,7 +4483,7 @@ describe("extractDocsTableOfContents", () => { [ "## { if (ready) foo(); else /don't/.test(value); }} /> Install", "## { do /don't/.test(value); while (ready); }} /> Install", - "## y'} /> Install", + "## y'} /> Install", "## Install", ].join("\n"), { @@ -4500,6 +4500,36 @@ describe("extractDocsTableOfContents", () => { ]); }); + it("tracks JavaScript block context without rescanning literal braces", () => { + const tags = [ + ` { if (ready) { const marker = "}"; } /don't/.test(value); }} />`, + ` { if (ready) { /* } { */ } /don't/.test(value); }} />`, + " { if (ready) { const marker = `}`; } /don't/.test(value); }} />", + ` { if (ready) { const marker = /[{}]/; } /don't/.test(value); }} />`, + ` { function helper() {} /don't/.test(value); }} />`, + ` { class Helper {} /don't/.test(value); }} />`, + ` y"} />`, + ` y"} />`, + ` y"} />`, + ` { for await (const item of values) /don't/.test(item); }} />`, + ]; + + for (const tag of tags) { + const toc = extractDocsTableOfContents( + [`## ${tag} Install`, "## Install"].join("\n"), + { + urlPath: "/docs/example", + absoluteUrl: "https://leadtype.dev/docs/example", + } + ); + + expect(toc.map((item) => ({ id: item.id, title: item.title }))).toEqual([ + { id: "install", title: "Install" }, + { id: "install-1", title: "Install" }, + ]); + } + }); + it("does not treat a thematic break after a blank line as a Setext heading", () => { const toc = extractDocsTableOfContents( ["A paragraph.", "", "---", "## After"].join("\n"), diff --git a/packages/leadtype/src/search/search.test.ts b/packages/leadtype/src/search/search.test.ts index 07b69b5b..f46847d8 100644 --- a/packages/leadtype/src/search/search.test.ts +++ b/packages/leadtype/src/search/search.test.ts @@ -912,7 +912,7 @@ describe("createDocsSearchIndex and searchDocs", () => { "Else regex section covers widgets.", "## { do /don't/.test(value); while (ready); }} /> Install", "Do regex section covers sprockets.", - "## y'} /> Install", + "## y'} /> Install", "Function division section covers calipers.", "## Install", "Plain section covers gadgets.", @@ -956,6 +956,61 @@ describe("createDocsSearchIndex and searchDocs", () => { ); }); + it("keeps forward JavaScript block context aligned with rendered anchors", () => { + const tags = [ + ` { if (ready) { const marker = "}"; } /don't/.test(value); }} />`, + ` { if (ready) { /* } { */ } /don't/.test(value); }} />`, + " { if (ready) { const marker = `}`; } /don't/.test(value); }} />", + ` { if (ready) { const marker = /[{}]/; } /don't/.test(value); }} />`, + ` { function helper() {} /don't/.test(value); }} />`, + ` { class Helper {} /don't/.test(value); }} />`, + ` y"} />`, + ` y"} />`, + ` y"} />`, + ` { for await (const item of values) /don't/.test(item); }} />`, + ]; + + for (const tag of tags) { + const content = [ + `## ${tag} Install`, + "First section covers widgets.", + "## Install", + "Second section covers sprockets.", + ].join("\n"); + const index = createDocsSearchIndex( + [ + { + id: "fixture", + title: "Fixture", + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + relativePath: "fixture.mdx", + content, + }, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + const tocIds = flattenTocIds( + extractDocsTableOfContents(content, { + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + }) + ); + const searchAnchors = index.chunks.map( + (chunk) => chunk[CHUNK_ANCHOR_INDEX] + ); + + expect(tocIds).toEqual(["install", "install-1"]); + expect(searchAnchors).toEqual(tocIds); + expect(searchDocs(index, "widgets")[0]?.urlWithHash).toBe( + "/docs/fixture#install" + ); + expect(searchDocs(index, "sprockets")[0]?.urlWithHash).toBe( + "/docs/fixture#install-1" + ); + } + }); + it("keeps MDX fragment flow and inline heading anchors aligned", () => { const content = [ "<>", From 21535948b271accc3f5966432aaecbc70803c8c4 Mon Sep 17 00:00:00 2001 From: Kaylee <65376239+KayleeWilliams@users.noreply.github.com> Date: Mon, 24 Aug 2026 10:35:57 +0100 Subject: [PATCH 19/29] fix(llm): stack JavaScript statement contexts --- .../leadtype/src/internal/docs-heading.ts | 155 ++++++++++++++---- packages/leadtype/src/llm/llm.test.ts | 28 ++++ packages/leadtype/src/search/search.test.ts | 53 ++++++ 3 files changed, 208 insertions(+), 28 deletions(-) diff --git a/packages/leadtype/src/internal/docs-heading.ts b/packages/leadtype/src/internal/docs-heading.ts index 91bba771..e2465b55 100644 --- a/packages/leadtype/src/internal/docs-heading.ts +++ b/packages/leadtype/src/internal/docs-heading.ts @@ -131,7 +131,17 @@ type JavaScriptBraceContext = { statementBody: boolean; }; -type JavaScriptParenthesisKind = "control" | "function" | "other"; +type JavaScriptParenthesisContext = + | { kind: "control" } + | { declaration: boolean; kind: "function" } + | { kind: "other" }; + +type JavaScriptCaseColonContext = { + braceDepth: number; + bracketDepth: number; + conditionalDepth: number; + parenthesisDepth: number; +}; function getHtmlConstruct(input: string): HtmlConstruct | null { if (input.startsWith("" | "?>" | "]]>"; tracksQuotes: false } | { closingSequence: ">"; tracksQuotes: boolean }; @@ -344,15 +350,18 @@ function findHtmlConstructEnd( } if ( braceDepth > 0 && - character !== undefined && - JAVASCRIPT_IDENTIFIER_START_PATTERN.test(character) + JAVASCRIPT_IDENTIFIER_START_PATTERN.test( + getUnicodeCharacterAt(input, index) + ) ) { - let identifierEnd = index + 1; - while ( - identifierEnd < input.length && - JAVASCRIPT_IDENTIFIER_PART_PATTERN.test(input[identifierEnd] ?? "") - ) { - identifierEnd += 1; + const identifierStart = getUnicodeCharacterAt(input, index); + let identifierEnd = index + identifierStart.length; + while (identifierEnd < input.length) { + const identifierPart = getUnicodeCharacterAt(input, identifierEnd); + if (!JAVASCRIPT_IDENTIFIER_PART_PATTERN.test(identifierPart)) { + break; + } + identifierEnd += identifierPart.length; } const identifier = input.slice(index, identifierEnd); const wasStatementStart: boolean = javascriptStatementStart; @@ -565,12 +574,16 @@ function findHtmlConstructEnd( continue; } if (braceDepth > 0 && character === ".") { - javascriptRegexAllowed = false; + const isSpreadOperator = input.startsWith("...", index); + javascriptRegexAllowed = isSpreadOperator; javascriptStatementStart = false; pendingLabelColon = false; - nextIdentifierIsProperty = true; + nextIdentifierIsProperty = !isSpreadOperator; nextBraceContext = null; pendingControlParenthesis = null; + if (isSpreadOperator) { + index += 2; + } continue; } if (braceDepth > 0 && character === "=" && nextCharacter === ">") { diff --git a/packages/leadtype/src/llm/llm.test.ts b/packages/leadtype/src/llm/llm.test.ts index d47f2447..ac6812b4 100644 --- a/packages/leadtype/src/llm/llm.test.ts +++ b/packages/leadtype/src/llm/llm.test.ts @@ -4586,6 +4586,35 @@ describe("extractDocsTableOfContents", () => { } }); + it("recognizes Unicode identifiers and spread operands", () => { + const tags = [ + ' y"} />', + ' y"} />', + ' y"} />', + ' y"} />', + ' y"} />', + ' y"} />', + ' y"} />', + ``, + ' y" }} />', + ]; + + for (const tag of tags) { + const toc = extractDocsTableOfContents( + [`## ${tag} Install`, "## Install"].join("\n"), + { + urlPath: "/docs/example", + absoluteUrl: "https://leadtype.dev/docs/example", + } + ); + + expect(toc.map((item) => ({ id: item.id, title: item.title }))).toEqual([ + { id: "install", title: "Install" }, + { id: "install-1", title: "Install" }, + ]); + } + }); + it("does not treat a thematic break after a blank line as a Setext heading", () => { const toc = extractDocsTableOfContents( ["A paragraph.", "", "---", "## After"].join("\n"), diff --git a/packages/leadtype/src/search/search.test.ts b/packages/leadtype/src/search/search.test.ts index 44001c88..44195daf 100644 --- a/packages/leadtype/src/search/search.test.ts +++ b/packages/leadtype/src/search/search.test.ts @@ -1117,6 +1117,60 @@ describe("createDocsSearchIndex and searchDocs", () => { } }); + it("keeps Unicode identifiers and spread operands aligned with rendered anchors", () => { + const tags = [ + ' y"} />', + ' y"} />', + ' y"} />', + ' y"} />', + ' y"} />', + ' y"} />', + ' y"} />', + ``, + ' y" }} />', + ]; + + for (const tag of tags) { + const content = [ + `## ${tag} Install`, + "First section covers widgets.", + "## Install", + "Second section covers sprockets.", + ].join("\n"); + const index = createDocsSearchIndex( + [ + { + id: "fixture", + title: "Fixture", + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + relativePath: "fixture.mdx", + content, + }, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + const tocIds = flattenTocIds( + extractDocsTableOfContents(content, { + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + }) + ); + const searchAnchors = index.chunks.map( + (chunk) => chunk[CHUNK_ANCHOR_INDEX] + ); + + expect(tocIds).toEqual(["install", "install-1"]); + expect(searchAnchors).toEqual(tocIds); + expect(searchDocs(index, "widgets")[0]?.urlWithHash).toBe( + "/docs/fixture#install" + ); + expect(searchDocs(index, "sprockets")[0]?.urlWithHash).toBe( + "/docs/fixture#install-1" + ); + } + }); + it("keeps MDX fragment flow and inline heading anchors aligned", () => { const content = [ "<>", From 58986cf0cdb9992fb9808cd6f54b503fdc4d5388 Mon Sep 17 00:00:00 2001 From: Kaylee <65376239+KayleeWilliams@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:05:01 +0100 Subject: [PATCH 23/29] fix(llm): classify class heritage operands --- .../leadtype/src/internal/docs-heading.ts | 2 + packages/leadtype/src/llm/llm.test.ts | 28 ++++++++++ packages/leadtype/src/search/search.test.ts | 53 +++++++++++++++++++ 3 files changed, 83 insertions(+) diff --git a/packages/leadtype/src/internal/docs-heading.ts b/packages/leadtype/src/internal/docs-heading.ts index 51c6256a..66038e4f 100644 --- a/packages/leadtype/src/internal/docs-heading.ts +++ b/packages/leadtype/src/internal/docs-heading.ts @@ -106,9 +106,11 @@ const JAVASCRIPT_CONTROL_KEYWORDS = new Set([ const JAVASCRIPT_REGEX_PREFIX_KEYWORDS = new Set([ "await", "case", + "default", "delete", "do", "else", + "extends", "in", "instanceof", "new", diff --git a/packages/leadtype/src/llm/llm.test.ts b/packages/leadtype/src/llm/llm.test.ts index ac6812b4..178bb80d 100644 --- a/packages/leadtype/src/llm/llm.test.ts +++ b/packages/leadtype/src/llm/llm.test.ts @@ -4615,6 +4615,34 @@ describe("extractDocsTableOfContents", () => { } }); + it("recognizes regex operands in class heritage", () => { + const tags = [ + ``, + ``, + ' y") {}} />', + ' y") {}} />', + ' y"} />', + ' y"} />', + ' y"} />', + ` { class Named {} /don't/.test(value); }} />`, + ]; + + for (const tag of tags) { + const toc = extractDocsTableOfContents( + [`## ${tag} Install`, "## Install"].join("\n"), + { + urlPath: "/docs/example", + absoluteUrl: "https://leadtype.dev/docs/example", + } + ); + + expect(toc.map((item) => ({ id: item.id, title: item.title }))).toEqual([ + { id: "install", title: "Install" }, + { id: "install-1", title: "Install" }, + ]); + } + }); + it("does not treat a thematic break after a blank line as a Setext heading", () => { const toc = extractDocsTableOfContents( ["A paragraph.", "", "---", "## After"].join("\n"), diff --git a/packages/leadtype/src/search/search.test.ts b/packages/leadtype/src/search/search.test.ts index 44195daf..b8c281ca 100644 --- a/packages/leadtype/src/search/search.test.ts +++ b/packages/leadtype/src/search/search.test.ts @@ -1171,6 +1171,59 @@ describe("createDocsSearchIndex and searchDocs", () => { } }); + it("keeps class-heritage regex operands aligned with rendered anchors", () => { + const tags = [ + ``, + ``, + ' y") {}} />', + ' y") {}} />', + ' y"} />', + ' y"} />', + ' y"} />', + ` { class Named {} /don't/.test(value); }} />`, + ]; + + for (const tag of tags) { + const content = [ + `## ${tag} Install`, + "First section covers widgets.", + "## Install", + "Second section covers sprockets.", + ].join("\n"); + const index = createDocsSearchIndex( + [ + { + id: "fixture", + title: "Fixture", + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + relativePath: "fixture.mdx", + content, + }, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + const tocIds = flattenTocIds( + extractDocsTableOfContents(content, { + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + }) + ); + const searchAnchors = index.chunks.map( + (chunk) => chunk[CHUNK_ANCHOR_INDEX] + ); + + expect(tocIds).toEqual(["install", "install-1"]); + expect(searchAnchors).toEqual(tocIds); + expect(searchDocs(index, "widgets")[0]?.urlWithHash).toBe( + "/docs/fixture#install" + ); + expect(searchDocs(index, "sprockets")[0]?.urlWithHash).toBe( + "/docs/fixture#install-1" + ); + } + }); + it("keeps MDX fragment flow and inline heading anchors aligned", () => { const content = [ "<>", From d2a6bdf169168d1c37bc3085fe0555b64b453c36 Mon Sep 17 00:00:00 2001 From: Kaylee <65376239+KayleeWilliams@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:07:36 +0100 Subject: [PATCH 24/29] refactor(llm): clarify update operator state --- packages/leadtype/src/internal/docs-heading.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/leadtype/src/internal/docs-heading.ts b/packages/leadtype/src/internal/docs-heading.ts index 66038e4f..d6b7f63a 100644 --- a/packages/leadtype/src/internal/docs-heading.ts +++ b/packages/leadtype/src/internal/docs-heading.ts @@ -606,8 +606,8 @@ function findHtmlConstructEnd( (character === "+" || character === "-") && nextCharacter === character ) { - const isPostfixUpdate: boolean = !javascriptRegexAllowed; - javascriptRegexAllowed = !isPostfixUpdate; + // Prefix updates still expect an operand and postfix updates yield a + // value, so a following slash keeps the meaning it already had. javascriptStatementStart = false; pendingLabelColon = false; nextIdentifierIsProperty = false; From b9d34d82fceb0df5b7d6794e60eae8cef872ca25 Mon Sep 17 00:00:00 2001 From: Kaylee <65376239+KayleeWilliams@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:19:24 +0100 Subject: [PATCH 25/29] fix(llm): recognize class static blocks --- .../leadtype/src/internal/docs-heading.ts | 25 ++++++++-- packages/leadtype/src/llm/llm.test.ts | 25 ++++++++++ packages/leadtype/src/search/search.test.ts | 50 +++++++++++++++++++ 3 files changed, 95 insertions(+), 5 deletions(-) diff --git a/packages/leadtype/src/internal/docs-heading.ts b/packages/leadtype/src/internal/docs-heading.ts index d6b7f63a..5d743fdf 100644 --- a/packages/leadtype/src/internal/docs-heading.ts +++ b/packages/leadtype/src/internal/docs-heading.ts @@ -136,6 +136,10 @@ type HtmlConstruct = type JavaScriptBraceContext = { allowsRegexAfterClose: boolean; + classBody?: { + bracketDepth: number; + parenthesisDepth: number; + }; statementBody: boolean; }; @@ -368,6 +372,12 @@ function findHtmlConstructEnd( const identifier = input.slice(index, identifierEnd); const wasStatementStart: boolean = javascriptStatementStart; const isKeywordPosition = !nextIdentifierIsProperty; + const activeClassBody = braceContexts.at(-1)?.classBody; + const startsStaticBlock = + isKeywordPosition && + identifier === "static" && + activeClassBody?.bracketDepth === bracketDepth && + activeClassBody.parenthesisDepth === parenthesisContexts.length; const startsCaseClause = isKeywordPosition && wasStatementStart && @@ -413,11 +423,12 @@ function findHtmlConstructEnd( wasStatementStart && isKeywordPosition && !startsCaseClause; if ( - isKeywordPosition && - (identifier === "do" || - identifier === "else" || - identifier === "finally" || - identifier === "try") + startsStaticBlock || + (isKeywordPosition && + (identifier === "do" || + identifier === "else" || + identifier === "finally" || + identifier === "try")) ) { nextBraceContext = { allowsRegexAfterClose: true, @@ -512,6 +523,10 @@ function findHtmlConstructEnd( if (startsClassBody && pendingClass) { braceContext = { allowsRegexAfterClose: pendingClass.allowsRegexAfterClose, + classBody: { + bracketDepth, + parenthesisDepth: parenthesisContexts.length, + }, statementBody: false, }; pendingClasses.pop(); diff --git a/packages/leadtype/src/llm/llm.test.ts b/packages/leadtype/src/llm/llm.test.ts index 178bb80d..9c7caa0a 100644 --- a/packages/leadtype/src/llm/llm.test.ts +++ b/packages/leadtype/src/llm/llm.test.ts @@ -4643,6 +4643,31 @@ describe("extractDocsTableOfContents", () => { } }); + it("recognizes statement bodies in class static blocks", () => { + const tags = [ + ``, + ``, + ' y"} />', + ' y"} />', + ' y"} />', + ]; + + for (const tag of tags) { + const toc = extractDocsTableOfContents( + [`## ${tag} Install`, "## Install"].join("\n"), + { + urlPath: "/docs/example", + absoluteUrl: "https://leadtype.dev/docs/example", + } + ); + + expect(toc.map((item) => ({ id: item.id, title: item.title }))).toEqual([ + { id: "install", title: "Install" }, + { id: "install-1", title: "Install" }, + ]); + } + }); + it("does not treat a thematic break after a blank line as a Setext heading", () => { const toc = extractDocsTableOfContents( ["A paragraph.", "", "---", "## After"].join("\n"), diff --git a/packages/leadtype/src/search/search.test.ts b/packages/leadtype/src/search/search.test.ts index b8c281ca..d69bc4f1 100644 --- a/packages/leadtype/src/search/search.test.ts +++ b/packages/leadtype/src/search/search.test.ts @@ -1224,6 +1224,56 @@ describe("createDocsSearchIndex and searchDocs", () => { } }); + it("keeps class static blocks aligned with rendered anchors", () => { + const tags = [ + ``, + ``, + ' y"} />', + ' y"} />', + ' y"} />', + ]; + + for (const tag of tags) { + const content = [ + `## ${tag} Install`, + "First section covers widgets.", + "## Install", + "Second section covers sprockets.", + ].join("\n"); + const index = createDocsSearchIndex( + [ + { + id: "fixture", + title: "Fixture", + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + relativePath: "fixture.mdx", + content, + }, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + const tocIds = flattenTocIds( + extractDocsTableOfContents(content, { + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + }) + ); + const searchAnchors = index.chunks.map( + (chunk) => chunk[CHUNK_ANCHOR_INDEX] + ); + + expect(tocIds).toEqual(["install", "install-1"]); + expect(searchAnchors).toEqual(tocIds); + expect(searchDocs(index, "widgets")[0]?.urlWithHash).toBe( + "/docs/fixture#install" + ); + expect(searchDocs(index, "sprockets")[0]?.urlWithHash).toBe( + "/docs/fixture#install-1" + ); + } + }); + it("keeps MDX fragment flow and inline heading anchors aligned", () => { const content = [ "<>", From cb08d5e1eac55f62c34f2dc193734ab748cedea1 Mon Sep 17 00:00:00 2001 From: Kaylee <65376239+KayleeWilliams@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:36:38 +0100 Subject: [PATCH 26/29] fix(llm): distinguish class member names --- .../leadtype/src/internal/docs-heading.ts | 39 ++++++++++++++- packages/leadtype/src/llm/llm.test.ts | 24 +++++++++ packages/leadtype/src/search/search.test.ts | 49 +++++++++++++++++++ 3 files changed, 111 insertions(+), 1 deletion(-) diff --git a/packages/leadtype/src/internal/docs-heading.ts b/packages/leadtype/src/internal/docs-heading.ts index 5d743fdf..3e136c39 100644 --- a/packages/leadtype/src/internal/docs-heading.ts +++ b/packages/leadtype/src/internal/docs-heading.ts @@ -95,6 +95,7 @@ const WHITESPACE_PATTERN = /\s+/g; const JAVASCRIPT_IDENTIFIER_START_PATTERN = /^[$_\p{ID_Start}]$/u; const JAVASCRIPT_IDENTIFIER_PART_PATTERN = /^(?:[$_\p{ID_Continue}]|\u200C|\u200D)$/u; +const JAVASCRIPT_WHITESPACE_PATTERN = /^\s$/; const JAVASCRIPT_CONTROL_KEYWORDS = new Set([ "catch", "for", @@ -130,6 +131,33 @@ const getUnicodeCharacterAt = (input: string, index: number): string => { return codePoint === undefined ? "" : String.fromCodePoint(codePoint); }; +function getNextJavaScriptTokenStart(input: string, start: number): string { + let cursor = start; + while (cursor < input.length) { + const character = input[cursor]; + const nextCharacter = input[cursor + 1]; + if ( + character !== undefined && + JAVASCRIPT_WHITESPACE_PATTERN.test(character) + ) { + cursor += 1; + continue; + } + if (character === "/" && nextCharacter === "/") { + const lineEnd = input.indexOf("\n", cursor + 2); + cursor = lineEnd < 0 ? input.length : lineEnd + 1; + continue; + } + if (character === "/" && nextCharacter === "*") { + const commentEnd = input.indexOf("*/", cursor + 2); + cursor = commentEnd < 0 ? input.length : commentEnd + 2; + continue; + } + return getUnicodeCharacterAt(input, cursor); + } + return ""; +} + type HtmlConstruct = | { closingSequence: "-->" | "?>" | "]]>"; tracksQuotes: false } | { closingSequence: ">"; tracksQuotes: boolean }; @@ -402,7 +430,16 @@ function findHtmlConstructEnd( pendingAsyncDeclaration = null; } - if (isKeywordPosition && identifier === "class") { + const nextClassTokenStart = + isKeywordPosition && identifier === "class" + ? getNextJavaScriptTokenStart(input, identifierEnd) + : ""; + const startsClass = + isKeywordPosition && + identifier === "class" && + (nextClassTokenStart === "{" || + JAVASCRIPT_IDENTIFIER_START_PATTERN.test(nextClassTokenStart)); + if (startsClass) { pendingClasses.push({ allowsRegexAfterClose: wasStatementStart, braceDepth, diff --git a/packages/leadtype/src/llm/llm.test.ts b/packages/leadtype/src/llm/llm.test.ts index 9c7caa0a..4f7e1b4e 100644 --- a/packages/leadtype/src/llm/llm.test.ts +++ b/packages/leadtype/src/llm/llm.test.ts @@ -4668,6 +4668,30 @@ describe("extractDocsTableOfContents", () => { } }); + it("does not classify member names as class keywords", () => { + const tags = [ + ``, + ` { function nested() {} /don't/.test(value); } }} />`, + ``, + ``, + ]; + + for (const tag of tags) { + const toc = extractDocsTableOfContents( + [`## ${tag} Install`, "## Install"].join("\n"), + { + urlPath: "/docs/example", + absoluteUrl: "https://leadtype.dev/docs/example", + } + ); + + expect(toc.map((item) => ({ id: item.id, title: item.title }))).toEqual([ + { id: "install", title: "Install" }, + { id: "install-1", title: "Install" }, + ]); + } + }); + it("does not treat a thematic break after a blank line as a Setext heading", () => { const toc = extractDocsTableOfContents( ["A paragraph.", "", "---", "## After"].join("\n"), diff --git a/packages/leadtype/src/search/search.test.ts b/packages/leadtype/src/search/search.test.ts index d69bc4f1..cb538ee2 100644 --- a/packages/leadtype/src/search/search.test.ts +++ b/packages/leadtype/src/search/search.test.ts @@ -1274,6 +1274,55 @@ describe("createDocsSearchIndex and searchDocs", () => { } }); + it("keeps class-named members aligned with rendered anchors", () => { + const tags = [ + ``, + ` { function nested() {} /don't/.test(value); } }} />`, + ``, + ``, + ]; + + for (const tag of tags) { + const content = [ + `## ${tag} Install`, + "First section covers widgets.", + "## Install", + "Second section covers sprockets.", + ].join("\n"); + const index = createDocsSearchIndex( + [ + { + id: "fixture", + title: "Fixture", + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + relativePath: "fixture.mdx", + content, + }, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + const tocIds = flattenTocIds( + extractDocsTableOfContents(content, { + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + }) + ); + const searchAnchors = index.chunks.map( + (chunk) => chunk[CHUNK_ANCHOR_INDEX] + ); + + expect(tocIds).toEqual(["install", "install-1"]); + expect(searchAnchors).toEqual(tocIds); + expect(searchDocs(index, "widgets")[0]?.urlWithHash).toBe( + "/docs/fixture#install" + ); + expect(searchDocs(index, "sprockets")[0]?.urlWithHash).toBe( + "/docs/fixture#install-1" + ); + } + }); + it("keeps MDX fragment flow and inline heading anchors aligned", () => { const content = [ "<>", From cf5b07e2c5369a245aa14ad3b5ce59d055e8251b Mon Sep 17 00:00:00 2001 From: Kaylee <65376239+KayleeWilliams@users.noreply.github.com> Date: Mon, 24 Aug 2026 11:44:49 +0100 Subject: [PATCH 27/29] fix(llm): parse escaped class identifiers --- .../src/internal/docs-heading.test.ts | 24 +++++ .../leadtype/src/internal/docs-heading.ts | 101 +++++++++++++++--- packages/leadtype/src/llm/llm.test.ts | 22 ++++ packages/leadtype/src/search/search.test.ts | 47 ++++++++ 4 files changed, 177 insertions(+), 17 deletions(-) diff --git a/packages/leadtype/src/internal/docs-heading.test.ts b/packages/leadtype/src/internal/docs-heading.test.ts index 778b030f..cee5568b 100644 --- a/packages/leadtype/src/internal/docs-heading.test.ts +++ b/packages/leadtype/src/internal/docs-heading.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it } from "vitest"; import { createDocsHeadingSlugger, docsHtmlBlockTagNames, + scanDocsMarkdown, slugifyDocsHeading, } from "./docs-heading"; @@ -57,3 +58,26 @@ describe("createDocsHeadingSlugger", () => { expect(slugger.slug("Foo")).toBe("foo-2"); }); }); + +describe("scanDocsMarkdown", () => { + it("rejects malformed escaped class binding starts", () => { + const invalidBindingStarts = [ + "\\x61", + "\\u{}", + "\\u{110000}", + "\\u0030", + "\\uD800", + ]; + + for (const bindingStart of invalidBindingStarts) { + const [heading] = scanDocsMarkdown( + `## Install` + ); + + expect(heading).toMatchObject({ kind: "heading", level: 2 }); + expect(heading?.kind === "heading" ? heading.title : "Install").not.toBe( + "Install" + ); + } + }); +}); diff --git a/packages/leadtype/src/internal/docs-heading.ts b/packages/leadtype/src/internal/docs-heading.ts index 3e136c39..e8889055 100644 --- a/packages/leadtype/src/internal/docs-heading.ts +++ b/packages/leadtype/src/internal/docs-heading.ts @@ -95,7 +95,10 @@ const WHITESPACE_PATTERN = /\s+/g; const JAVASCRIPT_IDENTIFIER_START_PATTERN = /^[$_\p{ID_Start}]$/u; const JAVASCRIPT_IDENTIFIER_PART_PATTERN = /^(?:[$_\p{ID_Continue}]|\u200C|\u200D)$/u; +const JAVASCRIPT_FIXED_UNICODE_ESCAPE_PATTERN = /^[0-9A-Fa-f]{4}$/; +const JAVASCRIPT_CODE_POINT_ESCAPE_PATTERN = /^[0-9A-Fa-f]+$/; const JAVASCRIPT_WHITESPACE_PATTERN = /^\s$/; +const MAX_UNICODE_CODE_POINT = 0x10_ff_ff; const JAVASCRIPT_CONTROL_KEYWORDS = new Set([ "catch", "for", @@ -131,7 +134,64 @@ const getUnicodeCharacterAt = (input: string, index: number): string => { return codePoint === undefined ? "" : String.fromCodePoint(codePoint); }; -function getNextJavaScriptTokenStart(input: string, start: number): string { +type JavaScriptIdentifierCharacter = { + character: string; + end: number; +}; + +function getEscapedJavaScriptIdentifierCharacterAt( + input: string, + start: number +): JavaScriptIdentifierCharacter | null { + if (input[start] !== "\\" || input[start + 1] !== "u") { + return null; + } + + let hexadecimalDigits: string; + let end: number; + if (input[start + 2] === "{") { + const escapeEnd = input.indexOf("}", start + 3); + if (escapeEnd < 0) { + return null; + } + hexadecimalDigits = input.slice(start + 3, escapeEnd); + if (!JAVASCRIPT_CODE_POINT_ESCAPE_PATTERN.test(hexadecimalDigits)) { + return null; + } + end = escapeEnd + 1; + } else { + hexadecimalDigits = input.slice(start + 2, start + 6); + if (!JAVASCRIPT_FIXED_UNICODE_ESCAPE_PATTERN.test(hexadecimalDigits)) { + return null; + } + end = start + 6; + } + + const codePoint = Number.parseInt(hexadecimalDigits, 16); + return codePoint <= MAX_UNICODE_CODE_POINT + ? { character: String.fromCodePoint(codePoint), end } + : null; +} + +function getJavaScriptIdentifierCharacterAt( + input: string, + start: number, + pattern: RegExp +): JavaScriptIdentifierCharacter | null { + const sourceCharacter = getUnicodeCharacterAt(input, start); + if (pattern.test(sourceCharacter)) { + return { character: sourceCharacter, end: start + sourceCharacter.length }; + } + const escapedCharacter = getEscapedJavaScriptIdentifierCharacterAt( + input, + start + ); + return escapedCharacter !== null && pattern.test(escapedCharacter.character) + ? escapedCharacter + : null; +} + +function getNextJavaScriptTokenStart(input: string, start: number): number { let cursor = start; while (cursor < input.length) { const character = input[cursor]; @@ -153,9 +213,9 @@ function getNextJavaScriptTokenStart(input: string, start: number): string { cursor = commentEnd < 0 ? input.length : commentEnd + 2; continue; } - return getUnicodeCharacterAt(input, cursor); + return cursor; } - return ""; + return -1; } type HtmlConstruct = @@ -382,20 +442,23 @@ function findHtmlConstructEnd( nextBraceContext = null; continue; } - if ( - braceDepth > 0 && - JAVASCRIPT_IDENTIFIER_START_PATTERN.test( - getUnicodeCharacterAt(input, index) - ) - ) { - const identifierStart = getUnicodeCharacterAt(input, index); - let identifierEnd = index + identifierStart.length; + const identifierStart = getJavaScriptIdentifierCharacterAt( + input, + index, + JAVASCRIPT_IDENTIFIER_START_PATTERN + ); + if (braceDepth > 0 && identifierStart !== null) { + let identifierEnd = identifierStart.end; while (identifierEnd < input.length) { - const identifierPart = getUnicodeCharacterAt(input, identifierEnd); - if (!JAVASCRIPT_IDENTIFIER_PART_PATTERN.test(identifierPart)) { + const identifierPart = getJavaScriptIdentifierCharacterAt( + input, + identifierEnd, + JAVASCRIPT_IDENTIFIER_PART_PATTERN + ); + if (identifierPart === null) { break; } - identifierEnd += identifierPart.length; + identifierEnd = identifierPart.end; } const identifier = input.slice(index, identifierEnd); const wasStatementStart: boolean = javascriptStatementStart; @@ -433,12 +496,16 @@ function findHtmlConstructEnd( const nextClassTokenStart = isKeywordPosition && identifier === "class" ? getNextJavaScriptTokenStart(input, identifierEnd) - : ""; + : -1; const startsClass = isKeywordPosition && identifier === "class" && - (nextClassTokenStart === "{" || - JAVASCRIPT_IDENTIFIER_START_PATTERN.test(nextClassTokenStart)); + (input[nextClassTokenStart] === "{" || + getJavaScriptIdentifierCharacterAt( + input, + nextClassTokenStart, + JAVASCRIPT_IDENTIFIER_START_PATTERN + ) !== null); if (startsClass) { pendingClasses.push({ allowsRegexAfterClose: wasStatementStart, diff --git a/packages/leadtype/src/llm/llm.test.ts b/packages/leadtype/src/llm/llm.test.ts index 4f7e1b4e..99376b33 100644 --- a/packages/leadtype/src/llm/llm.test.ts +++ b/packages/leadtype/src/llm/llm.test.ts @@ -4692,6 +4692,28 @@ describe("extractDocsTableOfContents", () => { } }); + it("recognizes escaped class binding identifiers", () => { + const tags = [ + ``, + ``, + ]; + + for (const tag of tags) { + const toc = extractDocsTableOfContents( + [`## ${tag} Install`, "## Install"].join("\n"), + { + urlPath: "/docs/example", + absoluteUrl: "https://leadtype.dev/docs/example", + } + ); + + expect(toc.map((item) => ({ id: item.id, title: item.title }))).toEqual([ + { id: "install", title: "Install" }, + { id: "install-1", title: "Install" }, + ]); + } + }); + it("does not treat a thematic break after a blank line as a Setext heading", () => { const toc = extractDocsTableOfContents( ["A paragraph.", "", "---", "## After"].join("\n"), diff --git a/packages/leadtype/src/search/search.test.ts b/packages/leadtype/src/search/search.test.ts index cb538ee2..1652dee2 100644 --- a/packages/leadtype/src/search/search.test.ts +++ b/packages/leadtype/src/search/search.test.ts @@ -1323,6 +1323,53 @@ describe("createDocsSearchIndex and searchDocs", () => { } }); + it("keeps escaped class bindings aligned with rendered anchors", () => { + const tags = [ + ``, + ``, + ]; + + for (const tag of tags) { + const content = [ + `## ${tag} Install`, + "First section covers widgets.", + "## Install", + "Second section covers sprockets.", + ].join("\n"); + const index = createDocsSearchIndex( + [ + { + id: "fixture", + title: "Fixture", + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + relativePath: "fixture.mdx", + content, + }, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + const tocIds = flattenTocIds( + extractDocsTableOfContents(content, { + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + }) + ); + const searchAnchors = index.chunks.map( + (chunk) => chunk[CHUNK_ANCHOR_INDEX] + ); + + expect(tocIds).toEqual(["install", "install-1"]); + expect(searchAnchors).toEqual(tocIds); + expect(searchDocs(index, "widgets")[0]?.urlWithHash).toBe( + "/docs/fixture#install" + ); + expect(searchDocs(index, "sprockets")[0]?.urlWithHash).toBe( + "/docs/fixture#install-1" + ); + } + }); + it("keeps MDX fragment flow and inline heading anchors aligned", () => { const content = [ "<>", From bdf6510198b127c832455696075788eeb26cd0d3 Mon Sep 17 00:00:00 2001 From: Kaylee <65376239+KayleeWilliams@users.noreply.github.com> Date: Mon, 24 Aug 2026 12:04:32 +0100 Subject: [PATCH 28/29] fix(llm): preserve lazy identifier scanning --- .../src/internal/docs-heading.test.ts | 38 ++++++++++++++----- .../leadtype/src/internal/docs-heading.ts | 15 +++++--- 2 files changed, 37 insertions(+), 16 deletions(-) diff --git a/packages/leadtype/src/internal/docs-heading.test.ts b/packages/leadtype/src/internal/docs-heading.test.ts index cee5568b..dd592d7d 100644 --- a/packages/leadtype/src/internal/docs-heading.test.ts +++ b/packages/leadtype/src/internal/docs-heading.test.ts @@ -62,22 +62,40 @@ describe("createDocsHeadingSlugger", () => { describe("scanDocsMarkdown", () => { it("rejects malformed escaped class binding starts", () => { const invalidBindingStarts = [ - "\\x61", - "\\u{}", - "\\u{110000}", - "\\u0030", - "\\uD800", + [ + "\\x61", + " Install", + ], + [ + "\\u{}", + " Install", + ], + // This also guards the range check before String.fromCodePoint, which + // would throw for an out-of-range escape instead of rejecting it. + [ + "\\u{110000}", + " Install", + ], + [ + "\\u0030", + " Install", + ], + [ + "\\uD800", + " Install", + ], ]; - for (const bindingStart of invalidBindingStarts) { + for (const [bindingStart, expectedTitle] of invalidBindingStarts) { const [heading] = scanDocsMarkdown( `## Install` ); - expect(heading).toMatchObject({ kind: "heading", level: 2 }); - expect(heading?.kind === "heading" ? heading.title : "Install").not.toBe( - "Install" - ); + expect(heading).toEqual({ + kind: "heading", + level: 2, + title: expectedTitle, + }); } }); }); diff --git a/packages/leadtype/src/internal/docs-heading.ts b/packages/leadtype/src/internal/docs-heading.ts index e8889055..3802a528 100644 --- a/packages/leadtype/src/internal/docs-heading.ts +++ b/packages/leadtype/src/internal/docs-heading.ts @@ -442,12 +442,15 @@ function findHtmlConstructEnd( nextBraceContext = null; continue; } - const identifierStart = getJavaScriptIdentifierCharacterAt( - input, - index, - JAVASCRIPT_IDENTIFIER_START_PATTERN - ); - if (braceDepth > 0 && identifierStart !== null) { + const identifierStart = + braceDepth > 0 + ? getJavaScriptIdentifierCharacterAt( + input, + index, + JAVASCRIPT_IDENTIFIER_START_PATTERN + ) + : null; + if (identifierStart !== null) { let identifierEnd = identifierStart.end; while (identifierEnd < input.length) { const identifierPart = getJavaScriptIdentifierCharacterAt( From 4fb0caf9c58989b077fa5c8188977389690653bc Mon Sep 17 00:00:00 2001 From: Kaylee <65376239+KayleeWilliams@users.noreply.github.com> Date: Mon, 24 Aug 2026 12:15:38 +0100 Subject: [PATCH 29/29] fix(llm): recognize binding-less catch blocks --- packages/leadtype/src/internal/docs-heading.test.ts | 8 ++++++++ packages/leadtype/src/internal/docs-heading.ts | 3 ++- packages/leadtype/src/llm/llm.test.ts | 1 + packages/leadtype/src/search/search.test.ts | 1 + 4 files changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/leadtype/src/internal/docs-heading.test.ts b/packages/leadtype/src/internal/docs-heading.test.ts index dd592d7d..384ace96 100644 --- a/packages/leadtype/src/internal/docs-heading.test.ts +++ b/packages/leadtype/src/internal/docs-heading.test.ts @@ -60,6 +60,14 @@ describe("createDocsHeadingSlugger", () => { }); describe("scanDocsMarkdown", () => { + it("recognizes binding-less catch statement bodies", () => { + const [heading] = scanDocsMarkdown( + `## { try {} catch { function helper() {} /don't/.test(value); } }} /> Install` + ); + + expect(heading).toEqual({ kind: "heading", level: 2, title: "Install" }); + }); + it("rejects malformed escaped class binding starts", () => { const invalidBindingStarts = [ [ diff --git a/packages/leadtype/src/internal/docs-heading.ts b/packages/leadtype/src/internal/docs-heading.ts index 3802a528..b68cb307 100644 --- a/packages/leadtype/src/internal/docs-heading.ts +++ b/packages/leadtype/src/internal/docs-heading.ts @@ -532,7 +532,8 @@ function findHtmlConstructEnd( if ( startsStaticBlock || (isKeywordPosition && - (identifier === "do" || + (identifier === "catch" || + identifier === "do" || identifier === "else" || identifier === "finally" || identifier === "try")) diff --git a/packages/leadtype/src/llm/llm.test.ts b/packages/leadtype/src/llm/llm.test.ts index 99376b33..ba39d23f 100644 --- a/packages/leadtype/src/llm/llm.test.ts +++ b/packages/leadtype/src/llm/llm.test.ts @@ -4508,6 +4508,7 @@ describe("extractDocsTableOfContents", () => { ` { if (ready) { const marker = /[{}]/; } /don't/.test(value); }} />`, ` { function helper() {} /don't/.test(value); }} />`, ` { class Helper {} /don't/.test(value); }} />`, + ` { try {} catch { function helper() {} /don't/.test(value); } }} />`, ` y"} />`, ` y"} />`, ` y"} />`, diff --git a/packages/leadtype/src/search/search.test.ts b/packages/leadtype/src/search/search.test.ts index 1652dee2..ef803a11 100644 --- a/packages/leadtype/src/search/search.test.ts +++ b/packages/leadtype/src/search/search.test.ts @@ -964,6 +964,7 @@ describe("createDocsSearchIndex and searchDocs", () => { ` { if (ready) { const marker = /[{}]/; } /don't/.test(value); }} />`, ` { function helper() {} /don't/.test(value); }} />`, ` { class Helper {} /don't/.test(value); }} />`, + ` { try {} catch { function helper() {} /don't/.test(value); } }} />`, ` y"} />`, ` y"} />`, ` y"} />`,