diff --git a/.changeset/setext-heading-scanner.md b/.changeset/setext-heading-scanner.md new file mode 100644 index 00000000..2ff1d9e3 --- /dev/null +++ b/.changeset/setext-heading-scanner.md @@ -0,0 +1,5 @@ +--- +"leadtype": patch +--- + +Keep table-of-contents and search anchors aligned with rendered Setext and ATX headings containing inline HTML or MDX markup, including indentation and line-ending edge cases. diff --git a/bun.lock b/bun.lock index 32fc3b0a..2d7d3d2a 100644 --- a/bun.lock +++ b/bun.lock @@ -255,6 +255,7 @@ "mdast-util-mdx": "3.0.0", "mdast-util-mdx-jsx": "3.2.0", "mdast-util-mdxjs-esm": "2.0.1", + "micromark-util-html-tag-name": "2.0.1", "react": "^19.0.0", "rollup": "^4.40.0", "rollup-plugin-dts": "^6.2.1", diff --git a/packages/leadtype/package.json b/packages/leadtype/package.json index c9ab3c9f..44daabaf 100644 --- a/packages/leadtype/package.json +++ b/packages/leadtype/package.json @@ -233,6 +233,7 @@ "mdast-util-mdx": "3.0.0", "mdast-util-mdx-jsx": "3.2.0", "mdast-util-mdxjs-esm": "2.0.1", + "micromark-util-html-tag-name": "2.0.1", "react": "^19.0.0", "rollup": "^4.40.0", "rollup-plugin-dts": "^6.2.1", diff --git a/packages/leadtype/src/internal/docs-heading.test.ts b/packages/leadtype/src/internal/docs-heading.test.ts index 17f7b2ff..384ace96 100644 --- a/packages/leadtype/src/internal/docs-heading.test.ts +++ b/packages/leadtype/src/internal/docs-heading.test.ts @@ -1,5 +1,17 @@ +import { htmlBlockNames } from "micromark-util-html-tag-name"; import { describe, expect, it } from "vitest"; -import { createDocsHeadingSlugger, slugifyDocsHeading } from "./docs-heading"; +import { + createDocsHeadingSlugger, + docsHtmlBlockTagNames, + scanDocsMarkdown, + slugifyDocsHeading, +} from "./docs-heading"; + +describe("docs HTML block tags", () => { + it("matches the tag list used by the Markdown parser", () => { + expect(docsHtmlBlockTagNames).toEqual(htmlBlockNames); + }); +}); describe("createDocsHeadingSlugger", () => { it("suffixes duplicate slugs the way extractDocsTableOfContents does", () => { @@ -46,3 +58,52 @@ describe("createDocsHeadingSlugger", () => { expect(slugger.slug("Foo")).toBe("foo-2"); }); }); + +describe("scanDocsMarkdown", () => { + it("recognizes binding-less catch statement bodies", () => { + const [heading] = scanDocsMarkdown( + `## { try {} catch { function helper() {} /don't/.test(value); } }} /> Install` + ); + + expect(heading).toEqual({ kind: "heading", level: 2, title: "Install" }); + }); + + it("rejects malformed escaped class binding starts", () => { + const invalidBindingStarts = [ + [ + "\\x61", + " Install", + ], + [ + "\\u{}", + " Install", + ], + // This also guards the range check before String.fromCodePoint, which + // would throw for an out-of-range escape instead of rejecting it. + [ + "\\u{110000}", + " Install", + ], + [ + "\\u0030", + " Install", + ], + [ + "\\uD800", + " Install", + ], + ]; + + for (const [bindingStart, expectedTitle] of invalidBindingStarts) { + const [heading] = scanDocsMarkdown( + `## Install` + ); + + expect(heading).toEqual({ + kind: "heading", + level: 2, + title: expectedTitle, + }); + } + }); +}); diff --git a/packages/leadtype/src/internal/docs-heading.ts b/packages/leadtype/src/internal/docs-heading.ts index e5788d45..b68cb307 100644 --- a/packages/leadtype/src/internal/docs-heading.ts +++ b/packages/leadtype/src/internal/docs-heading.ts @@ -1,36 +1,855 @@ const DIACRITIC_PATTERN = /[\u0300-\u036f]/g; const FRONTMATTER_PATTERN = /^---\s*\n[\s\S]*?\n---\s*\n?/; const HEADING_PATTERN = /^(#{1,6})(?:\s+(.*))?$/; -const SETEXT_H1_PATTERN = /^=+\s*$/; -const SETEXT_H2_PATTERN = /^-+\s*$/; +const SETEXT_H1_PATTERN = /^ {0,3}=+\s*$/; +const SETEXT_H2_PATTERN = /^ {0,3}-+\s*$/; const FENCE_PATTERN = /^(`{3,}|~{3,})/; const INDENTED_CODE_PATTERN = /^(?: {4}|\t)/; const BLOCKQUOTE_PATTERN = /^ {0,3}>/; const LIST_ITEM_PATTERN = /^ {0,3}(?:[*+-]|\d{1,9}[.)])(?:[ \t]+|$)/; -const HTML_OR_MDX_BLOCK_PATTERN = /^ {0,3}[<{]/; +const HTML_BLOCK_START_PATTERN = + /^ {0,3}(?:<(?:pre|script|style|textarea)(?:[ \t\r>]|$)|" | "?>" | "]]>"; tracksQuotes: false } + | { closingSequence: ">"; tracksQuotes: boolean }; + +type JavaScriptBraceContext = { + allowsRegexAfterClose: boolean; + classBody?: { + bracketDepth: number; + parenthesisDepth: number; + }; + statementBody: boolean; +}; + +type JavaScriptParenthesisContext = + | { kind: "control" } + | { declaration: boolean; kind: "function" } + | { kind: "other" }; + +type JavaScriptCaseColonContext = { + braceDepth: number; + bracketDepth: number; + conditionalDepth: number; + parenthesisDepth: number; +}; + +function getHtmlConstruct(input: string): HtmlConstruct | null { + if (input.startsWith("", tracksQuotes: false }; + } + if (input.startsWith("", tracksQuotes: false }; + } + if (input.startsWith("", tracksQuotes: false }; + } + if (HTML_DECLARATION_START_PATTERN.test(input)) { + return { closingSequence: ">", tracksQuotes: false }; + } + if ( + input.startsWith("<>") || + input.startsWith("") || + HTML_OR_MDX_TAG_START_PATTERN.test(input) + ) { + return { closingSequence: ">", tracksQuotes: true }; + } + return null; +} + +function findHtmlConstructEnd( + input: string, + start: number, + construct: HtmlConstruct +): number { + if (!construct.tracksQuotes) { + const closingIndex = input.indexOf(construct.closingSequence, start); + return closingIndex < 0 + ? -1 + : closingIndex + construct.closingSequence.length; + } + + let braceDepth = 0; + let escaped = false; + let javascriptComment: "block" | "line" | null = null; + let javascriptRegex = false; + let javascriptRegexAllowed = true; + let javascriptStatementStart = false; + let bracketDepth = 0; + const caseColonContexts: JavaScriptCaseColonContext[] = []; + let nextIdentifierIsProperty = false; + let nextBraceContext: JavaScriptBraceContext | null = null; + let pendingAsyncDeclaration: boolean | null = null; + const pendingClasses: Array<{ + allowsRegexAfterClose: boolean; + braceDepth: number; + parenthesisDepth: number; + }> = []; + let pendingControlParenthesis: "for" | "other" | null = null; + let pendingLabelColon = false; + let awaitingFunctionParameters: boolean | null = null; + const braceContexts: JavaScriptBraceContext[] = []; + const parenthesisContexts: JavaScriptParenthesisContext[] = []; + let quote: '"' | "'" | null = null; + let regexCharacterClass = false; + const templateInterpolationDepths: Array = []; + for (let index = start; index < input.length; index += 1) { + const previousCharacter = input[index - 1]; + const character = input[index]; + const nextCharacter = input[index + 1]; + if (javascriptComment === "line") { + if (character === "\n" || character === "\r") { + javascriptComment = null; + } + continue; + } + if (javascriptComment === "block") { + if (character === "*" && nextCharacter === "/") { + javascriptComment = null; + index += 1; + } + continue; + } + if (javascriptRegex) { + if (escaped) { + escaped = false; + continue; + } + if (character === "\\") { + escaped = true; + continue; + } + if (character === "[") { + regexCharacterClass = true; + continue; + } + if (character === "]") { + regexCharacterClass = false; + continue; + } + if (character === "/" && !regexCharacterClass) { + javascriptRegex = false; + javascriptRegexAllowed = false; + javascriptStatementStart = false; + nextIdentifierIsProperty = false; + } + continue; + } + const templateInterpolationDepth = templateInterpolationDepths.at(-1); + if ( + templateInterpolationDepths.length > 0 && + templateInterpolationDepth === null + ) { + if (escaped) { + escaped = false; + continue; + } + if (character === "\\") { + escaped = true; + continue; + } + if (character === "`") { + templateInterpolationDepths.pop(); + javascriptRegexAllowed = false; + javascriptStatementStart = false; + nextIdentifierIsProperty = false; + continue; + } + if (character === "$" && nextCharacter === "{") { + braceDepth += 1; + braceContexts.push({ + allowsRegexAfterClose: false, + statementBody: false, + }); + templateInterpolationDepths[templateInterpolationDepths.length - 1] = + braceDepth; + javascriptRegexAllowed = true; + javascriptStatementStart = false; + nextIdentifierIsProperty = false; + nextBraceContext = null; + index += 1; + } + continue; + } + if (quote !== null) { + if (escaped) { + escaped = false; + continue; + } + if (braceDepth > 0 && character === "\\") { + escaped = true; + continue; + } + if (character === quote) { + quote = null; + if (braceDepth > 0) { + javascriptRegexAllowed = false; + javascriptStatementStart = false; + nextIdentifierIsProperty = false; + } + } + continue; + } + if (braceDepth > 0 && character === "/") { + if (nextCharacter === "/") { + javascriptComment = "line"; + index += 1; + continue; + } + if (nextCharacter === "*") { + javascriptComment = "block"; + index += 1; + continue; + } + if (javascriptRegexAllowed) { + javascriptRegex = true; + regexCharacterClass = false; + javascriptStatementStart = false; + pendingLabelColon = false; + nextIdentifierIsProperty = false; + nextBraceContext = null; + continue; + } + javascriptRegexAllowed = true; + javascriptStatementStart = false; + pendingLabelColon = false; + nextIdentifierIsProperty = false; + nextBraceContext = null; + continue; + } + if (character === '"' || character === "'") { + quote = character; + javascriptStatementStart = false; + pendingLabelColon = false; + nextIdentifierIsProperty = false; + nextBraceContext = null; + continue; + } + if (braceDepth > 0 && character === "`") { + templateInterpolationDepths.push(null); + javascriptStatementStart = false; + pendingLabelColon = false; + nextIdentifierIsProperty = false; + nextBraceContext = null; + continue; + } + const identifierStart = + braceDepth > 0 + ? getJavaScriptIdentifierCharacterAt( + input, + index, + JAVASCRIPT_IDENTIFIER_START_PATTERN + ) + : null; + if (identifierStart !== null) { + let identifierEnd = identifierStart.end; + while (identifierEnd < input.length) { + const identifierPart = getJavaScriptIdentifierCharacterAt( + input, + identifierEnd, + JAVASCRIPT_IDENTIFIER_PART_PATTERN + ); + if (identifierPart === null) { + break; + } + identifierEnd = identifierPart.end; + } + const identifier = input.slice(index, identifierEnd); + const wasStatementStart: boolean = javascriptStatementStart; + const isKeywordPosition = !nextIdentifierIsProperty; + const activeClassBody = braceContexts.at(-1)?.classBody; + const startsStaticBlock = + isKeywordPosition && + identifier === "static" && + activeClassBody?.bracketDepth === bracketDepth && + activeClassBody.parenthesisDepth === parenthesisContexts.length; + const startsCaseClause = + isKeywordPosition && + wasStatementStart && + (identifier === "case" || identifier === "default"); + const isControlKeyword = + isKeywordPosition && JAVASCRIPT_CONTROL_KEYWORDS.has(identifier); + const preservesForAwait = + identifier === "await" && pendingControlParenthesis === "for"; + if (isControlKeyword) { + pendingControlParenthesis = identifier === "for" ? "for" : "other"; + } else if (!preservesForAwait) { + pendingControlParenthesis = null; + } + + if (isKeywordPosition && identifier === "async") { + pendingAsyncDeclaration = wasStatementStart; + } else if (isKeywordPosition && identifier === "function") { + awaitingFunctionParameters = + wasStatementStart || Boolean(pendingAsyncDeclaration); + pendingAsyncDeclaration = null; + } else { + pendingAsyncDeclaration = null; + } + + const nextClassTokenStart = + isKeywordPosition && identifier === "class" + ? getNextJavaScriptTokenStart(input, identifierEnd) + : -1; + const startsClass = + isKeywordPosition && + identifier === "class" && + (input[nextClassTokenStart] === "{" || + getJavaScriptIdentifierCharacterAt( + input, + nextClassTokenStart, + JAVASCRIPT_IDENTIFIER_START_PATTERN + ) !== null); + if (startsClass) { + pendingClasses.push({ + allowsRegexAfterClose: wasStatementStart, + braceDepth, + parenthesisDepth: parenthesisContexts.length, + }); + } + + if (startsCaseClause) { + caseColonContexts.push({ + braceDepth, + bracketDepth, + conditionalDepth: 0, + parenthesisDepth: parenthesisContexts.length, + }); + } + + pendingLabelColon = + wasStatementStart && isKeywordPosition && !startsCaseClause; + + if ( + startsStaticBlock || + (isKeywordPosition && + (identifier === "catch" || + identifier === "do" || + identifier === "else" || + identifier === "finally" || + identifier === "try")) + ) { + nextBraceContext = { + allowsRegexAfterClose: true, + statementBody: true, + }; + javascriptStatementStart = true; + } else { + nextBraceContext = null; + javascriptStatementStart = false; + } + javascriptRegexAllowed = + isKeywordPosition && JAVASCRIPT_REGEX_PREFIX_KEYWORDS.has(identifier); + nextIdentifierIsProperty = false; + index = identifierEnd - 1; + continue; + } + if (braceDepth > 0 && character !== undefined && /[0-9]/.test(character)) { + let numberEnd = index + 1; + while ( + numberEnd < input.length && + /[A-Za-z0-9_.]/.test(input[numberEnd] ?? "") + ) { + numberEnd += 1; + } + javascriptRegexAllowed = false; + javascriptStatementStart = false; + pendingLabelColon = false; + nextIdentifierIsProperty = false; + nextBraceContext = null; + pendingControlParenthesis = null; + index = numberEnd - 1; + continue; + } + if (braceDepth > 0 && character === "(") { + let parenthesisContext: JavaScriptParenthesisContext = { kind: "other" }; + if (awaitingFunctionParameters !== null) { + parenthesisContext = { + declaration: awaitingFunctionParameters, + kind: "function", + }; + } else if (pendingControlParenthesis) { + parenthesisContext = { kind: "control" }; + } + parenthesisContexts.push(parenthesisContext); + awaitingFunctionParameters = null; + pendingControlParenthesis = null; + pendingLabelColon = false; + javascriptRegexAllowed = true; + javascriptStatementStart = false; + nextIdentifierIsProperty = false; + nextBraceContext = null; + continue; + } + if (braceDepth > 0 && character === ")") { + const parenthesisContext = parenthesisContexts.pop() ?? { + kind: "other", + }; + if (parenthesisContext.kind === "control") { + nextBraceContext = { + allowsRegexAfterClose: true, + statementBody: true, + }; + javascriptRegexAllowed = true; + javascriptStatementStart = true; + } else if (parenthesisContext.kind === "function") { + nextBraceContext = { + allowsRegexAfterClose: parenthesisContext.declaration, + statementBody: true, + }; + javascriptRegexAllowed = false; + javascriptStatementStart = false; + } else { + nextBraceContext = { + allowsRegexAfterClose: false, + statementBody: true, + }; + javascriptRegexAllowed = false; + javascriptStatementStart = false; + } + pendingControlParenthesis = null; + pendingLabelColon = false; + nextIdentifierIsProperty = false; + continue; + } + if (character === "{") { + let braceContext: JavaScriptBraceContext | null = nextBraceContext; + const pendingClass = pendingClasses.at(-1); + const startsClassBody = + pendingClass !== undefined && + pendingClass.braceDepth === braceDepth && + pendingClass.parenthesisDepth === parenthesisContexts.length; + if (startsClassBody && pendingClass) { + braceContext = { + allowsRegexAfterClose: pendingClass.allowsRegexAfterClose, + classBody: { + bracketDepth, + parenthesisDepth: parenthesisContexts.length, + }, + statementBody: false, + }; + pendingClasses.pop(); + } + braceContext ??= javascriptStatementStart + ? { allowsRegexAfterClose: true, statementBody: true } + : { allowsRegexAfterClose: false, statementBody: false }; + braceDepth += 1; + braceContexts.push(braceContext); + javascriptRegexAllowed = true; + javascriptStatementStart = braceContext.statementBody; + nextIdentifierIsProperty = false; + nextBraceContext = null; + pendingControlParenthesis = null; + pendingLabelColon = false; + continue; + } + if (character === "}" && braceDepth > 0) { + if (templateInterpolationDepth === braceDepth) { + braceDepth -= 1; + braceContexts.pop(); + templateInterpolationDepths[templateInterpolationDepths.length - 1] = + null; + javascriptStatementStart = false; + nextIdentifierIsProperty = false; + continue; + } + braceDepth -= 1; + const braceContext = braceContexts.pop(); + while ( + caseColonContexts.at(-1)?.braceDepth !== undefined && + (caseColonContexts.at(-1)?.braceDepth ?? 0) > braceDepth + ) { + caseColonContexts.pop(); + } + javascriptRegexAllowed = braceContext?.allowsRegexAfterClose ?? false; + javascriptStatementStart = braceContext?.allowsRegexAfterClose ?? false; + nextIdentifierIsProperty = false; + nextBraceContext = null; + pendingControlParenthesis = null; + pendingLabelColon = false; + continue; + } + if (braceDepth > 0 && character === "[") { + bracketDepth += 1; + javascriptRegexAllowed = true; + javascriptStatementStart = false; + pendingLabelColon = false; + nextIdentifierIsProperty = false; + nextBraceContext = null; + pendingControlParenthesis = null; + continue; + } + if (braceDepth > 0 && character === "]") { + bracketDepth = Math.max(0, bracketDepth - 1); + javascriptRegexAllowed = false; + javascriptStatementStart = false; + pendingLabelColon = false; + nextIdentifierIsProperty = false; + nextBraceContext = null; + pendingControlParenthesis = null; + continue; + } + if (braceDepth > 0 && character === ".") { + const isSpreadOperator = input.startsWith("...", index); + javascriptRegexAllowed = isSpreadOperator; + javascriptStatementStart = false; + pendingLabelColon = false; + nextIdentifierIsProperty = !isSpreadOperator; + nextBraceContext = null; + pendingControlParenthesis = null; + if (isSpreadOperator) { + index += 2; + } + continue; + } + if (braceDepth > 0 && character === "=" && nextCharacter === ">") { + javascriptRegexAllowed = true; + javascriptStatementStart = false; + pendingLabelColon = false; + nextIdentifierIsProperty = false; + nextBraceContext = { + allowsRegexAfterClose: false, + statementBody: true, + }; + pendingControlParenthesis = null; + index += 1; + continue; + } + if ( + braceDepth > 0 && + (character === "+" || character === "-") && + nextCharacter === character + ) { + // Prefix updates still expect an operand and postfix updates yield a + // value, so a following slash keeps the meaning it already had. + javascriptStatementStart = false; + pendingLabelColon = false; + nextIdentifierIsProperty = false; + nextBraceContext = null; + pendingControlParenthesis = null; + index += 1; + continue; + } + if ( + braceDepth > 0 && + character !== undefined && + ",:;?=.!&|+-*%^~<>".includes(character) + ) { + const activeCaseColonContext = caseColonContexts.at(-1); + const atCaseColonDepth = + activeCaseColonContext !== undefined && + activeCaseColonContext.braceDepth === braceDepth && + activeCaseColonContext.bracketDepth === bracketDepth && + activeCaseColonContext.parenthesisDepth === parenthesisContexts.length; + if ( + character === "?" && + atCaseColonDepth && + activeCaseColonContext !== undefined && + previousCharacter !== "?" && + nextCharacter !== "?" && + nextCharacter !== "." + ) { + activeCaseColonContext.conditionalDepth += 1; + } + + let startsStatement = character === ";"; + if (character === ":" && pendingLabelColon) { + startsStatement = true; + } else if ( + character === ":" && + atCaseColonDepth && + activeCaseColonContext !== undefined + ) { + if (activeCaseColonContext.conditionalDepth > 0) { + activeCaseColonContext.conditionalDepth -= 1; + } else { + caseColonContexts.pop(); + startsStatement = true; + } + } + + javascriptRegexAllowed = true; + javascriptStatementStart = startsStatement; + pendingLabelColon = false; + nextIdentifierIsProperty = false; + nextBraceContext = null; + pendingControlParenthesis = null; + continue; + } + if (character === ">" && braceDepth === 0) { + return index + 1; + } + } + return -1; +} + +function isStandaloneHtmlOrMdxTag(line: string): boolean { + const content = line.trimEnd().replace(/^ {0,3}/, ""); + if (!HTML_OR_MDX_TAG_START_PATTERN.test(content)) { + return false; + } + const construct: HtmlConstruct = { + closingSequence: ">", + tracksQuotes: true, + }; + return findHtmlConstructEnd(content, 0, construct) === content.length; +} + +function stripHtmlTags(input: string): string { + const output: string[] = []; + let cursor = 0; + + while (cursor < input.length) { + const tagStart = input.indexOf("<", cursor); + if (tagStart < 0) { + output.push(input.slice(cursor)); + break; + } + output.push(input.slice(cursor, tagStart)); + const construct = getHtmlConstruct(input.slice(tagStart)); + if (!construct) { + output.push("<"); + cursor = tagStart + 1; + continue; + } + const constructEnd = findHtmlConstructEnd(input, tagStart, construct); + if (constructEnd < 0) { + output.push(input.slice(tagStart)); + break; + } + output.push(" "); + cursor = constructEnd; + } + + return output.join(""); } function cleanHeadingText(input: string): string { - return input - .replace(HEADING_CLOSING_SEQUENCE_PATTERN, "") + return stripHtmlTags(input.replace(HEADING_CLOSING_SEQUENCE_PATTERN, "")) .replace(MARKDOWN_LINK_PATTERN, "$1") - .replace(HTML_TAG_PATTERN, " ") .replace(HEADING_INLINE_PATTERN, " ") .replace(WHITESPACE_PATTERN, " ") .trim(); } +function isHtmlOrMdxBlock(line: string): boolean { + return ( + HTML_BLOCK_START_PATTERN.test(line) || + HTML_BLOCK_TAG_PATTERN.test(line) || + isStandaloneHtmlOrMdxTag(line) || + MDX_BLOCK_START_PATTERN.test(line) + ); +} + function isSetextHeadingText(line: string): boolean { if (line.trim().length === 0) { return false; @@ -40,7 +859,7 @@ function isSetextHeadingText(line: string): boolean { INDENTED_CODE_PATTERN.test(line) || BLOCKQUOTE_PATTERN.test(line) || LIST_ITEM_PATTERN.test(line) || - HTML_OR_MDX_BLOCK_PATTERN.test(line) || + isHtmlOrMdxBlock(line) || LINK_DEFINITION_PATTERN.test(line) || THEMATIC_BREAK_PATTERN.test(line) ); @@ -111,8 +930,8 @@ export function scanDocsMarkdown(content: string): DocsMarkdownToken[] { continue; } - const isSetextH1 = SETEXT_H1_PATTERN.test(trimmedLine); - const isSetextH2 = SETEXT_H2_PATTERN.test(trimmedLine); + const isSetextH1 = SETEXT_H1_PATTERN.test(line); + const isSetextH2 = SETEXT_H2_PATTERN.test(line); if (pendingSetextTitle !== null && (isSetextH1 || isSetextH2)) { tokens.push({ kind: "heading", diff --git a/packages/leadtype/src/llm/llm.test.ts b/packages/leadtype/src/llm/llm.test.ts index 2c6455e6..ba39d23f 100644 --- a/packages/leadtype/src/llm/llm.test.ts +++ b/packages/leadtype/src/llm/llm.test.ts @@ -4303,6 +4303,418 @@ describe("extractDocsTableOfContents", () => { ]); }); + it("does not treat a four-space-indented underline as Setext", () => { + const toc = extractDocsTableOfContents( + ["Install", " ---", "## Install"].join("\n"), + { + urlPath: "/docs/example", + absoluteUrl: "https://leadtype.dev/docs/example", + } + ); + + expect(toc.map((item) => ({ id: item.id, title: item.title }))).toEqual([ + { id: "install", title: "Install" }, + ]); + }); + + it("allows a Setext underline to use three spaces and CRLF", () => { + const toc = extractDocsTableOfContents( + ["Install", " ---", "## Install"].join("\r\n"), + { + urlPath: "/docs/example", + absoluteUrl: "https://leadtype.dev/docs/example", + } + ); + + expect(toc.map((item) => ({ id: item.id, title: item.title }))).toEqual([ + { id: "install", title: "Install" }, + { id: "install-1", title: "Install" }, + ]); + }); + + it("allows inline HTML at the start of Setext heading text", () => { + const fixtures = [ + { + heading: 'Install', + title: "Install", + }, + { heading: "Install ", title: "Install" }, + { heading: "
Note
", title: "Note" }, + ]; + + for (const { heading, title } of fixtures) { + const toc = extractDocsTableOfContents( + [heading, "---", `## ${title}`].join("\n"), + { + urlPath: "/docs/example", + absoluteUrl: "https://leadtype.dev/docs/example", + } + ); + + const id = title.toLowerCase(); + expect(toc.map((item) => ({ id: item.id, title: item.title }))).toEqual([ + { id, title }, + { id: `${id}-1`, title }, + ]); + } + }); + + it("keeps a trailing marker wrapped in inline HTML in the heading title", () => { + const toc = extractDocsTableOfContents("## Anchors and #", { + urlPath: "/docs/example", + absoluteUrl: "https://leadtype.dev/docs/example", + }); + + expect(toc.map((item) => ({ id: item.id, title: item.title }))).toEqual([ + { id: "anchors-and", title: "Anchors and #" }, + ]); + }); + + it("strips MDX JSX tags from ATX heading text", () => { + for (const tag of [ + "", + "", + "<_Icon />", + "<$Icon />", + "", + "", + ]) { + const toc = extractDocsTableOfContents( + [`## ${tag} Install`, "## Install"].join("\n"), + { + urlPath: "/docs/example", + absoluteUrl: "https://leadtype.dev/docs/example", + } + ); + + expect(toc.map((item) => ({ id: item.id, title: item.title }))).toEqual([ + { id: "install", title: "Install" }, + { id: "install-1", title: "Install" }, + ]); + } + }); + + it("preserves malformed MDX JSX names in heading text", () => { + for (const { id, tag } of [ + { id: "9icon-install", tag: "<9Icon />" }, + { id: "my-icon-install", tag: "" }, + { id: "my-icon-part-install", tag: "" }, + ]) { + const toc = extractDocsTableOfContents(`## ${tag} Install`, { + urlPath: "/docs/example", + absoluteUrl: "https://leadtype.dev/docs/example", + }); + + expect(toc.map((item) => ({ id: item.id, title: item.title }))).toEqual([ + { id, title: `${tag} Install` }, + ]); + } + }); + + it("handles balanced MDX expression attributes in flow and heading text", () => { + for (const tag of [ + " 0}>", + " 0 }}>", + ]) { + const toc = extractDocsTableOfContents( + [tag, "---", "", "## 0"].join("\n"), + { + urlPath: "/docs/example", + absoluteUrl: "https://leadtype.dev/docs/example", + } + ); + + expect(toc.map((item) => item.id)).toEqual(["0"]); + } + + const toc = extractDocsTableOfContents( + ["## 0}>Install", "## Install"].join("\n"), + { + urlPath: "/docs/example", + absoluteUrl: "https://leadtype.dev/docs/example", + } + ); + expect(toc.map((item) => ({ id: item.id, title: item.title }))).toEqual([ + { id: "install", title: "Install" }, + { id: "install-1", title: "Install" }, + ]); + }); + + it("parses JavaScript literals in MDX expression attributes", () => { + for (const lineEnding of ["\n", "\r\n"]) { + const toc = extractDocsTableOfContents( + [ + " */ 1 > 0}>", + "---", + "", + " 0}>", + "---", + "", + "## Install", + "## { if (ready) {} /don't/.test(value); }} /> Install", + "## { if (ready) /don't/.test(value); }} /> Install", + "## { while (ready) /don't/.test(value); }} /> Install", + `## Install`, + "## 0} /> Install", + "## 0} /> Install", + "## Install", + ].join(lineEnding), + { + urlPath: "/docs/example", + absoluteUrl: "https://leadtype.dev/docs/example", + } + ); + + expect(toc.map((item) => ({ id: item.id, title: item.title }))).toEqual([ + { id: "install", title: "Install" }, + { id: "install-1", title: "Install" }, + { id: "install-2", title: "Install" }, + { id: "install-3", title: "Install" }, + { id: "install-4", title: "Install" }, + { id: "install-5", title: "Install" }, + { id: "install-6", title: "Install" }, + { id: "install-7", title: "Install" }, + ]); + } + }); + + it("distinguishes regex statements from division around JavaScript bodies", () => { + const toc = extractDocsTableOfContents( + [ + "## { if (ready) foo(); else /don't/.test(value); }} /> Install", + "## { do /don't/.test(value); while (ready); }} /> Install", + "## y'} /> Install", + "## Install", + ].join("\n"), + { + urlPath: "/docs/example", + absoluteUrl: "https://leadtype.dev/docs/example", + } + ); + + expect(toc.map((item) => ({ id: item.id, title: item.title }))).toEqual([ + { id: "install", title: "Install" }, + { id: "install-1", title: "Install" }, + { id: "install-2", title: "Install" }, + { id: "install-3", title: "Install" }, + ]); + }); + + it("tracks JavaScript block context without rescanning literal braces", () => { + const tags = [ + ` { if (ready) { const marker = "}"; } /don't/.test(value); }} />`, + ` { if (ready) { /* } { */ } /don't/.test(value); }} />`, + " { if (ready) { const marker = `}`; } /don't/.test(value); }} />", + ` { if (ready) { const marker = /[{}]/; } /don't/.test(value); }} />`, + ` { function helper() {} /don't/.test(value); }} />`, + ` { class Helper {} /don't/.test(value); }} />`, + ` { try {} catch { function helper() {} /don't/.test(value); } }} />`, + ` y"} />`, + ` y"} />`, + ` y"} />`, + ` { for await (const item of values) /don't/.test(item); }} />`, + ]; + + for (const tag of tags) { + const toc = extractDocsTableOfContents( + [`## ${tag} Install`, "## Install"].join("\n"), + { + urlPath: "/docs/example", + absoluteUrl: "https://leadtype.dev/docs/example", + } + ); + + expect(toc.map((item) => ({ id: item.id, title: item.title }))).toEqual([ + { id: "install", title: "Install" }, + { id: "install-1", title: "Install" }, + ]); + } + }); + + it("stacks declaration contexts and recognizes statement colons", () => { + const tags = [ + ` { function helper(callback = function nested() {}) {} /don't/.test(value); }} />`, + ` { class Outer extends (class Inner {}) {} /don't/.test(value); }} />`, + ` { label: {} /don't/.test(value); }} />`, + ` { switch (value) { case 1: {} /don't/.test(value); } }} />`, + ` { switch (value) { case ready ? one : two: {} /don't/.test(value); } }} />`, + ` { switch (value) { case (() => { switch (inner) { case 1: return 2; } return 3; })(): {} /don't/.test(value); } }} />`, + ` y"} />`, + ` y"} />`, + ` y" }} />`, + ]; + + for (const tag of tags) { + const toc = extractDocsTableOfContents( + [`## ${tag} Install`, "## Install"].join("\n"), + { + urlPath: "/docs/example", + absoluteUrl: "https://leadtype.dev/docs/example", + } + ); + + expect(toc.map((item) => ({ id: item.id, title: item.title }))).toEqual([ + { id: "install", title: "Install" }, + { id: "install-1", title: "Install" }, + ]); + } + }); + + it("distinguishes postfix updates from prefix and binary operators", () => { + const tags = [ + "", + "", + ``, + "", + "", + ``, + ``, + ]; + + for (const tag of tags) { + const toc = extractDocsTableOfContents( + [`## ${tag} Install`, "## Install"].join("\n"), + { + urlPath: "/docs/example", + absoluteUrl: "https://leadtype.dev/docs/example", + } + ); + + expect(toc.map((item) => ({ id: item.id, title: item.title }))).toEqual([ + { id: "install", title: "Install" }, + { id: "install-1", title: "Install" }, + ]); + } + }); + + it("recognizes Unicode identifiers and spread operands", () => { + const tags = [ + ' y"} />', + ' y"} />', + ' y"} />', + ' y"} />', + ' y"} />', + ' y"} />', + ' y"} />', + ``, + ' y" }} />', + ]; + + for (const tag of tags) { + const toc = extractDocsTableOfContents( + [`## ${tag} Install`, "## Install"].join("\n"), + { + urlPath: "/docs/example", + absoluteUrl: "https://leadtype.dev/docs/example", + } + ); + + expect(toc.map((item) => ({ id: item.id, title: item.title }))).toEqual([ + { id: "install", title: "Install" }, + { id: "install-1", title: "Install" }, + ]); + } + }); + + it("recognizes regex operands in class heritage", () => { + const tags = [ + ``, + ``, + ' y") {}} />', + ' y") {}} />', + ' y"} />', + ' y"} />', + ' y"} />', + ` { class Named {} /don't/.test(value); }} />`, + ]; + + for (const tag of tags) { + const toc = extractDocsTableOfContents( + [`## ${tag} Install`, "## Install"].join("\n"), + { + urlPath: "/docs/example", + absoluteUrl: "https://leadtype.dev/docs/example", + } + ); + + expect(toc.map((item) => ({ id: item.id, title: item.title }))).toEqual([ + { id: "install", title: "Install" }, + { id: "install-1", title: "Install" }, + ]); + } + }); + + it("recognizes statement bodies in class static blocks", () => { + const tags = [ + ``, + ``, + ' y"} />', + ' y"} />', + ' y"} />', + ]; + + for (const tag of tags) { + const toc = extractDocsTableOfContents( + [`## ${tag} Install`, "## Install"].join("\n"), + { + urlPath: "/docs/example", + absoluteUrl: "https://leadtype.dev/docs/example", + } + ); + + expect(toc.map((item) => ({ id: item.id, title: item.title }))).toEqual([ + { id: "install", title: "Install" }, + { id: "install-1", title: "Install" }, + ]); + } + }); + + it("does not classify member names as class keywords", () => { + const tags = [ + ``, + ` { function nested() {} /don't/.test(value); } }} />`, + ``, + ``, + ]; + + for (const tag of tags) { + const toc = extractDocsTableOfContents( + [`## ${tag} Install`, "## Install"].join("\n"), + { + urlPath: "/docs/example", + absoluteUrl: "https://leadtype.dev/docs/example", + } + ); + + expect(toc.map((item) => ({ id: item.id, title: item.title }))).toEqual([ + { id: "install", title: "Install" }, + { id: "install-1", title: "Install" }, + ]); + } + }); + + it("recognizes escaped class binding identifiers", () => { + const tags = [ + ``, + ``, + ]; + + for (const tag of tags) { + const toc = extractDocsTableOfContents( + [`## ${tag} Install`, "## Install"].join("\n"), + { + urlPath: "/docs/example", + absoluteUrl: "https://leadtype.dev/docs/example", + } + ); + + expect(toc.map((item) => ({ id: item.id, title: item.title }))).toEqual([ + { id: "install", title: "Install" }, + { id: "install-1", title: "Install" }, + ]); + } + }); + it("does not treat a thematic break after a blank line as a Setext heading", () => { const toc = extractDocsTableOfContents( ["A paragraph.", "", "---", "## After"].join("\n"), @@ -4321,6 +4733,7 @@ describe("extractDocsTableOfContents", () => { "- Note", " Note", "", + "", "Note", "{note}", "[note]: /docs/note", @@ -4341,6 +4754,124 @@ describe("extractDocsTableOfContents", () => { } }); + it("recognizes lowercase MDX member-expression flow tags", () => { + const toc = extractDocsTableOfContents( + [ + "", + "---", + "", + "## Components Note", + ].join("\n"), + { + urlPath: "/docs/example", + absoluteUrl: "https://leadtype.dev/docs/example", + } + ); + + expect(toc.map((item) => item.id)).toEqual(["components-note"]); + }); + + it("recognizes MDX fragments in flow and ATX heading text", () => { + const toc = extractDocsTableOfContents( + ["<>", "Install", "", "---", "## <>Install", "## Install"].join( + "\n" + ), + { + urlPath: "/docs/example", + absoluteUrl: "https://leadtype.dev/docs/example", + } + ); + + expect(toc.map((item) => ({ id: item.id, title: item.title }))).toEqual([ + { id: "install", title: "Install" }, + { id: "install-1", title: "Install" }, + ]); + }); + + it("keeps a one-line MDX fragment as Setext heading text", () => { + const toc = extractDocsTableOfContents( + ["<>Install", "---", "## Install"].join("\n"), + { + urlPath: "/docs/example", + absoluteUrl: "https://leadtype.dev/docs/example", + } + ); + + expect(toc.map((item) => ({ id: item.id, title: item.title }))).toEqual([ + { id: "install", title: "Install" }, + { id: "install-1", title: "Install" }, + ]); + }); + + it("recognizes bare block starters for LF and CRLF", () => { + for (const blockStart of [" item.id)).toEqual(["after"]); + } + } + }); + + it("recognizes standalone HTML tags with quoted delimiters for LF and CRLF", () => { + for (const lineEnding of ["\n", "\r\n"]) { + const toc = extractDocsTableOfContents( + ['', "---", "", "## !!!", "## !!!"].join( + lineEnding + ), + { + urlPath: "/docs/example", + absoluteUrl: "https://leadtype.dev/docs/example", + } + ); + + expect(toc.map((item) => item.id)).toEqual(["", "-1"]); + } + }); + + it("preserves comparisons and strips declarations from heading text", () => { + const toc = extractDocsTableOfContents( + [ + "## 1 < 2 > 0", + "## Install ", + "## Install ", + "## Install", + ].join("\n"), + { + urlPath: "/docs/example", + absoluteUrl: "https://leadtype.dev/docs/example", + } + ); + + expect(toc.map((item) => ({ id: item.id, title: item.title }))).toEqual([ + { id: "1-2-0", title: "1 < 2 > 0" }, + { id: "install", title: "Install" }, + { id: "install-1", title: "Install" }, + { id: "install-2", title: "Install" }, + ]); + }); + + it("strips CDATA and preserves unterminated HTML constructs", () => { + const toc = extractDocsTableOfContents( + ["## Install ", '## Install ({ id: item.id, title: item.title }))).toEqual([ + { id: "install", title: "Install" }, + { id: "install-em-title-x", title: 'Install { const toc = extractDocsTableOfContents( ["# Page", "## Section", "### Child", "#### Detail"].join("\n"), diff --git a/packages/leadtype/src/search/search.test.ts b/packages/leadtype/src/search/search.test.ts index 181c0ecf..ef803a11 100644 --- a/packages/leadtype/src/search/search.test.ts +++ b/packages/leadtype/src/search/search.test.ts @@ -520,6 +520,59 @@ describe("createDocsSearchIndex and searchDocs", () => { ); }); + it("does not reserve a Setext anchor for an indented underline", () => { + const content = [ + "Install", + " ---", + "## Install", + "The ATX section covers widgets.", + ].join("\n"); + const index = createDocsSearchIndex( + [ + { + id: "install", + title: "Install", + urlPath: "/docs/install", + absoluteUrl: "https://leadtype.dev/docs/install", + relativePath: "install.mdx", + content, + }, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + + expect(searchDocs(index, "widgets")[0]?.urlWithHash).toBe( + "/docs/install#install" + ); + }); + + it("reserves inline-marked Setext headings before later ATX anchors", () => { + const content = [ + 'Install', + "---", + "The Setext section covers widgets.", + "## Install", + "The ATX section covers sprockets.", + ].join("\n"); + const index = createDocsSearchIndex( + [ + { + id: "install", + title: "Install", + urlPath: "/docs/install", + absoluteUrl: "https://leadtype.dev/docs/install", + relativePath: "install.mdx", + content, + }, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + + expect(searchDocs(index, "sprockets")[0]?.urlWithHash).toBe( + "/docs/install#install-1" + ); + }); + it("does not reserve headings inside tilde code fences", () => { const content = [ "# Reference", @@ -581,6 +634,27 @@ describe("createDocsSearchIndex and searchDocs", () => { "## Example", "The real example covers widgets.", ].join("\n"), + [ + 'Install', + " ---", + "Inline markup section covers widgets.", + "## Install", + "The ATX section covers sprockets.", + ].join("\r\n"), + [ + "
Note
", + "---", + "Inline HTML section covers widgets.", + "## Note", + "The ATX section covers sprockets.", + ].join("\n"), + [ + "Install ", + "---", + "Inline comment section covers widgets.", + "## Install", + "The ATX section covers sprockets.", + ].join("\n"), ]; for (const content of fixtures) { @@ -616,6 +690,861 @@ describe("createDocsSearchIndex and searchDocs", () => { } }); + it("keeps inline HTML text in search heading labels", () => { + const index = createDocsSearchIndex( + [ + { + id: "anchors", + title: "Anchors", + urlPath: "/docs/anchors", + absoluteUrl: "https://leadtype.dev/docs/anchors", + relativePath: "anchors.mdx", + content: [ + "## Anchors and #", + "Literal marker covers widgets.", + ].join("\n"), + }, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + + const result = searchDocs(index, "widgets")[0]; + expect(result?.headingPath.at(-1)).toBe("Anchors and #"); + expect(result?.urlWithHash).toBe("/docs/anchors#anchors-and"); + }); + + it("keeps MDX member-expression heading labels aligned with anchors", () => { + const content = [ + "## Install", + "Icon section covers widgets.", + "## Install", + "Plain section covers sprockets.", + "## Install", + "Namespaced section covers calipers.", + "## <_Icon /> Setup", + "Underscore section covers ratchets.", + "## <$Icon /> Configure", + "Dollar section covers spanners.", + "", + "---", + "", + "## Components Note", + "Member flow section covers gadgets.", + " 0}>", + "---", + "", + "## 0", + "Expression section covers levels.", + ].join("\n"); + const index = createDocsSearchIndex( + [ + { + id: "fixture", + title: "Fixture", + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + relativePath: "fixture.mdx", + content, + }, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + const tocIds = flattenTocIds( + extractDocsTableOfContents(content, { + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + }) + ); + const searchAnchors = index.chunks.map( + (chunk) => chunk[CHUNK_ANCHOR_INDEX] + ); + + expect(searchAnchors).toEqual(tocIds); + expect(searchDocs(index, "widgets")[0]?.urlWithHash).toBe( + "/docs/fixture#install" + ); + expect(searchDocs(index, "sprockets")[0]?.urlWithHash).toBe( + "/docs/fixture#install-1" + ); + expect(searchDocs(index, "calipers")[0]?.urlWithHash).toBe( + "/docs/fixture#install-2" + ); + expect(searchDocs(index, "ratchets")[0]?.urlWithHash).toBe( + "/docs/fixture#setup" + ); + expect(searchDocs(index, "spanners")[0]?.urlWithHash).toBe( + "/docs/fixture#configure" + ); + expect(searchDocs(index, "gadgets")[0]?.urlWithHash).toBe( + "/docs/fixture#components-note" + ); + expect(searchDocs(index, "levels")[0]?.urlWithHash).toBe("/docs/fixture#0"); + }); + + it("keeps letter-started MDX JSX names aligned with rendered anchors", () => { + const content = [ + "## Install", + "Underscore section covers widgets.", + "## Install", + "Dollar section covers sprockets.", + "## Install", + "Plain section covers gadgets.", + ].join("\n"); + const index = createDocsSearchIndex( + [ + { + id: "fixture", + title: "Fixture", + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + relativePath: "fixture.mdx", + content, + }, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + const tocIds = flattenTocIds( + extractDocsTableOfContents(content, { + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + }) + ); + const searchAnchors = index.chunks.map( + (chunk) => chunk[CHUNK_ANCHOR_INDEX] + ); + + expect(tocIds).toEqual(["install", "install-1", "install-2"]); + expect(searchAnchors).toEqual(tocIds); + expect(searchDocs(index, "widgets")[0]?.urlWithHash).toBe( + "/docs/fixture#install" + ); + expect(searchDocs(index, "sprockets")[0]?.urlWithHash).toBe( + "/docs/fixture#install-1" + ); + expect(searchDocs(index, "gadgets")[0]?.urlWithHash).toBe( + "/docs/fixture#install-2" + ); + }); + + it("keeps JavaScript literal MDX attributes aligned with rendered anchors", () => { + const content = [ + "## Install", + "Regex section covers widgets.", + "## { if (ready) {} /don't/.test(value); }} /> Install", + "Statement section covers calipers.", + "## { if (ready) /don't/.test(value); }} /> Install", + "Control section covers saws.", + `## Install`, + "Template section covers drills.", + "## 0} /> Install", + "Division section covers ratchets.", + "## 0} /> Install", + "Call division section covers levels.", + "## Install", + "Plain section covers sprockets.", + " */ 1 > 0}>", + "---", + "", + "## 0", + "Comparison section covers gauges.", + ].join("\n"); + const index = createDocsSearchIndex( + [ + { + id: "fixture", + title: "Fixture", + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + relativePath: "fixture.mdx", + content, + }, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + const tocIds = flattenTocIds( + extractDocsTableOfContents(content, { + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + }) + ); + const searchAnchors = index.chunks.map( + (chunk) => chunk[CHUNK_ANCHOR_INDEX] + ); + + expect(tocIds).toEqual([ + "install", + "install-1", + "install-2", + "install-3", + "install-4", + "install-5", + "install-6", + "0", + ]); + expect(searchAnchors).toEqual(tocIds); + expect(searchDocs(index, "widgets")[0]?.urlWithHash).toBe( + "/docs/fixture#install" + ); + expect(searchDocs(index, "sprockets")[0]?.urlWithHash).toBe( + "/docs/fixture#install-6" + ); + expect(searchDocs(index, "calipers")[0]?.urlWithHash).toBe( + "/docs/fixture#install-1" + ); + expect(searchDocs(index, "ratchets")[0]?.urlWithHash).toBe( + "/docs/fixture#install-4" + ); + expect(searchDocs(index, "saws")[0]?.urlWithHash).toBe( + "/docs/fixture#install-2" + ); + expect(searchDocs(index, "drills")[0]?.urlWithHash).toBe( + "/docs/fixture#install-3" + ); + expect(searchDocs(index, "levels")[0]?.urlWithHash).toBe( + "/docs/fixture#install-5" + ); + expect(searchDocs(index, "gauges")[0]?.urlWithHash).toBe("/docs/fixture#0"); + }); + + it("keeps regex statements and division around JavaScript bodies aligned", () => { + const content = [ + "## { if (ready) foo(); else /don't/.test(value); }} /> Install", + "Else regex section covers widgets.", + "## { do /don't/.test(value); while (ready); }} /> Install", + "Do regex section covers sprockets.", + "## y'} /> Install", + "Function division section covers calipers.", + "## Install", + "Plain section covers gadgets.", + ].join("\n"); + const index = createDocsSearchIndex( + [ + { + id: "fixture", + title: "Fixture", + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + relativePath: "fixture.mdx", + content, + }, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + const tocIds = flattenTocIds( + extractDocsTableOfContents(content, { + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + }) + ); + const searchAnchors = index.chunks.map( + (chunk) => chunk[CHUNK_ANCHOR_INDEX] + ); + + expect(tocIds).toEqual(["install", "install-1", "install-2", "install-3"]); + expect(searchAnchors).toEqual(tocIds); + expect(searchDocs(index, "widgets")[0]?.urlWithHash).toBe( + "/docs/fixture#install" + ); + expect(searchDocs(index, "sprockets")[0]?.urlWithHash).toBe( + "/docs/fixture#install-1" + ); + expect(searchDocs(index, "calipers")[0]?.urlWithHash).toBe( + "/docs/fixture#install-2" + ); + expect(searchDocs(index, "gadgets")[0]?.urlWithHash).toBe( + "/docs/fixture#install-3" + ); + }); + + it("keeps forward JavaScript block context aligned with rendered anchors", () => { + const tags = [ + ` { if (ready) { const marker = "}"; } /don't/.test(value); }} />`, + ` { if (ready) { /* } { */ } /don't/.test(value); }} />`, + " { if (ready) { const marker = `}`; } /don't/.test(value); }} />", + ` { if (ready) { const marker = /[{}]/; } /don't/.test(value); }} />`, + ` { function helper() {} /don't/.test(value); }} />`, + ` { class Helper {} /don't/.test(value); }} />`, + ` { try {} catch { function helper() {} /don't/.test(value); } }} />`, + ` y"} />`, + ` y"} />`, + ` y"} />`, + ` { for await (const item of values) /don't/.test(item); }} />`, + ]; + + for (const tag of tags) { + const content = [ + `## ${tag} Install`, + "First section covers widgets.", + "## Install", + "Second section covers sprockets.", + ].join("\n"); + const index = createDocsSearchIndex( + [ + { + id: "fixture", + title: "Fixture", + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + relativePath: "fixture.mdx", + content, + }, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + const tocIds = flattenTocIds( + extractDocsTableOfContents(content, { + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + }) + ); + const searchAnchors = index.chunks.map( + (chunk) => chunk[CHUNK_ANCHOR_INDEX] + ); + + expect(tocIds).toEqual(["install", "install-1"]); + expect(searchAnchors).toEqual(tocIds); + expect(searchDocs(index, "widgets")[0]?.urlWithHash).toBe( + "/docs/fixture#install" + ); + expect(searchDocs(index, "sprockets")[0]?.urlWithHash).toBe( + "/docs/fixture#install-1" + ); + } + }); + + it("keeps nested declarations and statement colons aligned with rendered anchors", () => { + const tags = [ + ` { function helper(callback = function nested() {}) {} /don't/.test(value); }} />`, + ` { class Outer extends (class Inner {}) {} /don't/.test(value); }} />`, + ` { label: {} /don't/.test(value); }} />`, + ` { switch (value) { case 1: {} /don't/.test(value); } }} />`, + ` { switch (value) { case ready ? one : two: {} /don't/.test(value); } }} />`, + ` { switch (value) { case (() => { switch (inner) { case 1: return 2; } return 3; })(): {} /don't/.test(value); } }} />`, + ` y"} />`, + ` y"} />`, + ` y" }} />`, + ]; + + for (const tag of tags) { + const content = [ + `## ${tag} Install`, + "First section covers widgets.", + "## Install", + "Second section covers sprockets.", + ].join("\n"); + const index = createDocsSearchIndex( + [ + { + id: "fixture", + title: "Fixture", + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + relativePath: "fixture.mdx", + content, + }, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + const tocIds = flattenTocIds( + extractDocsTableOfContents(content, { + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + }) + ); + const searchAnchors = index.chunks.map( + (chunk) => chunk[CHUNK_ANCHOR_INDEX] + ); + + expect(tocIds).toEqual(["install", "install-1"]); + expect(searchAnchors).toEqual(tocIds); + expect(searchDocs(index, "widgets")[0]?.urlWithHash).toBe( + "/docs/fixture#install" + ); + expect(searchDocs(index, "sprockets")[0]?.urlWithHash).toBe( + "/docs/fixture#install-1" + ); + } + }); + + it("keeps postfix updates and prefix operators aligned with rendered anchors", () => { + const tags = [ + "", + "", + ``, + "", + "", + ``, + ``, + ]; + + for (const tag of tags) { + const content = [ + `## ${tag} Install`, + "First section covers widgets.", + "## Install", + "Second section covers sprockets.", + ].join("\n"); + const index = createDocsSearchIndex( + [ + { + id: "fixture", + title: "Fixture", + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + relativePath: "fixture.mdx", + content, + }, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + const tocIds = flattenTocIds( + extractDocsTableOfContents(content, { + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + }) + ); + const searchAnchors = index.chunks.map( + (chunk) => chunk[CHUNK_ANCHOR_INDEX] + ); + + expect(tocIds).toEqual(["install", "install-1"]); + expect(searchAnchors).toEqual(tocIds); + expect(searchDocs(index, "widgets")[0]?.urlWithHash).toBe( + "/docs/fixture#install" + ); + expect(searchDocs(index, "sprockets")[0]?.urlWithHash).toBe( + "/docs/fixture#install-1" + ); + } + }); + + it("keeps Unicode identifiers and spread operands aligned with rendered anchors", () => { + const tags = [ + ' y"} />', + ' y"} />', + ' y"} />', + ' y"} />', + ' y"} />', + ' y"} />', + ' y"} />', + ``, + ' y" }} />', + ]; + + for (const tag of tags) { + const content = [ + `## ${tag} Install`, + "First section covers widgets.", + "## Install", + "Second section covers sprockets.", + ].join("\n"); + const index = createDocsSearchIndex( + [ + { + id: "fixture", + title: "Fixture", + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + relativePath: "fixture.mdx", + content, + }, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + const tocIds = flattenTocIds( + extractDocsTableOfContents(content, { + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + }) + ); + const searchAnchors = index.chunks.map( + (chunk) => chunk[CHUNK_ANCHOR_INDEX] + ); + + expect(tocIds).toEqual(["install", "install-1"]); + expect(searchAnchors).toEqual(tocIds); + expect(searchDocs(index, "widgets")[0]?.urlWithHash).toBe( + "/docs/fixture#install" + ); + expect(searchDocs(index, "sprockets")[0]?.urlWithHash).toBe( + "/docs/fixture#install-1" + ); + } + }); + + it("keeps class-heritage regex operands aligned with rendered anchors", () => { + const tags = [ + ``, + ``, + ' y") {}} />', + ' y") {}} />', + ' y"} />', + ' y"} />', + ' y"} />', + ` { class Named {} /don't/.test(value); }} />`, + ]; + + for (const tag of tags) { + const content = [ + `## ${tag} Install`, + "First section covers widgets.", + "## Install", + "Second section covers sprockets.", + ].join("\n"); + const index = createDocsSearchIndex( + [ + { + id: "fixture", + title: "Fixture", + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + relativePath: "fixture.mdx", + content, + }, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + const tocIds = flattenTocIds( + extractDocsTableOfContents(content, { + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + }) + ); + const searchAnchors = index.chunks.map( + (chunk) => chunk[CHUNK_ANCHOR_INDEX] + ); + + expect(tocIds).toEqual(["install", "install-1"]); + expect(searchAnchors).toEqual(tocIds); + expect(searchDocs(index, "widgets")[0]?.urlWithHash).toBe( + "/docs/fixture#install" + ); + expect(searchDocs(index, "sprockets")[0]?.urlWithHash).toBe( + "/docs/fixture#install-1" + ); + } + }); + + it("keeps class static blocks aligned with rendered anchors", () => { + const tags = [ + ``, + ``, + ' y"} />', + ' y"} />', + ' y"} />', + ]; + + for (const tag of tags) { + const content = [ + `## ${tag} Install`, + "First section covers widgets.", + "## Install", + "Second section covers sprockets.", + ].join("\n"); + const index = createDocsSearchIndex( + [ + { + id: "fixture", + title: "Fixture", + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + relativePath: "fixture.mdx", + content, + }, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + const tocIds = flattenTocIds( + extractDocsTableOfContents(content, { + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + }) + ); + const searchAnchors = index.chunks.map( + (chunk) => chunk[CHUNK_ANCHOR_INDEX] + ); + + expect(tocIds).toEqual(["install", "install-1"]); + expect(searchAnchors).toEqual(tocIds); + expect(searchDocs(index, "widgets")[0]?.urlWithHash).toBe( + "/docs/fixture#install" + ); + expect(searchDocs(index, "sprockets")[0]?.urlWithHash).toBe( + "/docs/fixture#install-1" + ); + } + }); + + it("keeps class-named members aligned with rendered anchors", () => { + const tags = [ + ``, + ` { function nested() {} /don't/.test(value); } }} />`, + ``, + ``, + ]; + + for (const tag of tags) { + const content = [ + `## ${tag} Install`, + "First section covers widgets.", + "## Install", + "Second section covers sprockets.", + ].join("\n"); + const index = createDocsSearchIndex( + [ + { + id: "fixture", + title: "Fixture", + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + relativePath: "fixture.mdx", + content, + }, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + const tocIds = flattenTocIds( + extractDocsTableOfContents(content, { + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + }) + ); + const searchAnchors = index.chunks.map( + (chunk) => chunk[CHUNK_ANCHOR_INDEX] + ); + + expect(tocIds).toEqual(["install", "install-1"]); + expect(searchAnchors).toEqual(tocIds); + expect(searchDocs(index, "widgets")[0]?.urlWithHash).toBe( + "/docs/fixture#install" + ); + expect(searchDocs(index, "sprockets")[0]?.urlWithHash).toBe( + "/docs/fixture#install-1" + ); + } + }); + + it("keeps escaped class bindings aligned with rendered anchors", () => { + const tags = [ + ``, + ``, + ]; + + for (const tag of tags) { + const content = [ + `## ${tag} Install`, + "First section covers widgets.", + "## Install", + "Second section covers sprockets.", + ].join("\n"); + const index = createDocsSearchIndex( + [ + { + id: "fixture", + title: "Fixture", + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + relativePath: "fixture.mdx", + content, + }, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + const tocIds = flattenTocIds( + extractDocsTableOfContents(content, { + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + }) + ); + const searchAnchors = index.chunks.map( + (chunk) => chunk[CHUNK_ANCHOR_INDEX] + ); + + expect(tocIds).toEqual(["install", "install-1"]); + expect(searchAnchors).toEqual(tocIds); + expect(searchDocs(index, "widgets")[0]?.urlWithHash).toBe( + "/docs/fixture#install" + ); + expect(searchDocs(index, "sprockets")[0]?.urlWithHash).toBe( + "/docs/fixture#install-1" + ); + } + }); + + it("keeps MDX fragment flow and inline heading anchors aligned", () => { + const content = [ + "<>", + "Install", + "", + "---", + "<>Setup", + "---", + "Setext fragment section covers gadgets.", + "## <>Install", + "Fragment section covers widgets.", + "## Install", + "Plain section covers sprockets.", + ].join("\n"); + const index = createDocsSearchIndex( + [ + { + id: "fixture", + title: "Fixture", + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + relativePath: "fixture.mdx", + content, + }, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + const tocIds = flattenTocIds( + extractDocsTableOfContents(content, { + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + }) + ); + const searchAnchors = index.chunks.map( + (chunk) => chunk[CHUNK_ANCHOR_INDEX] + ); + + expect(tocIds).toEqual(["setup", "install", "install-1"]); + expect(searchAnchors).toEqual(["", ...tocIds]); + expect(searchDocs(index, "gadgets")[0]?.urlWithHash).toBe( + "/docs/fixture#setup" + ); + expect(searchDocs(index, "widgets")[0]?.headingPath.at(-1)).toBe("Install"); + expect(searchDocs(index, "widgets")[0]?.urlWithHash).toBe( + "/docs/fixture#install" + ); + expect(searchDocs(index, "sprockets")[0]?.urlWithHash).toBe( + "/docs/fixture#install-1" + ); + }); + + it("keeps bare block starters aligned for LF and CRLF", () => { + for (const blockStart of [" chunk[CHUNK_ANCHOR_INDEX] + ); + + expect(tocIds).toEqual(["after"]); + expect(searchAnchors).toEqual(["", ...tocIds]); + } + } + }); + + it("keeps standalone HTML tags from shifting LF or CRLF heading anchors", () => { + for (const lineEnding of ["\n", "\r\n"]) { + const content = [ + '', + "---", + "", + "## !!!", + "First punctuation section covers widgets.", + "## !!!", + "Second punctuation section covers sprockets.", + ].join(lineEnding); + const index = createDocsSearchIndex( + [ + { + id: "fixture", + title: "Fixture", + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + relativePath: "fixture.mdx", + content, + }, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + const tocIds = flattenTocIds( + extractDocsTableOfContents(content, { + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + }) + ); + const searchAnchors = index.chunks.map( + (chunk) => chunk[CHUNK_ANCHOR_INDEX] + ); + + expect(tocIds).toEqual(["", "-1"]); + expect(searchAnchors).toEqual(["", ...tocIds]); + } + }); + + it("keeps comparison and declaration heading labels aligned with anchors", () => { + const content = [ + "## 1 < 2 > 0", + "Comparison section covers widgets.", + "## Install ", + "Processing section covers sprockets.", + "## Install ", + "Declaration section covers gadgets.", + ].join("\n"); + const index = createDocsSearchIndex( + [ + { + id: "fixture", + title: "Fixture", + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + relativePath: "fixture.mdx", + content, + }, + ], + { generatedAt: "2026-01-01T00:00:00.000Z" } + ); + + const comparison = searchDocs(index, "widgets")[0]; + const processing = searchDocs(index, "sprockets")[0]; + const declaration = searchDocs(index, "gadgets")[0]; + const tocIds = flattenTocIds( + extractDocsTableOfContents(content, { + urlPath: "/docs/fixture", + absoluteUrl: "https://leadtype.dev/docs/fixture", + }) + ); + const searchAnchors = index.chunks.map( + (chunk) => chunk[CHUNK_ANCHOR_INDEX] + ); + + expect(searchAnchors).toEqual(tocIds); + expect(comparison?.headingPath.at(-1)).toBe("1 < 2 > 0"); + expect(comparison?.urlWithHash).toBe("/docs/fixture#1-2-0"); + expect(processing?.headingPath.at(-1)).toBe("Install"); + expect(processing?.urlWithHash).toBe("/docs/fixture#install"); + expect(declaration?.headingPath.at(-1)).toBe("Install"); + expect(declaration?.urlWithHash).toBe("/docs/fixture#install-1"); + }); + it("slugifies headings for hash links", () => { expect(slugifyDocsHeading("Café API: Quick Start!")).toBe( "cafe-api-quick-start"