diff --git a/packages/comark/SPEC/HTML/README.md b/packages/comark/SPEC/HTML/README.md index 64bc938f7..61907717b 100644 --- a/packages/comark/SPEC/HTML/README.md +++ b/packages/comark/SPEC/HTML/README.md @@ -1,6 +1,6 @@ # HTML parsing contract -Behavioral contract for embedded HTML. Fixtures in this directory are the user-facing cases. `test/html-block.test.ts` covers the option matrix and the cases deferred below. +Behavioral contract for embedded HTML. Fixtures in this directory are the user-facing cases. `test/html-tags.test.ts` covers `markdown: false`, streaming, and large inputs; `test/html-block.test.ts` covers the option matrix and the cases deferred below. HTML tags become element nodes with `$: { html: 1, block: 0 | 1 }`. A node without `$` is markdown-originated (`` from `**bold**` is `['strong', {}, 'bold']`). @@ -15,6 +15,8 @@ HTML tags become element nodes with `$: { html: 1, block: 0 | 1 }`. A node witho Void tags (`img`, `br`, …) are always `block: 0`. +A non-void open tag that follows text in its paragraph (`GET /apps/`, `- **a** c`) is prose, not a container. It ends with its paragraph, `block: 0`, unless a later paragraph of the same container holds its close tag. A close tag in a nested container (`::card`, a blockquote) or past the end of the container does not count. A tag that starts its paragraph keeps collecting the blocks after it. + Consecutive same-paragraph tags stay inline even if indented on following lines: ```md @@ -41,7 +43,13 @@ Blank-line bodies parse as markdown in both modes. CommonMark ends the HTML bloc ` +``` + +## AST + +```json +{ + "frontmatter": {}, + "meta": {}, + "nodes": [ + [ + "pre", + { + "$": { + "html": 1, + "block": 1 + } + }, + [ + "style", + { + "$": { + "html": 1, + "block": 0 + } + }, + "a < b" + ] + ] + ] +} +``` + +## HTML + +```html +
+``` + +## Markdown + +```md +
+``` diff --git a/packages/comark/SPEC/HTML/pre-unclosed-tag.md b/packages/comark/SPEC/HTML/pre-unclosed-tag.md new file mode 100644 index 000000000..58f5cd1d0 --- /dev/null +++ b/packages/comark/SPEC/HTML/pre-unclosed-tag.md @@ -0,0 +1,51 @@ +## Input + +```md +
const a: Array = []
+
+ +after +``` + +## AST + +```json +{ + "frontmatter": {}, + "meta": {}, + "nodes": [ + [ + "pre", + { + "$": { + "html": 1, + "block": 1 + } + }, + "const a: Array = []\n" + ], + [ + "p", + {}, + "after" + ] + ] +} +``` + +## HTML + +```html +
const a: Array<string> = []
+
+

after

+``` + +## Markdown + +```md +
const a: Array<string> = []
+
+ +after +``` diff --git a/packages/comark/SPEC/HTML/pre-verbatim.md b/packages/comark/SPEC/HTML/pre-verbatim.md new file mode 100644 index 000000000..4829d1729 --- /dev/null +++ b/packages/comark/SPEC/HTML/pre-verbatim.md @@ -0,0 +1,71 @@ +## Input + +```md +
+  const x = 1
+  **not**
+  b
+
+ +after +``` + +## AST + +```json +{ + "frontmatter": {}, + "meta": {}, + "nodes": [ + [ + "pre", + { + "$": { + "html": 1, + "block": 1 + } + }, + "\n const x = 1\n **not**\n ", + [ + "b", + { + "$": { + "html": 1, + "block": 0 + } + }, + "b" + ], + "\n" + ], + [ + "p", + {}, + "after" + ] + ] +} +``` + +## HTML + +```html +
+  const x = 1
+  **not**
+  b
+
+

after

+``` + +## Markdown + +```md +
+  const x = 1
+  **not**
+  b
+
+ +after +``` diff --git a/packages/comark/SPEC/HTML/raw-script-inline.md b/packages/comark/SPEC/HTML/raw-script-inline.md new file mode 100644 index 000000000..88e4b951a --- /dev/null +++ b/packages/comark/SPEC/HTML/raw-script-inline.md @@ -0,0 +1,46 @@ +## Input + +```md +a after +``` + +## AST + +```json +{ + "frontmatter": {}, + "meta": {}, + "nodes": [ + [ + "p", + {}, + "a ", + [ + "script", + { + "$": { + "html": 1, + "block": 0 + } + }, + "**x**" + ], + " after" + ] + ] +} +``` + +## HTML + +```html +

+ a after +

+``` + +## Markdown + +```md +a after +``` diff --git a/packages/comark/SPEC/HTML/raw-tag-exact-name-prefix.md b/packages/comark/SPEC/HTML/raw-tag-exact-name-prefix.md new file mode 100644 index 000000000..068bbff48 --- /dev/null +++ b/packages/comark/SPEC/HTML/raw-tag-exact-name-prefix.md @@ -0,0 +1,50 @@ +## Input + +```md +a **x** after +``` + +## AST + +```json +{ + "frontmatter": {}, + "meta": {}, + "nodes": [ + [ + "p", + {}, + "a ", + [ + "prelude", + { + "$": { + "html": 1, + "block": 0 + } + }, + [ + "strong", + {}, + "x" + ] + ], + " after" + ] + ] +} +``` + +## HTML + +```html +

+ a x after +

+``` + +## Markdown + +```md +a **x** after +``` diff --git a/packages/comark/SPEC/HTML/raw-tag-exact-name.md b/packages/comark/SPEC/HTML/raw-tag-exact-name.md new file mode 100644 index 000000000..930305fb7 --- /dev/null +++ b/packages/comark/SPEC/HTML/raw-tag-exact-name.md @@ -0,0 +1,50 @@ +## Input + +```md +a **x** after +``` + +## AST + +```json +{ + "frontmatter": {}, + "meta": {}, + "nodes": [ + [ + "p", + {}, + "a ", + [ + "script-x", + { + "$": { + "html": 1, + "block": 0 + } + }, + [ + "strong", + {}, + "x" + ] + ], + " after" + ] + ] +} +``` + +## HTML + +```html +

+ a x after +

+``` + +## Markdown + +```md +a **x** after +``` diff --git a/packages/comark/SPEC/HTML/raw-text-elements.md b/packages/comark/SPEC/HTML/raw-text-elements.md new file mode 100644 index 000000000..6791d158f --- /dev/null +++ b/packages/comark/SPEC/HTML/raw-text-elements.md @@ -0,0 +1,68 @@ +## Input + +```md + + + + + +``` + +## AST + +```json +{ + "frontmatter": {}, + "meta": {}, + "nodes": [ + [ + "textarea", + { + "$": { + "html": 1, + "block": 1 + } + }, + "a < b" + ], + [ + "script", + { + "$": { + "html": 1, + "block": 1 + } + }, + "a < b" + ], + [ + "style", + { + "$": { + "html": 1, + "block": 1 + } + }, + "a < b" + ] + ] +} +``` + +## HTML + +```html + + + +``` + +## Markdown + +```md + + + + + +``` diff --git a/packages/comark/SPEC/HTML/raw-textarea-references.md b/packages/comark/SPEC/HTML/raw-textarea-references.md new file mode 100644 index 000000000..da8fd3c3d --- /dev/null +++ b/packages/comark/SPEC/HTML/raw-textarea-references.md @@ -0,0 +1,38 @@ +## Input + +```md + +``` + +## AST + +```json +{ + "frontmatter": {}, + "meta": {}, + "nodes": [ + [ + "textarea", + { + "$": { + "html": 1, + "block": 1 + } + }, + "a & b" + ] + ] +} +``` + +## HTML + +```html + +``` + +## Markdown + +```md + +``` diff --git a/packages/comark/SPEC/HTML/real-life-sample-1.md b/packages/comark/SPEC/HTML/real-life-sample-1.md index d6d500bf3..cc7065b62 100644 --- a/packages/comark/SPEC/HTML/real-life-sample-1.md +++ b/packages/comark/SPEC/HTML/real-life-sample-1.md @@ -128,5 +128,5 @@ ## Markdown ```md -

Discord  Twitter  GitHub  Bluesky

+

Discord  Twitter  GitHub  Bluesky

``` diff --git a/packages/comark/SPEC/HTML/unwrap-author-paragraph.md b/packages/comark/SPEC/HTML/unwrap-author-paragraph.md new file mode 100644 index 000000000..b4a52d366 --- /dev/null +++ b/packages/comark/SPEC/HTML/unwrap-author-paragraph.md @@ -0,0 +1,52 @@ +## Input + +```md +

Check

+``` + +## AST + +```json +{ + "frontmatter": {}, + "meta": {}, + "nodes": [ + [ + "section", + { + "$": { + "html": 1, + "block": 1 + } + }, + [ + "p", + { + "$": { + "html": 1, + "block": 0 + }, + "class": "c" + }, + "Check" + ] + ] + ] +} +``` + +## HTML + +```html +
+

Check

+
+``` + +## Markdown + +```md +
+

Check

+
+``` diff --git a/packages/comark/SPEC/HTML/unwrap-markdown-paragraph.md b/packages/comark/SPEC/HTML/unwrap-markdown-paragraph.md new file mode 100644 index 000000000..f8a3c0287 --- /dev/null +++ b/packages/comark/SPEC/HTML/unwrap-markdown-paragraph.md @@ -0,0 +1,46 @@ +## Input + +```md +
+ +Check + +
+``` + +## AST + +```json +{ + "frontmatter": {}, + "meta": {}, + "nodes": [ + [ + "section", + { + "$": { + "html": 1, + "block": 1 + } + }, + "Check" + ] + ] +} +``` + +## HTML + +```html +
+ Check +
+``` + +## Markdown + +```md +
+Check +
+``` diff --git a/packages/comark/src/internal/entities.ts b/packages/comark/src/internal/entities.ts index 7bdbbe3e0..b2ff49edc 100644 --- a/packages/comark/src/internal/entities.ts +++ b/packages/comark/src/internal/entities.ts @@ -61,6 +61,15 @@ function fromNumericReference(value: number): string { * alphanumeric follows them (`?a=1©=2` stays literal). */ export function decodeHTMLAttribute(input: string): string { + return decodeReferences(input, false) +} + +/** Decode only references that end with `;`, the way markdown-exit reads them in text. */ +export function decodeHTMLStrict(input: string): string { + return decodeReferences(input, true) +} + +function decodeReferences(input: string, strict: boolean): string { let amp = input.indexOf('&') if (amp === -1) return input @@ -84,9 +93,10 @@ export function decodeHTMLAttribute(input: string): string { value = Math.min(value * (hex ? 16 : 10) + Number.parseInt(input[end], 16), 0x110000) end++ } - if (end > start) { + const semicolon = input.charCodeAt(end) === 59 /* ; */ + if (end > start && (semicolon || !strict)) { decoded = fromNumericReference(value) - if (input.charCodeAt(end) === 59 /* ; */) end++ + if (semicolon) end++ } } else { while (end < length && isAlphanumeric(input.charCodeAt(end))) end++ @@ -95,7 +105,7 @@ export function decodeHTMLAttribute(input: string): string { if (next === 59 /* ; */ && own.call(characterEntities, name)) { decoded = characterEntities[name] end++ - } else if (next !== 61 /* = */ && legacy.has(name)) { + } else if (!strict && next !== 61 /* = */ && legacy.has(name)) { // Any shorter legacy prefix of `name` is followed by an alphanumeric, which keeps it literal. decoded = characterEntities[name] } diff --git a/packages/comark/src/internal/parse/html-tag.ts b/packages/comark/src/internal/parse/html-tag.ts new file mode 100644 index 000000000..ccf23a90c --- /dev/null +++ b/packages/comark/src/internal/parse/html-tag.ts @@ -0,0 +1,28 @@ +// Adapted from markdown-exit's html_re, with two changes: attribute values may be +// a balanced `{…}` expression (JSX style), and a comment may end in `--->` as in HTML. + +const attr_name = '[a-zA-Z_:][a-zA-Z0-9:._-]*' + +const braces = (depth: number): string => (depth === 0 ? '[^{}]*' : `(?:[^{}]|\\{${braces(depth - 1)}\\})*`) + +/** Source of a `{…}` attribute value. Nesting is limited to four levels. */ +export const braced_value = `\\{${braces(3)}\\}` + +const unquoted = '[^"\'=<>`\\x00-\\x20]+' +const single_quoted = "'[^']*'" +const double_quoted = '"[^"]*"' + +// `(?=(…))\1` is atomic: `{…}` and unquoted overlap, and backtracking between them is exponential. +const attr_value = `(?:(?=(${braced_value}(?:${unquoted})?|${unquoted}))\\1|${single_quoted}|${double_quoted})` + +const attribute = `(?:\\s+${attr_name}(?:\\s*=\\s*${attr_value})?)` + +const open_tag = `<[A-Za-z][A-Za-z0-9\\-]*${attribute}*\\s*\\/?>` + +const close_tag = '<\\/[A-Za-z][A-Za-z0-9\\-]*\\s*>' +const comment = ')' +const processing = '<[?][\\s\\S]*?[?]>' +const declaration = ']*>' +const cdata = '' + +export const HTML_TAG_RE = new RegExp(`^(?:${open_tag}|${close_tag}|${comment}|${processing}|${declaration}|${cdata})`) diff --git a/packages/comark/src/internal/parse/tree.ts b/packages/comark/src/internal/parse/tree.ts index 53b1ffd16..d28c3457b 100644 --- a/packages/comark/src/internal/parse/tree.ts +++ b/packages/comark/src/internal/parse/tree.ts @@ -36,6 +36,9 @@ type HtmlOpenFrame = { block: boolean /** Bare-inline nodes waiting to be flushed as a single

on a block frame. */ pendingInline: Node[] + afterText?: boolean + /** Stack index of the nearest `p` or scope boundary at or below this frame, or -1. */ + pScope: number } /** Shared mutable context for a whole document walk. */ @@ -46,6 +49,9 @@ interface ProcessState { startLine: number /** Unclosed HTML open tags, outer → inner. Lives across blocks like the heading id stack. */ htmlStack: HtmlOpenFrame[] + closers?: CloserIndex + /** Set once a paragraph closed a tag whose closer may still arrive in a later stream chunk. */ + provisional?: boolean } type Processor = (tokens: Token[], start: number, state: ProcessState) => ProcessorResult @@ -69,10 +75,23 @@ function isInlineish(node: Node): boolean { return tag === null || inlineTags.has(tag as string) } +function isInlineHtml(node: Node): boolean { + if (typeof node === 'string') return false + const tag = node[0] + if (tag === null || inlineTags.has(tag) || tag === 'img' || tag === 'br') return true + return tag.includes('-') && node[1].$?.block === 0 +} + function flushPendingInline(frame: HtmlOpenFrame) { if (frame.pendingInline.length === 0) return const pending = frame.pendingInline frame.pendingInline = [] + const only = pending.length === 1 ? pending[0] : undefined + // A lone author-written element is a block of its own, not a paragraph. + if (Array.isArray(only) && only[1].$?.html && !isInlineish(only)) { + frame.children.push(only) + return + } frame.children.push(['p', {}, ...pending] as ElementNode) } @@ -112,11 +131,11 @@ function deliverBlock(state: ProcessState, node: Node | undefined): Node | undef } /** Nest an inline/finished node into the open HTML frame during processInline. */ -function deliverInline(state: ProcessState, node: Node | undefined): Node | undefined { +function deliverInline(state: ProcessState, node: Node | undefined, inline?: boolean): Node | undefined { if (node === undefined) return undefined if (state.htmlStack.length === 0) return node const top = state.htmlStack[state.htmlStack.length - 1] - if (top.block && isInlineish(node)) { + if (top.block && (inline ?? isInlineish(node))) { top.pendingInline.push(node) } else { flushPendingInline(top) @@ -144,7 +163,14 @@ function tryCommentTag(content: string, state: ProcessState): ProcessorResult | // Open comment (no close in this chunk) if (content.startsWith('')) { const body = content.slice(4) - state.htmlStack.push({ tag: null, attrs: {}, children: body ? [body] : [], block: true, pendingInline: [] }) + state.htmlStack.push({ + tag: null, + attrs: {}, + children: body ? [body] : [], + block: true, + pendingInline: [], + pScope: state.htmlStack.at(-1)?.pScope ?? -1, + }) return { nextIndex: -1, node: undefined } } @@ -157,6 +183,89 @@ function tryCommentTag(content: string, state: ProcessState): ProcessorResult | return null } +function closeFramesTo(state: ProcessState, idx: number): Node | undefined { + let closed: Node | undefined + while (state.htmlStack.length > idx) { + const node = frameToNode(state.htmlStack.pop()!) + if (state.htmlStack.length > idx) deliverInline(state, node) + else closed = node + } + return closed +} + +// Elements that stop a `

` from being closed by a following `

` (HTML "button scope"). +const P_SCOPE_BOUNDARY = new Set(['table', 'td', 'th', 'caption', 'button', 'template', 'object', 'marquee', 'applet']) + +function closeImpliedParagraph(token: Token, state: ProcessState): Node | undefined { + if ((token.content.charCodeAt(1) | 0x20) !== 0x70 /* p */ || !/^]/i.test(token.content)) return undefined + const parsed = parseHtmlInline(token.content) + if (parsed.kind !== 'open' || parsed.tag !== 'p' || parsed.selfClosing) return undefined + const idx = state.htmlStack.at(-1)?.pScope ?? -1 + return idx >= 0 && state.htmlStack[idx].tag === 'p' ? closeFramesTo(state, idx) : undefined +} + +const firstProse = new WeakMap() + +function followsText(tokens: Token[], start: number): boolean { + if (start === 0) return false + if (tokens[start - 1].type === 'text' && tokens[start - 1].content.trim()) return true + let first = firstProse.get(tokens) + if (first === undefined) { + first = tokens.findIndex( + ({ type, content }) => !(type === 'softbreak' || type === 'html_inline' || (type === 'text' && !content.trim())) + ) + if (first < 0) first = tokens.length + firstProse.set(tokens, first) + } + return first < start +} + +/** Where each `` sits, so "is it closed later?" is a lookup and not a scan of the rest of the document. */ +interface CloserIndex { + /** Token indexes of the paragraphs holding a close tag, by `tag|level`. */ + at: Map + /** First later token with a lower level, i.e. where the container of token `i` ends. */ + end: number[] +} + +function buildCloserIndex(tokens: Token[]): CloserIndex { + const at = new Map() + const end: number[] = new Array(tokens.length).fill(tokens.length) + const open: number[] = [] + for (let i = 0; i < tokens.length; i++) { + while (open.length > 0 && tokens[open[open.length - 1]].level > tokens[i].level) end[open.pop()!] = i + open.push(i) + for (const child of tokens[i].type === 'inline' ? (tokens[i].children ?? []) : []) { + if (child.type !== 'html_inline' || child.content.charCodeAt(1) !== 0x2f /* / */) continue + const parsed = parseHtmlInline(child.content) + if (parsed.kind !== 'close') continue + const key = `${parsed.tag.toLowerCase()}|${tokens[i].level}` + const list = at.get(key) + if (list && list[list.length - 1] !== i) list.push(i) + else if (!list) at.set(key, [i]) + } + } + return { at, end } +} + +/** + * True when a later paragraph of the same container holds ``. A close tag inside a + * nested container, or past the end of this one, cannot pair with an opener out here. + */ +function hasCloserAhead(tokens: Token[], state: ProcessState, paragraph: number, from: number, tag: string): boolean { + const index = (state.closers ??= buildCloserIndex(tokens)) + const list = index.at.get(`${tag.toLowerCase()}|${tokens[paragraph].level + 1}`) + if (!list) return false + let lo = 0 + let hi = list.length + while (lo < hi) { + const mid = (lo + hi) >> 1 + if (list[mid] < from) lo = mid + 1 + else hi = mid + } + return lo < list.length && list[lo] < index.end[paragraph] +} + function processToken(tokens: Token[], i: number, state: ProcessState): ProcessorResult | undefined { return processors[tokens[i].type]?.(tokens, i, state) } @@ -175,12 +284,23 @@ function processInline(inlineTokens: Token[], state: ProcessState, nestHtml = tr let i = 0 while (i < inlineTokens.length) { + const fromHtml = inlineTokens[i].type === 'html_inline' + if (nestHtml && fromHtml) { + const closed = closeImpliedParagraph(inlineTokens[i], state) + if (closed !== undefined) { + const free = deliverInline(state, closed) + if (free !== undefined) pushNode(nodes, free) + } + } const result = processToken(inlineTokens, i, state) if (!result) { i += 1 continue } - const node = nestHtml || state.htmlStack.length > depth ? deliverInline(state, result.node) : result.node + const node = + nestHtml || state.htmlStack.length > depth + ? deliverInline(state, result.node, !fromHtml || (result.node !== undefined && isInlineHtml(result.node))) + : result.node if (node !== undefined) pushNode(nodes, node) i = result.nextIndex } @@ -232,7 +352,10 @@ function processChildren( if (result) { // Same nesting rule as processInline: free nodes join the open HTML // frame when nestHtml is on (e.g. block children under bare HTML). - const node = nestHtml || state.htmlStack.length > depth ? deliverInline(state, result.node) : result.node + const node = + (nestHtml || state.htmlStack.length > depth) && !result.siblingOfHtmlStack + ? deliverInline(state, result.node) + : result.node if (node !== undefined) pushNode(children, node) i = result.nextIndex } else { @@ -493,6 +616,13 @@ const processors: Record = { const inline = tokens[start + 1] const depthBefore = state.htmlStack.length const { children, nextIndex } = processChildren(tokens, start, 'paragraph_close', state, true) + // `text ` with no `` anywhere after is prose, not a container: it ends with its paragraph. + const opened = state.htmlStack[depthBefore] + if (opened?.afterText && opened.tag !== null && !hasCloserAhead(tokens, state, start, nextIndex, opened.tag)) { + state.provisional = true + const free = deliverInline(state, closeFramesTo(state, depthBefore), true) + if (free !== undefined) pushNode(children, free) + } const asParagraph = (): ProcessorResult => children.length === 0 ? { nextIndex, node: undefined } @@ -577,7 +707,7 @@ const processors: Record = { // `` are HTML open/close with tag null (may arrive as plain text // when blank lines split a multi-line comment). - const comment = tryCommentTag(content, state) + const comment = tokens[start].meta?.literal ? null : tryCommentTag(content, state) if (comment) return { nextIndex: start + 1, node: comment.node } return { nextIndex: start + 1, node: content } @@ -603,32 +733,19 @@ const processors: Record = { } if (parsed.kind === 'close') { + const closeTag = parsed.tag.toLowerCase() + if (closeTag === 'br') return { nextIndex: start + 1, node: ['br', makeHtmlAttrs({}, false)] } // Find the matching open frame (case-insensitive), auto-closing any // nested frames above it the way a real HTML parser would. - const closeTag = parsed.tag.toLowerCase() let idx = state.htmlStack.length - 1 while (idx >= 0) { const frameTag = state.htmlStack[idx].tag if (frameTag !== null && frameTag.toLowerCase() === closeTag) break idx-- } - if (idx < 0) { - // HTML parsers read a stray `
` as `
` - const node: Node | undefined = closeTag === 'br' ? ['br', makeHtmlAttrs({}, false)] : undefined - return { nextIndex: start + 1, node } - } + if (idx < 0) return { nextIndex: start + 1, node: undefined } - let closed: Node | undefined - while (state.htmlStack.length > idx) { - const node = frameToNode(state.htmlStack.pop()!) - // Nested auto-closed frames nest into the next outer open frame. - if (state.htmlStack.length > idx) { - deliverInline(state, node) - } else { - closed = node - } - } - return { nextIndex: start + 1, node: closed } + return { nextIndex: start + 1, node: closeFramesTo(state, idx) } } if (parsed.selfClosing) { @@ -644,6 +761,11 @@ const processors: Record = { children: [], block: false, pendingInline: [], + afterText: followsText(tokens, start), + pScope: + parsed.tag === 'p' || P_SCOPE_BOUNDARY.has(parsed.tag.toLowerCase()) + ? state.htmlStack.length + : (state.htmlStack.at(-1)?.pScope ?? -1), }) return { nextIndex: start + 1, node: undefined } }, @@ -680,7 +802,8 @@ export function tokenListToTree(tokens: Token[], options: ProcessorOptions = {}) for (const frame of state.htmlStack) frame.block = true if (result.node !== undefined) { - if (state.preservePositions) { + // Without a line, streaming reparses from before this node, so a closer arriving later still pairs. + if (state.preservePositions && !state.provisional) { preserveLineNumber(tokens, result.node, i, result.nextIndex, state) } // Sibling free nodes from close-then-open (`

`) sit beside the diff --git a/packages/comark/src/internal/parse/utils.ts b/packages/comark/src/internal/parse/utils.ts index 377bcb3b1..526ee72d4 100644 --- a/packages/comark/src/internal/parse/utils.ts +++ b/packages/comark/src/internal/parse/utils.ts @@ -1,4 +1,5 @@ import type { Node, ElementNode, ElementNodeAttributes, MarkdownDocument } from 'comark' +import { braced_value } from './html-tag.ts' import { isHtmlVoidElement, TREE_WALK_MAX_DEPTH } from 'comark/utils' import { decodeHTMLAttribute } from '../entities.ts' @@ -67,7 +68,13 @@ function applyAutoUnwrapAt(node: Node, depth: number): Node { return node } - if (significantCount === 1 && Array.isArray(significant) && significant[0] === 'p') { + // An author-written `

` (`$.html`) is content, not a wrapper. + if ( + significantCount === 1 && + Array.isArray(significant) && + significant[0] === 'p' && + !(significant[1] as ElementNodeAttributes | undefined)?.$?.html + ) { // Lift the paragraph's attrs onto the parent so trailing `{attr}` survives the unwrap. // Parent attrs take precedence so explicit component props aren't overridden. const paragraphAttrs = significant[1] as Record | undefined @@ -186,6 +193,11 @@ export function parseHtmlInline(content: string): ParsedHtmlTag { return { kind: 'other', content: trimmed } } +const ATTRIBUTE_RE = new RegExp( + `([:\\w.-]+)(?:\\s*=\\s*(?:"([^"]*)"|'([^']*)'|(${braced_value})(?=\\s|$)|([^\\s"'=<>\`]+)))?`, + 'g' +) + /** * Decode character references in a raw attribute value the way browsers do: * legacy semicolon-less references (`&`) are decoded, but not when followed by @@ -204,11 +216,11 @@ export function parseHtmlAttributes(attrsStr: string): Record { const attrs: Record = {} if (!attrsStr || !attrsStr.trim()) return attrs - const re = /([:\w.-]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?/g - let match: RegExpExecArray | null - while ((match = re.exec(attrsStr)) !== null) { + ATTRIBUTE_RE.lastIndex = 0 + for (let match = ATTRIBUTE_RE.exec(attrsStr); match; match = ATTRIBUTE_RE.exec(attrsStr)) { const key = match[1].toLowerCase() - const value = match[2] ?? match[3] ?? match[4] + if (Object.hasOwn(attrs, key)) continue + const value = match[2] ?? match[3] ?? match[4] ?? match[5] attrs[key] = value === undefined ? true : decodeAttributeValue(value) } diff --git a/packages/comark/src/internal/sanitize.ts b/packages/comark/src/internal/sanitize.ts index 87e031d11..96a08da09 100644 --- a/packages/comark/src/internal/sanitize.ts +++ b/packages/comark/src/internal/sanitize.ts @@ -288,7 +288,12 @@ function validateStyle(value: string, options: PropsValidationOptions): string | // The browser decodes entities before CSS sees them: `url("…")` const css = decodeHtmlEntities(value) if (!css.includes('(')) return value - for (const url of styleUrls(css)) { + const urls = styleUrls(css) + // `style={{…}}` is a JavaScript object whose strings are CSS: `{{background: 'url("…")'}}` + if (css.trimStart().startsWith('{')) { + for (const text of urls.slice()) if (text.includes('(')) urls.push(...styleUrls(text)) + } + for (const url of urls) { if (url && validateUrl(url, 'image', options) !== url) return REJECTED_PROP } return value diff --git a/packages/comark/src/internal/stringify/handlers/comment.ts b/packages/comark/src/internal/stringify/handlers/comment.ts index 4c999b219..6eb59a115 100644 --- a/packages/comark/src/internal/stringify/handlers/comment.ts +++ b/packages/comark/src/internal/stringify/handlers/comment.ts @@ -3,6 +3,7 @@ import type { ElementNode } from 'comark' export function comment(node: ElementNode, state: State, parent?: ElementNode) { if (node[0] === null) { + if (state.context.rawText) return `` const inline = state.context.inline === true || parent?.[0] === 'p' || diff --git a/packages/comark/src/internal/stringify/handlers/html.ts b/packages/comark/src/internal/stringify/handlers/html.ts index 0a7dc61ae..7fc60c8b1 100644 --- a/packages/comark/src/internal/stringify/handlers/html.ts +++ b/packages/comark/src/internal/stringify/handlers/html.ts @@ -48,7 +48,16 @@ export async function html(node: ElementNode, state: State, parent?: ElementNode if (!oneLiner && inlineTags.has(String(tag)) && hasOnlyTextChildren) { oneLiner = true } - if (tag === 'pre') { + // Every character in an authored `

` is content, nested tags included.
+  const authoredRaw = state.context.rawText === true && $.html === 1
+  if (tag === 'pre' || tag === 'textarea' || tag === 'script' || tag === 'style' || authoredRaw) {
+    oneLiner = true
+  }
+  // Re-indenting a paragraph would put spaces into the `
` written inside it.
+  if (
+    tag === 'p' &&
+    children.some((child) => typeof child !== 'string' && (child?.[0] === 'pre' || child?.[0] === 'textarea'))
+  ) {
     oneLiner = true
   }
 
@@ -63,16 +72,28 @@ export async function html(node: ElementNode, state: State, parent?: ElementNode
 
   const isSelfClose = selfCloseTags.has(String(tag))
 
-  // Do not modify context if we are already in html mode
-  const revert = state.applyContext({ inline: oneLiner })
+  const rawText = state.context.rawText === true || tag === 'pre' || tag === 'textarea'
+  const literal = $.html === 1 && ($.block === 1 || state.context.literal === true)
+  // Only changed keys go into the edit: every element renders through here.
+  const edit: Record = { inline: oneLiner }
+  if (rawText !== (state.context.rawText === true)) edit.rawText = rawText
+  if (literal !== (state.context.literal === true)) edit.literal = literal
+  const revert = state.applyContext(edit)
 
   const childrenContent: string[] = []
-  for (const child of children) {
-    childrenContent.push(await state.one(child, state, node))
+  for (let i = 0; i < children.length; i++) {
+    childrenContent.push(await state.one(children[i], state, node, i === 0 && !oneLiner && !state.context.html))
   }
 
   // A blank line inside a raw-HTML element would terminate it on reparse
-  const childSeparator = state.context.html ? state.context.blockSeparator : oneLiner ? '' : '\n'
+  const childSeparator =
+    state.context.rawText && $.html === 1
+      ? ''
+      : state.context.html
+        ? state.context.blockSeparator
+        : oneLiner
+          ? ''
+          : '\n'
 
   let content = ''
   let isPrevBlock = true
@@ -81,6 +102,7 @@ export async function html(node: ElementNode, state: State, parent?: ElementNode
     const child = children[i]
     const isBlock =
       typeof child !== 'string' &&
+      !(state.context.rawText && child[0] === null) &&
       (blockTags.has(String(child?.[0])) || (!inlineTags.has(String(child?.[0])) && !hasTextSibling))
 
     if (i > 0 && !isPrevBlock && isBlock) {
diff --git a/packages/comark/src/internal/stringify/state.ts b/packages/comark/src/internal/stringify/state.ts
index b9f84aa51..87b4e74b4 100644
--- a/packages/comark/src/internal/stringify/state.ts
+++ b/packages/comark/src/internal/stringify/state.ts
@@ -36,15 +36,25 @@ function findHandler(ctx: Context, node: ElementNode): NodeHandler | undefined {
  */
 export async function one(node: Node, state: State, parent?: ElementNode, atLineStart = false): Promise {
   if (typeof node === 'string') {
+    const inElement = parent?.[1].$?.html === 1
+    const rawTag = inElement ? String(parent[0]).toLowerCase() : ''
+    const rawBody = rawTag === 'script' || rawTag === 'style'
     if (state.context.html) {
-      // Do not convert ampersands to entities in raw HTML blocks
-      return escapeHtml(node, { '&': undefined, '"': undefined })
+      // Text in an HTML element was decoded on parse, so a `&` that could start a reference
+      // (`<`, `A`) is written as `&`. Markdown text keeps its entities as it always has.
+      return escapeHtml(inElement && !rawBody ? node.replace(/&(?=[a-z#])/gi, '&') : node, {
+        '&': undefined,
+        '"': undefined,
+      })
     }
-    // The content of a raw HTML block is copied verbatim on parse, so markdown
-    // syntax inside it must not be escaped (inline HTML, `$.block === 0`, has
-    // its markdown parsed into children and is escaped normally).
-    if (parent?.[1].$?.html === 1 && parent[1].$?.block === 1) {
-      return state.context.rawHtmlText === 'verbatim' ? node : escapeRawHtmlText(node, String(parent[0]))
+    const verbatim = state.context.rawHtmlText === 'verbatim'
+    if (rawBody) return verbatim ? node : escapeRawHtmlText(node, rawTag)
+    if (state.context.rawText === true) {
+      return verbatim ? node : node.replace(/&/g, '&').replace(/'])
 
+/** Write ` = { script: '<\\/', style: '<\\/', textarea: '</' }
-
-/**
- * Escape a text node inside a raw HTML block for markdown output.
- *
- * Renderers that read the block as raw HTML (CommonMark, GitHub, markdown-it)
- * parse any tag in it, so a text node such as ``, which the
- * parser kept as text, must not come back as markup. Only a `<` that would
- * start a tag is escaped, as `<`, which reads back as the same character.
- * Markdown syntax and existing entities are kept, like the HTML renderer does.
- * In `

Hello **World**

` → strong). * Implemented by narrowing markdown-exit's `html_block` rule to `
', + '', + '
 a < b
', + '
ac
', + '', + 'before
**x** < y
after', +])('parses the saved Markdown of %s back to the same tree', async (source) => { + const { first, again } = await roundTrip(source) + + expect(again.nodes).toEqual(first.nodes) +}) + +it('does not treat a name that only starts like a raw tag as one', async () => { + for (const [name, raw] of [ + ['style-x', 'style'], + ['pre2', 'pre'], + ['textarea-1', 'textarea'], + ]) { + const result = await parseMarkdown(`a <${name}>**x** after`) + + expect(JSON.stringify(result.nodes)).toContain('"strong"') + } +}) + +describe('large inputs', () => { + // Correctness at a size where a quadratic scan would time out the suite; scaling is measured by hand. + it('parses many unpaired raw openers in one paragraph', async () => { + const result = await parseMarkdown('a ', + '
&lt;b&gt;
', + ]) { + const { first, markdown, again, html: output } = await roundTrip(source, literal) + + expect(output).toContain('&lt;b&gt;') + expect(output).not.toContain('<b>') + expect(again.nodes).toEqual(first.nodes) + expect(markdown).toContain('&lt;b&gt;') + } + }) + + it('writes decoded < and & in a literal block back as references', async () => { + const { first, markdown, again, html: output } = await roundTrip('
\n<b> & x\n
', literal) + + expect(first.nodes).toEqual([['div', { $: block }, ' & x']]) + expect(markdown).toBe('
\n<b> & x\n
') + expect(again.nodes).toEqual(first.nodes) + expect(output).toBe('
\n <b> & x\n
') + }) + + it('keeps reference-looking text and a decoded AT&T']]) + expect(again.nodes).toEqual(first.nodes) + expect(output).toBe('
\n &foo; &copy &amp; &#65; <!-- c --> AT&T\n
') + expect(markdown).not.toContain('