From cf7e39c75ee581c0230017d3cfddea420c4898ce Mon Sep 17 00:00:00 2001 From: xiaolai Date: Tue, 15 Sep 2026 10:27:06 +0800 Subject: [PATCH 01/13] fix(markdown): settle hostile ], backtick and emphasis runs in linear time The weekly pathological soak (#1407) was killed at its 300s wall ceiling on link-closers-with-openers. It was not a regression: the same input costs the same at 0ce4d2e54, when the suite was added. Earlier soak runs never reached this class, because nested-strong-emph overflowed the stack first (#1374) and, before the matrix split, the fuzz step failed ahead of it. Measured at soak scale 8, the class alone took 60s to parse and 276s to serialize. The CPU profile names two backward scans that micromark runs at EVERY ] in a paragraph: label-end walks back to the nearest unbalanced label start, and gfm-footnote's potential call walks back to the nearest image or link. With no opener both reach the paragraph start and fail, so a ] x n paragraph is O(n^2). Growth measured 4.4x parse per 2x input. The same shape repeats in two more micromark constructs. codeText scans to the paragraph end for a closing run of the same length, so runs of distinct lengths cost O(openers x paragraph); backtick-runs had not finished after twelve minutes. attention's resolver walks every closer back to the start when no opener exists; emph-closers-without-openers took 27s to parse. The fix is micromark constructs that act only on a PROOF that the stock construct would fail, and defer to it otherwise: - at ], consume the bracket as data when no unbalanced label start exists, found from an incremental index. The footnote call cannot succeed at such a ]: its label would contain the unescaped ] that balanced the image, which no footnote definition label can hold. Splices by a stock success are caught by watching the event at the opener index; - at a backtick run, remember per run length where the last run is (cmark's fix for the same class) and consume a run that no later run can close; - for * and _, lower _close on closers that cannot open and have no same-marker opener anywhere before them, for the duration of the stock resolver. Resolving only consumes openers, so their walk can match nothing. inlineFastPaths.test.ts diffs the mdast, positions included, against a parser without the plugin over every vendored spec corpus (947 examples), hand-picked boundary inputs and 3000 seeded fuzz documents. It caught a real error while this was built: a wrapper added in front of gfm's email autolink turned (_A_@_.A into emphasis, so the attention wrapper is appended and refuses to run anywhere but directly behind the disabled stock construct. Each fast path's decision was also mutated to confirm the test fails on it. The plugin is registered in every parse mode, including the details-body chain that is built from leaf imports. --- src/utils/markdownPipeline/dialect.test.ts | 1 + .../markdownPipeline/dialectDescriptors.ts | 8 + .../parser/fastPaths/attentionFastPath.ts | 136 ++++++++++++++ .../parser/fastPaths/codeTextFastPath.ts | 137 +++++++++++++++ .../parser/fastPaths/inlineFastPaths.test.ts | 129 ++++++++++++++ .../parser/fastPaths/labelEndFastPath.ts | 166 ++++++++++++++++++ .../parser/fastPaths/micromarkTypes.ts | 105 +++++++++++ .../parser/fastPaths/remarkInlineFastPaths.ts | 51 ++++++ .../plugins/detailsBodyParser.ts | 2 + 9 files changed, 735 insertions(+) create mode 100644 src/utils/markdownPipeline/parser/fastPaths/attentionFastPath.ts create mode 100644 src/utils/markdownPipeline/parser/fastPaths/codeTextFastPath.ts create mode 100644 src/utils/markdownPipeline/parser/fastPaths/inlineFastPaths.test.ts create mode 100644 src/utils/markdownPipeline/parser/fastPaths/labelEndFastPath.ts create mode 100644 src/utils/markdownPipeline/parser/fastPaths/micromarkTypes.ts create mode 100644 src/utils/markdownPipeline/parser/fastPaths/remarkInlineFastPaths.ts diff --git a/src/utils/markdownPipeline/dialect.test.ts b/src/utils/markdownPipeline/dialect.test.ts index 82e24a405..1052b75d4 100644 --- a/src/utils/markdownPipeline/dialect.test.ts +++ b/src/utils/markdownPipeline/dialect.test.ts @@ -131,6 +131,7 @@ describe("the deltas between modes are the declared ones", () => { expect(namesFor("inline-summary")).toEqual([ "remarkParse", "remarkGfm", + "remarkInlineFastPaths", "remarkDepthLimit", "remarkCustomInline", ]); diff --git a/src/utils/markdownPipeline/dialectDescriptors.ts b/src/utils/markdownPipeline/dialectDescriptors.ts index 9316e520f..c89a4660f 100644 --- a/src/utils/markdownPipeline/dialectDescriptors.ts +++ b/src/utils/markdownPipeline/dialectDescriptors.ts @@ -56,6 +56,7 @@ import { remarkDepthLimit, type ContentAnalysis, } from "./parser/remarkPlugins"; +import { remarkInlineFastPaths } from "./parser/fastPaths/remarkInlineFastPaths"; /** * The parse dialects. Each answers a different question, so each runs a @@ -136,6 +137,13 @@ export const DIALECT: readonly PluginDescriptor[] = [ "every mode, or `~x~` would become deletion in one dialect and subscript " + "in another for the same text.", }, + { + name: "remarkInlineFastPaths", + plugin: remarkInlineFastPaths, + modes: { document: "always", "source-position": "always", "details-body": "always", "inline-summary": "always" }, + reason: "Linear-time `]`, backtick and emphasis closers that cannot match (#1407). Tree-identical, " + + "so every mode. Right after GFM: its emphasis wrapper must sit directly behind micromark's own.", + }, { name: "remarkDisableSetextHeadings", plugin: remarkDisableSetextHeadings, diff --git a/src/utils/markdownPipeline/parser/fastPaths/attentionFastPath.ts b/src/utils/markdownPipeline/parser/fastPaths/attentionFastPath.ts new file mode 100644 index 000000000..2511a81da --- /dev/null +++ b/src/utils/markdownPipeline/parser/fastPaths/attentionFastPath.ts @@ -0,0 +1,136 @@ +/** + * Purpose: stop an emphasis closer that has no possible opener from walking + * back to the start of the paragraph. + * + * micromark resolves `*`/`_` in one `resolveAll` pass: for every sequence that + * can close, it walks back event by event looking for one that can open. A + * closer with nothing to find walks all the way — so `a_ ` repeated n times, + * where every `_` closes and none opens, is O(n²). Measured at the soak's scale + * (#1407): 27s to parse and 44s to serialize. + * + * The exact, cheap observation: that walk can only ever match a sequence that + * can OPEN, with the SAME marker, EARLIER in the paragraph — and resolving never + * creates an opener, it only consumes them. So a closer with no same-marker + * opener anywhere before it is guaranteed to match nothing. One forward pass + * finds those closers; their `_close` flag is lowered for the duration of the + * stock resolver, so its outer loop skips the walk; then it is restored. + * + * Only closers that CANNOT also open are touched. A both-flanking sequence's + * `_close` flag is also read when it acts as an opener (the rule of three), so + * lowering it could change a later match. + * + * Mechanism: `attention` is disabled by name and this wrapper takes its place + * at `*` and `_`. The wrapper tokenizes with micromark's own `attention` + * tokenizer — looked up from the parser, not imported — and resolves with its + * own resolver around the prefilter. Link text inside a label is still resolved + * by the stock resolver through `insideSpan`, exactly as before. + * + * PLACE matters, unlike for the `]` and backtick fast paths: this wrapper + * SUCCEEDS, so whichever construct micromark tries first at `_` wins. GFM's + * email autolink is registered ahead of `attention` there, and a wrapper added + * in front of both turned `(_A_@_.A` from a link into emphasis — the + * equivalence test (`inlineFastPaths.test.ts`) caught exactly that. So the + * wrapper is added AFTER, and refuses loudly to run anywhere but directly + * behind the disabled stock construct, where the order of everything + * micromark tries is unchanged. + * + * Known limitation: sequences that DO pair up still cost O(span) per match — + * micromark re-resolves the text between opener and closer and splices the + * event list — so deeply nested emphasis stays quadratic in its depth. + * + * @coordinates-with micromarkTypes.ts — the tokenizer API slice used here + * @coordinates-with remarkInlineFastPaths.ts — registers this construct + * @module utils/markdownPipeline/parser/fastPaths/attentionFastPath + */ +import type { + Construct, + Effects, + Event, + ParseContext, + State, + Token, + TokenizeContext, +} from "./micromarkTypes"; + +const ASTERISK = 42; +const UNDERSCORE = 95; +export const STOCK_ATTENTION = "attention"; +const NAME = "vmarkAttention"; + +const placementChecked = new WeakSet(); + +/** Everything micromark registered at `code`, disabled or not, in try order. */ +function registeredAt(parser: ParseContext, code: number): Construct[] { + const registered = parser.constructs.text[code]; + return Array.isArray(registered) ? registered : registered ? [registered] : []; +} + +/** + * micromark's own `attention` construct, found by name in the parser — after + * confirming, once per parser, that the wrapper sits in its place. + * + * Returns undefined when there is no stock construct to wrap: then nothing was + * disabled by that name either, and declining is exactly the stock behaviour. + */ +function stockAttention(parser: ParseContext): Construct | undefined { + const stock = registeredAt(parser, ASTERISK).find((c) => c.name === STOCK_ATTENTION); + if (!stock || placementChecked.has(parser)) return stock; + const disabled = parser.constructs.disable.null ?? []; + for (const code of [ASTERISK, UNDERSCORE]) { + const list = registeredAt(parser, code); + const from = list.indexOf(stock); + const to = list.findIndex((c) => c.name === NAME); + const between = list.slice(from + 1, to); + if (from < 0 || to < from || between.some((c) => !(c.name && disabled.includes(c.name)))) { + throw new Error( + `[MarkdownPipeline] ${NAME} is not directly behind micromark's attention at ` + + `character ${code}; it would change which construct wins there. Register ` + + `remarkInlineFastPaths before any plugin that appends constructs at * or _.`, + ); + } + } + placementChecked.add(parser); + return stock; +} + +function tokenizeAttention( + this: TokenizeContext, + effects: Effects, + ok: State, + nok: State, +): State { + const stock = stockAttention(this.parser); + if (!stock) return nok; + return stock.tokenize.call(this, effects, ok, nok); +} + +function resolveAllAttention(events: Event[], context: TokenizeContext): Event[] { + const stock = stockAttention(context.parser); + if (!stock?.resolveAll) return events; + + const muted: Token[] = []; + const opened = new Set(); + for (const [kind, token] of events) { + if (kind !== "enter" || token.type !== "attentionSequence") continue; + const marker = context.sliceSerialize(token).charCodeAt(0); + if (token._close && !token._open && !opened.has(marker)) { + token._close = false; + muted.push(token); + } + if (token._open) opened.add(marker); + } + + try { + return stock.resolveAll(events, context); + } finally { + for (const token of muted) token._close = true; + } +} + +/** Takes `attention`'s place at `*` and `_`; see the header. */ +export const attentionWithoutFutileWalks: Construct = { + name: NAME, + add: "after", + tokenize: tokenizeAttention, + resolveAll: resolveAllAttention, +}; diff --git a/src/utils/markdownPipeline/parser/fastPaths/codeTextFastPath.ts b/src/utils/markdownPipeline/parser/fastPaths/codeTextFastPath.ts new file mode 100644 index 000000000..bc1a95175 --- /dev/null +++ b/src/utils/markdownPipeline/parser/fastPaths/codeTextFastPath.ts @@ -0,0 +1,137 @@ +/** + * Purpose: settle a backtick run that no later run can close in O(run length), + * instead of letting micromark scan to the end of the paragraph to find that + * out. + * + * micromark's `codeText` starts at a run of k backticks and consumes forward + * until a run of EXACTLY k backticks, or the end of the paragraph. When no such + * run exists it fails — and every other unmatched opener pays the same scan. + * Runs of distinct lengths therefore cost O(openers × paragraph), which for + * the soak's `backtick-runs` class is cubic in the number of runs: it had not + * finished after twelve minutes at soak scale (#1407). cmark fixed the same + * class by remembering, per run length, where the last such run was. + * + * So does this construct, registered ahead of `codeText`: + * + * 1. The FIRST opener in a paragraph scans to the end once, recording the + * start offset of the last run of each length, and then defers — so + * `codeText` runs exactly as before on that opener. + * 2. Every LATER opener of length k looks up the last run of length k. If it + * starts after the opener, `codeText` will close on some run of length k, + * so this defers. If not, `codeText` must fail, so the run is consumed as + * plain data — which is what the text tokenizer does with it anyway once + * `codeText` fails. + * + * Why the memo is exact: `codeText` closes on the first MAXIMAL run of exactly + * the opener's length after the opener, and nothing inside a code span can end + * it early. Runs after the opener are delimited by non-backtick characters, so + * a scan from any earlier point segments them identically. The one run whose + * segmentation can differ is the opener's own (after an escaped backtick), and + * `start > opener` excludes it. + * + * `previous` is copied from `codeText` so this construct never changes which + * characters the text tokenizer treats as a break. `inlineFastPaths.test.ts` + * diffs the mdast, positions included, against the stock parser. + * + * @coordinates-with micromarkTypes.ts — the tokenizer API slice used here + * @coordinates-with remarkInlineFastPaths.ts — registers this construct + * @module utils/markdownPipeline/parser/fastPaths/codeTextFastPath + */ +import { + onlyModelledConstructs, + type Code, + type Construct, + type Effects, + type Event, + type ParseContext, + type State, + type TokenizeContext, +} from "./micromarkTypes"; + +const GRAVE_ACCENT = 96; +const NAME = "vmarkInertCodeTextSequence"; +const CODE_TEXT = "codeText"; +const MODELLED: ReadonlySet = new Set([NAME, CODE_TEXT]); + +/** Backtick runs after the paragraph's first opener, by length. */ +interface RunMemo { + /** Offset of the opener whose scan built this memo. */ + from: number; + /** Run length → start offset of the LAST run of that length. */ + lastStart: Map; +} + +const memos = new WeakMap(); +const modelledAtGrave = new WeakMap(); + +/** `codeText`'s own `previous`: a run starts where the previous character is + * not a backtick, or where that backtick was escaped. */ +function previousAllowsCodeText(this: TokenizeContext, code: Code): boolean { + return code !== GRAVE_ACCENT || this.events[this.events.length - 1][1].type === "characterEscape"; +} + +function tokenizeInertCodeTextSequence( + this: TokenizeContext, + effects: Effects, + ok: State, + nok: State, +): State { + if (!onlyModelledConstructs(modelledAtGrave, this.parser, GRAVE_ACCENT, MODELLED)) return nok; + + // `now` is a closure over the tokenizer, not a method, so it needs no `this`. + const { events, now } = this; + const opener = now().offset; + const known = memos.get(events); + const memo = known && known.from <= opener ? known : undefined; + const recording: RunMemo = { from: opener, lastStart: new Map() }; + let size = 0; + let runStart = 0; + let runSize = 0; + return openingRun; + + function openingRun(code: Code): State | undefined { + if (code === GRAVE_ACCENT) { + if (size === 0) effects.enter("data"); + effects.consume(code); + size += 1; + return openingRun; + } + if (!memo) return scanBetween(code); + const last = memo.lastStart.get(size); + if (last !== undefined && last > opener) return nok(code); // codeText will close + effects.exit("data"); + return ok(code); + } + + /** Record every later run to the end of the paragraph, then defer. */ + function scanBetween(code: Code): State | undefined { + if (code === null) { + memos.set(events, recording); + return nok(code); + } + if (code === GRAVE_ACCENT) { + runStart = now().offset; + runSize = 0; + return scanRun(code); + } + effects.consume(code); + return scanBetween; + } + + function scanRun(code: Code): State | undefined { + if (code === GRAVE_ACCENT) { + effects.consume(code); + runSize += 1; + return scanRun; + } + recording.lastStart.set(runSize, runStart); + return scanBetween(code); + } +} + +/** Registered at a backtick, ahead of micromark's `codeText`. */ +export const inertCodeTextSequence: Construct = { + name: NAME, + previous: previousAllowsCodeText, + tokenize: tokenizeInertCodeTextSequence, +}; diff --git a/src/utils/markdownPipeline/parser/fastPaths/inlineFastPaths.test.ts b/src/utils/markdownPipeline/parser/fastPaths/inlineFastPaths.test.ts new file mode 100644 index 000000000..8e02a39c0 --- /dev/null +++ b/src/utils/markdownPipeline/parser/fastPaths/inlineFastPaths.test.ts @@ -0,0 +1,129 @@ +// @vitest-environment node +/** + * #1407 — the inline fast paths must never change a parse. + * + * Each fast path settles a character only on a proof that micromark's own + * construct would fail there, so a parser with the fast paths must produce the + * SAME mdast as one without — node for node, and position for position. This + * file checks exactly that, three ways: + * + * 1. every example of every vendored spec corpus (CommonMark, GFM, and the + * rest of the registry) — the constructs' documented behaviour; + * 2. hand-picked inputs at each fast path's decision boundary — the cases a + * wrong proof would get wrong first; + * 3. a seeded fuzz over an alphabet dense in `[`, `]`, `![`, `^`, backticks, + * `*` and `_` — the interactions nobody thought to write down. + * + * That they make anything FASTER is the other half of the contract, and lives + * in `__tests__/pathological/pathologicalScaling.test.ts`. + * + * @coordinates-with remarkInlineFastPaths.ts — the plugin under test + * @coordinates-with ../../__tests__/spec/corpusRegistry.ts — the corpora + * @module utils/markdownPipeline/parser/fastPaths/inlineFastPaths.test + */ +import { describe, it, expect } from "vitest"; +import fc from "fast-check"; +import { unified } from "unified"; +import remarkParse from "remark-parse"; +import remarkGfm from "remark-gfm"; +import remarkMath from "remark-math"; +import remarkFrontmatter from "remark-frontmatter"; +import { remarkInlineFastPaths } from "./remarkInlineFastPaths"; +import { CORPORA, loadExamples } from "../../__tests__/spec/corpusRegistry"; + +const stock = unified() + .use(remarkParse) + .use(remarkGfm, { singleTilde: false }) + .use(remarkMath) + .use(remarkFrontmatter, ["yaml"]); + +const fast = unified() + .use(remarkParse) + .use(remarkGfm, { singleTilde: false }) + .use(remarkInlineFastPaths) + .use(remarkMath) + .use(remarkFrontmatter, ["yaml"]); + +/** The syntax tree with positions, as a comparable string. */ +function tree(processor: typeof stock, markdown: string): string { + return JSON.stringify(processor.parse(markdown)); +} + +function expectSameTree(markdown: string): void { + expect(tree(fast, markdown), JSON.stringify(markdown)).toBe(tree(stock, markdown)); +} + +const BRACKET_BOUNDARIES = [ + "[a](b)", "[a]", "[a][]", "[a][b]\n\n[b]: /u", "[a]\n\n[a]: /u", "![a](b)", "![a]", + "[![a](b)](c)", "[a [b](c) d](e)", "[a](b]c)", "\\[a](b)", "a]", "]", "]]]", "[]", "[](x)", + "![]()", "[a\nb](c)", "[`a]`](b)", "`[a`](b)", "x]", "", + "[a]()", "*[a*](b)", "[中文](链接)", "[😀]\n\n[😀]: /e", "a](a](a](", "[a]( [a\n> ](b)", "- [a\n ](b)", "| a | [b |\n|---|---|\n| ] | c](d) |", "[a] ]\n\n[a]: /u", + "[a](b) ] [c](d) ]", "[a [b] c](d)", "[[a]](b)", "![[a](b)](c) ]", "![a] ] ]", + // Footnote calls and the potential call behind a balanced image label. + "[^1]\n\n[^1]: note", "![^1]\n\n[^1]: note", "![ ^1]\n\n[^1]: note", "![\t^1]\n\n[^1]: note", + "![\n^1]\n\n[^1]: note", "![x] a] b]\n\n[^1]: note", "![^1] ] ]\n\n[^1]: note", + "![^2]\n\n[^1]: note", "![^1\n\n[^1]: note", "![ ]\n\n[^1]: note", "![^1] [^1]\n\n[^1]: note", + "[^1] ] ![^1]\n\n[^1]: note", "![x] [^1]\n\n[^1]: note", + // A LATER `]` behind a balanced `![^…` can never complete a footnote call — + // its label would hold a bare `]`, and no definition label can. These are the + // spellings that argument has to survive: escaped brackets and backslash + // runs in the definition, in the call, and between them. + "![^y] a] b]\n\n[^x]: note", "![^a] ]\n\n[^a\\]]: note", "![^a\\] ]\n\n[^a\\]]: note", + "![^a\\\\] ]\n\n[^a\\\\]: note", "![^a] b]\n\n[^a]b]: note", "![^\\]] ]\n\n[^\\]]: note", + "![^a]\\] ]\n\n[^a]: note", "![^a `]` ] c]\n\n[^a]: note", "![^a] ]\n\n[^a]: note\n[^a] ]: n", +]; + +const BACKTICK_BOUNDARIES = [ + "`a`", "``a``", "`a``", "``a`", "` `` `", "\\`a`", "\\``a``", "`a\nb`", "e`e``e```", + "`` ` ``", "```a```", "a ` b ` c ` d", "`a` `b`", "``` `a` ```", "`[a](b)`", "\\\\`a`", + "` a`b` `", "> `a\n> b`", "`a`\n\n`b`", "`é`", "``中``", "`\t`", "`a` ``b`` `c", "``a` `b``", + "`a\\`b`", "\\`\\``a`", "e`e``e```e``e`", "`", "``", "a`", "`\n`", "- `a\n b`", + "| `a | b` |\n|---|---|\n| `c` | d |", "``", "[`](`)", "*`a*`", +]; + +const EMPHASIS_BOUNDARIES = [ + "a_ a_ a_", "*a*", "**a**", "_a_", "a*b*c", "a_b_c", "*a_", "_a*", "***a***", "*a **b** c*", + "**a*", "a** b*", "* a *", "a_ _b_", "_a_ b_", "*a* b* c*", "foo*bar*", "__a__", "_ a_", + "a* *b", "*a\n*", "[*a](b*)", "*[a*](b)", "`*a`*", "\\*a*", "a**b**c**", "*(*a*)*", + "_(_a_)_", "a ** b", "**a*b***", "a* b* *c", "a_ b_ _c_", "*a**b*", "a***b* c**", + "中*文*", "a*😀*b", "_a_\n\n_b", "a_ *b* c_", "a* _b_ c*", +]; + +describe("inline fast paths leave every parse unchanged (#1407)", () => { + it("on every vendored spec corpus example", () => { + let checked = 0; + for (const entry of CORPORA) { + for (const example of loadExamples(entry)) { + expectSameTree(example.markdown); + checked += 1; + } + } + // Non-vacuity: the registry is large; a loader that returned nothing + // would otherwise pass this with zero comparisons. + expect(checked).toBeGreaterThan(600); + }); + + it.each(BRACKET_BOUNDARIES)("at a `]` decision boundary: %j", expectSameTree); + + it.each(BACKTICK_BOUNDARIES)("at a backtick decision boundary: %j", expectSameTree); + + it.each(EMPHASIS_BOUNDARIES)("at an emphasis decision boundary: %j", expectSameTree); + + const SEED = Number(process.env.FAST_PATH_SEED ?? "1407"); + const TOKENS = [ + "[", "]", "![", "(", ")", "<", ">", "^", " ", "\t", "\n", "\n\n", "a", "b", "\\", "`", "``", + "*", "**", "_", "__", ":", "/", "中", "😀", "[^1]", "[^1]: n\n\n", "[a]: /u\n\n", "> ", "- ", + "\\\\", "\\]", "![^1", "[^1\\]]: n\n\n", + ]; + const document = fc.array(fc.constantFrom(...TOKENS), { maxLength: 40 }).map((t) => t.join("")); + + it(`on fuzzed delimiter-dense documents (seed ${SEED})`, () => { + fc.assert( + fc.property(document, (markdown) => { + expect(tree(fast, markdown)).toBe(tree(stock, markdown)); + }), + { numRuns: 3000, seed: SEED }, + ); + }, 120_000); +}); diff --git a/src/utils/markdownPipeline/parser/fastPaths/labelEndFastPath.ts b/src/utils/markdownPipeline/parser/fastPaths/labelEndFastPath.ts new file mode 100644 index 000000000..ac742a2e9 --- /dev/null +++ b/src/utils/markdownPipeline/parser/fastPaths/labelEndFastPath.ts @@ -0,0 +1,166 @@ +/** + * Purpose: settle a `]` that nothing can close in O(1) amortized, instead of + * letting micromark walk back through the whole paragraph to find that out. + * + * Two constructs are tried at every `]` in text, and both begin with a + * backward scan over every event tokenized so far: + * + * - `labelEnd` (micromark-core-commonmark) walks back to the nearest label + * start that is not yet balanced; + * - `gfmPotentialFootnoteCall` (micromark-extension-gfm-footnote) walks back + * to the nearest label image, link or footnote call. + * + * When the paragraph has no opener, both walks reach its start and fail — so + * `a](` repeated n times costs O(n²). Measured at the soak's scale (#1407): + * 60s to parse, and 276s to serialize, because the serializer's cosmetic pass + * re-parses its output twice. + * + * This construct runs FIRST at `]`. It keeps an incremental index of the label + * starts in the event list, and consumes the `]` as plain data exactly when no + * unbalanced `labelLink`/`labelImage` exists. Both stock constructs must then + * fail: + * + * - `labelEnd` needs an unbalanced label start, by definition. + * - The potential footnote call needs a balanced `![` whose label, from the + * `![` to this `]`, names a defined footnote. A `![` is balanced only by a + * `]` that label-end tried and rejected — an unescaped `]` that is then + * INSIDE the label of every later `]`. A footnote definition label cannot + * contain an unescaped `]` (gfm's own definition tokenizer ends the label + * there), so no later `]` can complete that call. The one `]` that can is + * the one that balances the `![` — and at that `]` the `![` is still an + * unbalanced opener, so this construct defers. + * + * Anything else defers to the stock constructs, which then run exactly as + * before. So the fast path can only make a parse FASTER; it cannot make one + * different — `inlineFastPaths.test.ts` diffs the mdast, positions included, + * against the stock parser over every vendored spec corpus, the spellings the + * footnote argument has to survive, and a seeded fuzz. + * + * Key decisions: + * - PROOF, NOT REIMPLEMENTATION. Resources, references, `_inactive` links, + * definitions — all of it stays in micromark. Only "can this `]` close + * anything at all?" is answered here, and only the NO answer is acted on. + * - WATCHES, NOT TRUST. A deferred `]` may let a stock construct succeed and + * splice the events from the label start it matched — label-end from the + * nearest unbalanced opener, the footnote call from that same `![`. Both + * resolvers put a NEW event at that index, so the index records the event + * there and re-indexes from it when it changes. Nothing else rewrites + * events below the scanned mark during text tokenization; a tail check + * backstops that claim by rebuilding from zero rather than trusting it. + * - UNKNOWN CONSTRUCTS DEFER. If anything but the two modelled constructs is + * registered at `]`, the fast path cannot prove inertness and never fires. + * + * Known limitation: when a candidate opener DOES exist, stock `labelEnd` + * serializes the whole label text to look up a definition — O(label length) + * per `]`, so `[` × n followed by `]` × n stays quadratic. That cost is the + * lookup itself, not the scan, and removing it would mean changing which + * labels can match a definition. + * + * @coordinates-with micromarkTypes.ts — the tokenizer API slice used here + * @coordinates-with remarkInlineFastPaths.ts — registers this construct + * @module utils/markdownPipeline/parser/fastPaths/labelEndFastPath + */ +import { + onlyModelledConstructs, + type Construct, + type Effects, + type Event, + type ParseContext, + type State, + type TokenizeContext, +} from "./micromarkTypes"; + +const RIGHT_SQUARE_BRACKET = 93; +const NAME = "vmarkInertLabelEnd"; +const MODELLED: ReadonlySet = new Set([NAME, "labelEnd", "gfmPotentialFootnoteCall"]); + +/** Tokens `labelEnd` accepts as an opener (when not `_balanced`). */ +const LABEL_STARTS: ReadonlySet = new Set(["labelLink", "labelImage"]); + +interface Watch { + index: number; + event: Event; +} + +/** Incremental index over ONE tokenizer's event list. */ +interface OpenerIndex { + /** Events `[0, scanned)` are reflected in `openers`. */ + scanned: number; + /** `events[scanned - 1]` when indexed — the backstop's sentinel. */ + tail: Event | undefined; + /** Ascending enter-event indices of label starts not yet seen balanced. */ + openers: number[]; + /** Where a deferred `]` could have let a stock resolver splice. */ + watches: Watch[]; +} + +const indexes = new WeakMap(); +const modelledAtBracket = new WeakMap(); + +/** Bring the index up to date with `events`, re-indexing whatever changed. */ +function syncIndex(events: Event[]): OpenerIndex { + let index = indexes.get(events); + if (!index) { + index = { scanned: 0, tail: undefined, openers: [], watches: [] }; + indexes.set(events, index); + } + + let from = index.scanned; + for (const watch of index.watches) { + if (events[watch.index] !== watch.event) from = Math.min(from, watch.index); + } + index.watches = []; + const untouched = + from === index.scanned && + (index.scanned === 0 || (index.scanned <= events.length && events[index.scanned - 1] === index.tail)); + if (!untouched && from === index.scanned) from = 0; // backstop: an unexplained rewrite + from = Math.min(from, events.length); + + if (from < index.scanned) { + while (index.openers.length > 0 && (index.openers.at(-1) as number) >= from) index.openers.pop(); + index.scanned = from; + } + for (let i = index.scanned; i < events.length; i += 1) { + const [kind, token] = events[i]; + if (kind === "enter" && LABEL_STARTS.has(token.type)) index.openers.push(i); + } + index.scanned = events.length; + index.tail = events[events.length - 1]; + return index; +} + +function tokenizeInertLabelEnd( + this: TokenizeContext, + effects: Effects, + ok: State, + nok: State, +): State { + if (!onlyModelledConstructs(modelledAtBracket, this.parser, RIGHT_SQUARE_BRACKET, MODELLED)) { + return nok; + } + const { events } = this; + const index = syncIndex(events); + + // A label start that failed to match is marked balanced, permanently. + while (index.openers.length > 0 && events[index.openers.at(-1) as number][1]._balanced) { + index.openers.pop(); + } + const opener = index.openers.at(-1); + if (opener !== undefined) { + index.watches.push({ index: opener, event: events[opener] }); + return nok; + } + + return function inertBracket(code) { + effects.enter("data"); + effects.consume(code); + effects.exit("data"); + return ok; + }; +} + +/** Registered at `]`, ahead of micromark's own constructs. */ +export const inertLabelEnd: Construct = { + name: NAME, + tokenize: tokenizeInertLabelEnd, +}; diff --git a/src/utils/markdownPipeline/parser/fastPaths/micromarkTypes.ts b/src/utils/markdownPipeline/parser/fastPaths/micromarkTypes.ts new file mode 100644 index 000000000..7f5277836 --- /dev/null +++ b/src/utils/markdownPipeline/parser/fastPaths/micromarkTypes.ts @@ -0,0 +1,105 @@ +/** + * Purpose: the slice of micromark's tokenizer API the inline fast paths touch, + * declared locally, plus the one lookup they all need — "which constructs + * would micromark try at this character?" + * + * Local rather than imported from `micromark-util-types`: that package is a + * transitive dependency, and adding it to reach six interfaces would put a + * version pin on the parser's internals for no runtime gain. These are + * STRUCTURAL: a field micromark renames shows up as a fast path that stops + * firing — which the scaling test catches — or one that fires wrongly — which + * the equivalence test catches. + * + * @coordinates-with labelEndFastPath.ts + * @coordinates-with codeTextFastPath.ts + * @coordinates-with attentionFastPath.ts + * @module utils/markdownPipeline/parser/fastPaths/micromarkTypes + */ + +/** A character code: a UTF-16 unit, a negative virtual code, or null (EOF). */ +export type Code = number | null; + +export interface Point { + line: number; + column: number; + offset: number; + _index: number; + _bufferIndex: number; +} + +export interface Token { + type: string; + start: Point; + end: Point; + _open?: boolean; + _close?: boolean; + _balanced?: boolean; +} + +export type Event = [kind: "enter" | "exit", token: Token, context: TokenizeContext]; + +export type State = (code: Code) => State | undefined; + +export interface Effects { + enter: (type: string) => Token; + exit: (type: string) => Token; + consume: (code: Code) => void; +} + +export interface Construct { + name?: string; + add?: "before" | "after"; + previous?: (this: TokenizeContext, code: Code) => boolean; + tokenize: (this: TokenizeContext, effects: Effects, ok: State, nok: State) => State; + resolveAll?: (events: Event[], context: TokenizeContext) => Event[]; +} + +export interface ParseContext { + constructs: { + text: Record; + disable: { null?: string[] }; + }; +} + +export interface TokenizeContext { + events: Event[]; + parser: ParseContext; + previous: Code; + now: () => Point; + sliceSerialize: (token: Pick) => string; +} + +/** A micromark syntax extension: constructs keyed by character code. */ +export interface SyntaxExtension { + text?: Record; + disable?: { null: string[] }; +} + +/** Constructs micromark would TRY at `code` in text: registered and not disabled. */ +function activeTextConstructs(parser: ParseContext, code: number): Construct[] { + const registered = parser.constructs.text[code]; + const list = Array.isArray(registered) ? registered : registered ? [registered] : []; + const disabled = parser.constructs.disable.null ?? []; + return list.filter((c) => !(c.name && disabled.includes(c.name))); +} + +/** + * Whether every construct micromark would try at `code` is one a fast path + * models. An unmodelled construct means a fast path cannot prove the character + * is inert, so it must defer — which is always correct, only slower. + * + * Cached per parser: the construct table is fixed once a parser exists. + */ +export function onlyModelledConstructs( + cache: WeakMap, + parser: ParseContext, + code: number, + modelled: ReadonlySet, +): boolean { + let answer = cache.get(parser); + if (answer === undefined) { + answer = activeTextConstructs(parser, code).every((c) => c.name !== undefined && modelled.has(c.name)); + cache.set(parser, answer); + } + return answer; +} diff --git a/src/utils/markdownPipeline/parser/fastPaths/remarkInlineFastPaths.ts b/src/utils/markdownPipeline/parser/fastPaths/remarkInlineFastPaths.ts new file mode 100644 index 000000000..e16c5a366 --- /dev/null +++ b/src/utils/markdownPipeline/parser/fastPaths/remarkInlineFastPaths.ts @@ -0,0 +1,51 @@ +/** + * Purpose: register micromark fast paths that make three classes of hostile + * inline markdown parse in linear time, without changing what any document + * parses to (#1407). + * + * | Fast path | Stock cost it removes | + * |---|---| + * | `inertLabelEnd` at `]` | a walk to the paragraph start per `]` that nothing can close | + * | `inertCodeTextSequence` at a backtick | a scan to the paragraph end per unclosable run | + * | `attentionWithoutFutileWalks` at `*`/`_` | a walk to the paragraph start per closer with no opener | + * + * Each one acts only on a PROOF that micromark's own construct would fail, and + * otherwise defers to it, so the tree is identical — the equivalence tests diff + * the mdast, positions included, against a parser without this plugin. The + * fast paths take the characters micromark would have left as plain data, and + * nothing else. + * + * Why this lives in VMark rather than being waited for upstream: the costs are + * inside micromark (latest versions at the time of writing), and a document + * that takes minutes to open is a frozen editor now. The pathological soak had + * never reached these classes before — an earlier stack overflow ended the run + * first — so they were never measured, not recently introduced. + * + * @coordinates-with labelEndFastPath.ts + * @coordinates-with codeTextFastPath.ts + * @coordinates-with attentionFastPath.ts + * @coordinates-with ../../dialectDescriptors.ts — registers this in every mode + * @module utils/markdownPipeline/parser/fastPaths/remarkInlineFastPaths + */ +import type { Plugin } from "unified"; +import type { Root } from "mdast"; +import { attentionWithoutFutileWalks, STOCK_ATTENTION } from "./attentionFastPath"; +import { inertCodeTextSequence } from "./codeTextFastPath"; +import { inertLabelEnd } from "./labelEndFastPath"; +import type { SyntaxExtension } from "./micromarkTypes"; + +/** The micromark syntax extension this plugin registers. */ +export const inlineFastPathsExtension: SyntaxExtension = { + text: { + 42: [attentionWithoutFutileWalks], + 93: [inertLabelEnd], + 95: [attentionWithoutFutileWalks], + 96: [inertCodeTextSequence], + }, + disable: { null: [STOCK_ATTENTION] }, +}; + +export const remarkInlineFastPaths: Plugin<[], Root> = function remarkInlineFastPaths() { + const data = this.data() as { micromarkExtensions?: unknown[] }; + (data.micromarkExtensions ??= []).push(inlineFastPathsExtension); +}; diff --git a/src/utils/markdownPipeline/plugins/detailsBodyParser.ts b/src/utils/markdownPipeline/plugins/detailsBodyParser.ts index 5d0c440ee..6952fe005 100644 --- a/src/utils/markdownPipeline/plugins/detailsBodyParser.ts +++ b/src/utils/markdownPipeline/plugins/detailsBodyParser.ts @@ -31,6 +31,7 @@ import remarkParse from "remark-parse"; import remarkGfm from "remark-gfm"; import remarkMath from "remark-math"; import remarkFrontmatter from "remark-frontmatter"; +import { remarkInlineFastPaths } from "../parser/fastPaths/remarkInlineFastPaths"; import { remarkDepthLimit } from "../parser/remarkPlugins"; import { remarkCustomInline } from "./customInline"; import { remarkResolveReferences } from "./resolveReferences"; @@ -61,6 +62,7 @@ function buildDefaultBodyProcessor(): DetailsBodyProcessor { return unified() .use(remarkParse) .use(remarkGfm, { singleTilde: false }) + .use(remarkInlineFastPaths) .use(remarkMath) .use(remarkDepthLimit) .use(remarkFrontmatter, ["yaml"]) From 57b74ca1cfb9607132d3a75d0749af3d968473da Mon Sep 17 00:00:00 2001 From: xiaolai Date: Tue, 15 Sep 2026 10:27:31 +0800 Subject: [PATCH 02/13] fix(deps): patch two quadratic loops in micromark and mdast-util-to-markdown With the inline fast paths in, two classes of the pathological soak (#1407) were still super-linear, and both costs sit where no extension API reaches. Both packages are at their latest releases. micromark 4.0.2, text initializer resolveAllText. It merges adjacent data tokens with one events.splice per run, and each splice moves every later event. That is O(runs x events), and it turns sharply worse once a paragraph's event list outgrows V8's regular heap objects: plain escaped prose, the shape of the serializer's own conservative output, parsed in 546ms at 72 KB and 5.5s at 96 KB. It is also what kept unclosed-inline-links quadratic, through the cosmetic pass re-parsing that output. The patch merges in one compacting pass that keeps the same events, sets the same end point on the merged token, and returns the same array. Both the lib and dev builds are patched. mdast-util-to-markdown 2.1.2, safe(). positions.includes(position) runs once per unsafe-pattern match, so escaping is O(escapes^2); a text node of 116k backticks took 15s to serialize. infos gains a key exactly when positions gains the position, so an own-key check is the same test in O(1). Verified against the unpatched copies still in the store: micromark event streams, positions included, were identical over all 947 spec corpus examples and 70000 seeded fuzz documents across the lib and dev builds, and safe() returned identical strings for 100000 fuzzed calls. A version bump of either package now fails install until the patch is re-made. That is deliberate: a patch that silently stopped applying would bring the quadratic back, and pathologicalScaling.test.ts would then fail. The lockfile diff is the patch records and hashes only. --- package.json | 4 + patches/mdast-util-to-markdown@2.1.2.patch | 17 +++ patches/micromark@4.0.2.patch | 121 +++++++++++++++++++++ pnpm-lock.yaml | 32 ++++-- 4 files changed, 162 insertions(+), 12 deletions(-) create mode 100644 patches/mdast-util-to-markdown@2.1.2.patch create mode 100644 patches/micromark@4.0.2.patch diff --git a/package.json b/package.json index 8de2015e4..1de3cafd5 100644 --- a/package.json +++ b/package.json @@ -218,6 +218,10 @@ "js-yaml@4": ">=4.3.2 <5", "baseline-browser-mapping": ">=2.11.0 <3", "smol-toml": ">=1.7.1 <2" + }, + "patchedDependencies": { + "micromark@4.0.2": "patches/micromark@4.0.2.patch", + "mdast-util-to-markdown@2.1.2": "patches/mdast-util-to-markdown@2.1.2.patch" } }, "devDependencies": { diff --git a/patches/mdast-util-to-markdown@2.1.2.patch b/patches/mdast-util-to-markdown@2.1.2.patch new file mode 100644 index 000000000..aff5925de --- /dev/null +++ b/patches/mdast-util-to-markdown@2.1.2.patch @@ -0,0 +1,17 @@ +diff --git a/lib/util/safe.js b/lib/util/safe.js +index 456fe215ae3d032ffaea4590f0d097bcb9195619..10f82c99cff8123142f230c087e6d09742f2b313 100644 +--- a/lib/util/safe.js ++++ b/lib/util/safe.js +@@ -56,7 +56,11 @@ export function safe(state, input, config) { + const after = 'after' in pattern + const position = match.index + (before ? match[1].length : 0) + +- if (positions.includes(position)) { ++ // VMark patch (#1407): `infos` gains a key exactly when `positions` ++ // gains the position, so an own-key check is the same test in O(1). ++ // `positions.includes` made escaping O(escapes²) — a text node of 116k ++ // backticks took 15s to serialize. ++ if (Object.hasOwn(infos, position)) { + if (infos[position].before && !before) { + infos[position].before = false + } diff --git a/patches/micromark@4.0.2.patch b/patches/micromark@4.0.2.patch new file mode 100644 index 000000000..aa6e4cff8 --- /dev/null +++ b/patches/micromark@4.0.2.patch @@ -0,0 +1,121 @@ +diff --git a/dev/lib/initialize/text.js b/dev/lib/initialize/text.js +index 7192e70a0972e808d779999e57153bf3ed7bfc3a..0eaba5658038c8f1885f1279b8f30ba720a72d65 100644 +--- a/dev/lib/initialize/text.js ++++ b/dev/lib/initialize/text.js +@@ -113,30 +113,39 @@ function createResolver(extraResolver) { + + /** @type {Resolver} */ + function resolveAllText(events, context) { +- let index = -1 +- /** @type {number | undefined} */ +- let enter +- + // A rather boring computation (to merge adjacent `data` events) which + // improves mm performance by 29%. +- while (++index <= events.length) { +- if (enter === undefined) { +- if (events[index] && events[index][1].type === types.data) { +- enter = index +- index++ +- } +- } else if (!events[index] || events[index][1].type !== types.data) { +- // Don’t do anything if there is one data token. +- if (index !== enter + 2) { +- events[enter][1].end = events[index - 1][1].end +- events.splice(enter + 2, index - enter - 2) +- index = enter + 2 +- } +- +- enter = undefined ++ // ++ // VMark patch (#1407): merge in ONE compacting pass. The original removed ++ // each run with `events.splice`, moving every later event once per run — ++ // O(runs × events), and ~10× slower per step once the list outgrows V8's ++ // regular heap objects. The kept events, the merged token's `end`, and the ++ // returned array are identical. ++ const length = events.length ++ let write = 0 ++ let index = 0 ++ ++ while (index < length) { ++ const event = events[index] ++ ++ if (event[1].type !== types.data) { ++ events[write++] = event ++ index++ ++ continue + } ++ ++ // The enter's own exit is at `index + 1`; a run continues from there. ++ let end = index + 2 ++ while (end < length && events[end][1].type === types.data) end++ ++ // Don’t do anything if there is one data token. ++ if (end !== index + 2) event[1].end = events[end - 1][1].end ++ events[write++] = event ++ if (index + 1 < length) events[write++] = events[index + 1] ++ index = end + } + ++ events.length = write ++ + return extraResolver ? extraResolver(events, context) : events + } + } +diff --git a/lib/initialize/text.js b/lib/initialize/text.js +index b9383cb721b95c68bd0e684da6cbf441ed82b8eb..eac1239a4263cd6945fc36b8db8bb1927659d6f7 100644 +--- a/lib/initialize/text.js ++++ b/lib/initialize/text.js +@@ -104,28 +104,35 @@ function createResolver(extraResolver) { + + /** @type {Resolver} */ + function resolveAllText(events, context) { +- let index = -1; +- /** @type {number | undefined} */ +- let enter; +- + // A rather boring computation (to merge adjacent `data` events) which + // improves mm performance by 29%. +- while (++index <= events.length) { +- if (enter === undefined) { +- if (events[index] && events[index][1].type === "data") { +- enter = index; +- index++; +- } +- } else if (!events[index] || events[index][1].type !== "data") { +- // Don’t do anything if there is one data token. +- if (index !== enter + 2) { +- events[enter][1].end = events[index - 1][1].end; +- events.splice(enter + 2, index - enter - 2); +- index = enter + 2; +- } +- enter = undefined; ++ // ++ // VMark patch (#1407): merge in ONE compacting pass. The original removed ++ // each run with `events.splice`, moving every later event once per run — ++ // O(runs × events), and ~10× slower per step once the list outgrows V8's ++ // regular heap objects. The kept events, the merged token's `end`, and the ++ // returned array are identical. ++ const length = events.length; ++ let write = 0; ++ let index = 0; ++ while (index < length) { ++ const event = events[index]; ++ if (event[1].type !== "data") { ++ events[write++] = event; ++ index++; ++ continue; + } ++ ++ // The enter's own exit is at `index + 1`; a run continues from there. ++ let end = index + 2; ++ while (end < length && events[end][1].type === "data") end++; ++ // Don’t do anything if there is one data token. ++ if (end !== index + 2) event[1].end = events[end - 1][1].end; ++ events[write++] = event; ++ if (index + 1 < length) events[write++] = events[index + 1]; ++ index = end; + } ++ events.length = write; + return extraResolver ? extraResolver(events, context) : events; + } + } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 911c870fb..1423b900c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -30,6 +30,14 @@ overrides: baseline-browser-mapping: '>=2.11.0 <3' smol-toml: '>=1.7.1 <2' +patchedDependencies: + mdast-util-to-markdown@2.1.2: + hash: 2dbee866e6cf244b099fa367885b1e8ad51ff315b895c1ad77581dee0f4206bf + path: patches/mdast-util-to-markdown@2.1.2.patch + micromark@4.0.2: + hash: c4162f5915a6a3308e12ec56d2a8ef6a3a9cbaa7abea611d0ce5957fc4f620ea + path: patches/micromark@4.0.2.patch + importers: .: @@ -16614,7 +16622,7 @@ snapshots: markdownlint@0.41.1: dependencies: - micromark: 4.0.2 + micromark: 4.0.2(patch_hash=c4162f5915a6a3308e12ec56d2a8ef6a3a9cbaa7abea611d0ce5957fc4f620ea) micromark-core-commonmark: 2.0.3 micromark-extension-directive: 4.0.0 micromark-extension-gfm-autolink-literal: 2.1.0 @@ -16687,7 +16695,7 @@ snapshots: decode-named-character-reference: 1.3.0 devlop: 1.1.0 mdast-util-to-string: 4.0.0 - micromark: 4.0.2 + micromark: 4.0.2(patch_hash=c4162f5915a6a3308e12ec56d2a8ef6a3a9cbaa7abea611d0ce5957fc4f620ea) micromark-util-decode-numeric-character-reference: 2.0.2 micromark-util-decode-string: 2.0.1 micromark-util-normalize-identifier: 2.0.1 @@ -16703,7 +16711,7 @@ snapshots: devlop: 1.1.0 escape-string-regexp: 5.0.0 mdast-util-from-markdown: 2.0.3 - mdast-util-to-markdown: 2.1.2 + mdast-util-to-markdown: 2.1.2(patch_hash=2dbee866e6cf244b099fa367885b1e8ad51ff315b895c1ad77581dee0f4206bf) micromark-extension-frontmatter: 2.0.0 transitivePeerDependencies: - supports-color @@ -16721,7 +16729,7 @@ snapshots: '@types/mdast': 4.0.4 devlop: 1.1.0 mdast-util-from-markdown: 2.0.3 - mdast-util-to-markdown: 2.1.2 + mdast-util-to-markdown: 2.1.2(patch_hash=2dbee866e6cf244b099fa367885b1e8ad51ff315b895c1ad77581dee0f4206bf) micromark-util-normalize-identifier: 2.0.1 transitivePeerDependencies: - supports-color @@ -16730,7 +16738,7 @@ snapshots: dependencies: '@types/mdast': 4.0.4 mdast-util-from-markdown: 2.0.3 - mdast-util-to-markdown: 2.1.2 + mdast-util-to-markdown: 2.1.2(patch_hash=2dbee866e6cf244b099fa367885b1e8ad51ff315b895c1ad77581dee0f4206bf) transitivePeerDependencies: - supports-color @@ -16740,7 +16748,7 @@ snapshots: devlop: 1.1.0 markdown-table: 3.0.4 mdast-util-from-markdown: 2.0.3 - mdast-util-to-markdown: 2.1.2 + mdast-util-to-markdown: 2.1.2(patch_hash=2dbee866e6cf244b099fa367885b1e8ad51ff315b895c1ad77581dee0f4206bf) transitivePeerDependencies: - supports-color @@ -16749,7 +16757,7 @@ snapshots: '@types/mdast': 4.0.4 devlop: 1.1.0 mdast-util-from-markdown: 2.0.3 - mdast-util-to-markdown: 2.1.2 + mdast-util-to-markdown: 2.1.2(patch_hash=2dbee866e6cf244b099fa367885b1e8ad51ff315b895c1ad77581dee0f4206bf) transitivePeerDependencies: - supports-color @@ -16761,7 +16769,7 @@ snapshots: mdast-util-gfm-strikethrough: 2.0.0 mdast-util-gfm-table: 2.0.0 mdast-util-gfm-task-list-item: 2.0.0 - mdast-util-to-markdown: 2.1.2 + mdast-util-to-markdown: 2.1.2(patch_hash=2dbee866e6cf244b099fa367885b1e8ad51ff315b895c1ad77581dee0f4206bf) transitivePeerDependencies: - supports-color @@ -16772,7 +16780,7 @@ snapshots: devlop: 1.1.0 longest-streak: 3.1.0 mdast-util-from-markdown: 2.0.3 - mdast-util-to-markdown: 2.1.2 + mdast-util-to-markdown: 2.1.2(patch_hash=2dbee866e6cf244b099fa367885b1e8ad51ff315b895c1ad77581dee0f4206bf) unist-util-remove-position: 5.0.0 transitivePeerDependencies: - supports-color @@ -16799,7 +16807,7 @@ snapshots: unist-util-visit: 5.1.0 vfile: 6.0.3 - mdast-util-to-markdown@2.1.2: + mdast-util-to-markdown@2.1.2(patch_hash=2dbee866e6cf244b099fa367885b1e8ad51ff315b895c1ad77581dee0f4206bf): dependencies: '@types/mdast': 4.0.4 '@types/unist': 3.0.3 @@ -17047,7 +17055,7 @@ snapshots: micromark-util-types@2.0.2: {} - micromark@4.0.2: + micromark@4.0.2(patch_hash=c4162f5915a6a3308e12ec56d2a8ef6a3a9cbaa7abea611d0ce5957fc4f620ea): dependencies: '@types/debug': 4.1.13 debug: 4.4.3 @@ -17976,7 +17984,7 @@ snapshots: remark-stringify@11.0.0: dependencies: '@types/mdast': 4.0.4 - mdast-util-to-markdown: 2.1.2 + mdast-util-to-markdown: 2.1.2(patch_hash=2dbee866e6cf244b099fa367885b1e8ad51ff315b895c1ad77581dee0f4206bf) unified: 11.0.5 require-directory@2.1.1: {} From 07ad4c4ef28c123ff95e3848b4d0c5420d491096 Mon Sep 17 00:00:00 2001 From: xiaolai Date: Tue, 15 Sep 2026 10:27:41 +0800 Subject: [PATCH 03/13] fix(markdown): find each escape's line context in one forward pass The serializer's cosmetic pass decides, for every defensive escape, whether only whitespace precedes it on its line. It did that with a lastIndexOf back to the line start plus a slice per escape: O(n^2) on a long line of escapes. Every hostile inline class in the pathological soak (#1407) serializes to exactly such a line, and on backtick-runs this loop was the largest single cost in the serializer profile. One forward pass now tracks the current line start and the last non-whitespace index as it moves between matches. \s is the set trimStart removes, so the decision is the same one as before. Adding the pass took serializerCosmetics.ts over the 300-line limit, so the code-range helpers move to serializerCodeRanges.ts. They are the natural seam: the hard-break pass in serializer.ts uses them too, and nothing in them depends on the cosmetic pass. --- src/utils/markdownPipeline/serializer.ts | 7 +- .../markdownPipeline/serializerCodeRanges.ts | 88 +++++++++++++++++ .../markdownPipeline/serializerCosmetics.ts | 96 ++++--------------- 3 files changed, 109 insertions(+), 82 deletions(-) create mode 100644 src/utils/markdownPipeline/serializerCodeRanges.ts diff --git a/src/utils/markdownPipeline/serializer.ts b/src/utils/markdownPipeline/serializer.ts index 70693b7ad..58ccba313 100644 --- a/src/utils/markdownPipeline/serializer.ts +++ b/src/utils/markdownPipeline/serializer.ts @@ -36,11 +36,8 @@ import { remarkCustomInline, remarkDetailsBlock, remarkWikiLinks, tocToMarkdown import { handleImage, handleLink, blankLinesJoin } from "./serializerHandlers"; import type { MarkdownPipelineOptions } from "./types"; import { parseMarkdownToMdast } from "./parser"; -import { - applyCosmeticPass, - buildCodeRanges, - replaceOutsideCode, -} from "./serializerCosmetics"; +import { applyCosmeticPass } from "./serializerCosmetics"; +import { buildCodeRanges, replaceOutsideCode } from "./serializerCodeRanges"; /** * Build the unified processor configured for VMark markdown serialization. diff --git a/src/utils/markdownPipeline/serializerCodeRanges.ts b/src/utils/markdownPipeline/serializerCodeRanges.ts new file mode 100644 index 000000000..02f07e0bf --- /dev/null +++ b/src/utils/markdownPipeline/serializerCodeRanges.ts @@ -0,0 +1,88 @@ +/** + * Serialized-markdown code ranges — where the post-stringify passes must not + * rewrite text. + * + * Purpose: the cosmetic pass and the hard-break pass both edit the string + * remark-stringify produced, and both must leave fenced code blocks and inline + * code spans alone. This module answers "is this offset inside code?" for + * them, from one sorted range list. + * + * Split out of `serializerCosmetics.ts` to keep it within its size budget. + * + * @coordinates-with serializerCosmetics.ts — skips escapes inside code + * @coordinates-with serializer.ts — the hard-break pass + * @module utils/markdownPipeline/serializerCodeRanges + */ + +/** + * Build sorted, merged character ranges for fenced code blocks and inline + * code spans. Ranges are non-overlapping and sorted by start, enabling + * O(log N) `isInsideCode` lookups during escape processing. + */ +export function buildCodeRanges(markdown: string): Array<[number, number]> { + const raw: Array<[number, number]> = []; + const fenceRe = /^(`{3,}|~{3,}).*\n([\s\S]*?\n)\1\s*$/gm; + let fm: RegExpExecArray | null; + while ((fm = fenceRe.exec(markdown))) { + raw.push([fm.index, fm.index + fm[0].length]); + } + // Only treat unescaped backticks as code-span boundaries. Without this, + // serialized plain text such as `[\`LICENSE\`]\(./LICENSE).` would falsely + // register `\`LICENSE\`` as an inline code range, blocking later escape + // stripping on the contained `\``. + const inlineRe = /(? a[0] - b[0]); + const merged: Array<[number, number]> = [raw[0]]; + for (let i = 1; i < raw.length; i++) { + const last = merged[merged.length - 1]; + const [s, e] = raw[i]; + if (s <= last[1]) { + if (e > last[1]) last[1] = e; + } else { + merged.push([s, e]); + } + } + return merged; +} + +/** + * Binary-search a sorted, non-overlapping ranges array for whether `offset` + * falls inside any range. O(log N) vs the previous O(N) `Array.some`. + */ +export function isInsideCodeRange( + ranges: Array<[number, number]>, + offset: number +): boolean { + let lo = 0; + let hi = ranges.length - 1; + while (lo <= hi) { + const mid = (lo + hi) >> 1; + const [s, e] = ranges[mid]; + if (s <= offset) { + if (offset < e) return true; + lo = mid + 1; + } else { + hi = mid - 1; + } + } + return false; +} + +/** Apply a regex replacement only outside code blocks and inline code. */ +export function replaceOutsideCode( + markdown: string, + re: RegExp, + replacement: string, + ranges: Array<[number, number]> +): string { + return markdown.replace(re, (match, ...args) => { + const offset = args[args.length - 2] as number; + if (isInsideCodeRange(ranges, offset)) return match; + return match.replace(re, replacement); + }); +} diff --git a/src/utils/markdownPipeline/serializerCosmetics.ts b/src/utils/markdownPipeline/serializerCosmetics.ts index dc500def7..7d5b3c770 100644 --- a/src/utils/markdownPipeline/serializerCosmetics.ts +++ b/src/utils/markdownPipeline/serializerCosmetics.ts @@ -10,10 +10,12 @@ * * @coordinates-with serializer.ts — the only caller * @coordinates-with parser.ts — the re-parse used to verify each edit + * @coordinates-with serializerCodeRanges.ts — where edits must not reach * @module utils/markdownPipeline/serializerCosmetics */ import { parseMarkdownToMdast } from "./parser"; +import { buildCodeRanges, isInsideCodeRange } from "./serializerCodeRanges"; /** * Strip unnecessary backslash escapes added by remark-stringify. @@ -60,79 +62,6 @@ export const BLOCK_START_GUARD: ReadonlySet = new Set( [...BLOCK_START_CHARS].filter((c) => UNESCAPABLE_CHARS.has(c)), ); -/** - * Build sorted, merged character ranges for fenced code blocks and inline - * code spans. Ranges are non-overlapping and sorted by start, enabling - * O(log N) `isInsideCode` lookups during escape processing. - */ -export function buildCodeRanges(markdown: string): Array<[number, number]> { - const raw: Array<[number, number]> = []; - const fenceRe = /^(`{3,}|~{3,}).*\n([\s\S]*?\n)\1\s*$/gm; - let fm: RegExpExecArray | null; - while ((fm = fenceRe.exec(markdown))) { - raw.push([fm.index, fm.index + fm[0].length]); - } - // Only treat unescaped backticks as code-span boundaries. Without this, - // serialized plain text such as `[\`LICENSE\`]\(./LICENSE).` would falsely - // register `\`LICENSE\`` as an inline code range, blocking later escape - // stripping on the contained `\``. - const inlineRe = /(? a[0] - b[0]); - const merged: Array<[number, number]> = [raw[0]]; - for (let i = 1; i < raw.length; i++) { - const last = merged[merged.length - 1]; - const [s, e] = raw[i]; - if (s <= last[1]) { - if (e > last[1]) last[1] = e; - } else { - merged.push([s, e]); - } - } - return merged; -} - -/** - * Binary-search a sorted, non-overlapping ranges array for whether `offset` - * falls inside any range. O(log N) vs the previous O(N) `Array.some`. - */ -function isInsideCodeRange( - ranges: Array<[number, number]>, - offset: number -): boolean { - let lo = 0; - let hi = ranges.length - 1; - while (lo <= hi) { - const mid = (lo + hi) >> 1; - const [s, e] = ranges[mid]; - if (s <= offset) { - if (offset < e) return true; - lo = mid + 1; - } else { - hi = mid - 1; - } - } - return false; -} - -/** Apply a regex replacement only outside code blocks and inline code. */ -export function replaceOutsideCode( - markdown: string, - re: RegExp, - replacement: string, - ranges: Array<[number, number]> -): string { - return markdown.replace(re, (match, ...args) => { - const offset = args[args.length - 2] as number; - if (isInsideCodeRange(ranges, offset)) return match; - return match.replace(re, replacement); - }); -} - /** One pending cosmetic replacement on the serialized string. */ interface CosmeticEdit { start: number; @@ -177,21 +106,34 @@ function collectEntityEdits( return edits; } -/** Collect candidate escape strips, applying the same guards as before. */ +/** + * Collect candidate escape strips, applying the same guards as before. + * + * "Only whitespace before it on its line" is tracked in ONE forward pass. It + * was a `lastIndexOf` + `slice` per escape, which walks back to the line start + * each time — O(n²) on a long line of escapes, and every hostile inline class + * serializes to exactly that (#1407). `\s` is the set `trimStart` removes. + */ function collectEscapeEdits( markdown: string, ranges: Array<[number, number]> ): CosmeticEdit[] { const edits: CosmeticEdit[] = []; const re = new RegExp(SAFE_UNESCAPE_RE.source, "g"); + let scanned = 0; + let lineStart = 0; + let lastInk = -1; // last non-whitespace index before `scanned` let m: RegExpExecArray | null; while ((m = re.exec(markdown))) { const offset = m.index; + for (; scanned < offset; scanned += 1) { + const ch = markdown[scanned]; + if (ch === "\n") lineStart = scanned + 1; + else if (!/\s/.test(ch)) lastInk = scanned; + } if (isInsideCodeRange(ranges, offset)) continue; const char = m[1]; - const lineStart = markdown.lastIndexOf("\n", offset - 1) + 1; - const beforeOnLine = markdown.slice(lineStart, offset).trimStart(); - if (beforeOnLine === "" && BLOCK_START_GUARD.has(char)) continue; + if (lastInk < lineStart && BLOCK_START_GUARD.has(char)) continue; edits.push({ start: offset, end: offset + 2, replacement: char }); } return edits; From 6db9f31ee9d477e958b6b991a180bba5df1aff61 Mon Sep 17 00:00:00 2001 From: xiaolai Date: Tue, 15 Sep 2026 10:28:08 +0800 Subject: [PATCH 04/13] test(markdown): assert hostile inline classes scale linearly in the PR tier The only place a super-linear parse or serialize ever showed up was the weekly soak, and it showed up as a hang (#1407). This brings the property into check:all. It asserts a growth EXPONENT, never a duration: wall-clock bounds flake under contention, which is why performance.test.ts is opt-in. Contention slows a small and a large run alike, but it cannot turn cost proportional to n into n^2, which for an 8x input is the gap between ~8x and ~64x the CPU time. Process CPU time, interleaved runs keeping the minimum, and a warm-up keep the measurement honest under load. Eight classes: link closers with no opener, after an image, after a footnote-like image with footnotes defined, and between real links; unclosed inline links; emphasis closers without openers; backtick runs; and escaped prose past V8's large-object threshold (parse only, for cost). Measured at a load average above 150, every class is 1.04 to 1.18 with the fixes. Each was RED first: 1.82 to 2.35 against the unfixed pipeline, 1.41 for backtick-runs, 1.61 for the footnote-like image against the first version of the ] fast path, and 1.55 for the between-links case with the fast path's re-index watch removed. backtick-runs carries a tighter bound of 1.25, because its broken cost is n^3 in the run count, which is size^1.5 rather than size^2. --- .../pathological/pathologicalScaling.test.ts | 195 ++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100644 src/utils/markdownPipeline/__tests__/pathological/pathologicalScaling.test.ts diff --git a/src/utils/markdownPipeline/__tests__/pathological/pathologicalScaling.test.ts b/src/utils/markdownPipeline/__tests__/pathological/pathologicalScaling.test.ts new file mode 100644 index 000000000..c1f7fd127 --- /dev/null +++ b/src/utils/markdownPipeline/__tests__/pathological/pathologicalScaling.test.ts @@ -0,0 +1,195 @@ +// @vitest-environment node +/** + * #1407 — the pathological classes must scale LINEARLY with input size. + * + * The soak tier runs these classes at cmark's full sizes inside a killable + * child, weekly. That is the only place a super-linear parse or serialize was + * ever visible, and it was visible as a HANG: `link-closers-with-openers` + * alone took 60s to parse and 276s to serialize at soak scale, against a 300s + * wall ceiling for the whole suite. This file brings the property into the PR + * tier, where it is cheap. + * + * It asserts a GROWTH EXPONENT, never a duration. Wall-clock bounds flake + * under contention (`../performance.test.ts` is opt-in for exactly that + * reason), and a busy machine slows a small run and a large run alike. What + * contention cannot do is turn cost ∝ n into cost ∝ n²: for an 8× larger input + * that is the difference between ~8× and ~64× the CPU time, so the exponent + * separates them with a wide margin on both sides. Three things keep the + * measurement honest under load: + * + * - CPU time of this process (`process.cpuUsage`), not wall time, so being + * descheduled costs nothing. Vitest runs each file in its own fork. + * - Small and large runs are INTERLEAVED and the minimum of each is kept, so + * a burst of contention or a GC pause inflates one sample, not the answer. + * - A warm-up parse first, so JIT compilation is not billed to the small run + * (which would make growth look better or worse than it is). + * + * Fixed overhead per call (processor lookup, content analysis) can only pull + * the exponent DOWN, so it can hide nothing the bound is meant to catch. + * + * @coordinates-with pathologicalCases.ts — the soak-tier shapes these mirror + * @coordinates-with ../../parser/fastPaths/ — the linear decisions under test + * @coordinates-with ../../serializerCosmetics.ts — the linear escape scan + * @coordinates-with patches/micromark@4.0.2.patch — linear data-token merge + * @coordinates-with patches/mdast-util-to-markdown@2.1.2.patch — linear escaping + * @module utils/markdownPipeline/__tests__/pathological/pathologicalScaling.test + */ +import { describe, it, expect } from "vitest"; +import { getSchema } from "@tiptap/core"; +import StarterKit from "@tiptap/starter-kit"; +import "../../dialect"; +import { parseMarkdown, serializeMarkdown } from "../../adapter"; + +const schema = getSchema([StarterKit]); + +/** Linear is 1, quadratic is 2. Measured with the fixes: 1.04–1.18 under a + * load average above 150. Without them: 1.82–2.35 for every case except + * `backtick-runs`, which carries its own bound below. */ +const MAX_EXPONENT = 1.35; + +interface ScalingCase { + name: string; + /** Build the input for size parameter `n`. */ + make: (n: number) => string; + small: number; + large: number; + /** Measure parse alone — for a parser property whose input must be big. */ + parseOnly?: boolean; + /** A tighter bound, for a class whose broken growth is below quadratic. */ + maxExponent?: number; +} + +const backtickRuns = (n: number): string => { + let out = ""; + for (let i = 1; i <= n; i += 1) out += `e${"`".repeat(i)}`; + return `${out}\n`; +}; + +const CASES: ScalingCase[] = [ + { + // `]` with no opener anywhere: every one used to walk the whole paragraph. + name: "link-closers-with-openers", + make: (n) => `${"a](".repeat(n)}\n`, + small: 500, + large: 4000, + }, + { + // Parses linearly, but its SERIALIZED form escapes every `[` — so the + // cosmetic pass re-parses a paragraph of openerless `]`. + name: "unclosed-inline-links", + make: (n) => `${"[a]( `![x] ${"a](".repeat(n)}\n`, + small: 500, + large: 4000, + }, + { + // …including an image label that LOOKS like a footnote call while + // footnotes are defined, so the footnote-call check has something to look + // up. A later `]` still cannot complete that call (its label would contain + // a bare `]`, which no definition label can), so it must not reintroduce + // the walk either. + name: "link-closers-after-a-footnote-like-image", + make: (n) => `![^y] ${"a](".repeat(n)}\n\n[^x]: note\n`, + small: 500, + large: 4000, + }, + { + // Real links between the stray `]`s: each one rewrites the event list, so + // the opener index must re-index from the link — not from the start of + // the paragraph, which would put the quadratic walk back one link at a time. + name: "link-closers-between-links", + make: (n) => `${"[a](b) ] ".repeat(n)}\n`, + small: 300, + large: 2400, + }, + { + // Every `_` closes and none opens: each used to walk back to the start. + name: "emph-closers-without-openers", + make: (n) => `${"a_ ".repeat(n)}\n`, + small: 500, + large: 4000, + }, + { + // Runs of distinct lengths never close, so each opener scanned to the end + // of the paragraph. And every backtick is escaped on the way out, which is + // where remark-stringify's escaping was quadratic in the escape count. + // Input size grows with n², so these sizes are a ~16× size step — and the + // broken scan costs n³, i.e. size^1.5, not size². Measured: 1.04–1.11 + // fixed, 1.41–1.48 broken, so the bound sits between them. + name: "backtick-runs", + make: backtickRuns, + small: 120, + large: 480, + maxExponent: 1.25, + }, + { + // Ordinary escaped prose — what the serializer's conservative output looks + // like. micromark merged adjacent data tokens with one `splice` per run, + // each moving the rest of the paragraph's events: harmless until the event + // list outgrows V8's regular heap objects, then ~10× per 1.3× more text. + // So this case needs a paragraph past that size (~100 KB), and measures + // parse alone to keep the cost of doing so down. + name: "escaped-prose", + make: (n) => `${"\\*ab c".repeat(n)}\n`, + small: 6000, + large: 24000, + parseOnly: true, + }, +]; + +function roundTrip(markdown: string): void { + serializeMarkdown(schema, parseMarkdown(schema, markdown)); +} + +function parseOnce(markdown: string): void { + parseMarkdown(schema, markdown); +} + +function cpuMs(fn: () => void): number { + const start = process.cpuUsage(); + fn(); + const used = process.cpuUsage(start); + return (used.user + used.system) / 1000; +} + +/** Minimum CPU cost of small and large inputs, interleaved. */ +function measure( + run: (markdown: string) => void, + small: string, + large: string, +): { small: number; large: number } { + run(small); // warm-up: JIT and processor caches + let bestSmall = Number.POSITIVE_INFINITY; + let bestLarge = Number.POSITIVE_INFINITY; + for (let round = 0; round < 3; round += 1) { + bestSmall = Math.min(bestSmall, cpuMs(() => run(small))); + bestLarge = Math.min(bestLarge, cpuMs(() => run(large))); + bestSmall = Math.min(bestSmall, cpuMs(() => run(small))); + } + return { small: bestSmall, large: bestLarge }; +} + +describe("pathological inputs scale linearly (#1407)", () => { + it.each(CASES)("$name: cost grows linearly with input size", (c) => { + const small = c.make(c.small); + const large = c.make(c.large); + const cost = measure(c.parseOnly ? parseOnce : roundTrip, small, large); + // A floor on the small sample keeps a sub-millisecond reading from + // manufacturing a huge ratio out of timer resolution. + const smallCost = Math.max(cost.small, 1); + const exponent = Math.log(cost.large / smallCost) / Math.log(large.length / small.length); + expect( + exponent, + `${c.name}: ${small.length} chars → ${cost.small.toFixed(1)}ms, ` + + `${large.length} chars → ${cost.large.toFixed(1)}ms ` + + `(exponent ${exponent.toFixed(2)}; 1 is linear, 2 is quadratic)`, + ).toBeLessThan(c.maxExponent ?? MAX_EXPONENT); + }, 600_000); +}); From 899e0c3a56c6110b153da670a4e67bb73fbf0746 Mon Sep 17 00:00:00 2001 From: xiaolai Date: Tue, 15 Sep 2026 10:28:55 +0800 Subject: [PATCH 05/13] test(soak): count the nesting refusal as handled in every hostile-input sweep The OSS-Fuzz leg of the corpora soak (#1407) failed on a corpus file nested 16382 levels deep, reproduced locally with the same error. The pipeline refuses such a document on purpose: past MAX_NESTING_DEPTH the parser's own dependencies overflow the stack, and 8c6bc00c4 made that a named NestingTooDeepError instead of a RangeError (#1374). The soak treated the designed refusal as a crash. The pathological runner already classified it, by hand. Both sweeps now go through one helper, parseArbitraryMarkdown, which returns a refusal as data and rethrows every other failure. The classification is by type along the cause chain, never by message: nestingRefusal() reads the depth and limit off the error, and an error that merely says it is a nesting refusal is not one. Only a sweep's INPUT may be refused. The soak's re-parses read the pipeline's own output, where a refusal would be a real defect, so they still throw. The soak reports refused inputs loudly, measures the oscillation fraction over the inputs it actually round-tripped, and requires more than 100 of them, so a refusal cannot eat the sample and pass. The same sweep had a second, latent failure the hang was hiding. At the soak's scale 8, deep-blockquotes (4000 levels) and deep-lists (1600) are refused, but the parent test only counted parsed classes as finished, so it would have failed on them next; reproduced at PATHOLOGICAL_SCALE=2.1. Each case now declares its container depth from its generator, and the parent requires exactly those above the limit to be refused and every other class to parse. A refusal nobody expected fails instead of passing as finished. The scale is read in one place, so child and parent cannot disagree. The other property and fuzz tests that parse markdown build inputs from fixed pools that cannot nest anywhere near the limit, so they need no classification. --- src/test/arbitraryMarkdown.test.ts | 54 +++++++++++++++++++ src/test/arbitraryMarkdown.ts | 41 ++++++++++++++ src/test/externalCorpora.soak.test.ts | 24 +++++++-- .../pathological/pathological.test.ts | 21 ++++++-- .../pathological/pathologicalCases.ts | 24 +++++++++ .../__tests__/pathological/runCases.ts | 33 ++++++------ .../markdownPipeline/nestingDepth.test.ts | 53 +++++++++++++++++- src/utils/markdownPipeline/nestingDepth.ts | 26 +++++++-- 8 files changed, 247 insertions(+), 29 deletions(-) create mode 100644 src/test/arbitraryMarkdown.test.ts create mode 100644 src/test/arbitraryMarkdown.ts diff --git a/src/test/arbitraryMarkdown.test.ts b/src/test/arbitraryMarkdown.test.ts new file mode 100644 index 000000000..217611650 --- /dev/null +++ b/src/test/arbitraryMarkdown.test.ts @@ -0,0 +1,54 @@ +// @vitest-environment node +/** + * #1407 — the sweep-side classification of the nesting refusal. + * + * The OSS-Fuzz soak died on a corpus file nested 16382 levels deep because it + * treated the pipeline's designed refusal as a crash. The helper under test is + * the one place a sweep decides which is which, so both halves are pinned: + * a refusal comes back as data, and every other failure still throws. + * + * @coordinates-with arbitraryMarkdown.ts + * @module test/arbitraryMarkdown.test + */ +import { describe, it, expect } from "vitest"; +import type { Schema } from "@tiptap/pm/model"; +import { parseArbitraryMarkdown } from "./arbitraryMarkdown"; +import { MAX_NESTING_DEPTH } from "@/utils/markdownPipeline/nestingDepth"; +import { testSchema } from "@/utils/markdownPipeline/testSchema"; + +describe("parseArbitraryMarkdown", () => { + it("returns the document for ordinary markdown", () => { + const outcome = parseArbitraryMarkdown(testSchema, "# Title\n\nbody 中文\n"); + expect(outcome.kind).toBe("parsed"); + expect(outcome.kind === "parsed" && outcome.doc.childCount).toBe(2); + }); + + it("returns an empty document for empty input", () => { + const outcome = parseArbitraryMarkdown(testSchema, ""); + expect(outcome.kind).toBe("parsed"); + }); + + it.each([ + ["blockquotes", `${"> ".repeat(MAX_NESTING_DEPTH + 1)}a\n`, MAX_NESTING_DEPTH + 1], + ["lists", Array.from({ length: MAX_NESTING_DEPTH + 5 }, (_, i) => `${" ".repeat(i)}- a`).join("\n"), MAX_NESTING_DEPTH + 5], + ])("returns a refusal, with its numbers, for too-deep %s", (_shape, markdown, depth) => { + expect(parseArbitraryMarkdown(testSchema, markdown)).toEqual({ + kind: "refused", + refusal: { depth, limit: MAX_NESTING_DEPTH }, + }); + }); + + it("parses a document exactly at the limit", () => { + const outcome = parseArbitraryMarkdown(testSchema, `${"> ".repeat(MAX_NESTING_DEPTH)}a\n`); + expect(outcome.kind).toBe("parsed"); + }); + + it("rethrows every other failure", () => { + const exploding = new Proxy({} as Schema, { + get() { + throw new Error("schema exploded"); + }, + }); + expect(() => parseArbitraryMarkdown(exploding, "a paragraph\n")).toThrow(/schema exploded/); + }); +}); diff --git a/src/test/arbitraryMarkdown.ts b/src/test/arbitraryMarkdown.ts new file mode 100644 index 000000000..1079d7c69 --- /dev/null +++ b/src/test/arbitraryMarkdown.ts @@ -0,0 +1,41 @@ +/** + * Purpose: parse ARBITRARY markdown in a fuzz, soak or pathological sweep, and + * tell the pipeline's one deliberate refusal apart from a crash. + * + * `parseMarkdown` refuses a document nested deeper than `MAX_NESTING_DEPTH` + * (#1374): past that depth the parser's own dependencies overflow the stack, so + * the refusal is the designed outcome, not a defect. A sweep over hostile input + * must therefore count a refusal as "handled" — the OSS-Fuzz soak failed on a + * 16382-level corpus file precisely because it treated the refusal as a crash + * (#1407) — while every OTHER throw stays a failure. + * + * Both sweeps share this one classification so they cannot drift apart. It is + * by TYPE along the `cause` chain (`nestingRefusal`), never by message text. + * + * Use it only on the INPUT of a sweep. A refusal when re-parsing the pipeline's + * own serialized output would be a real defect, so round-trip legs must call + * `parseMarkdown` directly and let it throw. + * + * @coordinates-with utils/markdownPipeline/nestingDepth.ts — the refusal + * @coordinates-with test/externalCorpora.soak.test.ts — the OSS-Fuzz sweep + * @coordinates-with utils/markdownPipeline/__tests__/pathological/runCases.ts + * @module test/arbitraryMarkdown + */ +import type { Node as PMNode, Schema } from "@tiptap/pm/model"; +import { parseMarkdown } from "@/utils/markdownPipeline/adapter"; +import { nestingRefusal, type NestingRefusal } from "@/utils/markdownPipeline/nestingDepth"; + +export type ArbitraryParseOutcome = + | { kind: "parsed"; doc: PMNode } + | { kind: "refused"; refusal: NestingRefusal }; + +/** Parse `markdown`; a nesting refusal is returned, anything else is thrown. */ +export function parseArbitraryMarkdown(schema: Schema, markdown: string): ArbitraryParseOutcome { + try { + return { kind: "parsed", doc: parseMarkdown(schema, markdown) }; + } catch (error) { + const refusal = nestingRefusal(error); + if (!refusal) throw error; + return { kind: "refused", refusal }; + } +} diff --git a/src/test/externalCorpora.soak.test.ts b/src/test/externalCorpora.soak.test.ts index e2affa46a..709ce3546 100644 --- a/src/test/externalCorpora.soak.test.ts +++ b/src/test/externalCorpora.soak.test.ts @@ -22,6 +22,7 @@ import StarterKit from "@tiptap/starter-kit"; import { gunzipSync } from "node:zlib"; import "@/utils/markdownPipeline/dialect"; import { parseMarkdown, serializeMarkdown } from "@/utils/markdownPipeline/adapter"; +import { parseArbitraryMarkdown } from "./arbitraryMarkdown"; const schema = getSchema([StarterKit]); @@ -108,6 +109,7 @@ describe("OSS-Fuzz cmark corpus soak (best-effort bucket)", () => { const files = readdirSync(join(dir, "corpus")).slice(0, 500); expect(files.length).toBeGreaterThan(100); const oscillating: string[] = []; + const refused: string[] = []; for (const file of files) { const bytes = readFileSync(join(dir, "corpus", file)); const text = bytes.toString("utf8"); @@ -117,17 +119,33 @@ describe("OSS-Fuzz cmark corpus soak (best-effort bucket)", () => { // inputs hit that class, so a small, LOUDLY-REPORTED tolerance keeps // the weekly soak signal instead of permanently red. Fixing the // ledgered defects shrinks this to zero. - const md1 = serializeMarkdown(schema, parseMarkdown(schema, text)); + // + // A document nested past MAX_NESTING_DEPTH is REFUSED by design (#1374), + // and the corpus has one 16382 levels deep (#1407) — that is handled, + // not a crash. Only the input may be refused: the re-parses below read + // the pipeline's own output, where a refusal would be a real defect. + const first = parseArbitraryMarkdown(schema, text); + if (first.kind === "refused") { + refused.push(`${file} (${first.refusal.depth} levels)`); + continue; + } + const md1 = serializeMarkdown(schema, first.doc); const md2 = serializeMarkdown(schema, parseMarkdown(schema, md1)); const md3 = serializeMarkdown(schema, parseMarkdown(schema, md2)); if (md3 !== md2) oscillating.push(file); } + if (refused.length > 0) { + console.warn(`[SOAK] ${refused.length}/${files.length} fuzz inputs refused as too deeply nested: ${refused.join(", ")}`); + } + const checked = files.length - refused.length; + // The sample is only worth its name if the refusal did not eat it. + expect(checked, "fuzz inputs actually round-tripped").toBeGreaterThan(100); if (oscillating.length > 0) { console.warn( - `[SOAK] ${oscillating.length}/${files.length} fuzz inputs oscillate (known escape-growth class): ` + + `[SOAK] ${oscillating.length}/${checked} fuzz inputs oscillate (known escape-growth class): ` + oscillating.slice(0, 10).join(", "), ); } - expect(oscillating.length / files.length, "oscillating fraction").toBeLessThanOrEqual(0.02); + expect(oscillating.length / checked, "oscillating fraction").toBeLessThanOrEqual(0.02); }); }); diff --git a/src/utils/markdownPipeline/__tests__/pathological/pathological.test.ts b/src/utils/markdownPipeline/__tests__/pathological/pathological.test.ts index 70db84afe..4ef1d498f 100644 --- a/src/utils/markdownPipeline/__tests__/pathological/pathological.test.ts +++ b/src/utils/markdownPipeline/__tests__/pathological/pathological.test.ts @@ -22,7 +22,8 @@ import { describe, it, expect } from "vitest"; import { spawnSync } from "node:child_process"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; -import { pathologicalCases } from "./pathologicalCases"; +import { pathologicalCases, pathologicalScale } from "./pathologicalCases"; +import { MAX_NESTING_DEPTH } from "../../nestingDepth"; import { LIVENESS_TIMEOUT_MS } from "../../../../../vitest.shared"; const here = dirname(fileURLToPath(import.meta.url)); @@ -62,6 +63,8 @@ interface CaseReport { starting?: boolean; parseMs?: number; serializeMs?: number; + refusedMs?: number; + refusedDepth?: number; done?: boolean; } @@ -118,7 +121,9 @@ describe("pathological inputs (killable child process)", () => { const { res, lines } = runChild({}, WALL_CEILING_MS); const started = lines.filter((l) => l.starting && l.name).map((l) => l.name); - const finished = new Set(lines.filter((l) => l.parseMs !== undefined).map((l) => l.name)); + const parsed = lines.filter((l) => l.parseMs !== undefined).map((l) => l.name); + const refused = lines.filter((l) => l.refusedMs !== undefined).map((l) => l.name); + const finished = new Set([...parsed, ...refused]); const lastStarted = started.at(-1); expect( @@ -129,8 +134,16 @@ describe("pathological inputs (killable child process)", () => { expect(res.status, `child failed\nstderr: ${res.stderr}`).toBe(0); expect(lines.some((l) => l.done)).toBe(true); - const expected = pathologicalCases(1).map((c) => c.name); - expect([...finished].sort()).toEqual([...expected].sort()); + // Every class must come back — parsed, or refused as too deeply nested. + // Which of the two is decided by the case's DECLARED container depth at + // the scale the child ran, so a refusal nobody expected fails here instead + // of passing as "finished". The soak's scale 8 refuses the two container + // classes; counting only parsed classes made that a failure (#1407). + const cases = pathologicalCases(pathologicalScale()); + const deep = (c: (typeof cases)[number]) => c.containerDepth > MAX_NESTING_DEPTH; + expect([...refused].sort()).toEqual(cases.filter(deep).map((c) => c.name).sort()); + expect([...parsed].sort()).toEqual(cases.filter((c) => !deep(c)).map((c) => c.name).sort()); + expect(finished.size).toBe(cases.length); }, WALL_CEILING_MS + 30_000); it("SELF-TEST: a deliberate busy loop is killed and reported, not hung", () => { diff --git a/src/utils/markdownPipeline/__tests__/pathological/pathologicalCases.ts b/src/utils/markdownPipeline/__tests__/pathological/pathologicalCases.ts index cc0bf25d4..7d8c402d4 100644 --- a/src/utils/markdownPipeline/__tests__/pathological/pathologicalCases.ts +++ b/src/utils/markdownPipeline/__tests__/pathological/pathologicalCases.ts @@ -21,6 +21,19 @@ export interface PathologicalCase { markdown: string; /** Also run the parse→ProseMirror→markdown leg. */ serialize: boolean; + /** + * How deeply the input nests CONTAINERS (blockquotes, lists), stated from + * the generator rather than measured by the guard — so a case above + * `MAX_NESTING_DEPTH` is one the parent EXPECTS to be refused, and a refusal + * of any other case is a failure rather than a quiet pass. + */ + containerDepth: number; +} + +/** The scale a run uses: `PATHOLOGICAL_SCALE`, read in ONE place so the child + * that generates the inputs and the parent that judges them cannot disagree. */ +export function pathologicalScale(): number { + return Number(process.env.PATHOLOGICAL_SCALE ?? "1"); } export function pathologicalCases(scale = 1): PathologicalCase[] { @@ -35,46 +48,55 @@ export function pathologicalCases(scale = 1): PathologicalCase[] { name: "nested-brackets", markdown: `${"[".repeat(n(2000))}a${"]".repeat(n(2000))}\n`, serialize: true, + containerDepth: 0, }, { name: "nested-strong-emph", markdown: `${"*a **a ".repeat(n(300))}b${" a** a*".repeat(n(300))}\n`, serialize: true, + containerDepth: 0, }, { name: "emph-closers-without-openers", markdown: `${"a_ ".repeat(n(3000))}\n`, serialize: true, + containerDepth: 0, }, { name: "emph-openers-without-closers", markdown: `${"_a ".repeat(n(3000))}\n`, serialize: true, + containerDepth: 0, }, { name: "link-closers-with-openers", markdown: `${"a](".repeat(n(3000))}\n`, serialize: true, + containerDepth: 0, }, { name: "backtick-runs", markdown: `${backtickRuns(n(250))}\n`, serialize: true, + containerDepth: 0, }, { name: "unclosed-inline-links", markdown: `${"[a]( ".repeat(n(500))}a\n`, serialize: true, + containerDepth: n(500), }, { name: "deep-lists", markdown: Array.from({ length: n(200) }, (_, i) => `${" ".repeat(i)}- a`).join("\n") + "\n", serialize: true, + containerDepth: n(200), }, { // Deep mdast NESTING (not just long delimiter runs): alternating @@ -84,6 +106,7 @@ export function pathologicalCases(scale = 1): PathologicalCase[] { name: "deep-inline-nesting", markdown: `${"*_".repeat(n(1500))}x${"_*".repeat(n(1500))}\n`, serialize: true, + containerDepth: 0, }, { name: "many-link-references", @@ -91,6 +114,7 @@ export function pathologicalCases(scale = 1): PathologicalCase[] { Array.from({ length: n(1000) }, (_, i) => `[ref${i}]: /url${i}`).join("\n") + `\n\n${Array.from({ length: n(1000) }, (_, i) => `[ref${i}]`).join(" ")}\n`, serialize: true, + containerDepth: 0, }, ]; } diff --git a/src/utils/markdownPipeline/__tests__/pathological/runCases.ts b/src/utils/markdownPipeline/__tests__/pathological/runCases.ts index e1776b2a4..287d70b05 100644 --- a/src/utils/markdownPipeline/__tests__/pathological/runCases.ts +++ b/src/utils/markdownPipeline/__tests__/pathological/runCases.ts @@ -18,7 +18,8 @@ * the spec gates on the (already-nasty, small) spec corpora. * * Protocol: one JSON line per completed case on stdout - * (`{"name","parseMs","serializeMs"}`), `{"done":true}` at the end. The + * (`{"name","parseMs","serializeMs"}`, or `{"name","refusedMs","refusedDepth"}` + * for a nesting refusal), `{"done":true}` at the end. The * PARENT owns all judgement; a case that hangs simply never prints, and the * last-started name (printed BEFORE running) identifies the culprit. * @@ -30,9 +31,9 @@ import { getSchema } from "@tiptap/core"; import StarterKit from "@tiptap/starter-kit"; import "../../dialect"; -import { parseMarkdown, serializeMarkdown } from "../../adapter"; -import { isNestingTooDeep } from "../../nestingDepth"; -import { pathologicalCases } from "./pathologicalCases"; +import { serializeMarkdown } from "../../adapter"; +import { parseArbitraryMarkdown } from "@/test/arbitraryMarkdown"; +import { pathologicalCases, pathologicalScale } from "./pathologicalCases"; if (process.env.HANG_PROBE === "1") { console.log(JSON.stringify({ name: "hang-probe", starting: true })); @@ -41,32 +42,30 @@ if (process.env.HANG_PROBE === "1") { } } -const scale = Number(process.env.PATHOLOGICAL_SCALE ?? "1"); +const scale = pathologicalScale(); const schema = getSchema([StarterKit]); for (const testCase of pathologicalCases(scale)) { console.log(JSON.stringify({ name: testCase.name, starting: true })); const t0 = performance.now(); - let doc; - try { - doc = parseMarkdown(schema, testCase.markdown); - } catch (error) { - // A nesting refusal is a PASS here, and the only tolerated throw. This - // suite's contract is liveness — no hang, no stack overflow — and the - // guard added for #1374 satisfies it deliberately rather than by luck. - // At scale 1 `deep-blockquotes` is 500 levels and parses; at the soak's - // scale 8 it is 4000 and is refused. Any OTHER error still propagates and - // fails the run, which is what keeps this from swallowing real defects. - if (!isNestingTooDeep(error)) throw error; + // A nesting refusal is a defined outcome here, and the only tolerated one. + // This suite's contract is liveness — no hang, no stack overflow — and the + // guard added for #1374 satisfies it deliberately rather than by luck. At + // scale 1 `deep-blockquotes` is 500 levels and parses; at the soak's scale 8 + // it is 4000 and is refused. Any OTHER error still propagates and fails the + // run. The PARENT decides whether a refusal was expected for this case. + const outcome = parseArbitraryMarkdown(schema, testCase.markdown); + if (outcome.kind === "refused") { console.log( JSON.stringify({ name: testCase.name, refusedMs: Math.round(performance.now() - t0), - refused: "nesting", + refusedDepth: outcome.refusal.depth, }), ); continue; } + const doc = outcome.doc; const parseMs = performance.now() - t0; let serializeMs = 0; if (testCase.serialize) { diff --git a/src/utils/markdownPipeline/nestingDepth.test.ts b/src/utils/markdownPipeline/nestingDepth.test.ts index 1b76fa04f..e09e6a68a 100644 --- a/src/utils/markdownPipeline/nestingDepth.test.ts +++ b/src/utils/markdownPipeline/nestingDepth.test.ts @@ -31,7 +31,15 @@ * @module utils/markdownPipeline/nestingDepth.test */ import { describe, it, expect } from "vitest"; -import { MAX_NESTING_DEPTH, maxContainerDepth, checkNestingDepth } from "./nestingDepth"; +import { + MAX_NESTING_DEPTH, + maxContainerDepth, + checkNestingDepth, + isNestingTooDeep, + nestingRefusal, +} from "./nestingDepth"; +import { parseMarkdown } from "./adapter"; +import { testSchema } from "./testSchema"; describe("maxContainerDepth", () => { it("is zero for ordinary prose", () => { @@ -118,3 +126,46 @@ describe("checkNestingDepth", () => { expect(MAX_NESTING_DEPTH).toBeGreaterThanOrEqual(500); }); }); + +describe("nestingRefusal (#1407)", () => { + const tooDeep = `${"> ".repeat(MAX_NESTING_DEPTH + 7)}a\n`; + + function thrownBy(fn: () => unknown): unknown { + try { + fn(); + } catch (error) { + return error; + } + throw new Error("expected a throw"); + } + + it("reads the depth and the limit off the guard's own error", () => { + const error = thrownBy(() => checkNestingDepth(tooDeep)); + expect(nestingRefusal(error)).toEqual({ depth: MAX_NESTING_DEPTH + 7, limit: MAX_NESTING_DEPTH }); + expect(isNestingTooDeep(error)).toBe(true); + }); + + it("finds it through parseMarkdown's wrapper, where callers actually meet it", () => { + const error = thrownBy(() => parseMarkdown(testSchema, tooDeep)); + expect(error).toBeInstanceOf(Error); + expect(nestingRefusal(error)).toEqual({ depth: MAX_NESTING_DEPTH + 7, limit: MAX_NESTING_DEPTH }); + }); + + it.each([ + ["an unrelated error", new Error("boom")], + ["an error that merely SAYS it is a nesting refusal", new Error("Nesting is 5000 levels deep")], + ["a thrown string", "Nesting is 5000 levels deep"], + ["undefined", undefined], + ["null", null], + ])("is undefined for %s", (_label, error) => { + expect(nestingRefusal(error)).toBeUndefined(); + expect(isNestingTooDeep(error)).toBe(false); + }); + + it("terminates on a cyclic cause chain", () => { + const a = new Error("a"); + const b = new Error("b", { cause: a }); + (a as { cause?: unknown }).cause = b; + expect(nestingRefusal(a)).toBeUndefined(); + }); +}); diff --git a/src/utils/markdownPipeline/nestingDepth.ts b/src/utils/markdownPipeline/nestingDepth.ts index 05224d5ef..72a38b772 100644 --- a/src/utils/markdownPipeline/nestingDepth.ts +++ b/src/utils/markdownPipeline/nestingDepth.ts @@ -170,16 +170,34 @@ class NestingTooDeepError extends Error { } } -/** Is `error` — or anything it wraps — a nesting refusal? */ -export function isNestingTooDeep(error: unknown): boolean { +/** What a nesting refusal reports: how deep the document was, and the limit. */ +export interface NestingRefusal { + depth: number; + limit: number; +} + +/** + * The nesting refusal `error` is — or wraps — or undefined. + * + * Classified by TYPE along the `cause` chain, never by message: an error that + * merely says "Nesting is 5000 levels deep" is not one. Callers that must tell + * a deliberate refusal from a crash (the fuzz soaks, the editor's parse + * failure path) need the numbers too, to say what happened. + */ +export function nestingRefusal(error: unknown): NestingRefusal | undefined { let cursor: unknown = error; // Bounded: a cause chain is short, and an accidental cycle must not hang the // caller that is already handling a failure. for (let i = 0; i < 10 && cursor instanceof Error; i += 1) { - if (cursor instanceof NestingTooDeepError) return true; + if (cursor instanceof NestingTooDeepError) return { depth: cursor.depth, limit: cursor.limit }; cursor = (cursor as { cause?: unknown }).cause; } - return false; + return undefined; +} + +/** Is `error` — or anything it wraps — a nesting refusal? */ +export function isNestingTooDeep(error: unknown): boolean { + return nestingRefusal(error) !== undefined; } /** Throw if `markdown` nests deeper than the parser can survive. */ From 3ac1a10d46001359262aa9deb0d96ca16577a77c Mon Sep 17 00:00:00 2001 From: xiaolai Date: Tue, 15 Sep 2026 10:29:11 +0800 Subject: [PATCH 06/13] fix(editor): open a document the parser refuses in Source mode, with a message Following the nesting refusal from #1407 into the app: when a user opens a file nested past MAX_NESTING_DEPTH, TiptapEditor caught the parse failure, logged it to the console with tiptapError, and carried on with an EMPTY document. That editor was live. Its first keystroke flushed the empty document to the store, so typing would have saved over the file the user had just opened, and nothing told them why the editor was blank. An external content change that failed to parse had the same hazard: the editor kept the old content, and the next edit would overwrite the new text. Both failure paths now go to reportUnparseableDocument, which puts the tab in Source mode through the existing per-tab forced-Source marker and shows an error toast. Source mode shows the text as it is and needs no parse, so it is the one place the document is both visible and safely editable. A nesting refusal says how deep the document is and what the limit is; any other parse failure gets a general message, because the blank editor is the same hazard whatever the cause. The marker now records why a tab is forced: large-file, as before, or unparseable. The status line says "Opened in Source mode (cannot be displayed in WYSIWYG)." for a refusal instead of claiming the file is large, and keeps Switch to WYSIWYG, which is how the user asks again after reducing the nesting. A split view parses one document in several editors, so the report happens once per tab, and a tab already forced for its size keeps that reason. Existing editor tests that asserted the old dev-only log now assert the report. One of them left its deferred parse pending with a one-shot throwing mock, which leaked into whichever later test ran timers next; it now runs its timers itself. The new strings are translated into all ten locales, and the Large Files guide documents the behavior in every language. --- .../Editor/TiptapEditor.callbacks.test.tsx | 15 ++- .../Editor/TiptapEditor.sync.test.tsx | 22 +++-- src/components/Editor/TiptapEditor.test.tsx | 16 +++- src/components/Editor/TiptapEditor.tsx | 10 +- .../Editor/tiptapEditorHelpers.test.ts | 42 +++++++++ src/components/Editor/tiptapEditorHelpers.ts | 10 +- src/components/Editor/useTiptapContentSync.ts | 4 +- .../StatusBar/SourceModeUpgrade.test.tsx | 17 ++++ .../StatusBar/SourceModeUpgrade.tsx | 17 ++-- src/lib/formats/markdownLargeFile.test.ts | 6 +- src/locales/de/editor.json | 4 +- src/locales/de/statusbar.json | 3 +- src/locales/en/editor.json | 4 +- src/locales/en/statusbar.json | 3 +- src/locales/es/editor.json | 4 +- src/locales/es/statusbar.json | 3 +- src/locales/fr/editor.json | 4 +- src/locales/fr/statusbar.json | 3 +- src/locales/it/editor.json | 4 +- src/locales/it/statusbar.json | 3 +- src/locales/ja/editor.json | 4 +- src/locales/ja/statusbar.json | 3 +- src/locales/ko/editor.json | 4 +- src/locales/ko/statusbar.json | 3 +- src/locales/pt-BR/editor.json | 4 +- src/locales/pt-BR/statusbar.json | 3 +- src/locales/zh-CN/editor.json | 4 +- src/locales/zh-CN/statusbar.json | 3 +- src/locales/zh-TW/editor.json | 4 +- src/locales/zh-TW/statusbar.json | 3 +- .../editor/unparseableDocument.test.ts | 93 +++++++++++++++++++ src/services/editor/unparseableDocument.ts | 54 +++++++++++ src/stores/documentStore/largeFileSession.ts | 26 ++++-- src/stores/largeFileSessionStore.test.ts | 23 +++++ website/de/guide/large-files.md | 1 + website/es/guide/large-files.md | 1 + website/fr/guide/large-files.md | 1 + website/guide/large-files.md | 1 + website/it/guide/large-files.md | 1 + website/ja/guide/large-files.md | 1 + website/ko/guide/large-files.md | 1 + website/pt-BR/guide/large-files.md | 1 + website/zh-CN/guide/large-files.md | 1 + website/zh-TW/guide/large-files.md | 1 + 44 files changed, 376 insertions(+), 59 deletions(-) create mode 100644 src/services/editor/unparseableDocument.test.ts create mode 100644 src/services/editor/unparseableDocument.ts diff --git a/src/components/Editor/TiptapEditor.callbacks.test.tsx b/src/components/Editor/TiptapEditor.callbacks.test.tsx index 84f719eba..cbc10638e 100644 --- a/src/components/Editor/TiptapEditor.callbacks.test.tsx +++ b/src/components/Editor/TiptapEditor.callbacks.test.tsx @@ -42,6 +42,7 @@ const mocks = vi.hoisted(() => ({ })), useWindowLabel: vi.fn(() => "main"), consumeWysiwygPendingNav: vi.fn(() => false), + reportUnparseableDocument: vi.fn(), // Mock editor returned by useEditor mockEditor: null as ReturnType | null, useEditor: vi.fn(), @@ -218,6 +219,10 @@ vi.mock("./wysiwygPendingNav", () => ({ consumeWysiwygPendingNav: (...args: unknown[]) => mocks.consumeWysiwygPendingNav(...args), })); +vi.mock("@/services/editor/unparseableDocument", () => ({ + reportUnparseableDocument: (...args: unknown[]) => mocks.reportUnparseableDocument(...args), +})); + vi.mock("./ImageContextMenu", () => ({ ImageContextMenu: ({ onAction }: { onAction: (a: string) => void }) => (