diff --git a/AGENTS.md b/AGENTS.md index 9d81bd25d..f785f4cff 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -552,6 +552,36 @@ Shared instructions for all AI agents (Claude, Codex, etc.). file would be minutes for nothing. Measured clean on adoption, so it is a hard failure with no allowlist. + - **The markdown parser's hostile-input cost is linear, and three layers keep + it that way (#1407).** The weekly pathological soak never got past its fifth + input class: `a](` × 24000 took 60s to parse and 276s to serialize, and + `backtick-runs` ran over twelve minutes. Every cause was an O(n²) or worse + walk inside micromark or mdast-util-to-markdown (latest releases), plus one + in VMark's own escape scan. + + - `src/utils/markdownPipeline/parser/fastPaths/` — micromark constructs that + settle `]` and backtick runs nothing can close, acting only on a PROOF + that micromark's own construct would fail, plus micromark's emphasis + resolver ported with a per-marker opener list in place of its event + walk. `inlineFastPaths.test.ts` diffs the mdast, positions included, + against a parser without them over every spec corpus and a fuzz. + - `patches/` — `pnpm` patches for the two costs no extension API can reach + (micromark's data-token merge, mdast-util-to-markdown's escape lookup). + Each is a few lines, commented at the change. **A Dependabot bump of + either package fails install until the patch is re-made** (`pnpm patch` + → port the change → `pnpm patch-commit`), and that failure is the point: + a patch that silently stopped applying would bring the quadratic back. + - `__tests__/pathological/pathologicalScaling.test.ts` — asserts a CPU-time + growth EXPONENT per class in the PR tier, so a regression fails + `check:all` instead of waiting for the soak. + + The whole scale-8 pathological run now takes ~11s (measured 10.8s idle). + Two classes stay super-linear, and are what is left of it: + `nested-strong-emph` (~5s — micromark re-resolves each matched span, and + each match splices the event list) and `nested-brackets` (~2s — label-end + serializes the label to look up a definition). Removing either changes + what the stock algorithm does, not how fast it finds it. + - **i18n gate has two halves.** `pnpm lint:i18n` checks that every key exists in every locale AND that values were actually translated. The second half exists because the first cannot see a key copied over with its English value — ~1,160 diff --git a/package.json b/package.json index 1b1c622dd..14ae816ea 100644 --- a/package.json +++ b/package.json @@ -218,6 +218,10 @@ "js-yaml@4": ">=4.3.2 <5", "baseline-browser-mapping": ">=2.11.0 <3", "smol-toml": ">=1.7.1 <2" + }, + "patchedDependencies": { + "micromark@4.0.2": "patches/micromark@4.0.2.patch", + "mdast-util-to-markdown@2.1.2": "patches/mdast-util-to-markdown@2.1.2.patch" } }, "devDependencies": { diff --git a/patches/mdast-util-to-markdown@2.1.2.patch b/patches/mdast-util-to-markdown@2.1.2.patch new file mode 100644 index 000000000..aff5925de --- /dev/null +++ b/patches/mdast-util-to-markdown@2.1.2.patch @@ -0,0 +1,17 @@ +diff --git a/lib/util/safe.js b/lib/util/safe.js +index 456fe215ae3d032ffaea4590f0d097bcb9195619..10f82c99cff8123142f230c087e6d09742f2b313 100644 +--- a/lib/util/safe.js ++++ b/lib/util/safe.js +@@ -56,7 +56,11 @@ export function safe(state, input, config) { + const after = 'after' in pattern + const position = match.index + (before ? match[1].length : 0) + +- if (positions.includes(position)) { ++ // VMark patch (#1407): `infos` gains a key exactly when `positions` ++ // gains the position, so an own-key check is the same test in O(1). ++ // `positions.includes` made escaping O(escapes²) — a text node of 116k ++ // backticks took 15s to serialize. ++ if (Object.hasOwn(infos, position)) { + if (infos[position].before && !before) { + infos[position].before = false + } diff --git a/patches/micromark@4.0.2.patch b/patches/micromark@4.0.2.patch new file mode 100644 index 000000000..aa6e4cff8 --- /dev/null +++ b/patches/micromark@4.0.2.patch @@ -0,0 +1,121 @@ +diff --git a/dev/lib/initialize/text.js b/dev/lib/initialize/text.js +index 7192e70a0972e808d779999e57153bf3ed7bfc3a..0eaba5658038c8f1885f1279b8f30ba720a72d65 100644 +--- a/dev/lib/initialize/text.js ++++ b/dev/lib/initialize/text.js +@@ -113,30 +113,39 @@ function createResolver(extraResolver) { + + /** @type {Resolver} */ + function resolveAllText(events, context) { +- let index = -1 +- /** @type {number | undefined} */ +- let enter +- + // A rather boring computation (to merge adjacent `data` events) which + // improves mm performance by 29%. +- while (++index <= events.length) { +- if (enter === undefined) { +- if (events[index] && events[index][1].type === types.data) { +- enter = index +- index++ +- } +- } else if (!events[index] || events[index][1].type !== types.data) { +- // Don’t do anything if there is one data token. +- if (index !== enter + 2) { +- events[enter][1].end = events[index - 1][1].end +- events.splice(enter + 2, index - enter - 2) +- index = enter + 2 +- } +- +- enter = undefined ++ // ++ // VMark patch (#1407): merge in ONE compacting pass. The original removed ++ // each run with `events.splice`, moving every later event once per run — ++ // O(runs × events), and ~10× slower per step once the list outgrows V8's ++ // regular heap objects. The kept events, the merged token's `end`, and the ++ // returned array are identical. ++ const length = events.length ++ let write = 0 ++ let index = 0 ++ ++ while (index < length) { ++ const event = events[index] ++ ++ if (event[1].type !== types.data) { ++ events[write++] = event ++ index++ ++ continue + } ++ ++ // The enter's own exit is at `index + 1`; a run continues from there. ++ let end = index + 2 ++ while (end < length && events[end][1].type === types.data) end++ ++ // Don’t do anything if there is one data token. ++ if (end !== index + 2) event[1].end = events[end - 1][1].end ++ events[write++] = event ++ if (index + 1 < length) events[write++] = events[index + 1] ++ index = end + } + ++ events.length = write ++ + return extraResolver ? extraResolver(events, context) : events + } + } +diff --git a/lib/initialize/text.js b/lib/initialize/text.js +index b9383cb721b95c68bd0e684da6cbf441ed82b8eb..eac1239a4263cd6945fc36b8db8bb1927659d6f7 100644 +--- a/lib/initialize/text.js ++++ b/lib/initialize/text.js +@@ -104,28 +104,35 @@ function createResolver(extraResolver) { + + /** @type {Resolver} */ + function resolveAllText(events, context) { +- let index = -1; +- /** @type {number | undefined} */ +- let enter; +- + // A rather boring computation (to merge adjacent `data` events) which + // improves mm performance by 29%. +- while (++index <= events.length) { +- if (enter === undefined) { +- if (events[index] && events[index][1].type === "data") { +- enter = index; +- index++; +- } +- } else if (!events[index] || events[index][1].type !== "data") { +- // Don’t do anything if there is one data token. +- if (index !== enter + 2) { +- events[enter][1].end = events[index - 1][1].end; +- events.splice(enter + 2, index - enter - 2); +- index = enter + 2; +- } +- enter = undefined; ++ // ++ // VMark patch (#1407): merge in ONE compacting pass. The original removed ++ // each run with `events.splice`, moving every later event once per run — ++ // O(runs × events), and ~10× slower per step once the list outgrows V8's ++ // regular heap objects. The kept events, the merged token's `end`, and the ++ // returned array are identical. ++ const length = events.length; ++ let write = 0; ++ let index = 0; ++ while (index < length) { ++ const event = events[index]; ++ if (event[1].type !== "data") { ++ events[write++] = event; ++ index++; ++ continue; + } ++ ++ // The enter's own exit is at `index + 1`; a run continues from there. ++ let end = index + 2; ++ while (end < length && events[end][1].type === "data") end++; ++ // Don’t do anything if there is one data token. ++ if (end !== index + 2) event[1].end = events[end - 1][1].end; ++ events[write++] = event; ++ if (index + 1 < length) events[write++] = events[index + 1]; ++ index = end; + } ++ events.length = write; + return extraResolver ? extraResolver(events, context) : events; + } + } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 911c870fb..1423b900c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -30,6 +30,14 @@ overrides: baseline-browser-mapping: '>=2.11.0 <3' smol-toml: '>=1.7.1 <2' +patchedDependencies: + mdast-util-to-markdown@2.1.2: + hash: 2dbee866e6cf244b099fa367885b1e8ad51ff315b895c1ad77581dee0f4206bf + path: patches/mdast-util-to-markdown@2.1.2.patch + micromark@4.0.2: + hash: c4162f5915a6a3308e12ec56d2a8ef6a3a9cbaa7abea611d0ce5957fc4f620ea + path: patches/micromark@4.0.2.patch + importers: .: @@ -16614,7 +16622,7 @@ snapshots: markdownlint@0.41.1: dependencies: - micromark: 4.0.2 + micromark: 4.0.2(patch_hash=c4162f5915a6a3308e12ec56d2a8ef6a3a9cbaa7abea611d0ce5957fc4f620ea) micromark-core-commonmark: 2.0.3 micromark-extension-directive: 4.0.0 micromark-extension-gfm-autolink-literal: 2.1.0 @@ -16687,7 +16695,7 @@ snapshots: decode-named-character-reference: 1.3.0 devlop: 1.1.0 mdast-util-to-string: 4.0.0 - micromark: 4.0.2 + micromark: 4.0.2(patch_hash=c4162f5915a6a3308e12ec56d2a8ef6a3a9cbaa7abea611d0ce5957fc4f620ea) micromark-util-decode-numeric-character-reference: 2.0.2 micromark-util-decode-string: 2.0.1 micromark-util-normalize-identifier: 2.0.1 @@ -16703,7 +16711,7 @@ snapshots: devlop: 1.1.0 escape-string-regexp: 5.0.0 mdast-util-from-markdown: 2.0.3 - mdast-util-to-markdown: 2.1.2 + mdast-util-to-markdown: 2.1.2(patch_hash=2dbee866e6cf244b099fa367885b1e8ad51ff315b895c1ad77581dee0f4206bf) micromark-extension-frontmatter: 2.0.0 transitivePeerDependencies: - supports-color @@ -16721,7 +16729,7 @@ snapshots: '@types/mdast': 4.0.4 devlop: 1.1.0 mdast-util-from-markdown: 2.0.3 - mdast-util-to-markdown: 2.1.2 + mdast-util-to-markdown: 2.1.2(patch_hash=2dbee866e6cf244b099fa367885b1e8ad51ff315b895c1ad77581dee0f4206bf) micromark-util-normalize-identifier: 2.0.1 transitivePeerDependencies: - supports-color @@ -16730,7 +16738,7 @@ snapshots: dependencies: '@types/mdast': 4.0.4 mdast-util-from-markdown: 2.0.3 - mdast-util-to-markdown: 2.1.2 + mdast-util-to-markdown: 2.1.2(patch_hash=2dbee866e6cf244b099fa367885b1e8ad51ff315b895c1ad77581dee0f4206bf) transitivePeerDependencies: - supports-color @@ -16740,7 +16748,7 @@ snapshots: devlop: 1.1.0 markdown-table: 3.0.4 mdast-util-from-markdown: 2.0.3 - mdast-util-to-markdown: 2.1.2 + mdast-util-to-markdown: 2.1.2(patch_hash=2dbee866e6cf244b099fa367885b1e8ad51ff315b895c1ad77581dee0f4206bf) transitivePeerDependencies: - supports-color @@ -16749,7 +16757,7 @@ snapshots: '@types/mdast': 4.0.4 devlop: 1.1.0 mdast-util-from-markdown: 2.0.3 - mdast-util-to-markdown: 2.1.2 + mdast-util-to-markdown: 2.1.2(patch_hash=2dbee866e6cf244b099fa367885b1e8ad51ff315b895c1ad77581dee0f4206bf) transitivePeerDependencies: - supports-color @@ -16761,7 +16769,7 @@ snapshots: mdast-util-gfm-strikethrough: 2.0.0 mdast-util-gfm-table: 2.0.0 mdast-util-gfm-task-list-item: 2.0.0 - mdast-util-to-markdown: 2.1.2 + mdast-util-to-markdown: 2.1.2(patch_hash=2dbee866e6cf244b099fa367885b1e8ad51ff315b895c1ad77581dee0f4206bf) transitivePeerDependencies: - supports-color @@ -16772,7 +16780,7 @@ snapshots: devlop: 1.1.0 longest-streak: 3.1.0 mdast-util-from-markdown: 2.0.3 - mdast-util-to-markdown: 2.1.2 + mdast-util-to-markdown: 2.1.2(patch_hash=2dbee866e6cf244b099fa367885b1e8ad51ff315b895c1ad77581dee0f4206bf) unist-util-remove-position: 5.0.0 transitivePeerDependencies: - supports-color @@ -16799,7 +16807,7 @@ snapshots: unist-util-visit: 5.1.0 vfile: 6.0.3 - mdast-util-to-markdown@2.1.2: + mdast-util-to-markdown@2.1.2(patch_hash=2dbee866e6cf244b099fa367885b1e8ad51ff315b895c1ad77581dee0f4206bf): dependencies: '@types/mdast': 4.0.4 '@types/unist': 3.0.3 @@ -17047,7 +17055,7 @@ snapshots: micromark-util-types@2.0.2: {} - micromark@4.0.2: + micromark@4.0.2(patch_hash=c4162f5915a6a3308e12ec56d2a8ef6a3a9cbaa7abea611d0ce5957fc4f620ea): dependencies: '@types/debug': 4.1.13 debug: 4.4.3 @@ -17976,7 +17984,7 @@ snapshots: remark-stringify@11.0.0: dependencies: '@types/mdast': 4.0.4 - mdast-util-to-markdown: 2.1.2 + mdast-util-to-markdown: 2.1.2(patch_hash=2dbee866e6cf244b099fa367885b1e8ad51ff315b895c1ad77581dee0f4206bf) unified: 11.0.5 require-directory@2.1.1: {} diff --git a/src/components/Editor/TiptapEditor.callbacks.test.tsx b/src/components/Editor/TiptapEditor.callbacks.test.tsx index 84f719eba..d977024a1 100644 --- a/src/components/Editor/TiptapEditor.callbacks.test.tsx +++ b/src/components/Editor/TiptapEditor.callbacks.test.tsx @@ -42,6 +42,7 @@ const mocks = vi.hoisted(() => ({ })), useWindowLabel: vi.fn(() => "main"), consumeWysiwygPendingNav: vi.fn(() => false), + reportUnparseableDocument: vi.fn(), // Mock editor returned by useEditor mockEditor: null as ReturnType | null, useEditor: vi.fn(), @@ -208,7 +209,7 @@ vi.mock("@/stores/documentStore", () => ({ }, useRevisionStore: { getState: () => ({ registerEdit: vi.fn(), setRevision: vi.fn(), getRevision: vi.fn(() => null) }) }, generateRevisionId: () => "rev-test-id", - useLargeFileSessionStore: { getState: () => ({ isForcedSource: () => false }), subscribe: () => () => {} }, + useLargeFileSessionStore: { getState: () => ({ isForcedSource: () => false, forcedSourceReason: () => undefined }), subscribe: () => () => {} }, useUnifiedHistoryStore: { getState: () => ({ documents: {}, createCheckpoint: vi.fn() }), subscribe: () => () => {} }, useLintStore: { getState: () => ({ diagnosticsByTab: {}, selectedIndexByTab: {}, clearDiagnostics: vi.fn() }), subscribe: () => () => {} }, useFileLoadStore: { getState: () => ({ active: false }) }, @@ -218,6 +219,10 @@ vi.mock("./wysiwygPendingNav", () => ({ consumeWysiwygPendingNav: (...args: unknown[]) => mocks.consumeWysiwygPendingNav(...args), })); +vi.mock("@/services/editor/unparseableDocument", () => ({ + reportUnparseableDocument: (...args: unknown[]) => mocks.reportUnparseableDocument(...args), +})); + vi.mock("./ImageContextMenu", () => ({ ImageContextMenu: ({ onAction }: { onAction: (a: string) => void }) => (