From 413593ba4abd00d5cd2a5bf5d2e13e47f9944237 Mon Sep 17 00:00:00 2001 From: Andrew Mikofalvy <5668128+amikofalvy@users.noreply.github.com> Date: Mon, 20 Jul 2026 12:36:32 -0700 Subject: [PATCH 1/2] refactor(app): FileTree pure-helper extraction and row lookup map (#2733) * refactor(app): extract FileTree pure helpers, icon sprite, and row lookup map Move the pure tree-path helper functions and the Lucide sprite-sheet generation out of FileTree.tsx into sibling modules, merge the three navigate*WithPulse variants into one discriminated-union function, and replace renderRowDecoration's per-row linear scans of the document list with an O(1) Map lookup derived once per document-list change. * test(app): add file-tree-path-helpers unit tests; fix purity JSDoc Address review feedback on the FileTree extraction: - Tighten the module-level purity claim in file-tree-path-helpers.ts to note isEditableKeyboardTarget performs a DOM read (target.closest). - Add co-located file-tree-path-helpers.test.ts covering the two load-bearing invariants the reviewer flagged: buildRowDecorationIndex first-entry-wins on key collision (matching the Array.find scans it replaced) and deleteTargetCoversPendingCreate's file/asset/folder branches, plus the pure string helpers (parse/extension/agent/alternate-sibling) and selectedTreePathsToDeleteTargets dedup/.ok-exclusion/folder-containment. GitOrigin-RevId: 20f65ccbf6006f3a48e0c19ec8b1e009b7f9577f --- knip.config.ts | 4 +- packages/app/src/components/FileTree.tsx | 360 +++++------------- .../src/components/file-tree-icon-sprite.ts | 50 +++ .../components/file-tree-path-helpers.test.ts | 189 +++++++++ .../src/components/file-tree-path-helpers.ts | 239 ++++++++++++ .../core/src/bridge/bind-frontmatter-doc.ts | 1 + packages/core/src/bridge/bridge-invariant.ts | 1 + .../core/src/bridge/doc-boundary-space.ts | 1 + packages/core/src/bridge/growth-detect.ts | 1 + packages/core/src/bridge/merge-three-way.ts | 1 + packages/core/src/bridge/normalize.ts | 1 + .../src/bridge/pm-structural-equivalence.ts | 3 + packages/core/src/bridge/subsequence.ts | 1 + .../core/src/markdown/callout-transformer.ts | 1 + .../core/src/markdown/comment-promoter.ts | 1 + .../src/markdown/dedent-block-jsx-close.ts | 1 + .../markdown/details-accordion-promoter.ts | 1 + packages/core/src/markdown/fence-regions.ts | 1 + packages/core/src/markdown/fixtures/index.ts | 6 + .../src/markdown/fixtures/perf/generate.ts | 5 + .../src/markdown/guard-flanking-matrix.ts | 1 + .../core/src/markdown/handler-shadow-audit.ts | 1 + .../core/src/markdown/highlight-promoter.ts | 1 + packages/core/src/markdown/html-to-mdast.ts | 4 +- packages/core/src/markdown/image-promoter.ts | 1 + packages/core/src/markdown/index.ts | 9 + .../core/src/markdown/lint/config-schemas.ts | 1 + .../core/src/markdown/lint/default-config.ts | 1 + packages/core/src/markdown/lint/index.ts | 1 + packages/core/src/markdown/lint/plugins.ts | 2 + packages/core/src/markdown/lint/types.ts | 2 + packages/core/src/markdown/math-promoter.ts | 1 + .../src/markdown/mdast-to-hast-handlers.ts | 1 + packages/core/src/markdown/mdast-to-html.ts | 1 + packages/core/src/markdown/merged-walker.ts | 1 + .../core/src/markdown/mermaid-promoter.ts | 1 + .../core/src/markdown/parse-with-fallback.ts | 5 + .../core/src/markdown/parser-drop-closure.ts | 1 + packages/core/src/markdown/pipeline.ts | 1 + .../core/src/markdown/position-aware-join.ts | 1 + packages/core/src/markdown/position-slice.ts | 4 +- .../rehype-plugins/skip-notion-whitespace.ts | 1 + .../rehype-plugins/strip-cocoa-meta.ts | 1 + .../rehype-plugins/strip-gdocs-wrapper.ts | 1 + .../rehype-plugins/strip-github-hovercard.ts | 1 + .../rehype-plugins/strip-gmail-classes.ts | 1 + .../rehype-plugins/strip-gsheets-wrapper.ts | 1 + .../rehype-plugins/strip-mso-styles.ts | 1 + .../rehype-plugins/strip-slack-classes.ts | 1 + .../rehype-plugins/strip-vscode-spans.ts | 1 + .../core/src/markdown/resolve-image-url.ts | 1 + packages/core/src/markdown/safe-url.ts | 1 + .../core/src/markdown/serialize-helpers.ts | 1 + .../markdown/single-dollar-math-promoter.ts | 1 + .../core/src/markdown/to-markdown-handlers.ts | 1 + .../core/src/markdown/unknown-mdast-guard.ts | 1 + .../core/src/markdown/whitespace-char-ref.ts | 1 + .../core/src/markdown/wiki-link-micromark.ts | 3 + 58 files changed, 651 insertions(+), 277 deletions(-) create mode 100644 packages/app/src/components/file-tree-icon-sprite.ts create mode 100644 packages/app/src/components/file-tree-path-helpers.test.ts create mode 100644 packages/app/src/components/file-tree-path-helpers.ts diff --git a/knip.config.ts b/knip.config.ts index 16d637b4b..9633755a0 100644 --- a/knip.config.ts +++ b/knip.config.ts @@ -119,7 +119,9 @@ export default { 'packages/server': { entry: ['src/**/*.test.ts'], project: 'src/**', - ignoreDependencies: ['@types/shell-quote'], + ignoreDependencies: [ + '@types/shell-quote', + ], }, 'packages/cli': { entry: ['src/**/*.test.ts', 'scripts/*.ts', 'tests/**/*.ts'], diff --git a/packages/app/src/components/FileTree.tsx b/packages/app/src/components/FileTree.tsx index a5a401d1a..9ef72868b 100644 --- a/packages/app/src/components/FileTree.tsx +++ b/packages/app/src/components/FileTree.tsx @@ -43,8 +43,6 @@ import { TriangleAlert, UnfoldVertical, } from 'lucide-react'; -import { __iconNode as botIcon } from 'lucide-react/dist/esm/icons/bot'; -import { __iconNode as link2Icon } from 'lucide-react/dist/esm/icons/link-2'; import { useTheme } from 'next-themes'; import { type DragEvent as ReactDragEvent, @@ -61,10 +59,7 @@ import { import { toast } from 'sonner'; import { DeleteConfirmationDialog } from '@/components/DeleteConfirmationDialog'; import { FileTreeFilteredToZeroNotice } from '@/components/FileTreeFilteredToZeroNotice'; -import { - MARKDOWN_FILE_ICON_PATH_D, - MARKDOWN_FILE_ICON_VIEWBOX, -} from '@/components/file-entry-icon'; +import { MARKDOWN_FILE_ICON_VIEWBOX } from '@/components/file-entry-icon'; import { appendSidebarUploadFields, collectTreeFolderPathsFromDocuments, @@ -101,6 +96,12 @@ import { applyExtensionBadges, FILE_TREE_EXT_BADGE_CSS, } from '@/components/file-tree-extension-badge'; +import { + AGENT_DECORATION_ICON_ID, + FILE_TREE_DECORATION_SPRITE_SHEET, + LINK_DECORATION_ICON_ID, + MARKDOWN_FILE_ICON_ID, +} from '@/components/file-tree-icon-sprite'; import { buildOkignorePatternFromTarget } from '@/components/file-tree-okignore'; import { applyDeleteToDocuments, @@ -116,6 +117,20 @@ import { type RenamedFolderMapping, remapActiveDocName, } from '@/components/file-tree-operations'; +import { + alternateMarkdownTreePath, + buildRowDecorationIndex, + collectTabsToCloseForDelete, + deleteTargetCoversPendingCreate, + hasSameStemMarkdownSiblingTreePath, + isAgentTreePath, + isEditableKeyboardTarget, + markdownTreeExtension, + parseAlreadyExistsRenamePath, + resolveDuplicableKeyboardTarget, + resolveKeyboardDeleteTargets, + selectedTreePathsToDeleteTargets, +} from '@/components/file-tree-path-helpers'; import { applyRenameInputAffordance, FILE_TREE_RENAME_INPUT_CSS, @@ -135,7 +150,6 @@ import { classifyEmptyTree, type DocumentEntry, type FileEntry, - type FolderEntry, filterVisibleEntries, hasOkPathSegment, isAssetEntry, @@ -228,18 +242,6 @@ import { useInstalledAgents } from './handoff/useInstalledAgents'; import { cancelHoverPrewarm, scheduleHoverPrewarm } from './sidebar-hover-prewarm'; import { useSidebar } from './ui/sidebar'; -const MARKDOWN_TREE_EXTENSION_PATTERN = /\.(md|mdx)$/i; - -function parseAlreadyExistsRenamePath(message: string): string | null { - const match = message.match(/^"(.+)" already exists\.$/); - return match ? match[1] : null; -} - -function markdownTreeExtension(path: string): string | null { - const match = path.match(MARKDOWN_TREE_EXTENSION_PATTERN); - return match ? match[0] : null; -} - function focusEditorAfterRename(docName: string): void { window.requestAnimationFrame(() => { const editor = getEditorForDoc(docName); @@ -300,43 +302,6 @@ async function copyToClipboard(text: string, kind: 'full' | 'relative'): Promise } } -const AGENT_FILE_NAMES = new Set(['agents', 'agent', 'claude', 'skill']); -const LINK_DECORATION_ICON_ID = 'ok-file-tree-link-decoration'; -const AGENT_DECORATION_ICON_ID = 'ok-file-tree-agent-decoration'; -const MARKDOWN_FILE_ICON_ID = 'ok-file-tree-markdown'; -// Custom Markdown file glyph (document with an "MD" label) overriding Pierre's -// built-in `complete`-set markdown glyph. `fill="currentColor"` lets -// `--trees-file-icon-color-markdown` (set in createFileTreeStyle, see -// file-tree-density.ts) color it. -const MARKDOWN_FILE_ICON_SYMBOL = ``; - -type IconNode = [string, Record][]; - -function iconNodeToSvg(iconNode: IconNode): string { - return ( - iconNode - // remove React key - .map(([tag, { key: _, ...attrs }]) => { - const attrString = Object.entries(attrs) - .map(([k, v]) => `${k}="${v}"`) - .join(' '); - return `<${tag} ${attrString} />`; - }) - .join('') - ); -} - -function createLucideSpriteSymbol(id: string, iconNode: IconNode): string { - const symbolContent = iconNodeToSvg(iconNode); - return `${symbolContent}`; -} - -const FILE_TREE_DECORATION_SPRITE_SHEET = ``; - // Drop-to-root affordance. The patched `@pierre/trees` sets // `data-file-tree-root-drag-target="true"` on the virtualized root while an // in-tree drag hovers empty content area (or a top-level file) — i.e. when the @@ -432,11 +397,6 @@ const FILE_TREE_CREATION_CLEARED_CSS = ` // CSS + DOM-mutation contract stays in one place. const FILE_TREE_UNSAFE_CSS = `${FILE_TREE_EXT_BADGE_CSS}\n${FILE_TREE_RENAME_INPUT_CSS}\n${FILE_TREE_ROOT_DROP_CSS}\n${FILE_TREE_EXTERNAL_FILE_DROP_CSS}\n${FILE_TREE_CREATION_CLEARED_CSS}\n${FILE_TREE_INDENT_GUIDE_CSS}\n${FILE_TREE_STICKY_HEADER_CSS}`; -function isAgentTreePath(treePath: string): boolean { - const name = treePath.split('/').pop()?.replace(/\.md$/i, '').toLowerCase(); - return !!name && AGENT_FILE_NAMES.has(name); -} - interface PendingCreate { kind: 'file' | 'folder'; renamePath: string; @@ -592,161 +552,6 @@ interface FileTreeMenuProps { documents: readonly FileEntry[]; } -function treePathToTarget(treePath: string, documents: readonly FileEntry[]): FileTreeTarget { - return treeItemToTarget( - { - kind: treePath.endsWith('/') ? 'directory' : 'file', - name: treePath, - path: treePath, - }, - documents, - ); -} - -function alternateMarkdownTreePath(treePath: string): string | null { - const match = treePath.match(/\.(md|mdx)$/i); - if (!match) return null; - const ext = match[0].toLowerCase(); - const alternateExt = ext === '.md' ? '.mdx' : '.md'; - return `${treePath.slice(0, -match[0].length)}${alternateExt}`; -} - -function hasSameStemMarkdownSiblingTreePath( - treePath: string, - treePaths: readonly string[], -): boolean { - const alternate = alternateMarkdownTreePath(treePath); - if (!alternate) return false; - return treePaths.includes(alternate); -} - -function isTreePathInsideFolder(treePath: string, folderTreePath: string): boolean { - return treePath !== folderTreePath && treePath.startsWith(folderTreePath); -} - -function selectedTreePathsToDeleteTargets( - selectedTreePaths: readonly string[], - documents: readonly FileEntry[], -): FileTreeTarget[] { - // Revealed `.ok` rows are read-only OK-managed state — they never become - // delete targets, even when swept into a multi-selection beside deletable - // rows (this also keeps the confirm dialog's item count honest). - const uniqueDeletablePaths = [...new Set(selectedTreePaths)].filter( - (treePath) => !hasOkPathSegment(treePath), - ); - const selectedFolderPaths = uniqueDeletablePaths.filter((treePath) => treePath.endsWith('/')); - return uniqueDeletablePaths - .filter( - (treePath) => - !selectedFolderPaths.some((folderPath) => isTreePathInsideFolder(treePath, folderPath)), - ) - .map((treePath) => treePathToTarget(treePath, documents)); -} - -function normalizeTreePathFromModel(model: PierreFileTreeModel, treePath: string): string { - const selectedItem = - model.getItem(treePath) ?? model.getItem(folderPathToTreeDirectoryPath(treePath)); - return selectedItem?.isDirectory() - ? folderPathToTreeDirectoryPath(treeDirectoryPathToFolderPath(selectedItem.getPath())) - : treePath; -} - -function focusedOrFirstSelectedTreePath(model: PierreFileTreeModel): string | null { - const selectedPath = model.getFocusedPath() ?? model.getSelectedPaths()[0] ?? null; - return selectedPath ? normalizeTreePathFromModel(model, selectedPath) : null; -} - -function resolveDuplicableKeyboardTarget( - model: PierreFileTreeModel, - documents: readonly FileEntry[], - assetTreePaths: ReadonlySet, -): FileTreeTarget | null { - const selectedPath = focusedOrFirstSelectedTreePath(model); - // Revealed `.ok` rows are read-only OK-managed state — no keyboard - // copy/paste/duplicate, matching their menu's suppressed affordances. - if (!selectedPath || assetTreePaths.has(selectedPath) || hasOkPathSegment(selectedPath)) { - return null; - } - return treePathToTarget(selectedPath, documents); -} - -function resolveKeyboardDeleteTargets( - model: PierreFileTreeModel, - documents: readonly FileEntry[], -): FileTreeTarget[] { - const selectedPaths = model.getSelectedPaths(); - const focusedPath = focusedOrFirstSelectedTreePath(model); - const paths = - selectedPaths.length > 0 - ? selectedPaths.map((treePath) => normalizeTreePathFromModel(model, treePath)) - : focusedPath - ? [focusedPath] - : []; - return selectedTreePathsToDeleteTargets(paths, documents); -} - -function isPathAtOrInsideFolder(path: string, folderPath: string): boolean { - return path === folderPath || path.startsWith(`${folderPath}/`); -} - -function isEditableKeyboardTarget(target: EventTarget | null): boolean { - if (!(target instanceof HTMLElement)) return false; - const tagName = target.tagName; - return ( - tagName === 'INPUT' || - tagName === 'TEXTAREA' || - tagName === 'SELECT' || - target.isContentEditable || - target.closest('[contenteditable="true"]') !== null - ); -} - -function collectTabsToCloseForDelete( - targets: readonly FileTreeTarget[], - documents: readonly FileEntry[], - folderTreePaths: readonly string[], -): { docNames: Set; folderPaths: Set; assetPaths: Set } { - const docNames = new Set(); - const folderPaths = new Set(); - const assetPaths = new Set(); - - for (const target of targets) { - if (target.kind === 'file') { - docNames.add(target.path); - continue; - } - if (target.kind === 'asset') { - assetPaths.add(target.path); - continue; - } - - folderPaths.add(target.path); - for (const entry of documents) { - if (isDocumentEntry(entry) && entry.docName.startsWith(`${target.path}/`)) { - docNames.add(entry.docName); - } else if (isAssetEntry(entry) && entry.path.startsWith(`${target.path}/`)) { - assetPaths.add(entry.path); - } - } - for (const treePath of folderTreePaths) { - const folderPath = treeDirectoryPathToFolderPath(treePath); - if (isPathAtOrInsideFolder(folderPath, target.path)) { - folderPaths.add(folderPath); - } - } - } - - return { docNames, folderPaths, assetPaths }; -} - -function deleteTargetCoversPendingCreate(target: FileTreeTarget, pending: PendingCreate): boolean { - if (target.kind === 'file') { - return pending.kind === 'file' && target.path === pending.createdPath; - } - if (target.kind === 'asset') return false; - return isPathAtOrInsideFolder(pending.createdPath, target.path); -} - function FileTreeMenu({ item, context, @@ -1386,25 +1191,6 @@ export function FileTree({ } ); } - function navigateToWithPulse( - targetPath: string, - size?: number, - options?: { registerPage?: boolean }, - ) { - if (options?.registerPage) addPage(targetPath); - openTarget(navigationTargetForDocument(targetPath, size), { tabBehavior: 'replace-active' }); - replaceHashWithoutNavigation(hashFromDocName(targetPath)); - notifySidebarFileSelected(); - } - function navigateToFolderWithPulse(folderPath: string) { - const nextHash = hashFromFolderPath(folderPath); - openTarget( - { kind: 'folder', target: folderPath, folderPath }, - { tabBehavior: 'replace-active' }, - ); - replaceHashWithoutNavigation(nextHash); - notifySidebarFileSelected(); - } const [documents, setDocuments] = useState([]); const [loading, setLoading] = useState(true); const [error, setError] = useState(null); @@ -1467,6 +1253,45 @@ export function FileTree({ const documentsRef = useRef(documents); const pageMetaRef = useRef(pageMeta); const pendingExactFileSelectionRef = useRef(null); + // Single navigation path for tree-initiated opens: resolve the target per + // kind, open it in the active tab, sync the hash, and pulse the sidebar. + function navigateWithPulse( + target: + | { kind: 'doc'; docName: string; size?: number; registerPage?: boolean } + | { kind: 'folder'; folderPath: string } + | { kind: 'asset'; assetPath: string; entries?: readonly FileEntry[] }, + ) { + if (target.kind === 'doc') { + if (target.registerPage) addPage(target.docName); + openTarget(navigationTargetForDocument(target.docName, target.size), { + tabBehavior: 'replace-active', + }); + replaceHashWithoutNavigation(hashFromDocName(target.docName)); + } else if (target.kind === 'folder') { + openTarget( + { kind: 'folder', target: target.folderPath, folderPath: target.folderPath }, + { tabBehavior: 'replace-active' }, + ); + replaceHashWithoutNavigation(hashFromFolderPath(target.folderPath)); + } else { + const currentEntries = target.entries ?? documentsRef.current; + const entry = currentEntries.find( + (item): item is Extract => + isAssetEntry(item) && item.path === target.assetPath, + ); + openTarget( + { + kind: 'asset', + target: target.assetPath, + assetPath: target.assetPath, + mediaKind: entry?.mediaKind ?? null, + }, + { tabBehavior: 'replace-active' }, + ); + replaceHashWithoutNavigation(hashFromAssetPath(target.assetPath)); + } + notifySidebarFileSelected(); + } function activateTreePath(treePath: string, entries: readonly FileEntry[] = documents) { const action = resolveFileTreeSelectionAction(treePath, entries); if (action.kind === 'none') { @@ -1491,7 +1316,7 @@ export function FileTree({ return; } if (action.kind === 'folder') { - navigateToFolderWithPulse(action.path); + navigateWithPulse({ kind: 'folder', folderPath: action.path }); return; } const docEntry = entries.find( @@ -1513,36 +1338,23 @@ export function FileTree({ return; } if (okTarget?.kind === 'doc') { - navigateToWithPulse(okTarget.docName); + navigateWithPulse({ kind: 'doc', docName: okTarget.docName }); return; } - navigateToWithPulse(action.path, docEntry?.size, { + navigateWithPulse({ + kind: 'doc', + docName: action.path, + size: docEntry?.size, registerPage: hasSupportedDocumentExtension(action.path), }); } - function navigateToAssetWithPulse(assetPath: string, entries?: readonly FileEntry[]) { - const currentEntries = entries ?? documentsRef.current; - const entry = currentEntries.find( - (item): item is Extract => - isAssetEntry(item) && item.path === assetPath, - ); - openTarget( - { - kind: 'asset', - target: assetPath, - assetPath, - mediaKind: entry?.mediaKind ?? null, - }, - { tabBehavior: 'replace-active' }, - ); - replaceHashWithoutNavigation(hashFromAssetPath(assetPath)); - notifySidebarFileSelected(); - } const activeDocNameRef = useRef(activeDocName); const assetTreePaths = new Set( documents.filter(isAssetEntry).map((entry) => fileEntryToTreePath(entry)), ); const assetTreePathsRef = useRef(assetTreePaths); + const rowDecorationIndex = buildRowDecorationIndex(documents); + const rowDecorationIndexRef = useRef(rowDecorationIndex); const activeAncestorTreePathsRef = useRef([]); const pendingCreateRef = useRef(null); const cleanupPendingCreateRef = useRef< @@ -1854,10 +1666,7 @@ export function FileTree({ onSelectionChange: (selectedPaths) => handleSelectionChangeRef.current(selectedPaths), renderRowDecoration: ({ item }) => { if (item.kind === 'file') { - const doc = documentsRef.current.find( - (entry): entry is DocumentEntry => - isDocumentEntry(entry) && docNameToTreePath(entry.docName, entry.docExt) === item.path, - ); + const doc = rowDecorationIndexRef.current.docsByTreePath.get(item.path); if (doc?.isSymlink) { const targetPath = doc.targetPath; return { @@ -1876,10 +1685,8 @@ export function FileTree({ // Symlinked directories carry isSymlink on their FolderEntry. Badge the // alias folder itself (Finder-style — its contents are not separately // marked, since they live behind the one symlink). - const folder = documentsRef.current.find( - (entry): entry is FolderEntry => - isFolderEntry(entry) && - folderPathToTreeDirectoryPath(entry.path) === folderPathToTreeDirectoryPath(item.path), + const folder = rowDecorationIndexRef.current.foldersByTreeDirectoryPath.get( + folderPathToTreeDirectoryPath(item.path), ); if (folder?.isSymlink) { const targetPath = folder.targetPath; @@ -2162,9 +1969,9 @@ export function FileTree({ if (success.path !== target.path) { if (success.kind === 'folder') { - navigateToFolderWithPulse(success.path); + navigateWithPulse({ kind: 'folder', folderPath: success.path }); } else { - navigateToWithPulse(success.path); + navigateWithPulse({ kind: 'doc', docName: success.path }); } } toast.success(success.kind === 'folder' ? t`Folder duplicated` : t`File duplicated`, { @@ -2830,15 +2637,19 @@ export function FileTree({ nextActiveFolderPath && nextActiveFolderPath !== currentActiveFolderPath ) { - navigateToFolderWithPulse(nextActiveFolderPath); + navigateWithPulse({ kind: 'folder', folderPath: nextActiveFolderPath }); } else if (nextActiveDocName && nextActiveDocName !== currentActiveDocName) { - navigateToWithPulse(nextActiveDocName); + navigateWithPulse({ kind: 'doc', docName: nextActiveDocName }); focusEditorAfterRename(nextActiveDocName); } else if ( nextActiveAssetPath && (activeDocToAssetPath || nextActiveAssetPath !== currentActiveAssetPath) ) { - navigateToAssetWithPulse(nextActiveAssetPath, nextDocumentsForRename ?? documentsRef.current); + navigateWithPulse({ + kind: 'asset', + assetPath: nextActiveAssetPath, + entries: nextDocumentsForRename ?? documentsRef.current, + }); } emitDocumentsChanged(['files', 'backlinks', 'graph']); }; @@ -3279,7 +3090,7 @@ export function FileTree({ return next; }); emitDocumentsChanged(['files', 'backlinks', 'graph']); - navigateToWithPulse(docName); + navigateWithPulse({ kind: 'doc', docName }); } else { const folderPath = treeDirectoryPathToFolderPath(placeholder.addPath); const res = await fetch('/api/create-folder', { @@ -3319,7 +3130,7 @@ export function FileTree({ return next; }); emitDocumentsChanged(['files']); - navigateToFolderWithPulse(createdPath); + navigateWithPulse({ kind: 'folder', folderPath: createdPath }); } let disposed = false; @@ -3403,6 +3214,7 @@ export function FileTree({ useLayoutEffect(() => { documentsRef.current = documents; + rowDecorationIndexRef.current = rowDecorationIndex; pageMetaRef.current = pageMeta; activeDocNameRef.current = activeDocName; activeTargetRef.current = activeTarget; @@ -4350,7 +4162,7 @@ export function FileTree({ event.preventDefault(); event.stopPropagation(); if (folderItem && !folderItem.isExpanded()) folderItem.expand(); - queueMicrotask(() => navigateToFolderWithPulse(folderPath)); + queueMicrotask(() => navigateWithPulse({ kind: 'folder', folderPath })); return; } if (model.getSelectedPaths().length !== 1) return; @@ -4358,7 +4170,7 @@ export function FileTree({ event.preventDefault(); event.stopPropagation(); if (folderItem && !folderItem.isExpanded()) folderItem.expand(); - queueMicrotask(() => navigateToFolderWithPulse(folderPath)); + queueMicrotask(() => navigateWithPulse({ kind: 'folder', folderPath })); return; } @@ -4372,7 +4184,7 @@ export function FileTree({ pendingExactFileSelectionRef.current = path; // Let handleSelectionChange's microtask consume the exact file selection // before navigation commits the extension-qualified URL. - setTimeout(() => navigateToWithPulse(path, undefined, { registerPage: true }), 0); + setTimeout(() => navigateWithPulse({ kind: 'doc', docName: path, registerPage: true }), 0); return; } queueMicrotask(() => activateTreePath(path)); diff --git a/packages/app/src/components/file-tree-icon-sprite.ts b/packages/app/src/components/file-tree-icon-sprite.ts new file mode 100644 index 000000000..d3f72644d --- /dev/null +++ b/packages/app/src/components/file-tree-icon-sprite.ts @@ -0,0 +1,50 @@ +/** + * Inline SVG sprite sheet fed to the @pierre/trees `icons.spriteSheet` option: + * row-decoration glyphs (symlink, agent file) plus the custom Markdown file + * icon. Lucide icons are imported as raw `__iconNode` data (not React + * components) so they can be serialized into `` markup once at module + * load. + */ + +import { __iconNode as botIcon } from 'lucide-react/dist/esm/icons/bot'; +import { __iconNode as link2Icon } from 'lucide-react/dist/esm/icons/link-2'; +import { + MARKDOWN_FILE_ICON_PATH_D, + MARKDOWN_FILE_ICON_VIEWBOX, +} from '@/components/file-entry-icon'; + +export const LINK_DECORATION_ICON_ID = 'ok-file-tree-link-decoration'; +export const AGENT_DECORATION_ICON_ID = 'ok-file-tree-agent-decoration'; +export const MARKDOWN_FILE_ICON_ID = 'ok-file-tree-markdown'; +// Custom Markdown file glyph (document with an "MD" label) overriding Pierre's +// built-in `complete`-set markdown glyph. `fill="currentColor"` lets +// `--trees-file-icon-color-markdown` (set in createFileTreeStyle, see +// file-tree-density.ts) color it. +const MARKDOWN_FILE_ICON_SYMBOL = ``; + +type IconNode = [string, Record][]; + +function iconNodeToSvg(iconNode: IconNode): string { + return ( + iconNode + // remove React key + .map(([tag, { key: _, ...attrs }]) => { + const attrString = Object.entries(attrs) + .map(([k, v]) => `${k}="${v}"`) + .join(' '); + return `<${tag} ${attrString} />`; + }) + .join('') + ); +} + +function createLucideSpriteSymbol(id: string, iconNode: IconNode): string { + const symbolContent = iconNodeToSvg(iconNode); + return `${symbolContent}`; +} + +export const FILE_TREE_DECORATION_SPRITE_SHEET = ``; diff --git a/packages/app/src/components/file-tree-path-helpers.test.ts b/packages/app/src/components/file-tree-path-helpers.test.ts new file mode 100644 index 000000000..65714cab0 --- /dev/null +++ b/packages/app/src/components/file-tree-path-helpers.test.ts @@ -0,0 +1,189 @@ +import { describe, expect, test } from 'bun:test'; +import type { FileTreeTarget } from './file-tree-operations'; +import { + alternateMarkdownTreePath, + buildRowDecorationIndex, + deleteTargetCoversPendingCreate, + hasSameStemMarkdownSiblingTreePath, + isAgentTreePath, + markdownTreeExtension, + parseAlreadyExistsRenamePath, + selectedTreePathsToDeleteTargets, +} from './file-tree-path-helpers'; +import type { FileEntry } from './file-tree-utils'; + +const doc = (docName: string, docExt?: string): FileEntry => ({ + kind: 'document', + docName, + docExt, + size: 1, + modified: '2026-04-13T00:00:00.000Z', +}); + +const folder = (path: string): FileEntry => ({ + kind: 'folder', + path, + size: 0, + modified: '2026-04-13T00:00:00.000Z', +}); + +const asset = (path: string): FileEntry => ({ + kind: 'asset', + path, + assetExt: '.png', + mediaKind: 'image', + size: 1, + modified: '2026-04-13T00:00:00.000Z', +}); + +describe('file-tree-path-helpers', () => { + test('parseAlreadyExistsRenamePath extracts the quoted path', () => { + expect(parseAlreadyExistsRenamePath('"docs/notes.md" already exists.')).toBe('docs/notes.md'); + expect(parseAlreadyExistsRenamePath('some other error')).toBeNull(); + }); + + test('markdownTreeExtension returns the matched extension or null', () => { + expect(markdownTreeExtension('docs/notes.md')).toBe('.md'); + expect(markdownTreeExtension('docs/notes.MDX')).toBe('.MDX'); + expect(markdownTreeExtension('docs/image.png')).toBeNull(); + }); + + test('isAgentTreePath flags agent/skill filenames case-insensitively', () => { + expect(isAgentTreePath('AGENTS.md')).toBe(true); + expect(isAgentTreePath('nested/claude.md')).toBe(true); + expect(isAgentTreePath('skill')).toBe(true); + expect(isAgentTreePath('docs/notes.md')).toBe(false); + }); + + test('alternateMarkdownTreePath swaps .md <-> .mdx and preserves the stem', () => { + expect(alternateMarkdownTreePath('docs/notes.md')).toBe('docs/notes.mdx'); + expect(alternateMarkdownTreePath('docs/notes.mdx')).toBe('docs/notes.md'); + expect(alternateMarkdownTreePath('docs/image.png')).toBeNull(); + }); + + test('hasSameStemMarkdownSiblingTreePath detects the alternate-extension sibling', () => { + const paths = ['docs/notes.mdx', 'docs/other.md']; + expect(hasSameStemMarkdownSiblingTreePath('docs/notes.md', paths)).toBe(true); + expect(hasSameStemMarkdownSiblingTreePath('docs/other.md', paths)).toBe(false); + }); + + describe('selectedTreePathsToDeleteTargets', () => { + const documents: FileEntry[] = [doc('docs/notes'), folder('docs'), doc('docs/nested/page')]; + + test('deduplicates repeated selections', () => { + const targets = selectedTreePathsToDeleteTargets( + ['docs/notes.md', 'docs/notes.md'], + documents, + ); + expect(targets).toHaveLength(1); + expect(targets[0].path).toBe('docs/notes'); + }); + + test('excludes read-only .ok rows', () => { + const targets = selectedTreePathsToDeleteTargets( + ['docs/notes.md', 'docs/.ok/templates/trip.md'], + documents, + ); + expect(targets.map((t) => t.path)).toEqual(['docs/notes']); + }); + + test('drops paths already covered by a selected ancestor folder', () => { + const targets = selectedTreePathsToDeleteTargets(['docs/', 'docs/notes.md'], documents); + // The folder subsumes the file inside it — only the folder survives. + expect(targets).toHaveLength(1); + expect(targets[0].kind).toBe('folder'); + expect(targets[0].path).toBe('docs'); + }); + }); + + describe('deleteTargetCoversPendingCreate', () => { + const fileTarget = (path: string): FileTreeTarget => ({ kind: 'file', name: path, path }); + const folderTarget = (path: string): FileTreeTarget => ({ kind: 'folder', name: path, path }); + const assetTarget = (path: string): FileTreeTarget => ({ kind: 'asset', name: path, path }); + + test('file target covers a pending file at the same path only', () => { + expect( + deleteTargetCoversPendingCreate(fileTarget('docs/new'), { + kind: 'file', + createdPath: 'docs/new', + }), + ).toBe(true); + expect( + deleteTargetCoversPendingCreate(fileTarget('docs/new'), { + kind: 'file', + createdPath: 'docs/other', + }), + ).toBe(false); + // A file target never covers a pending folder. + expect( + deleteTargetCoversPendingCreate(fileTarget('docs/new'), { + kind: 'folder', + createdPath: 'docs/new', + }), + ).toBe(false); + }); + + test('asset target never covers a pending create', () => { + expect( + deleteTargetCoversPendingCreate(assetTarget('docs/img.png'), { + kind: 'file', + createdPath: 'docs/img.png', + }), + ).toBe(false); + }); + + test('folder target covers a pending create at or inside the folder', () => { + expect( + deleteTargetCoversPendingCreate(folderTarget('docs'), { + kind: 'file', + createdPath: 'docs/new', + }), + ).toBe(true); + expect( + deleteTargetCoversPendingCreate(folderTarget('docs'), { + kind: 'folder', + createdPath: 'docs', + }), + ).toBe(true); + // A sibling folder that only shares a name prefix is not inside it. + expect( + deleteTargetCoversPendingCreate(folderTarget('docs'), { + kind: 'folder', + createdPath: 'docs-archive/new', + }), + ).toBe(false); + }); + }); + + describe('buildRowDecorationIndex', () => { + test('keys documents by tree path and folders by directory path; skips assets', () => { + const index = buildRowDecorationIndex([ + doc('docs/notes', '.md'), + doc('README'), + folder('docs/empty'), + asset('docs/image.png'), + ]); + + expect(index.docsByTreePath.get('docs/notes.md')?.kind).toBe('document'); + expect(index.docsByTreePath.get('README.md')?.kind).toBe('document'); + expect(index.foldersByTreeDirectoryPath.get('docs/empty/')?.kind).toBe('folder'); + // Assets are not indexed by either map. + expect(index.docsByTreePath.has('docs/image.png')).toBe(false); + expect(index.foldersByTreeDirectoryPath.has('docs/image.png')).toBe(false); + }); + + test('first entry wins on document key collision, matching Array.find', () => { + const first = doc('dup', '.md'); + const second = doc('dup', '.md'); + const index = buildRowDecorationIndex([first, second]); + expect(index.docsByTreePath.get('dup.md')).toBe(first); + }); + + test('first entry wins on folder key collision, matching Array.find', () => { + const first = folder('dup'); + const second = folder('dup'); + const index = buildRowDecorationIndex([first, second]); + expect(index.foldersByTreeDirectoryPath.get('dup/')).toBe(first); + }); + }); +}); diff --git a/packages/app/src/components/file-tree-path-helpers.ts b/packages/app/src/components/file-tree-path-helpers.ts new file mode 100644 index 000000000..560e4f985 --- /dev/null +++ b/packages/app/src/components/file-tree-path-helpers.ts @@ -0,0 +1,239 @@ +/** + * Pure tree-path helpers for the file sidebar: keyboard/selection target + * resolution, delete-target planning, markdown-sibling checks, and the + * row-decoration lookup index. Everything here is a pure function of its + * inputs — no React, no DOM mutation, no network — except + * `isEditableKeyboardTarget`, which performs a DOM read. + */ + +import type { FileTree as PierreFileTreeModel } from '@pierre/trees'; +import { + docNameToTreePath, + folderPathToTreeDirectoryPath, + treeDirectoryPathToFolderPath, + treeItemToTarget, +} from '@/components/file-tree-adapter'; +import type { FileTreeTarget } from '@/components/file-tree-operations'; +import { + type DocumentEntry, + type FileEntry, + type FolderEntry, + hasOkPathSegment, + isAssetEntry, + isDocumentEntry, + isFolderEntry, +} from '@/components/file-tree-utils'; + +const MARKDOWN_TREE_EXTENSION_PATTERN = /\.(md|mdx)$/i; + +export function parseAlreadyExistsRenamePath(message: string): string | null { + const match = message.match(/^"(.+)" already exists\.$/); + return match ? match[1] : null; +} + +export function markdownTreeExtension(path: string): string | null { + const match = path.match(MARKDOWN_TREE_EXTENSION_PATTERN); + return match ? match[0] : null; +} + +const AGENT_FILE_NAMES = new Set(['agents', 'agent', 'claude', 'skill']); + +export function isAgentTreePath(treePath: string): boolean { + const name = treePath.split('/').pop()?.replace(/\.md$/i, '').toLowerCase(); + return !!name && AGENT_FILE_NAMES.has(name); +} + +function treePathToTarget(treePath: string, documents: readonly FileEntry[]): FileTreeTarget { + return treeItemToTarget( + { + kind: treePath.endsWith('/') ? 'directory' : 'file', + name: treePath, + path: treePath, + }, + documents, + ); +} + +export function alternateMarkdownTreePath(treePath: string): string | null { + const match = treePath.match(/\.(md|mdx)$/i); + if (!match) return null; + const ext = match[0].toLowerCase(); + const alternateExt = ext === '.md' ? '.mdx' : '.md'; + return `${treePath.slice(0, -match[0].length)}${alternateExt}`; +} + +export function hasSameStemMarkdownSiblingTreePath( + treePath: string, + treePaths: readonly string[], +): boolean { + const alternate = alternateMarkdownTreePath(treePath); + if (!alternate) return false; + return treePaths.includes(alternate); +} + +function isTreePathInsideFolder(treePath: string, folderTreePath: string): boolean { + return treePath !== folderTreePath && treePath.startsWith(folderTreePath); +} + +export function selectedTreePathsToDeleteTargets( + selectedTreePaths: readonly string[], + documents: readonly FileEntry[], +): FileTreeTarget[] { + // Revealed `.ok` rows are read-only OK-managed state — they never become + // delete targets, even when swept into a multi-selection beside deletable + // rows (this also keeps the confirm dialog's item count honest). + const uniqueDeletablePaths = [...new Set(selectedTreePaths)].filter( + (treePath) => !hasOkPathSegment(treePath), + ); + const selectedFolderPaths = uniqueDeletablePaths.filter((treePath) => treePath.endsWith('/')); + return uniqueDeletablePaths + .filter( + (treePath) => + !selectedFolderPaths.some((folderPath) => isTreePathInsideFolder(treePath, folderPath)), + ) + .map((treePath) => treePathToTarget(treePath, documents)); +} + +function normalizeTreePathFromModel(model: PierreFileTreeModel, treePath: string): string { + const selectedItem = + model.getItem(treePath) ?? model.getItem(folderPathToTreeDirectoryPath(treePath)); + return selectedItem?.isDirectory() + ? folderPathToTreeDirectoryPath(treeDirectoryPathToFolderPath(selectedItem.getPath())) + : treePath; +} + +function focusedOrFirstSelectedTreePath(model: PierreFileTreeModel): string | null { + const selectedPath = model.getFocusedPath() ?? model.getSelectedPaths()[0] ?? null; + return selectedPath ? normalizeTreePathFromModel(model, selectedPath) : null; +} + +export function resolveDuplicableKeyboardTarget( + model: PierreFileTreeModel, + documents: readonly FileEntry[], + assetTreePaths: ReadonlySet, +): FileTreeTarget | null { + const selectedPath = focusedOrFirstSelectedTreePath(model); + // Revealed `.ok` rows are read-only OK-managed state — no keyboard + // copy/paste/duplicate, matching their menu's suppressed affordances. + if (!selectedPath || assetTreePaths.has(selectedPath) || hasOkPathSegment(selectedPath)) { + return null; + } + return treePathToTarget(selectedPath, documents); +} + +export function resolveKeyboardDeleteTargets( + model: PierreFileTreeModel, + documents: readonly FileEntry[], +): FileTreeTarget[] { + const selectedPaths = model.getSelectedPaths(); + const focusedPath = focusedOrFirstSelectedTreePath(model); + const paths = + selectedPaths.length > 0 + ? selectedPaths.map((treePath) => normalizeTreePathFromModel(model, treePath)) + : focusedPath + ? [focusedPath] + : []; + return selectedTreePathsToDeleteTargets(paths, documents); +} + +function isPathAtOrInsideFolder(path: string, folderPath: string): boolean { + return path === folderPath || path.startsWith(`${folderPath}/`); +} + +export function isEditableKeyboardTarget(target: EventTarget | null): boolean { + if (!(target instanceof HTMLElement)) return false; + const tagName = target.tagName; + return ( + tagName === 'INPUT' || + tagName === 'TEXTAREA' || + tagName === 'SELECT' || + target.isContentEditable || + target.closest('[contenteditable="true"]') !== null + ); +} + +export function collectTabsToCloseForDelete( + targets: readonly FileTreeTarget[], + documents: readonly FileEntry[], + folderTreePaths: readonly string[], +): { docNames: Set; folderPaths: Set; assetPaths: Set } { + const docNames = new Set(); + const folderPaths = new Set(); + const assetPaths = new Set(); + + for (const target of targets) { + if (target.kind === 'file') { + docNames.add(target.path); + continue; + } + if (target.kind === 'asset') { + assetPaths.add(target.path); + continue; + } + + folderPaths.add(target.path); + for (const entry of documents) { + if (isDocumentEntry(entry) && entry.docName.startsWith(`${target.path}/`)) { + docNames.add(entry.docName); + } else if (isAssetEntry(entry) && entry.path.startsWith(`${target.path}/`)) { + assetPaths.add(entry.path); + } + } + for (const treePath of folderTreePaths) { + const folderPath = treeDirectoryPathToFolderPath(treePath); + if (isPathAtOrInsideFolder(folderPath, target.path)) { + folderPaths.add(folderPath); + } + } + } + + return { docNames, folderPaths, assetPaths }; +} + +/** + * The slice of FileTree's pending-create state this module needs — kept + * structural so the component's richer lifecycle record stays local to it. + */ +export interface PendingCreateTarget { + kind: 'file' | 'folder'; + createdPath: string; +} + +export function deleteTargetCoversPendingCreate( + target: FileTreeTarget, + pending: PendingCreateTarget, +): boolean { + if (target.kind === 'file') { + return pending.kind === 'file' && target.path === pending.createdPath; + } + if (target.kind === 'asset') return false; + return isPathAtOrInsideFolder(pending.createdPath, target.path); +} + +export interface RowDecorationIndex { + docsByTreePath: ReadonlyMap; + foldersByTreeDirectoryPath: ReadonlyMap; +} + +/** + * O(1) lookup index for `renderRowDecoration`, which the tree invokes per + * visible row on every redraw — a linear scan of the document list there is + * O(rows × entries). First entry wins on key collision, matching the + * front-to-back `Array.find` this replaces. + */ +export function buildRowDecorationIndex(entries: readonly FileEntry[]): RowDecorationIndex { + const docsByTreePath = new Map(); + const foldersByTreeDirectoryPath = new Map(); + for (const entry of entries) { + if (isDocumentEntry(entry)) { + const treePath = docNameToTreePath(entry.docName, entry.docExt); + if (!docsByTreePath.has(treePath)) docsByTreePath.set(treePath, entry); + } else if (isFolderEntry(entry)) { + const directoryPath = folderPathToTreeDirectoryPath(entry.path); + if (!foldersByTreeDirectoryPath.has(directoryPath)) { + foldersByTreeDirectoryPath.set(directoryPath, entry); + } + } + } + return { docsByTreePath, foldersByTreeDirectoryPath }; +} diff --git a/packages/core/src/bridge/bind-frontmatter-doc.ts b/packages/core/src/bridge/bind-frontmatter-doc.ts index c014b1413..8f9d95287 100644 --- a/packages/core/src/bridge/bind-frontmatter-doc.ts +++ b/packages/core/src/bridge/bind-frontmatter-doc.ts @@ -1,3 +1,4 @@ + import type * as Y from 'yjs'; import type { Err, Ok, Result } from '../config/result.ts'; import { type FrontmatterValidationError, toFrontmatterIssue } from '../frontmatter/errors.ts'; diff --git a/packages/core/src/bridge/bridge-invariant.ts b/packages/core/src/bridge/bridge-invariant.ts index 005627195..8bfdb40cd 100644 --- a/packages/core/src/bridge/bridge-invariant.ts +++ b/packages/core/src/bridge/bridge-invariant.ts @@ -1,3 +1,4 @@ + import { fnv1aDigest } from './hash-util.ts'; export type BridgeInvariantSite = 'observer-b' | 'persistence' | 'test-harness'; diff --git a/packages/core/src/bridge/doc-boundary-space.ts b/packages/core/src/bridge/doc-boundary-space.ts index 8890f6739..c18c086d5 100644 --- a/packages/core/src/bridge/doc-boundary-space.ts +++ b/packages/core/src/bridge/doc-boundary-space.ts @@ -1,3 +1,4 @@ + import { FRONTMATTER_RE, stripFrontmatter } from '../extensions/frontmatter.ts'; const LEADING_BOUNDARY_RE = /^(?:\r?\n)+/; diff --git a/packages/core/src/bridge/growth-detect.ts b/packages/core/src/bridge/growth-detect.ts index 9d9c244b0..463e89595 100644 --- a/packages/core/src/bridge/growth-detect.ts +++ b/packages/core/src/bridge/growth-detect.ts @@ -1,3 +1,4 @@ + const DEFAULT_MIN_SUBSTANTIVE_LINE_LENGTH = 16; export const DUPLICATION_GATE_MIN_LINE_LENGTH = 8; diff --git a/packages/core/src/bridge/merge-three-way.ts b/packages/core/src/bridge/merge-three-way.ts index 3ab9cc6ef..3c927eb47 100644 --- a/packages/core/src/bridge/merge-three-way.ts +++ b/packages/core/src/bridge/merge-three-way.ts @@ -38,6 +38,7 @@ function mergeThreeWayImpl(baseline: string, userText: string, agentText: string return parts.join('\n'); } + export type BridgeMergeContentLossSide = 'user' | 'agent'; export type BridgeMergeContentLossWhich = 'substring' | 'order' | 'growth'; diff --git a/packages/core/src/bridge/normalize.ts b/packages/core/src/bridge/normalize.ts index 4b8890476..3f0880aa5 100644 --- a/packages/core/src/bridge/normalize.ts +++ b/packages/core/src/bridge/normalize.ts @@ -1,3 +1,4 @@ + const COMMONMARK_ESCAPE_RE = /\\([!"#$%&'()*+,\-./:;<=>?@[\]^_`{|}~])/g; const TABLE_ALIGN_ROW_RE = /^\s*\|?\s*:?-+:?\s*(\|\s*:?-+:?\s*)+\|?\s*$/; diff --git a/packages/core/src/bridge/pm-structural-equivalence.ts b/packages/core/src/bridge/pm-structural-equivalence.ts index 754dc5cc2..0be5203e4 100644 --- a/packages/core/src/bridge/pm-structural-equivalence.ts +++ b/packages/core/src/bridge/pm-structural-equivalence.ts @@ -32,6 +32,7 @@ const BR_SENTINEL: PmStructuralNode = { type: '__br__' }; const BR_LITERAL_RE = /^$/i; + interface DegradeEntry { readonly label: string; readonly severity: ToleranceClassSeverity; @@ -127,6 +128,7 @@ export interface ComparePmStructuralOptions { ignoreAttrs?: (attrKey: string) => boolean; } + /** Rebuild a node with `fn` applied to each child; identity when childless. * Shared by the degrade canonicalizers so none re-implements the walk. */ function mapChildren( @@ -248,6 +250,7 @@ function applyDegrades(tree: PmStructuralNode): { return { normalized: current, fired }; } + export function comparePmStructural( expected: PmStructuralNode, actual: PmStructuralNode, diff --git a/packages/core/src/bridge/subsequence.ts b/packages/core/src/bridge/subsequence.ts index af20de1e2..dca644141 100644 --- a/packages/core/src/bridge/subsequence.ts +++ b/packages/core/src/bridge/subsequence.ts @@ -1,3 +1,4 @@ + /** True when `needle` is a subsequence of `haystack` (two-pointer, O(n+m)): * insertions in `haystack` are free; any dropped or substituted `needle` byte * fails. */ diff --git a/packages/core/src/markdown/callout-transformer.ts b/packages/core/src/markdown/callout-transformer.ts index 83d47404a..a363dfd54 100644 --- a/packages/core/src/markdown/callout-transformer.ts +++ b/packages/core/src/markdown/callout-transformer.ts @@ -1,3 +1,4 @@ + import type { Blockquote, Paragraph, Root } from 'mdast'; import type { MdxJsxAttribute, MdxJsxFlowElement } from 'mdast-util-mdx'; import { visit } from 'unist-util-visit'; diff --git a/packages/core/src/markdown/comment-promoter.ts b/packages/core/src/markdown/comment-promoter.ts index 193e286b0..47f0d07e4 100644 --- a/packages/core/src/markdown/comment-promoter.ts +++ b/packages/core/src/markdown/comment-promoter.ts @@ -1,3 +1,4 @@ + import type { Nodes, Paragraph, PhrasingContent, Root, RootContent, Text } from 'mdast'; import { SKIP, visit } from 'unist-util-visit'; import type { VFile } from 'vfile'; diff --git a/packages/core/src/markdown/dedent-block-jsx-close.ts b/packages/core/src/markdown/dedent-block-jsx-close.ts index ece5c02a1..b6299a948 100644 --- a/packages/core/src/markdown/dedent-block-jsx-close.ts +++ b/packages/core/src/markdown/dedent-block-jsx-close.ts @@ -1,3 +1,4 @@ + import { findFencedRegions, isInsideFence } from './fence-regions.ts'; const INDENTED_BLOCK_JSX_CLOSE_RE = /^([ ]{1,3})(<\/[A-Z][A-Za-z0-9_]*\s*>)([ \t]*)$/gm; diff --git a/packages/core/src/markdown/details-accordion-promoter.ts b/packages/core/src/markdown/details-accordion-promoter.ts index 54774764a..acd398aa6 100644 --- a/packages/core/src/markdown/details-accordion-promoter.ts +++ b/packages/core/src/markdown/details-accordion-promoter.ts @@ -1,3 +1,4 @@ + import type { Nodes, Paragraph, Parent, Root, Text } from 'mdast'; import type { MdxJsxAttribute, MdxJsxFlowElement } from 'mdast-util-mdx'; import { visit } from 'unist-util-visit'; diff --git a/packages/core/src/markdown/fence-regions.ts b/packages/core/src/markdown/fence-regions.ts index c7518d614..fb5ac8e8a 100644 --- a/packages/core/src/markdown/fence-regions.ts +++ b/packages/core/src/markdown/fence-regions.ts @@ -1,3 +1,4 @@ + const FENCE_RE = /^(`{3,}|~{3,})/gm; export function findFencedRegions(src: string): Array<[number, number]> { diff --git a/packages/core/src/markdown/fixtures/index.ts b/packages/core/src/markdown/fixtures/index.ts index 00f5771ee..6931623c7 100644 --- a/packages/core/src/markdown/fixtures/index.ts +++ b/packages/core/src/markdown/fixtures/index.ts @@ -1,3 +1,4 @@ + import { readFileSync } from 'node:fs'; import { dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -8,6 +9,7 @@ function fixturePath(...segments: string[]): string { return resolve(FIXTURES_DIR, ...segments); } + interface GfmExample { section: string; markdown: string; @@ -17,6 +19,7 @@ export function loadGfmExamples(): GfmExample[] { return JSON.parse(readFileSync(fixturePath('gfm', 'examples.json'), 'utf8')) as GfmExample[]; } + interface MdxCrashEntry { id: string; input: string; @@ -68,6 +71,7 @@ export function loadLargeEmbedFixtures(): LargeEmbedFixture[] { ) as LargeEmbedFixture[]; } + export function loadPrd6955Before(): string { return readFileSync(fixturePath('regression', 'prd-6955-before.md'), 'utf8'); } @@ -76,6 +80,7 @@ export function loadPrd6955CorruptedTriplicated(): string { return readFileSync(fixturePath('regression', 'prd-6955-corrupted-triplicated.md'), 'utf8'); } + export interface NgPinnedCase { id: string; name: string; @@ -92,6 +97,7 @@ export function loadNgPinnedCases(): NgPinnedCase[] { ) as NgPinnedCase[]; } + export function loadLargeRealistic(): string { return readFileSync(fixturePath('perf', 'large-realistic.md'), 'utf8'); } diff --git a/packages/core/src/markdown/fixtures/perf/generate.ts b/packages/core/src/markdown/fixtures/perf/generate.ts index 8b1eb8519..daa3ef82b 100644 --- a/packages/core/src/markdown/fixtures/perf/generate.ts +++ b/packages/core/src/markdown/fixtures/perf/generate.ts @@ -1,3 +1,4 @@ + import { writeFileSync } from 'node:fs'; import { dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -10,6 +11,7 @@ const BASELINE_ONLY_COUNTS = [500, 2500] as const; const SEED = 0xf1de1117; + function mulberry32(seed: number): () => number { let a = seed >>> 0; return () => { @@ -36,6 +38,7 @@ function pickWeighted(rand: () => number, items: readonly [T, number][]): T { return items[items.length - 1][0]; } + const WORDS = [ 'lorem', 'ipsum', @@ -153,6 +156,7 @@ function mdxBlock(rand: () => number): string { return `<${name}>\n\n${sentence(rand)}\n\n`; } + type BlockKind = 'paragraph' | 'heading' | 'list' | 'code' | 'table' | 'mdx'; const MIX: readonly [BlockKind, number][] = [ @@ -191,6 +195,7 @@ function generateDocument(blockCount: number, seed: number): string { return `${blocks.join('\n\n')}\n`; } + function main(): void { for (const count of [...BLOCK_COUNTS, ...BASELINE_ONLY_COUNTS]) { const doc = generateDocument(count, SEED); diff --git a/packages/core/src/markdown/guard-flanking-matrix.ts b/packages/core/src/markdown/guard-flanking-matrix.ts index 420dc0fe5..32c055c5b 100644 --- a/packages/core/src/markdown/guard-flanking-matrix.ts +++ b/packages/core/src/markdown/guard-flanking-matrix.ts @@ -7,6 +7,7 @@ import { import { BACKSLASH_GUARD_SUBSTITUTIONS, encodeBackslashEscapes } from './backslash-escape-guard.ts'; import { ENTITY_REF_GUARD_SUBSTITUTIONS, encodeEntityRefs } from './entity-ref-guard.ts'; + export const ATTENTION_DELIMITERS = ['*', '_', '**', '~~', '=='] as const; export type FlankClass = 'whitespace' | 'punctuation' | 'other'; diff --git a/packages/core/src/markdown/handler-shadow-audit.ts b/packages/core/src/markdown/handler-shadow-audit.ts index 2653473da..283913255 100644 --- a/packages/core/src/markdown/handler-shadow-audit.ts +++ b/packages/core/src/markdown/handler-shadow-audit.ts @@ -1,3 +1,4 @@ + export interface HandlerShadowWitness { input: string; expect: 'byte' | 'byte-and-reparse-type'; diff --git a/packages/core/src/markdown/highlight-promoter.ts b/packages/core/src/markdown/highlight-promoter.ts index f55912407..6319f264f 100644 --- a/packages/core/src/markdown/highlight-promoter.ts +++ b/packages/core/src/markdown/highlight-promoter.ts @@ -1,3 +1,4 @@ + import type { Nodes, Parent, PhrasingContent, Root, Text } from 'mdast'; import type { MdxJsxTextElement } from 'mdast-util-mdx'; import { SKIP, visit } from 'unist-util-visit'; diff --git a/packages/core/src/markdown/html-to-mdast.ts b/packages/core/src/markdown/html-to-mdast.ts index 78fdfeb02..2f082e6d0 100644 --- a/packages/core/src/markdown/html-to-mdast.ts +++ b/packages/core/src/markdown/html-to-mdast.ts @@ -1,3 +1,4 @@ + import type { Root as HastRoot } from 'hast'; import type { Root as MdastRoot } from 'mdast'; import rehypeParse from 'rehype-parse'; @@ -75,7 +76,8 @@ export function htmlToMdast(html: string, options?: HtmlToMdastOptions): MdastRo throw new HtmlPayloadTooLargeError(html.length, maxBytes); } - const processor = unified().use(rehypeParse, { fragment: true }); + const processor = unified() + .use(rehypeParse, { fragment: true }); for (const plugin of cleanupPlugins) { processor.use(plugin); diff --git a/packages/core/src/markdown/image-promoter.ts b/packages/core/src/markdown/image-promoter.ts index 706fa9afb..994eb6917 100644 --- a/packages/core/src/markdown/image-promoter.ts +++ b/packages/core/src/markdown/image-promoter.ts @@ -1,3 +1,4 @@ + import type { Image, Paragraph, Root } from 'mdast'; import type { MdxJsxAttribute, MdxJsxFlowElement } from 'mdast-util-mdx'; import { visit } from 'unist-util-visit'; diff --git a/packages/core/src/markdown/index.ts b/packages/core/src/markdown/index.ts index 8b36807fd..fc30bda02 100644 --- a/packages/core/src/markdown/index.ts +++ b/packages/core/src/markdown/index.ts @@ -1,3 +1,4 @@ + import { type FromProseMirrorOptions, fromPmMark, @@ -196,6 +197,7 @@ export class MarkdownManager { } } + const registry = createRegistry(); function destructureAttrs( @@ -244,6 +246,7 @@ function destructureAttrs( return result; } + function hasDirtyDescendant(node: PmNode): boolean { let found = false; node.descendants((child) => { @@ -263,6 +266,7 @@ function effectiveDirty(node: PmNode, freshnessChecker?: StructuralFreshnessChec return freshnessChecker ? freshnessChecker.isDiverged(node.toJSON()) : false; } + function isEmptyMdastParagraph(node: MdastNodes): boolean { if (node.type !== 'paragraph') return false; const children = node.children ?? []; @@ -331,6 +335,7 @@ function extractTextFromMdastNodes(nodes: MdastNodes[]): string { return out; } + import { AUDIO_EXTENSIONS, FILE_ATTACHMENT_EXTENSIONS, @@ -577,6 +582,7 @@ function buildMdastToPmHandlers( })); } + if (m.emphasis) { handlers.emphasis = toPmMark(m.emphasis, (node: Emphasis) => ({ sourceDelimiter: node.data?.sourceDelimiter ?? '*', @@ -657,6 +663,7 @@ function buildMdastToPmHandlers( })); } + if (m.link) { const sourceLiteralMark = m.sourceLiteral; handlers.link = (node: Link, _parent: MdastParent, state: MdastToPmState) => { @@ -935,6 +942,7 @@ function buildMdastToPmHandlers( }; } + const blockUnknownHandler = (node: { type: string; position?: { start: { offset: number }; end: { offset: number } }; @@ -1060,6 +1068,7 @@ function buildMdastToPmHandlers( return handlers as RemarkProseMirrorOptions['handlers']; } + function buildPmToMdastHandlers( schema: Schema, freshness: FreshnessCheckerHolder, diff --git a/packages/core/src/markdown/lint/config-schemas.ts b/packages/core/src/markdown/lint/config-schemas.ts index e78a1381d..207cd95c2 100644 --- a/packages/core/src/markdown/lint/config-schemas.ts +++ b/packages/core/src/markdown/lint/config-schemas.ts @@ -29,6 +29,7 @@ const fullPluginShape = Object.fromEntries( LINT_PLUGINS.map((plugin) => [plugin.id, plugin.sliceSchema]), ) as z.ZodRawShape; + export const LinterConfigSchema = z.object({ enabled: z.boolean(), plugins: z.object(fullPluginShape), diff --git a/packages/core/src/markdown/lint/default-config.ts b/packages/core/src/markdown/lint/default-config.ts index d48eea5e4..a51f2ffb0 100644 --- a/packages/core/src/markdown/lint/default-config.ts +++ b/packages/core/src/markdown/lint/default-config.ts @@ -6,6 +6,7 @@ export const DEFAULT_MARKDOWNLINT_CONFIG: Record | undefined, ): Configuration { diff --git a/packages/core/src/markdown/lint/index.ts b/packages/core/src/markdown/lint/index.ts index d1a5ab4dc..3baacdeb7 100644 --- a/packages/core/src/markdown/lint/index.ts +++ b/packages/core/src/markdown/lint/index.ts @@ -1,3 +1,4 @@ + import { LINT_PLUGINS, type LinterConfig } from './plugins.ts'; import type { LintDiagnostic } from './types.ts'; diff --git a/packages/core/src/markdown/lint/plugins.ts b/packages/core/src/markdown/lint/plugins.ts index 1cd40ef3a..65a1b564c 100644 --- a/packages/core/src/markdown/lint/plugins.ts +++ b/packages/core/src/markdown/lint/plugins.ts @@ -1,3 +1,4 @@ + import { z } from 'zod'; import { DEFAULT_MARKDOWNLINT_CONFIG, resolveMarkdownlintConfig } from './default-config.ts'; import { fixMarkdownText, runMarkdownlint } from './markdownlint-runner.ts'; @@ -8,6 +9,7 @@ import { type MarkdownlintSlice, } from './types.ts'; + const MarkdownlintRuleSettingSchema = z.union([ z.boolean(), z.enum(MARKDOWNLINT_RULE_SEVERITIES), diff --git a/packages/core/src/markdown/lint/types.ts b/packages/core/src/markdown/lint/types.ts index 617f6e5b9..3651bd7eb 100644 --- a/packages/core/src/markdown/lint/types.ts +++ b/packages/core/src/markdown/lint/types.ts @@ -1,3 +1,4 @@ + export const LINT_PLUGIN_IDS = ['markdownlint'] as const; export type LintPluginId = (typeof LINT_PLUGIN_IDS)[number]; @@ -41,6 +42,7 @@ export interface MarkdownlintSlice { rules: Record; } + interface RuleOptionSpecBase { key: string; description: string; diff --git a/packages/core/src/markdown/math-promoter.ts b/packages/core/src/markdown/math-promoter.ts index a5b511f25..bd79424c7 100644 --- a/packages/core/src/markdown/math-promoter.ts +++ b/packages/core/src/markdown/math-promoter.ts @@ -1,3 +1,4 @@ + import type { Code, Root } from 'mdast'; import type { Math as MdastMath } from 'mdast-util-math'; import type { MdxJsxAttribute, MdxJsxFlowElement } from 'mdast-util-mdx'; diff --git a/packages/core/src/markdown/mdast-to-hast-handlers.ts b/packages/core/src/markdown/mdast-to-hast-handlers.ts index 679c72ef7..f84967798 100644 --- a/packages/core/src/markdown/mdast-to-hast-handlers.ts +++ b/packages/core/src/markdown/mdast-to-hast-handlers.ts @@ -1,3 +1,4 @@ + import type { Comment, Element, ElementContent, Properties } from 'hast'; import type { FootnoteDefinition, FootnoteReference } from 'mdast'; import type { MdxJsxFlowElement, MdxJsxTextElement } from 'mdast-util-mdx'; diff --git a/packages/core/src/markdown/mdast-to-html.ts b/packages/core/src/markdown/mdast-to-html.ts index 8ce2b2492..df788a80d 100644 --- a/packages/core/src/markdown/mdast-to-html.ts +++ b/packages/core/src/markdown/mdast-to-html.ts @@ -1,3 +1,4 @@ + import type { Element, Root as HastRoot, Text as HastText } from 'hast'; import type { Root as MdastRoot, Text as MdastText } from 'mdast'; import type { Handler } from 'mdast-util-to-hast'; diff --git a/packages/core/src/markdown/merged-walker.ts b/packages/core/src/markdown/merged-walker.ts index bfd7e3109..fa3cfcd86 100644 --- a/packages/core/src/markdown/merged-walker.ts +++ b/packages/core/src/markdown/merged-walker.ts @@ -1,3 +1,4 @@ + import type { Nodes, Parent, Root } from 'mdast'; import { SKIP, visit } from 'unist-util-visit'; import type { VFile } from 'vfile'; diff --git a/packages/core/src/markdown/mermaid-promoter.ts b/packages/core/src/markdown/mermaid-promoter.ts index 1a897fca8..17ed2b829 100644 --- a/packages/core/src/markdown/mermaid-promoter.ts +++ b/packages/core/src/markdown/mermaid-promoter.ts @@ -1,3 +1,4 @@ + import type { Code, Root } from 'mdast'; import type { MdxJsxAttribute, MdxJsxFlowElement } from 'mdast-util-mdx'; import { visit } from 'unist-util-visit'; diff --git a/packages/core/src/markdown/parse-with-fallback.ts b/packages/core/src/markdown/parse-with-fallback.ts index 9864b3b60..7e4a9a48b 100644 --- a/packages/core/src/markdown/parse-with-fallback.ts +++ b/packages/core/src/markdown/parse-with-fallback.ts @@ -168,6 +168,7 @@ export function parseRecursive( } } + interface VFilePlace { offset?: number; start?: { offset?: number }; @@ -186,6 +187,7 @@ function extractErrorOffset(err: unknown): number | undefined { return undefined; } + interface Region { start: number; end: number; @@ -209,6 +211,7 @@ function nearestBlankLineAfter(src: string, offset: number): number | null { return null; } + export interface TagEvent { kind: 'open' | 'close' | 'self-close'; name: string; @@ -355,6 +358,7 @@ function findFallbackRegion(src: string, errorOffset: number): Region { return { start: blockStart, end: blockEnd }; } + interface SourceBlock { src: string; start: number; @@ -452,6 +456,7 @@ function tryPerBlockFallback( }; } + function wholeDocRawText(source: string): JSONContent { return { type: 'doc', diff --git a/packages/core/src/markdown/parser-drop-closure.ts b/packages/core/src/markdown/parser-drop-closure.ts index e6ab4fd57..43d420187 100644 --- a/packages/core/src/markdown/parser-drop-closure.ts +++ b/packages/core/src/markdown/parser-drop-closure.ts @@ -1,3 +1,4 @@ + export type DroppedTokenAdjudication = | { kind: 'format-dof-axis'; diff --git a/packages/core/src/markdown/pipeline.ts b/packages/core/src/markdown/pipeline.ts index 7f2d5adca..e9e507469 100644 --- a/packages/core/src/markdown/pipeline.ts +++ b/packages/core/src/markdown/pipeline.ts @@ -1,3 +1,4 @@ + import { type FromProseMirrorOptions, fromProseMirror, diff --git a/packages/core/src/markdown/position-aware-join.ts b/packages/core/src/markdown/position-aware-join.ts index 5a5e4c42a..f8f5e8530 100644 --- a/packages/core/src/markdown/position-aware-join.ts +++ b/packages/core/src/markdown/position-aware-join.ts @@ -1,3 +1,4 @@ + import type { Join } from 'mdast-util-to-markdown'; import type { Position } from 'unist'; diff --git a/packages/core/src/markdown/position-slice.ts b/packages/core/src/markdown/position-slice.ts index 0f399d34c..e5b4c9eba 100644 --- a/packages/core/src/markdown/position-slice.ts +++ b/packages/core/src/markdown/position-slice.ts @@ -1,3 +1,4 @@ + import type { Nodes, Root } from 'mdast'; import { visit } from 'unist-util-visit'; import type { VFile } from 'vfile'; @@ -476,7 +477,8 @@ export function applyPositionSliceToNode( const first = source[startOff]; if (first !== '[' && first !== '<' && !node.title) { node.data.sourceStyle = 'gfm-autolink'; - } else if (first === '[') { + } + else if (first === '[') { const slice = source.slice(startOff, endOff); const closeBracketIdx = slice.lastIndexOf(']('); if (closeBracketIdx !== -1) { diff --git a/packages/core/src/markdown/rehype-plugins/skip-notion-whitespace.ts b/packages/core/src/markdown/rehype-plugins/skip-notion-whitespace.ts index db10fa54b..ca225ce9b 100644 --- a/packages/core/src/markdown/rehype-plugins/skip-notion-whitespace.ts +++ b/packages/core/src/markdown/rehype-plugins/skip-notion-whitespace.ts @@ -1,3 +1,4 @@ + import type { Comment, Element, Root, Text } from 'hast'; import type { Plugin } from 'unified'; diff --git a/packages/core/src/markdown/rehype-plugins/strip-cocoa-meta.ts b/packages/core/src/markdown/rehype-plugins/strip-cocoa-meta.ts index 34068d103..3a2524a9a 100644 --- a/packages/core/src/markdown/rehype-plugins/strip-cocoa-meta.ts +++ b/packages/core/src/markdown/rehype-plugins/strip-cocoa-meta.ts @@ -1,3 +1,4 @@ + import type { Element, ElementContent, Root } from 'hast'; import type { Plugin } from 'unified'; diff --git a/packages/core/src/markdown/rehype-plugins/strip-gdocs-wrapper.ts b/packages/core/src/markdown/rehype-plugins/strip-gdocs-wrapper.ts index 93151d43b..2279a0189 100644 --- a/packages/core/src/markdown/rehype-plugins/strip-gdocs-wrapper.ts +++ b/packages/core/src/markdown/rehype-plugins/strip-gdocs-wrapper.ts @@ -1,3 +1,4 @@ + import type { Element, Root } from 'hast'; import type { Plugin } from 'unified'; diff --git a/packages/core/src/markdown/rehype-plugins/strip-github-hovercard.ts b/packages/core/src/markdown/rehype-plugins/strip-github-hovercard.ts index 188f02d83..a0f467c42 100644 --- a/packages/core/src/markdown/rehype-plugins/strip-github-hovercard.ts +++ b/packages/core/src/markdown/rehype-plugins/strip-github-hovercard.ts @@ -1,3 +1,4 @@ + import type { Element, Root } from 'hast'; import type { Plugin } from 'unified'; diff --git a/packages/core/src/markdown/rehype-plugins/strip-gmail-classes.ts b/packages/core/src/markdown/rehype-plugins/strip-gmail-classes.ts index 4ba2c3418..4e8542a21 100644 --- a/packages/core/src/markdown/rehype-plugins/strip-gmail-classes.ts +++ b/packages/core/src/markdown/rehype-plugins/strip-gmail-classes.ts @@ -1,3 +1,4 @@ + import type { Element, Root } from 'hast'; import type { Plugin } from 'unified'; diff --git a/packages/core/src/markdown/rehype-plugins/strip-gsheets-wrapper.ts b/packages/core/src/markdown/rehype-plugins/strip-gsheets-wrapper.ts index 1a74b1272..a3c96b24a 100644 --- a/packages/core/src/markdown/rehype-plugins/strip-gsheets-wrapper.ts +++ b/packages/core/src/markdown/rehype-plugins/strip-gsheets-wrapper.ts @@ -1,3 +1,4 @@ + import type { Element, ElementContent, Root } from 'hast'; import type { Plugin } from 'unified'; diff --git a/packages/core/src/markdown/rehype-plugins/strip-mso-styles.ts b/packages/core/src/markdown/rehype-plugins/strip-mso-styles.ts index de35e7b79..e95cf43d6 100644 --- a/packages/core/src/markdown/rehype-plugins/strip-mso-styles.ts +++ b/packages/core/src/markdown/rehype-plugins/strip-mso-styles.ts @@ -1,3 +1,4 @@ + import type { Comment, Element, ElementContent, Root } from 'hast'; import type { Plugin } from 'unified'; diff --git a/packages/core/src/markdown/rehype-plugins/strip-slack-classes.ts b/packages/core/src/markdown/rehype-plugins/strip-slack-classes.ts index b4be1b2ea..ffdb4d5b5 100644 --- a/packages/core/src/markdown/rehype-plugins/strip-slack-classes.ts +++ b/packages/core/src/markdown/rehype-plugins/strip-slack-classes.ts @@ -1,3 +1,4 @@ + import type { Element, ElementContent, Root } from 'hast'; import type { Plugin } from 'unified'; diff --git a/packages/core/src/markdown/rehype-plugins/strip-vscode-spans.ts b/packages/core/src/markdown/rehype-plugins/strip-vscode-spans.ts index 3c8d88b7c..e2a96d1a7 100644 --- a/packages/core/src/markdown/rehype-plugins/strip-vscode-spans.ts +++ b/packages/core/src/markdown/rehype-plugins/strip-vscode-spans.ts @@ -1,3 +1,4 @@ + import type { Element, Root } from 'hast'; import type { Plugin } from 'unified'; diff --git a/packages/core/src/markdown/resolve-image-url.ts b/packages/core/src/markdown/resolve-image-url.ts index cf2c9da8e..0389575ac 100644 --- a/packages/core/src/markdown/resolve-image-url.ts +++ b/packages/core/src/markdown/resolve-image-url.ts @@ -1,3 +1,4 @@ + import { isRelativeUrl } from './safe-url.ts'; function isDevDiagnosticContext(): boolean { diff --git a/packages/core/src/markdown/safe-url.ts b/packages/core/src/markdown/safe-url.ts index 4d3250d3a..8be632ba4 100644 --- a/packages/core/src/markdown/safe-url.ts +++ b/packages/core/src/markdown/safe-url.ts @@ -1,3 +1,4 @@ + export const SAFE_URL_SCHEMES = ['https', 'http', 'mailto', 'tel', 'ftp', 'sms'] as const; const SCHEME_ALT = SAFE_URL_SCHEMES.map((s) => `${s}:`).join('|'); diff --git a/packages/core/src/markdown/serialize-helpers.ts b/packages/core/src/markdown/serialize-helpers.ts index 46cc6a074..542204403 100644 --- a/packages/core/src/markdown/serialize-helpers.ts +++ b/packages/core/src/markdown/serialize-helpers.ts @@ -1,3 +1,4 @@ + import type { Node as PmNode } from '@tiptap/pm/model'; import type { MdxJsxAttribute, MdxJsxExpressionAttribute, MdxJsxFlowElement } from 'mdast-util-mdx'; import type { PropDef, SerializeContext } from '../registry/types.ts'; diff --git a/packages/core/src/markdown/single-dollar-math-promoter.ts b/packages/core/src/markdown/single-dollar-math-promoter.ts index be97d4d62..fe73659e6 100644 --- a/packages/core/src/markdown/single-dollar-math-promoter.ts +++ b/packages/core/src/markdown/single-dollar-math-promoter.ts @@ -1,3 +1,4 @@ + import type { PhrasingContent, Root, Text } from 'mdast'; import type { InlineMath } from 'mdast-util-math'; import { SKIP, visit } from 'unist-util-visit'; diff --git a/packages/core/src/markdown/to-markdown-handlers.ts b/packages/core/src/markdown/to-markdown-handlers.ts index 8f4e9a41c..7bce04302 100644 --- a/packages/core/src/markdown/to-markdown-handlers.ts +++ b/packages/core/src/markdown/to-markdown-handlers.ts @@ -1,3 +1,4 @@ + import type { Nodes, Parents } from 'mdast'; import type { MdxJsxAttribute, MdxJsxExpressionAttribute, MdxJsxFlowElement } from 'mdast-util-mdx'; import type { Handle, Info, State } from 'mdast-util-to-markdown'; diff --git a/packages/core/src/markdown/unknown-mdast-guard.ts b/packages/core/src/markdown/unknown-mdast-guard.ts index cd245c38c..c5bfd2386 100644 --- a/packages/core/src/markdown/unknown-mdast-guard.ts +++ b/packages/core/src/markdown/unknown-mdast-guard.ts @@ -1,3 +1,4 @@ + import type { Root as MdastRoot } from 'mdast'; import type { VFile } from 'vfile'; diff --git a/packages/core/src/markdown/whitespace-char-ref.ts b/packages/core/src/markdown/whitespace-char-ref.ts index d578476a7..ac9f5239b 100644 --- a/packages/core/src/markdown/whitespace-char-ref.ts +++ b/packages/core/src/markdown/whitespace-char-ref.ts @@ -1,3 +1,4 @@ + const INLINE_WHITESPACE_BY_CODE: ReadonlyMap = new Map([ [0x20, ' '], [0x09, '\t'], diff --git a/packages/core/src/markdown/wiki-link-micromark.ts b/packages/core/src/markdown/wiki-link-micromark.ts index 6dad26200..afd9936fe 100644 --- a/packages/core/src/markdown/wiki-link-micromark.ts +++ b/packages/core/src/markdown/wiki-link-micromark.ts @@ -17,6 +17,7 @@ declare module 'micromark-util-types' { } } + const CODE_BANG = 33; // ! const CODE_LBRACKET = 91; // [ const CODE_RBRACKET = 93; // ] @@ -260,6 +261,7 @@ export function wikiLinkSyntax(): Extension { }; } + function enterWikiLink(this: CompileContext, token: Token) { this.enter( { @@ -385,6 +387,7 @@ export const wikiLinkToMarkdown: { unsafe: [{ character: '[', inConstruct: ['phrasing'] }], }; + const MICROMARK_EXT = wikiLinkSyntax(); export function remarkWikiLink(this: Processor) { From 5f5c5727bf24dd39fcccedb26fff42240aedbc26 Mon Sep 17 00:00:00 2001 From: Andrew Mikofalvy <5668128+amikofalvy@users.noreply.github.com> Date: Mon, 20 Jul 2026 12:46:44 -0700 Subject: [PATCH 2/2] feat(open-knowledge): bundle checkpoint refs, agent effects, drop counts (#2735) * feat(diagnose): bundle checkpoint refs, agent effects, log drop counts Full-tier bundles now stage the shadow repo's refs/checkpoints listing (state/checkpoint-refs.txt, subjects only) and a state/agent-effects.json snapshot from the new loopback-only GET /api/metrics/agent-effects endpoint, which summarizes the per-doc agent-effects ring buffers as character counts across loaded docs. Doc names in the new artifacts ride the doc.name key so the existing --redact pass anonymizes them. The web client log forwarder counts entries lost to buffer overflow or failed POSTs and carries droppedSinceLastFlush on the next delivered batch; the server persists it as an explicit gap-marker log line. A failed recovery-checkpoint write in the server observers also emits a pino line with doc and branch context alongside the existing warn. * fix(diagnose): register agent-effects handler, redact presence, clamp drop count Address review feedback and the shard2 conflict-gate regression: - Register handleMetricsAgentEffects as exempt in the conflict-gate route registry meta-test (GET, non-mutating; mirrors handleMetricsAgentPresence). This is the integration:shard2 failure. - Hash the agent-presence bundle's currentDoc path in the redactor so a redacted state/agent-presence.json no longer leaks the doc a person edited. - Clamp droppedSinceLastFlush at zero on the success path so concurrent in-flight flushes cannot drive the counter negative and suppress the next gap marker. - Include the failing doc name in the agent-effects handler catch log. - Update the stale "Eight handlers" inventory comment to nine. - Add tests: currentDoc redaction (real presence wire shape + null case) and a sustained-failing-burst drop-count storm case. GitOrigin-RevId: 636f941afb2bfd899d3389963d31e64066247ed3 --- .../bundle-checkpoint-effects-drop-counts.md | 5 + knip.config.ts | 4 +- .../lib/install-client-log-forwarder.test.ts | 103 +++++++++++++++++ .../src/lib/install-client-log-forwarder.ts | 45 +++++++- .../metrics-agent-effects.test.ts | 107 ++++++++++++++++++ .../attribution-sweep-coverage.test.ts | 4 + .../integration/client-logs-endpoint.test.ts | 17 +++ .../conflict-gate-coverage.test.ts | 5 + .../cli/src/diagnose/bundle-redact.test.ts | 72 ++++++++++++ packages/cli/src/diagnose/bundle-redact.ts | 16 ++- packages/cli/src/diagnose/bundle.test.ts | 66 +++++++++++ packages/cli/src/diagnose/bundle.ts | 76 +++++++++++++ .../core/src/bridge/bind-frontmatter-doc.ts | 1 - packages/core/src/bridge/bridge-invariant.ts | 1 - .../core/src/bridge/doc-boundary-space.ts | 1 - packages/core/src/bridge/growth-detect.ts | 1 - packages/core/src/bridge/merge-three-way.ts | 1 - packages/core/src/bridge/normalize.ts | 1 - .../src/bridge/pm-structural-equivalence.ts | 3 - packages/core/src/bridge/subsequence.ts | 1 - packages/core/src/index.ts | 6 + .../core/src/markdown/callout-transformer.ts | 1 - .../core/src/markdown/comment-promoter.ts | 1 - .../src/markdown/dedent-block-jsx-close.ts | 1 - .../markdown/details-accordion-promoter.ts | 1 - packages/core/src/markdown/fence-regions.ts | 1 - packages/core/src/markdown/fixtures/index.ts | 6 - .../src/markdown/fixtures/perf/generate.ts | 5 - .../src/markdown/guard-flanking-matrix.ts | 1 - .../core/src/markdown/handler-shadow-audit.ts | 1 - .../core/src/markdown/highlight-promoter.ts | 1 - packages/core/src/markdown/html-to-mdast.ts | 4 +- packages/core/src/markdown/image-promoter.ts | 1 - packages/core/src/markdown/index.ts | 9 -- .../core/src/markdown/lint/config-schemas.ts | 1 - .../core/src/markdown/lint/default-config.ts | 1 - packages/core/src/markdown/lint/index.ts | 1 - packages/core/src/markdown/lint/plugins.ts | 2 - packages/core/src/markdown/lint/types.ts | 2 - packages/core/src/markdown/math-promoter.ts | 1 - .../src/markdown/mdast-to-hast-handlers.ts | 1 - packages/core/src/markdown/mdast-to-html.ts | 1 - packages/core/src/markdown/merged-walker.ts | 1 - .../core/src/markdown/mermaid-promoter.ts | 1 - .../core/src/markdown/parse-with-fallback.ts | 5 - .../core/src/markdown/parser-drop-closure.ts | 1 - packages/core/src/markdown/pipeline.ts | 1 - .../core/src/markdown/position-aware-join.ts | 1 - packages/core/src/markdown/position-slice.ts | 4 +- .../rehype-plugins/skip-notion-whitespace.ts | 1 - .../rehype-plugins/strip-cocoa-meta.ts | 1 - .../rehype-plugins/strip-gdocs-wrapper.ts | 1 - .../rehype-plugins/strip-github-hovercard.ts | 1 - .../rehype-plugins/strip-gmail-classes.ts | 1 - .../rehype-plugins/strip-gsheets-wrapper.ts | 1 - .../rehype-plugins/strip-mso-styles.ts | 1 - .../rehype-plugins/strip-slack-classes.ts | 1 - .../rehype-plugins/strip-vscode-spans.ts | 1 - .../core/src/markdown/resolve-image-url.ts | 1 - packages/core/src/markdown/safe-url.ts | 1 - .../core/src/markdown/serialize-helpers.ts | 1 - .../markdown/single-dollar-math-promoter.ts | 1 - .../core/src/markdown/to-markdown-handlers.ts | 1 - .../core/src/markdown/unknown-mdast-guard.ts | 1 - .../core/src/markdown/whitespace-char-ref.ts | 1 - .../core/src/markdown/wiki-link-micromark.ts | 3 - .../core/src/schemas/api/client-logs.test.ts | 11 ++ packages/core/src/schemas/api/client-logs.ts | 9 +- packages/core/src/schemas/api/metrics.test.ts | 44 +++++++ packages/core/src/schemas/api/metrics.ts | 50 +++++++- packages/server/src/api-extension.ts | 106 ++++++++++++++++- packages/server/src/server-observers.ts | 21 ++++ 72 files changed, 750 insertions(+), 103 deletions(-) create mode 100644 .changeset/bundle-checkpoint-effects-drop-counts.md create mode 100644 packages/app/tests/integration/api-error-envelope/metrics-agent-effects.test.ts diff --git a/.changeset/bundle-checkpoint-effects-drop-counts.md b/.changeset/bundle-checkpoint-effects-drop-counts.md new file mode 100644 index 000000000..cb9ee7930 --- /dev/null +++ b/.changeset/bundle-checkpoint-effects-drop-counts.md @@ -0,0 +1,5 @@ +--- +"@inkeep/open-knowledge": patch +--- + +Diagnostics bundles now capture more of what matters for content-loss triage. The full bundle stages the shadow repo's recovery-checkpoint refs (`state/checkpoint-refs.txt` — ref name, date, and content-free subject per checkpoint) and a new `state/agent-effects.json` snapshot of the per-document agent activity ring buffers, served by a new loopback-only `GET /api/metrics/agent-effects` endpoint that summarizes agent writes as character counts (never raw text). Doc names in the new artifacts are anonymized by the existing `--redact` pass, which now also hashes the agent-presence bundle's `currentDoc` path so a redacted `state/agent-presence.json` no longer leaks the document a person was editing. The web client's log forwarder also stops losing entries silently: drops from buffer overflow or failed uploads are counted and recorded as an explicit `droppedSinceLastFlush` gap marker in the server log on the next delivered batch, and a failed recovery-checkpoint write now also emits a structured server-log line with doc and branch context. diff --git a/knip.config.ts b/knip.config.ts index 9633755a0..16d637b4b 100644 --- a/knip.config.ts +++ b/knip.config.ts @@ -119,9 +119,7 @@ export default { 'packages/server': { entry: ['src/**/*.test.ts'], project: 'src/**', - ignoreDependencies: [ - '@types/shell-quote', - ], + ignoreDependencies: ['@types/shell-quote'], }, 'packages/cli': { entry: ['src/**/*.test.ts', 'scripts/*.ts', 'tests/**/*.ts'], diff --git a/packages/app/src/lib/install-client-log-forwarder.test.ts b/packages/app/src/lib/install-client-log-forwarder.test.ts index a39695e58..d43d85e55 100644 --- a/packages/app/src/lib/install-client-log-forwarder.test.ts +++ b/packages/app/src/lib/install-client-log-forwarder.test.ts @@ -228,6 +228,109 @@ describe('installClientLogForwarder', () => { expect((entry?.message as string).length).toBeLessThanOrEqual(8192); }); + test('a failed POST is counted and carried as droppedSinceLastFlush on the next batch', async () => { + let failNext = true; + const fetchSpy = mock((_url: string, _init?: RequestInit) => + failNext + ? Promise.reject(new Error('network down')) + : Promise.resolve(new Response(null, { status: 200 })), + ); + const { con } = install(fetchSpy); + + con.warn('lost-1'); + con.warn('lost-2'); + handle?.flushNow(); + await new Promise((r) => setTimeout(r, 0)); // let the rejection handler run + + failNext = false; + con.warn('delivered'); + handle?.flushNow(); + expect(fetchSpy).toHaveBeenCalledTimes(2); + const body = bodyOf(fetchSpy) as { entries: unknown[]; droppedSinceLastFlush?: number }; + expect(body.entries).toHaveLength(1); + expect(body.droppedSinceLastFlush).toBe(2); + }); + + test('a non-2xx response counts the batch as dropped', async () => { + let reject = true; + const fetchSpy = mock((_url: string, _init?: RequestInit) => + Promise.resolve(new Response(null, { status: reject ? 500 : 200 })), + ); + const { con } = install(fetchSpy); + + con.warn('rejected'); + handle?.flushNow(); + await new Promise((r) => setTimeout(r, 0)); + + reject = false; + con.warn('delivered'); + handle?.flushNow(); + const body = bodyOf(fetchSpy) as { droppedSinceLastFlush?: number }; + expect(body.droppedSinceLastFlush).toBe(1); + }); + + test('a sustained failing burst past the entry cap preserves the accumulated drop count on recovery', async () => { + // The reconnect-storm scenario the feature targets: entries arrive faster + // than they can be delivered while the server is unreachable. Each auto-flush + // at the entry cap drains the queue and fails its POST — the failed-batch + // accounting must accumulate across cycles and ride the first delivered batch + // once the server recovers. (The ring-overflow `shift` in `enqueue` is not + // exercised here: the flush-at-cap drains the queue at exactly the cap, so it + // never grows past it — the failed-POST path is the real bound.) + let failNext = true; + const fetchSpy = mock((_url: string, _init?: RequestInit) => + failNext + ? Promise.reject(new Error('network down')) + : Promise.resolve(new Response(null, { status: 200 })), + ); + const { con } = install(fetchSpy); + + // Two full cap-sized batches, all failing → two auto-flush cycles. + for (let i = 0; i < RENDERER_LOG_MAX_ENTRIES * 2; i++) con.warn(`storm-${i}`); + await new Promise((r) => setTimeout(r, 0)); // let both rejection handlers run + + failNext = false; + con.warn('delivered'); + handle?.flushNow(); + const body = bodyOf(fetchSpy) as { entries: unknown[]; droppedSinceLastFlush?: number }; + expect(body.entries).toHaveLength(1); + expect(body.droppedSinceLastFlush).toBe(RENDERER_LOG_MAX_ENTRIES * 2); + }); + + test('drop count resets after a delivered batch (field absent when zero)', async () => { + let failNext = true; + const fetchSpy = mock((_url: string, _init?: RequestInit) => + failNext + ? Promise.reject(new Error('network down')) + : Promise.resolve(new Response(null, { status: 200 })), + ); + const { con } = install(fetchSpy); + + con.warn('lost'); + handle?.flushNow(); + await new Promise((r) => setTimeout(r, 0)); + + failNext = false; + con.warn('carries the count'); + handle?.flushNow(); + await new Promise((r) => setTimeout(r, 0)); // let the success handler subtract + + con.warn('clean batch'); + handle?.flushNow(); + const body = bodyOf(fetchSpy) as { entries: unknown[]; droppedSinceLastFlush?: number }; + expect(body.entries).toHaveLength(1); + expect(body.droppedSinceLastFlush).toBeUndefined(); + }); + + test('no droppedSinceLastFlush field when nothing was dropped', () => { + const fetchSpy = makeFetchSpy(); + const { con } = install(fetchSpy); + con.warn('all good'); + handle?.flushNow(); + const body = bodyOf(fetchSpy) as { droppedSinceLastFlush?: number }; + expect(body.droppedSinceLastFlush).toBeUndefined(); + }); + test('uninstall restores console and clears the marker (fresh install works)', () => { const fetchSpy = makeFetchSpy(); const con = makeFakeConsole(); diff --git a/packages/app/src/lib/install-client-log-forwarder.ts b/packages/app/src/lib/install-client-log-forwarder.ts index 5237b5db6..8a14be77d 100644 --- a/packages/app/src/lib/install-client-log-forwarder.ts +++ b/packages/app/src/lib/install-client-log-forwarder.ts @@ -145,6 +145,12 @@ export function installClientLogForwarder( // Set while flushing so neither the flush path nor any transitive console // call it triggers gets re-captured (recursion guard). let inForward = false; + // Entries lost since the last delivered batch (ring overflow + failed + // POSTs). Carried as `droppedSinceLastFlush` on the next successful batch so + // the persisted log records that a gap exists — the drops cluster exactly + // when diagnostics matter most (reconnect storms), and a silent gap reads + // as "nothing happened". + let droppedSinceLastFlush = 0; const original: ConsoleLike = { log: con.log.bind(con), @@ -161,19 +167,43 @@ export function installClientLogForwarder( if (queue.length === 0) return; const entries = queue.splice(0, RENDERER_LOG_MAX_ENTRIES); pendingBytes = 0; + // Snapshot the drop count at send time: on delivery only this snapshot is + // subtracted (not a reset to 0), so drops that accrue while the POST is in + // flight survive to the next batch. + const droppedAtSend = droppedSinceLastFlush; inForward = true; try { void doFetch('/api/client-logs', { method: 'POST', headers: { 'content-type': 'application/json' }, keepalive: true, - body: JSON.stringify({ entries }), - }).catch(() => { - // Network failure — drop. Never surface; never console.* here (would - // re-capture). The entries are already removed from the queue. - }); + body: JSON.stringify( + droppedAtSend > 0 ? { entries, droppedSinceLastFlush: droppedAtSend } : { entries }, + ), + }).then( + (res) => { + if (res.ok) { + // Clamp at zero: `inForward` is released in the `finally` before this + // promise settles, so a second flush can fire and snapshot the same + // non-zero `droppedAtSend`. Two overlapping 2xx deliveries would then + // each subtract it and drive the counter negative, suppressing the + // next genuine gap marker. The clamp keeps double-reporting from + // going below zero. + droppedSinceLastFlush = Math.max(0, droppedSinceLastFlush - droppedAtSend); + } else { + // Non-2xx: the server rejected the batch; these entries are gone. + droppedSinceLastFlush += entries.length; + } + }, + () => { + // Network failure — drop. Never surface; never console.* here (would + // re-capture). The entries are already removed from the queue. + droppedSinceLastFlush += entries.length; + }, + ); } catch { // Synchronous failure (e.g. serialization) — drop the batch. + droppedSinceLastFlush += entries.length; } finally { inForward = false; } @@ -185,7 +215,10 @@ export function installClientLogForwarder( // Bounded ring — drop oldest under sustained backpressure. if (queue.length > RENDERER_LOG_MAX_ENTRIES) { const dropped = queue.shift(); - if (dropped) pendingBytes -= estimateEntryBytes(dropped); + if (dropped) { + pendingBytes -= estimateEntryBytes(dropped); + droppedSinceLastFlush += 1; + } } // Flush on entry count OR byte budget — the byte cap keeps each POST under // the browser's ~64KB `keepalive` limit so unload-time flushes aren't diff --git a/packages/app/tests/integration/api-error-envelope/metrics-agent-effects.test.ts b/packages/app/tests/integration/api-error-envelope/metrics-agent-effects.test.ts new file mode 100644 index 000000000..c44e16182 --- /dev/null +++ b/packages/app/tests/integration/api-error-envelope/metrics-agent-effects.test.ts @@ -0,0 +1,107 @@ +/** + * Per-handler narrow-integration smoke test for `handleMetricsAgentEffects`. + * + * Asserts the canonical RFC 9457 wire shape for + * `GET /api/metrics/agent-effects`. This handler shares the same + * auth-before-method-dispatch ordering as `handleMetricsAgentPresence` + * (loopback gate → host-allowlist gate → method check) so a bad Host never + * leaks "verb the endpoint expects" via 405. + * + * Coverage: + * - happy path: 200 + `application/json` + body parses against + * `MetricsAgentEffectsSuccessSchema`, no `ok` discriminator. + * - populated path: an agent write is captured into the doc's ring buffer + * and surfaces as a summarized per-doc block (character counts only — + * never the raw delta text). + * - DNS-rebinding Host → 403 `urn:ok:error:host-not-allowed` (must + * emit BEFORE the method check). + * - method-not-allowed on POST → 405 `urn:ok:error:method-not-allowed` + * with `Allow: GET`. + */ + +import { afterAll, beforeAll, describe, expect, test } from 'bun:test'; +import { + MetricsAgentEffectsSuccessSchema, + ProblemDetailsSchema, +} from '@inkeep/open-knowledge-core'; +import { HARNESS_BOOT_TIMEOUT_MS } from '../harness-boot-timeout'; +import { fetchWithHostHeader } from '../host-header-request.test-helper'; +import { createTestServer, type TestServer } from '../test-harness'; + +let server: TestServer; + +beforeAll(async () => { + server = await createTestServer(); +}, HARNESS_BOOT_TIMEOUT_MS); + +afterAll(async () => { + await server.cleanup(); +}); + +describe('metrics-agent-effects envelope (RFC 9457)', () => { + test('happy path emits flat success body with application/json', async () => { + const res = await fetch(`http://127.0.0.1:${server.port}/api/metrics/agent-effects`); + expect(res.status).toBe(200); + expect(res.headers.get('content-type')).toBe('application/json'); + + const body = await res.json(); + expect(MetricsAgentEffectsSuccessSchema.safeParse(body).success).toBe(true); + expect((body as Record).ok).toBeUndefined(); + }); + + test('an agent write surfaces as a summarized per-doc block (no raw delta text)', async () => { + const docName = `agent-effects-populated-${crypto.randomUUID().slice(0, 8)}`; + const written = '# Agent effects probe\n'; + const writeRes = await fetch(`http://127.0.0.1:${server.port}/api/agent-write-md`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ markdown: written, position: 'replace', docName }), + }); + expect(writeRes.status).toBe(200); + + const res = await fetch(`http://127.0.0.1:${server.port}/api/metrics/agent-effects`); + expect(res.status).toBe(200); + const parsed = MetricsAgentEffectsSuccessSchema.safeParse(await res.json()); + expect(parsed.success).toBe(true); + if (!parsed.success) return; + + const block = parsed.data.effects.find((b) => b['doc.name'] === docName); + expect(block).toBeDefined(); + expect(block?.entries.length).toBeGreaterThan(0); + const entry = block?.entries[0]; + expect(entry?.insertedChars).toBeGreaterThan(0); + // Summaries only: the response must never carry the written text. + expect(JSON.stringify(parsed.data)).not.toContain('Agent effects probe'); + }); + + test('DNS-rebinding Host emits 403 urn:ok:error:host-not-allowed BEFORE method check', async () => { + const res = await fetchWithHostHeader( + `http://127.0.0.1:${server.port}/api/metrics/agent-effects`, + 'evil.example.com', + { method: 'POST' }, + ); + // Auth-before-method-dispatch ordering: bad Host → 403, NOT 405. + expect(res.status).toBe(403); + expect(res.headers.get('content-type')).toBe('application/problem+json'); + + const parsed = ProblemDetailsSchema.safeParse(await res.json()); + expect(parsed.success).toBe(true); + if (parsed.success) { + expect(parsed.data.type).toBe('urn:ok:error:host-not-allowed'); + } + }); + + test('method-not-allowed on POST (with valid Host) emits problem+json with Allow: GET', async () => { + const res = await fetch(`http://127.0.0.1:${server.port}/api/metrics/agent-effects`, { + method: 'POST', + }); + expect(res.status).toBe(405); + expect(res.headers.get('allow')).toBe('GET'); + + const parsed = ProblemDetailsSchema.safeParse(await res.json()); + expect(parsed.success).toBe(true); + if (parsed.success) { + expect(parsed.data.type).toBe('urn:ok:error:method-not-allowed'); + } + }); +}); diff --git a/packages/app/tests/integration/attribution-sweep-coverage.test.ts b/packages/app/tests/integration/attribution-sweep-coverage.test.ts index da42fb579..eaa92a3a7 100644 --- a/packages/app/tests/integration/attribution-sweep-coverage.test.ts +++ b/packages/app/tests/integration/attribution-sweep-coverage.test.ts @@ -188,6 +188,10 @@ const EXEMPT_HANDLERS = new Set([ 'handleMetricsReconciliation', 'handleMetricsParseHealth', 'handleMetricsAgentPresence', + // `/api/metrics/agent-effects` — GET-only loopback + host-gated diagnostic + // (per-doc agent-effects ring-buffer summaries). Read path, no writes, + // nothing to attribute — same posture as `handleMetricsAgentPresence`. + 'handleMetricsAgentEffects', // `/api/client-logs` — web/browser renderer console-log ingest. Writes only // to the `renderer` pino log (diagnostics), no Y.Docs / agent content; gated // by `checkLocalOpSecurity` like the local-op handlers. No identity needed. diff --git a/packages/app/tests/integration/client-logs-endpoint.test.ts b/packages/app/tests/integration/client-logs-endpoint.test.ts index 9d7b04777..902cb33db 100644 --- a/packages/app/tests/integration/client-logs-endpoint.test.ts +++ b/packages/app/tests/integration/client-logs-endpoint.test.ts @@ -57,6 +57,23 @@ describe('POST /api/client-logs', () => { expect(body.accepted).toBe(2); }); + test('accepts a batch carrying a droppedSinceLastFlush gap marker', async () => { + const res = await postLogs({ + entries: [{ level: 'warn', message: 'after a reconnect storm' }], + droppedSinceLastFlush: 7, + }); + expect(res.status).toBe(200); + expect(((await res.json()) as { accepted: number }).accepted).toBe(1); + }); + + test('rejects a negative droppedSinceLastFlush with 400', async () => { + const res = await postLogs({ + entries: [{ level: 'warn', message: 'x' }], + droppedSinceLastFlush: -1, + }); + expect(res.status).toBe(400); + }); + test('accepts an empty batch (accepted: 0)', async () => { const res = await postLogs({ entries: [] }); expect(res.status).toBe(200); diff --git a/packages/app/tests/integration/conflict-gate-coverage.test.ts b/packages/app/tests/integration/conflict-gate-coverage.test.ts index 935b0ac2d..8f5fa0023 100644 --- a/packages/app/tests/integration/conflict-gate-coverage.test.ts +++ b/packages/app/tests/integration/conflict-gate-coverage.test.ts @@ -125,6 +125,11 @@ const EXEMPT_HANDLERS = new Set([ 'handleMetricsReconciliation', 'handleMetricsParseHealth', 'handleMetricsAgentPresence', + // `/api/metrics/agent-effects` — GET-only loopback + host-gated diagnostic + // summarizing the per-doc `Y.Map('agent-effects')` ring buffers. Reads only; + // targets no Y.Doc write, so the per-doc conflict gate does not apply — same + // posture as its `handleMetricsAgentPresence` sibling. + 'handleMetricsAgentEffects', // `/api/__embed-detect` — read-only loopback + host-gated diagnostic for the // embedded-viewer detection spikes; reads the in-process UA ring buffer and // returns boolean signals, targets no Y.Doc, so the per-doc conflict gate diff --git a/packages/cli/src/diagnose/bundle-redact.test.ts b/packages/cli/src/diagnose/bundle-redact.test.ts index 948553238..2fe6333b7 100644 --- a/packages/cli/src/diagnose/bundle-redact.test.ts +++ b/packages/cli/src/diagnose/bundle-redact.test.ts @@ -358,6 +358,78 @@ describe('redactStagedBundle — contentDir substitution', () => { expect(Object.values(result.docNameMap)).toContain('live-doc'); }); + test('hashes the real `currentDoc` field the presence endpoint emits', () => { + // The live `/api/metrics/agent-presence` wire body carries the doc path + // under `currentDoc`, not `doc.name`. The walker must hash that exact key + // so a staged presence artifact never leaks the doc a human is editing. + const stagingDir = makeStagingDir(); + const contentDir = '/Users/test/notes'; + writeStaged( + stagingDir, + 'state/agent-presence.json', + `${JSON.stringify({ + presence: { + 'agent-a1': { + displayName: 'Claude', + icon: 'claude', + color: '#abc', + currentDoc: 'meetings/secret-standup', + mode: 'writing', + ts: 1, + }, + }, + })}`, + ); + + const result = redactStagedBundle({ stagingDir, contentDir }); + + const after = JSON.parse(readStaged(stagingDir, 'state/agent-presence.json')); + expect(after.presence['agent-a1'].currentDoc).toMatch(/^doc:[a-f0-9]{8}$/); + expect(JSON.stringify(after)).not.toContain('meetings/secret-standup'); + expect(Object.values(result.docNameMap)).toContain('meetings/secret-standup'); + }); + + test('leaves a null `currentDoc` untouched (only strings hash)', () => { + const stagingDir = makeStagingDir(); + const contentDir = '/Users/test/notes'; + writeStaged( + stagingDir, + 'state/agent-presence.json', + `${JSON.stringify({ + presence: { 'agent-a1': { currentDoc: null, mode: 'idle', ts: 1 } }, + })}`, + ); + + redactStagedBundle({ stagingDir, contentDir }); + + const after = JSON.parse(readStaged(stagingDir, 'state/agent-presence.json')); + expect(after.presence['agent-a1'].currentDoc).toBeNull(); + }); + + test('hashes doc.name values in state/agent-effects.json via the walker pass', () => { + const stagingDir = makeStagingDir(); + const contentDir = '/Users/test/notes'; + writeStaged( + stagingDir, + 'state/agent-effects.json', + `${JSON.stringify({ + effects: [ + { + 'doc.name': 'meetings/standup', + entries: [{ sessionId: 'agent-a1', agentType: 'claude', ts: 1, insertedChars: 4 }], + }, + ], + })}`, + ); + + const result = redactStagedBundle({ stagingDir, contentDir }); + + const after = JSON.parse(readStaged(stagingDir, 'state/agent-effects.json')); + expect(after.effects[0]['doc.name']).toMatch(/^doc:[a-f0-9]{8}$/); + expect(after.effects[0].entries[0].sessionId).toBe('agent-a1'); + expect(Object.values(result.docNameMap)).toContain('meetings/standup'); + }); + test('strings that do not include contentDir are passed through verbatim', () => { const stagingDir = makeStagingDir(); const contentDir = '/Users/test/notes'; diff --git a/packages/cli/src/diagnose/bundle-redact.ts b/packages/cli/src/diagnose/bundle-redact.ts index 045ad5d32..79e95bdc7 100644 --- a/packages/cli/src/diagnose/bundle-redact.ts +++ b/packages/cli/src/diagnose/bundle-redact.ts @@ -31,8 +31,12 @@ const HASH_HEX_LEN = 8; // Keys whose values are treated as doc-name-shaped. The OTLP attribute pair // form sets `key` to one of these and stores the value under `value.stringValue`; // the Pino flat form sets the key directly on the log record. Single source -// for both shapes — extending DOC_NAME_KEYS covers both at once. -const DOC_NAME_KEYS = new Set(['doc.name']); +// for both shapes — extending DOC_NAME_KEYS covers both at once. `currentDoc` +// is the workspace-relative doc path carried by `agent-presence.json` entries; +// hashing it keeps a redacted presence artifact from leaking the doc a human +// was editing (same anonymization guarantee the `doc.name`-keyed effects +// artifact already gets). +const DOC_NAME_KEYS = new Set(['doc.name', 'currentDoc']); export interface RedactStagedBundleOpts { /** Absolute path to the staging dir (contains telemetry/, logs/, state/). */ @@ -263,10 +267,10 @@ function walkDirFiles(dir: string): string[] { } } -// State files that are JSON-shaped get a full walker pass (agent-presence may -// carry per-agent doc.name fields; runtime.json may carry the contentDir). -// Other state files get substring-only substitution. -const STATE_JSON_FILES = new Set(['agent-presence.json', 'runtime.json']); +// State files that are JSON-shaped get a full walker pass (agent-presence and +// agent-effects may carry per-doc `doc.name` fields; runtime.json may carry +// the contentDir). Other state files get substring-only substitution. +const STATE_JSON_FILES = new Set(['agent-presence.json', 'agent-effects.json', 'runtime.json']); export function redactStagedBundle(opts: RedactStagedBundleOpts): RedactStagedBundleResult { const ctx: RedactCtx = { diff --git a/packages/cli/src/diagnose/bundle.test.ts b/packages/cli/src/diagnose/bundle.test.ts index 7f8b60c76..37385c8db 100644 --- a/packages/cli/src/diagnose/bundle.test.ts +++ b/packages/cli/src/diagnose/bundle.test.ts @@ -44,7 +44,9 @@ afterEach(() => { function makeDeterministicDeps(over: Partial = {}): CollectBundleDeps { return { fetchAgentPresence: async () => null, + fetchAgentEffects: async () => null, readShadowHead: () => null, + readCheckpointRefs: () => null, now: () => new Date('2026-05-28T14:22:01.000Z'), okVersion: () => '0.7.99', readDesktopEnv: () => null, @@ -310,6 +312,48 @@ describe('collectBundle — server status', () => { collected.cleanup(); }); + test('lock present + agent-effects 2xx → agent-effects.json staged', async () => { + const contentDir = makeTmpDir(); + writeAt( + contentDir, + '.ok/local/server.lock', + JSON.stringify({ pid: 1, port: 4711, hostname: 'h', startedAt: 't', worktreeRoot: '/' }), + ); + let queriedPort = -1; + const deps = makeDeterministicDeps({ + fetchAgentPresence: async () => JSON.stringify({ presence: {} }), + fetchAgentEffects: async (port) => { + queriedPort = port; + return JSON.stringify({ effects: [] }); + }, + }); + const collected = await collectBundle({ contentDir, deps }); + + expect(queriedPort).toBe(4711); + const effectsPath = join(collected.stagingDir, 'state', 'agent-effects.json'); + expect(existsSync(effectsPath)).toBe(true); + expect(JSON.parse(readFileSync(effectsPath, 'utf-8'))).toEqual({ effects: [] }); + collected.cleanup(); + }); + + test('agent-effects failure never affects serverStatus (presence is the probe)', async () => { + const contentDir = makeTmpDir(); + writeAt( + contentDir, + '.ok/local/server.lock', + JSON.stringify({ pid: 1, port: 4711, hostname: 'h', startedAt: 't', worktreeRoot: '/' }), + ); + const deps = makeDeterministicDeps({ + fetchAgentPresence: async () => JSON.stringify({ presence: {} }), + fetchAgentEffects: async () => null, + }); + const collected = await collectBundle({ contentDir, deps }); + + expect(collected.manifest.serverStatus).toBe('running'); + expect(existsSync(join(collected.stagingDir, 'state', 'agent-effects.json'))).toBe(false); + collected.cleanup(); + }); + test('corrupt lock → not-running, lock still staged for forensics', async () => { const contentDir = makeTmpDir(); writeAt(contentDir, '.ok/local/server.lock', 'not json {'); @@ -345,6 +389,28 @@ describe('collectBundle — state files', () => { collected.cleanup(); }); + test('checkpoint-refs.txt is written when readCheckpointRefs returns content', async () => { + const contentDir = makeTmpDir(); + const listing = + 'refs/checkpoints/main/deadbee\t2026-05-28T14:00:00+00:00\tcheckpoint: Before concurrent merge @ 2026-05-28T14:00:00.000Z\n'; + const deps = makeDeterministicDeps({ + readCheckpointRefs: () => listing, + }); + const collected = await collectBundle({ contentDir, deps }); + expect(readFileSync(join(collected.stagingDir, 'state', 'checkpoint-refs.txt'), 'utf-8')).toBe( + listing, + ); + expect(collected.manifest.files.map((f) => f.path)).toContain('state/checkpoint-refs.txt'); + collected.cleanup(); + }); + + test('checkpoint-refs.txt is omitted when readCheckpointRefs returns null', async () => { + const contentDir = makeTmpDir(); + const collected = await collectBundle({ contentDir, deps: makeDeterministicDeps() }); + expect(existsSync(join(collected.stagingDir, 'state', 'checkpoint-refs.txt'))).toBe(false); + collected.cleanup(); + }); + test('last-server-exit.json is staged when the desktop host wrote one', async () => { const contentDir = makeTmpDir(); const body = `${JSON.stringify( diff --git a/packages/cli/src/diagnose/bundle.ts b/packages/cli/src/diagnose/bundle.ts index ba63ba839..26f5e037e 100644 --- a/packages/cli/src/diagnose/bundle.ts +++ b/packages/cli/src/diagnose/bundle.ts @@ -202,12 +202,28 @@ export interface CollectBundleDeps { * on any failure (network, timeout, non-2xx). */ fetchAgentPresence?: (port: number) => Promise; + /** + * Fetch the running server's `GET /api/metrics/agent-effects` response + * (per-doc agent activity ring-buffer summaries), within the 1s budget. + * Returns the response body string on 2xx, or `null` on any failure. + * Best-effort like agent-presence; a `null` never affects `serverStatus`. + */ + fetchAgentEffects?: (port: number) => Promise; /** * Read the last 50 entries of the shadow repo log at * `/.git/ok/`. Returns the stdout string or `null` if the * shadow repo is missing or git fails. */ readShadowHead?: (contentDir: string) => string | null; + /** + * List the most recent `refs/checkpoints/*` refs of the shadow repo at + * `/.git/ok/` — the content-loss / rescue recovery snapshots. + * One tab-separated line per ref: ref name, creation date, commit subject + * (the subject is a content-free label; the commit BODY, which can carry + * doc names and lost content, is deliberately not read). Returns the + * listing string or `null` if the shadow repo is missing or git fails. + */ + readCheckpointRefs?: (contentDir: string) => string | null; /** Returns the current timestamp. Override in tests for determinism. */ now?: () => Date; /** Returns the OK CLI's package version. */ @@ -322,6 +338,7 @@ export const _pathHelpersForTests = { const AGENT_PRESENCE_TIMEOUT_MS = 1000; const SHADOW_GIT_LOG_LIMIT = 50; +const CHECKPOINT_REF_LIMIT = 50; async function defaultFetchAgentPresence(port: number): Promise { try { @@ -338,6 +355,21 @@ async function defaultFetchAgentPresence(port: number): Promise { } } +async function defaultFetchAgentEffects(port: number): Promise { + try { + const response = await fetch(`http://127.0.0.1:${port}/api/metrics/agent-effects`, { + signal: AbortSignal.timeout(AGENT_PRESENCE_TIMEOUT_MS), + }); + if (!response.ok) return null; + return await response.text(); + } catch { + // Same all-failures-collapse-to-null posture as agent-presence — a + // server without the endpoint (older version) or not running must not + // fail the bundle. + return null; + } +} + function defaultReadShadowHead(contentDir: string): string | null { const shadowDir = join(contentDir, '.git', 'ok'); if (!existsSync(shadowDir)) return null; @@ -350,6 +382,31 @@ function defaultReadShadowHead(contentDir: string): string | null { return result.stdout ?? ''; } +function defaultReadCheckpointRefs(contentDir: string): string | null { + const shadowDir = join(contentDir, '.git', 'ok'); + if (!existsSync(shadowDir)) return null; + // `%(contents:subject)` only: checkpoint commit bodies carry an + // `ok-checkpoint-v1` JSON line with the doc name and (for merge-loss + // checkpoints) lost-content substrings — the subject is a content-free + // label, so restricting the format keeps the artifact safe to stage as + // plain text under the same redaction rules as shadow-head.txt. + const result = spawnSync( + 'git', + [ + '-C', + shadowDir, + 'for-each-ref', + '--sort=-creatordate', + `--count=${CHECKPOINT_REF_LIMIT}`, + '--format=%(refname)%09%(creatordate:iso-strict)%09%(contents:subject)', + 'refs/checkpoints/', + ], + withHiddenWindowsConsole({ encoding: 'utf-8', timeout: 2000 }), + ); + if (result.error || result.status !== 0) return null; + return result.stdout ?? ''; +} + export function defaultReadDesktopEnv(): DesktopMetadata | null { const electronVersion = process.env.OK_DESKTOP_VERSION; const packagedRaw = process.env.OK_DESKTOP_PACKAGED; @@ -569,7 +626,9 @@ export async function collectBundle(opts: CollectBundleOpts): Promise new Date()); const okVersion = deps.okVersion ?? (() => PACKAGE_VERSION); const readDesktopEnv = deps.readDesktopEnv ?? defaultReadDesktopEnv; @@ -621,6 +680,13 @@ export async function collectBundle(opts: CollectBundleOpts): Promise?@[\]^_`{|}~])/g; const TABLE_ALIGN_ROW_RE = /^\s*\|?\s*:?-+:?\s*(\|\s*:?-+:?\s*)+\|?\s*$/; diff --git a/packages/core/src/bridge/pm-structural-equivalence.ts b/packages/core/src/bridge/pm-structural-equivalence.ts index 0be5203e4..754dc5cc2 100644 --- a/packages/core/src/bridge/pm-structural-equivalence.ts +++ b/packages/core/src/bridge/pm-structural-equivalence.ts @@ -32,7 +32,6 @@ const BR_SENTINEL: PmStructuralNode = { type: '__br__' }; const BR_LITERAL_RE = /^$/i; - interface DegradeEntry { readonly label: string; readonly severity: ToleranceClassSeverity; @@ -128,7 +127,6 @@ export interface ComparePmStructuralOptions { ignoreAttrs?: (attrKey: string) => boolean; } - /** Rebuild a node with `fn` applied to each child; identity when childless. * Shared by the degrade canonicalizers so none re-implements the walk. */ function mapChildren( @@ -250,7 +248,6 @@ function applyDegrades(tree: PmStructuralNode): { return { normalized: current, fired }; } - export function comparePmStructural( expected: PmStructuralNode, actual: PmStructuralNode, diff --git a/packages/core/src/bridge/subsequence.ts b/packages/core/src/bridge/subsequence.ts index dca644141..af20de1e2 100644 --- a/packages/core/src/bridge/subsequence.ts +++ b/packages/core/src/bridge/subsequence.ts @@ -1,4 +1,3 @@ - /** True when `needle` is a subsequence of `haystack` (two-pointer, O(n+m)): * insertions in `haystack` are free; any dropped or substituted `needle` byte * fails. */ diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 950792556..666c3e892 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -561,6 +561,10 @@ export { AgentActivitySuccessSchema, type AgentBurstDiffSuccess, AgentBurstDiffSuccessSchema, + AgentEffectEntrySchema, + type AgentEffectEntryWire, + AgentEffectsDocSchema, + type AgentEffectsDocWire, type AgentPatchRequest, AgentPatchRequestSchema, type AgentPatchSuccess, @@ -736,6 +740,8 @@ export { LocalOpOkInitRequestSchema, type LocalOpOkInitResponse, LocalOpOkInitResponseSchema, + type MetricsAgentEffectsSuccess, + MetricsAgentEffectsSuccessSchema, type MetricsAgentPresenceSuccess, MetricsAgentPresenceSuccessSchema, type MetricsParseHealthSuccess, diff --git a/packages/core/src/markdown/callout-transformer.ts b/packages/core/src/markdown/callout-transformer.ts index a363dfd54..83d47404a 100644 --- a/packages/core/src/markdown/callout-transformer.ts +++ b/packages/core/src/markdown/callout-transformer.ts @@ -1,4 +1,3 @@ - import type { Blockquote, Paragraph, Root } from 'mdast'; import type { MdxJsxAttribute, MdxJsxFlowElement } from 'mdast-util-mdx'; import { visit } from 'unist-util-visit'; diff --git a/packages/core/src/markdown/comment-promoter.ts b/packages/core/src/markdown/comment-promoter.ts index 47f0d07e4..193e286b0 100644 --- a/packages/core/src/markdown/comment-promoter.ts +++ b/packages/core/src/markdown/comment-promoter.ts @@ -1,4 +1,3 @@ - import type { Nodes, Paragraph, PhrasingContent, Root, RootContent, Text } from 'mdast'; import { SKIP, visit } from 'unist-util-visit'; import type { VFile } from 'vfile'; diff --git a/packages/core/src/markdown/dedent-block-jsx-close.ts b/packages/core/src/markdown/dedent-block-jsx-close.ts index b6299a948..ece5c02a1 100644 --- a/packages/core/src/markdown/dedent-block-jsx-close.ts +++ b/packages/core/src/markdown/dedent-block-jsx-close.ts @@ -1,4 +1,3 @@ - import { findFencedRegions, isInsideFence } from './fence-regions.ts'; const INDENTED_BLOCK_JSX_CLOSE_RE = /^([ ]{1,3})(<\/[A-Z][A-Za-z0-9_]*\s*>)([ \t]*)$/gm; diff --git a/packages/core/src/markdown/details-accordion-promoter.ts b/packages/core/src/markdown/details-accordion-promoter.ts index acd398aa6..54774764a 100644 --- a/packages/core/src/markdown/details-accordion-promoter.ts +++ b/packages/core/src/markdown/details-accordion-promoter.ts @@ -1,4 +1,3 @@ - import type { Nodes, Paragraph, Parent, Root, Text } from 'mdast'; import type { MdxJsxAttribute, MdxJsxFlowElement } from 'mdast-util-mdx'; import { visit } from 'unist-util-visit'; diff --git a/packages/core/src/markdown/fence-regions.ts b/packages/core/src/markdown/fence-regions.ts index fb5ac8e8a..c7518d614 100644 --- a/packages/core/src/markdown/fence-regions.ts +++ b/packages/core/src/markdown/fence-regions.ts @@ -1,4 +1,3 @@ - const FENCE_RE = /^(`{3,}|~{3,})/gm; export function findFencedRegions(src: string): Array<[number, number]> { diff --git a/packages/core/src/markdown/fixtures/index.ts b/packages/core/src/markdown/fixtures/index.ts index 6931623c7..00f5771ee 100644 --- a/packages/core/src/markdown/fixtures/index.ts +++ b/packages/core/src/markdown/fixtures/index.ts @@ -1,4 +1,3 @@ - import { readFileSync } from 'node:fs'; import { dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -9,7 +8,6 @@ function fixturePath(...segments: string[]): string { return resolve(FIXTURES_DIR, ...segments); } - interface GfmExample { section: string; markdown: string; @@ -19,7 +17,6 @@ export function loadGfmExamples(): GfmExample[] { return JSON.parse(readFileSync(fixturePath('gfm', 'examples.json'), 'utf8')) as GfmExample[]; } - interface MdxCrashEntry { id: string; input: string; @@ -71,7 +68,6 @@ export function loadLargeEmbedFixtures(): LargeEmbedFixture[] { ) as LargeEmbedFixture[]; } - export function loadPrd6955Before(): string { return readFileSync(fixturePath('regression', 'prd-6955-before.md'), 'utf8'); } @@ -80,7 +76,6 @@ export function loadPrd6955CorruptedTriplicated(): string { return readFileSync(fixturePath('regression', 'prd-6955-corrupted-triplicated.md'), 'utf8'); } - export interface NgPinnedCase { id: string; name: string; @@ -97,7 +92,6 @@ export function loadNgPinnedCases(): NgPinnedCase[] { ) as NgPinnedCase[]; } - export function loadLargeRealistic(): string { return readFileSync(fixturePath('perf', 'large-realistic.md'), 'utf8'); } diff --git a/packages/core/src/markdown/fixtures/perf/generate.ts b/packages/core/src/markdown/fixtures/perf/generate.ts index daa3ef82b..8b1eb8519 100644 --- a/packages/core/src/markdown/fixtures/perf/generate.ts +++ b/packages/core/src/markdown/fixtures/perf/generate.ts @@ -1,4 +1,3 @@ - import { writeFileSync } from 'node:fs'; import { dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -11,7 +10,6 @@ const BASELINE_ONLY_COUNTS = [500, 2500] as const; const SEED = 0xf1de1117; - function mulberry32(seed: number): () => number { let a = seed >>> 0; return () => { @@ -38,7 +36,6 @@ function pickWeighted(rand: () => number, items: readonly [T, number][]): T { return items[items.length - 1][0]; } - const WORDS = [ 'lorem', 'ipsum', @@ -156,7 +153,6 @@ function mdxBlock(rand: () => number): string { return `<${name}>\n\n${sentence(rand)}\n\n`; } - type BlockKind = 'paragraph' | 'heading' | 'list' | 'code' | 'table' | 'mdx'; const MIX: readonly [BlockKind, number][] = [ @@ -195,7 +191,6 @@ function generateDocument(blockCount: number, seed: number): string { return `${blocks.join('\n\n')}\n`; } - function main(): void { for (const count of [...BLOCK_COUNTS, ...BASELINE_ONLY_COUNTS]) { const doc = generateDocument(count, SEED); diff --git a/packages/core/src/markdown/guard-flanking-matrix.ts b/packages/core/src/markdown/guard-flanking-matrix.ts index 32c055c5b..420dc0fe5 100644 --- a/packages/core/src/markdown/guard-flanking-matrix.ts +++ b/packages/core/src/markdown/guard-flanking-matrix.ts @@ -7,7 +7,6 @@ import { import { BACKSLASH_GUARD_SUBSTITUTIONS, encodeBackslashEscapes } from './backslash-escape-guard.ts'; import { ENTITY_REF_GUARD_SUBSTITUTIONS, encodeEntityRefs } from './entity-ref-guard.ts'; - export const ATTENTION_DELIMITERS = ['*', '_', '**', '~~', '=='] as const; export type FlankClass = 'whitespace' | 'punctuation' | 'other'; diff --git a/packages/core/src/markdown/handler-shadow-audit.ts b/packages/core/src/markdown/handler-shadow-audit.ts index 283913255..2653473da 100644 --- a/packages/core/src/markdown/handler-shadow-audit.ts +++ b/packages/core/src/markdown/handler-shadow-audit.ts @@ -1,4 +1,3 @@ - export interface HandlerShadowWitness { input: string; expect: 'byte' | 'byte-and-reparse-type'; diff --git a/packages/core/src/markdown/highlight-promoter.ts b/packages/core/src/markdown/highlight-promoter.ts index 6319f264f..f55912407 100644 --- a/packages/core/src/markdown/highlight-promoter.ts +++ b/packages/core/src/markdown/highlight-promoter.ts @@ -1,4 +1,3 @@ - import type { Nodes, Parent, PhrasingContent, Root, Text } from 'mdast'; import type { MdxJsxTextElement } from 'mdast-util-mdx'; import { SKIP, visit } from 'unist-util-visit'; diff --git a/packages/core/src/markdown/html-to-mdast.ts b/packages/core/src/markdown/html-to-mdast.ts index 2f082e6d0..78fdfeb02 100644 --- a/packages/core/src/markdown/html-to-mdast.ts +++ b/packages/core/src/markdown/html-to-mdast.ts @@ -1,4 +1,3 @@ - import type { Root as HastRoot } from 'hast'; import type { Root as MdastRoot } from 'mdast'; import rehypeParse from 'rehype-parse'; @@ -76,8 +75,7 @@ export function htmlToMdast(html: string, options?: HtmlToMdastOptions): MdastRo throw new HtmlPayloadTooLargeError(html.length, maxBytes); } - const processor = unified() - .use(rehypeParse, { fragment: true }); + const processor = unified().use(rehypeParse, { fragment: true }); for (const plugin of cleanupPlugins) { processor.use(plugin); diff --git a/packages/core/src/markdown/image-promoter.ts b/packages/core/src/markdown/image-promoter.ts index 994eb6917..706fa9afb 100644 --- a/packages/core/src/markdown/image-promoter.ts +++ b/packages/core/src/markdown/image-promoter.ts @@ -1,4 +1,3 @@ - import type { Image, Paragraph, Root } from 'mdast'; import type { MdxJsxAttribute, MdxJsxFlowElement } from 'mdast-util-mdx'; import { visit } from 'unist-util-visit'; diff --git a/packages/core/src/markdown/index.ts b/packages/core/src/markdown/index.ts index fc30bda02..8b36807fd 100644 --- a/packages/core/src/markdown/index.ts +++ b/packages/core/src/markdown/index.ts @@ -1,4 +1,3 @@ - import { type FromProseMirrorOptions, fromPmMark, @@ -197,7 +196,6 @@ export class MarkdownManager { } } - const registry = createRegistry(); function destructureAttrs( @@ -246,7 +244,6 @@ function destructureAttrs( return result; } - function hasDirtyDescendant(node: PmNode): boolean { let found = false; node.descendants((child) => { @@ -266,7 +263,6 @@ function effectiveDirty(node: PmNode, freshnessChecker?: StructuralFreshnessChec return freshnessChecker ? freshnessChecker.isDiverged(node.toJSON()) : false; } - function isEmptyMdastParagraph(node: MdastNodes): boolean { if (node.type !== 'paragraph') return false; const children = node.children ?? []; @@ -335,7 +331,6 @@ function extractTextFromMdastNodes(nodes: MdastNodes[]): string { return out; } - import { AUDIO_EXTENSIONS, FILE_ATTACHMENT_EXTENSIONS, @@ -582,7 +577,6 @@ function buildMdastToPmHandlers( })); } - if (m.emphasis) { handlers.emphasis = toPmMark(m.emphasis, (node: Emphasis) => ({ sourceDelimiter: node.data?.sourceDelimiter ?? '*', @@ -663,7 +657,6 @@ function buildMdastToPmHandlers( })); } - if (m.link) { const sourceLiteralMark = m.sourceLiteral; handlers.link = (node: Link, _parent: MdastParent, state: MdastToPmState) => { @@ -942,7 +935,6 @@ function buildMdastToPmHandlers( }; } - const blockUnknownHandler = (node: { type: string; position?: { start: { offset: number }; end: { offset: number } }; @@ -1068,7 +1060,6 @@ function buildMdastToPmHandlers( return handlers as RemarkProseMirrorOptions['handlers']; } - function buildPmToMdastHandlers( schema: Schema, freshness: FreshnessCheckerHolder, diff --git a/packages/core/src/markdown/lint/config-schemas.ts b/packages/core/src/markdown/lint/config-schemas.ts index 207cd95c2..e78a1381d 100644 --- a/packages/core/src/markdown/lint/config-schemas.ts +++ b/packages/core/src/markdown/lint/config-schemas.ts @@ -29,7 +29,6 @@ const fullPluginShape = Object.fromEntries( LINT_PLUGINS.map((plugin) => [plugin.id, plugin.sliceSchema]), ) as z.ZodRawShape; - export const LinterConfigSchema = z.object({ enabled: z.boolean(), plugins: z.object(fullPluginShape), diff --git a/packages/core/src/markdown/lint/default-config.ts b/packages/core/src/markdown/lint/default-config.ts index a51f2ffb0..d48eea5e4 100644 --- a/packages/core/src/markdown/lint/default-config.ts +++ b/packages/core/src/markdown/lint/default-config.ts @@ -6,7 +6,6 @@ export const DEFAULT_MARKDOWNLINT_CONFIG: Record | undefined, ): Configuration { diff --git a/packages/core/src/markdown/lint/index.ts b/packages/core/src/markdown/lint/index.ts index 3baacdeb7..d1a5ab4dc 100644 --- a/packages/core/src/markdown/lint/index.ts +++ b/packages/core/src/markdown/lint/index.ts @@ -1,4 +1,3 @@ - import { LINT_PLUGINS, type LinterConfig } from './plugins.ts'; import type { LintDiagnostic } from './types.ts'; diff --git a/packages/core/src/markdown/lint/plugins.ts b/packages/core/src/markdown/lint/plugins.ts index 65a1b564c..1cd40ef3a 100644 --- a/packages/core/src/markdown/lint/plugins.ts +++ b/packages/core/src/markdown/lint/plugins.ts @@ -1,4 +1,3 @@ - import { z } from 'zod'; import { DEFAULT_MARKDOWNLINT_CONFIG, resolveMarkdownlintConfig } from './default-config.ts'; import { fixMarkdownText, runMarkdownlint } from './markdownlint-runner.ts'; @@ -9,7 +8,6 @@ import { type MarkdownlintSlice, } from './types.ts'; - const MarkdownlintRuleSettingSchema = z.union([ z.boolean(), z.enum(MARKDOWNLINT_RULE_SEVERITIES), diff --git a/packages/core/src/markdown/lint/types.ts b/packages/core/src/markdown/lint/types.ts index 3651bd7eb..617f6e5b9 100644 --- a/packages/core/src/markdown/lint/types.ts +++ b/packages/core/src/markdown/lint/types.ts @@ -1,4 +1,3 @@ - export const LINT_PLUGIN_IDS = ['markdownlint'] as const; export type LintPluginId = (typeof LINT_PLUGIN_IDS)[number]; @@ -42,7 +41,6 @@ export interface MarkdownlintSlice { rules: Record; } - interface RuleOptionSpecBase { key: string; description: string; diff --git a/packages/core/src/markdown/math-promoter.ts b/packages/core/src/markdown/math-promoter.ts index bd79424c7..a5b511f25 100644 --- a/packages/core/src/markdown/math-promoter.ts +++ b/packages/core/src/markdown/math-promoter.ts @@ -1,4 +1,3 @@ - import type { Code, Root } from 'mdast'; import type { Math as MdastMath } from 'mdast-util-math'; import type { MdxJsxAttribute, MdxJsxFlowElement } from 'mdast-util-mdx'; diff --git a/packages/core/src/markdown/mdast-to-hast-handlers.ts b/packages/core/src/markdown/mdast-to-hast-handlers.ts index f84967798..679c72ef7 100644 --- a/packages/core/src/markdown/mdast-to-hast-handlers.ts +++ b/packages/core/src/markdown/mdast-to-hast-handlers.ts @@ -1,4 +1,3 @@ - import type { Comment, Element, ElementContent, Properties } from 'hast'; import type { FootnoteDefinition, FootnoteReference } from 'mdast'; import type { MdxJsxFlowElement, MdxJsxTextElement } from 'mdast-util-mdx'; diff --git a/packages/core/src/markdown/mdast-to-html.ts b/packages/core/src/markdown/mdast-to-html.ts index df788a80d..8ce2b2492 100644 --- a/packages/core/src/markdown/mdast-to-html.ts +++ b/packages/core/src/markdown/mdast-to-html.ts @@ -1,4 +1,3 @@ - import type { Element, Root as HastRoot, Text as HastText } from 'hast'; import type { Root as MdastRoot, Text as MdastText } from 'mdast'; import type { Handler } from 'mdast-util-to-hast'; diff --git a/packages/core/src/markdown/merged-walker.ts b/packages/core/src/markdown/merged-walker.ts index fa3cfcd86..bfd7e3109 100644 --- a/packages/core/src/markdown/merged-walker.ts +++ b/packages/core/src/markdown/merged-walker.ts @@ -1,4 +1,3 @@ - import type { Nodes, Parent, Root } from 'mdast'; import { SKIP, visit } from 'unist-util-visit'; import type { VFile } from 'vfile'; diff --git a/packages/core/src/markdown/mermaid-promoter.ts b/packages/core/src/markdown/mermaid-promoter.ts index 17ed2b829..1a897fca8 100644 --- a/packages/core/src/markdown/mermaid-promoter.ts +++ b/packages/core/src/markdown/mermaid-promoter.ts @@ -1,4 +1,3 @@ - import type { Code, Root } from 'mdast'; import type { MdxJsxAttribute, MdxJsxFlowElement } from 'mdast-util-mdx'; import { visit } from 'unist-util-visit'; diff --git a/packages/core/src/markdown/parse-with-fallback.ts b/packages/core/src/markdown/parse-with-fallback.ts index 7e4a9a48b..9864b3b60 100644 --- a/packages/core/src/markdown/parse-with-fallback.ts +++ b/packages/core/src/markdown/parse-with-fallback.ts @@ -168,7 +168,6 @@ export function parseRecursive( } } - interface VFilePlace { offset?: number; start?: { offset?: number }; @@ -187,7 +186,6 @@ function extractErrorOffset(err: unknown): number | undefined { return undefined; } - interface Region { start: number; end: number; @@ -211,7 +209,6 @@ function nearestBlankLineAfter(src: string, offset: number): number | null { return null; } - export interface TagEvent { kind: 'open' | 'close' | 'self-close'; name: string; @@ -358,7 +355,6 @@ function findFallbackRegion(src: string, errorOffset: number): Region { return { start: blockStart, end: blockEnd }; } - interface SourceBlock { src: string; start: number; @@ -456,7 +452,6 @@ function tryPerBlockFallback( }; } - function wholeDocRawText(source: string): JSONContent { return { type: 'doc', diff --git a/packages/core/src/markdown/parser-drop-closure.ts b/packages/core/src/markdown/parser-drop-closure.ts index 43d420187..e6ab4fd57 100644 --- a/packages/core/src/markdown/parser-drop-closure.ts +++ b/packages/core/src/markdown/parser-drop-closure.ts @@ -1,4 +1,3 @@ - export type DroppedTokenAdjudication = | { kind: 'format-dof-axis'; diff --git a/packages/core/src/markdown/pipeline.ts b/packages/core/src/markdown/pipeline.ts index e9e507469..7f2d5adca 100644 --- a/packages/core/src/markdown/pipeline.ts +++ b/packages/core/src/markdown/pipeline.ts @@ -1,4 +1,3 @@ - import { type FromProseMirrorOptions, fromProseMirror, diff --git a/packages/core/src/markdown/position-aware-join.ts b/packages/core/src/markdown/position-aware-join.ts index f8f5e8530..5a5e4c42a 100644 --- a/packages/core/src/markdown/position-aware-join.ts +++ b/packages/core/src/markdown/position-aware-join.ts @@ -1,4 +1,3 @@ - import type { Join } from 'mdast-util-to-markdown'; import type { Position } from 'unist'; diff --git a/packages/core/src/markdown/position-slice.ts b/packages/core/src/markdown/position-slice.ts index e5b4c9eba..0f399d34c 100644 --- a/packages/core/src/markdown/position-slice.ts +++ b/packages/core/src/markdown/position-slice.ts @@ -1,4 +1,3 @@ - import type { Nodes, Root } from 'mdast'; import { visit } from 'unist-util-visit'; import type { VFile } from 'vfile'; @@ -477,8 +476,7 @@ export function applyPositionSliceToNode( const first = source[startOff]; if (first !== '[' && first !== '<' && !node.title) { node.data.sourceStyle = 'gfm-autolink'; - } - else if (first === '[') { + } else if (first === '[') { const slice = source.slice(startOff, endOff); const closeBracketIdx = slice.lastIndexOf(']('); if (closeBracketIdx !== -1) { diff --git a/packages/core/src/markdown/rehype-plugins/skip-notion-whitespace.ts b/packages/core/src/markdown/rehype-plugins/skip-notion-whitespace.ts index ca225ce9b..db10fa54b 100644 --- a/packages/core/src/markdown/rehype-plugins/skip-notion-whitespace.ts +++ b/packages/core/src/markdown/rehype-plugins/skip-notion-whitespace.ts @@ -1,4 +1,3 @@ - import type { Comment, Element, Root, Text } from 'hast'; import type { Plugin } from 'unified'; diff --git a/packages/core/src/markdown/rehype-plugins/strip-cocoa-meta.ts b/packages/core/src/markdown/rehype-plugins/strip-cocoa-meta.ts index 3a2524a9a..34068d103 100644 --- a/packages/core/src/markdown/rehype-plugins/strip-cocoa-meta.ts +++ b/packages/core/src/markdown/rehype-plugins/strip-cocoa-meta.ts @@ -1,4 +1,3 @@ - import type { Element, ElementContent, Root } from 'hast'; import type { Plugin } from 'unified'; diff --git a/packages/core/src/markdown/rehype-plugins/strip-gdocs-wrapper.ts b/packages/core/src/markdown/rehype-plugins/strip-gdocs-wrapper.ts index 2279a0189..93151d43b 100644 --- a/packages/core/src/markdown/rehype-plugins/strip-gdocs-wrapper.ts +++ b/packages/core/src/markdown/rehype-plugins/strip-gdocs-wrapper.ts @@ -1,4 +1,3 @@ - import type { Element, Root } from 'hast'; import type { Plugin } from 'unified'; diff --git a/packages/core/src/markdown/rehype-plugins/strip-github-hovercard.ts b/packages/core/src/markdown/rehype-plugins/strip-github-hovercard.ts index a0f467c42..188f02d83 100644 --- a/packages/core/src/markdown/rehype-plugins/strip-github-hovercard.ts +++ b/packages/core/src/markdown/rehype-plugins/strip-github-hovercard.ts @@ -1,4 +1,3 @@ - import type { Element, Root } from 'hast'; import type { Plugin } from 'unified'; diff --git a/packages/core/src/markdown/rehype-plugins/strip-gmail-classes.ts b/packages/core/src/markdown/rehype-plugins/strip-gmail-classes.ts index 4e8542a21..4ba2c3418 100644 --- a/packages/core/src/markdown/rehype-plugins/strip-gmail-classes.ts +++ b/packages/core/src/markdown/rehype-plugins/strip-gmail-classes.ts @@ -1,4 +1,3 @@ - import type { Element, Root } from 'hast'; import type { Plugin } from 'unified'; diff --git a/packages/core/src/markdown/rehype-plugins/strip-gsheets-wrapper.ts b/packages/core/src/markdown/rehype-plugins/strip-gsheets-wrapper.ts index a3c96b24a..1a74b1272 100644 --- a/packages/core/src/markdown/rehype-plugins/strip-gsheets-wrapper.ts +++ b/packages/core/src/markdown/rehype-plugins/strip-gsheets-wrapper.ts @@ -1,4 +1,3 @@ - import type { Element, ElementContent, Root } from 'hast'; import type { Plugin } from 'unified'; diff --git a/packages/core/src/markdown/rehype-plugins/strip-mso-styles.ts b/packages/core/src/markdown/rehype-plugins/strip-mso-styles.ts index e95cf43d6..de35e7b79 100644 --- a/packages/core/src/markdown/rehype-plugins/strip-mso-styles.ts +++ b/packages/core/src/markdown/rehype-plugins/strip-mso-styles.ts @@ -1,4 +1,3 @@ - import type { Comment, Element, ElementContent, Root } from 'hast'; import type { Plugin } from 'unified'; diff --git a/packages/core/src/markdown/rehype-plugins/strip-slack-classes.ts b/packages/core/src/markdown/rehype-plugins/strip-slack-classes.ts index ffdb4d5b5..b4be1b2ea 100644 --- a/packages/core/src/markdown/rehype-plugins/strip-slack-classes.ts +++ b/packages/core/src/markdown/rehype-plugins/strip-slack-classes.ts @@ -1,4 +1,3 @@ - import type { Element, ElementContent, Root } from 'hast'; import type { Plugin } from 'unified'; diff --git a/packages/core/src/markdown/rehype-plugins/strip-vscode-spans.ts b/packages/core/src/markdown/rehype-plugins/strip-vscode-spans.ts index e2a96d1a7..3c8d88b7c 100644 --- a/packages/core/src/markdown/rehype-plugins/strip-vscode-spans.ts +++ b/packages/core/src/markdown/rehype-plugins/strip-vscode-spans.ts @@ -1,4 +1,3 @@ - import type { Element, Root } from 'hast'; import type { Plugin } from 'unified'; diff --git a/packages/core/src/markdown/resolve-image-url.ts b/packages/core/src/markdown/resolve-image-url.ts index 0389575ac..cf2c9da8e 100644 --- a/packages/core/src/markdown/resolve-image-url.ts +++ b/packages/core/src/markdown/resolve-image-url.ts @@ -1,4 +1,3 @@ - import { isRelativeUrl } from './safe-url.ts'; function isDevDiagnosticContext(): boolean { diff --git a/packages/core/src/markdown/safe-url.ts b/packages/core/src/markdown/safe-url.ts index 8be632ba4..4d3250d3a 100644 --- a/packages/core/src/markdown/safe-url.ts +++ b/packages/core/src/markdown/safe-url.ts @@ -1,4 +1,3 @@ - export const SAFE_URL_SCHEMES = ['https', 'http', 'mailto', 'tel', 'ftp', 'sms'] as const; const SCHEME_ALT = SAFE_URL_SCHEMES.map((s) => `${s}:`).join('|'); diff --git a/packages/core/src/markdown/serialize-helpers.ts b/packages/core/src/markdown/serialize-helpers.ts index 542204403..46cc6a074 100644 --- a/packages/core/src/markdown/serialize-helpers.ts +++ b/packages/core/src/markdown/serialize-helpers.ts @@ -1,4 +1,3 @@ - import type { Node as PmNode } from '@tiptap/pm/model'; import type { MdxJsxAttribute, MdxJsxExpressionAttribute, MdxJsxFlowElement } from 'mdast-util-mdx'; import type { PropDef, SerializeContext } from '../registry/types.ts'; diff --git a/packages/core/src/markdown/single-dollar-math-promoter.ts b/packages/core/src/markdown/single-dollar-math-promoter.ts index fe73659e6..be97d4d62 100644 --- a/packages/core/src/markdown/single-dollar-math-promoter.ts +++ b/packages/core/src/markdown/single-dollar-math-promoter.ts @@ -1,4 +1,3 @@ - import type { PhrasingContent, Root, Text } from 'mdast'; import type { InlineMath } from 'mdast-util-math'; import { SKIP, visit } from 'unist-util-visit'; diff --git a/packages/core/src/markdown/to-markdown-handlers.ts b/packages/core/src/markdown/to-markdown-handlers.ts index 7bce04302..8f4e9a41c 100644 --- a/packages/core/src/markdown/to-markdown-handlers.ts +++ b/packages/core/src/markdown/to-markdown-handlers.ts @@ -1,4 +1,3 @@ - import type { Nodes, Parents } from 'mdast'; import type { MdxJsxAttribute, MdxJsxExpressionAttribute, MdxJsxFlowElement } from 'mdast-util-mdx'; import type { Handle, Info, State } from 'mdast-util-to-markdown'; diff --git a/packages/core/src/markdown/unknown-mdast-guard.ts b/packages/core/src/markdown/unknown-mdast-guard.ts index c5bfd2386..cd245c38c 100644 --- a/packages/core/src/markdown/unknown-mdast-guard.ts +++ b/packages/core/src/markdown/unknown-mdast-guard.ts @@ -1,4 +1,3 @@ - import type { Root as MdastRoot } from 'mdast'; import type { VFile } from 'vfile'; diff --git a/packages/core/src/markdown/whitespace-char-ref.ts b/packages/core/src/markdown/whitespace-char-ref.ts index ac9f5239b..d578476a7 100644 --- a/packages/core/src/markdown/whitespace-char-ref.ts +++ b/packages/core/src/markdown/whitespace-char-ref.ts @@ -1,4 +1,3 @@ - const INLINE_WHITESPACE_BY_CODE: ReadonlyMap = new Map([ [0x20, ' '], [0x09, '\t'], diff --git a/packages/core/src/markdown/wiki-link-micromark.ts b/packages/core/src/markdown/wiki-link-micromark.ts index afd9936fe..6dad26200 100644 --- a/packages/core/src/markdown/wiki-link-micromark.ts +++ b/packages/core/src/markdown/wiki-link-micromark.ts @@ -17,7 +17,6 @@ declare module 'micromark-util-types' { } } - const CODE_BANG = 33; // ! const CODE_LBRACKET = 91; // [ const CODE_RBRACKET = 93; // ] @@ -261,7 +260,6 @@ export function wikiLinkSyntax(): Extension { }; } - function enterWikiLink(this: CompileContext, token: Token) { this.enter( { @@ -387,7 +385,6 @@ export const wikiLinkToMarkdown: { unsafe: [{ character: '[', inConstruct: ['phrasing'] }], }; - const MICROMARK_EXT = wikiLinkSyntax(); export function remarkWikiLink(this: Processor) { diff --git a/packages/core/src/schemas/api/client-logs.test.ts b/packages/core/src/schemas/api/client-logs.test.ts index 837ba659a..b4e9bf065 100644 --- a/packages/core/src/schemas/api/client-logs.test.ts +++ b/packages/core/src/schemas/api/client-logs.test.ts @@ -38,6 +38,17 @@ describe('ClientLogsRequestSchema', () => { expect(ClientLogsRequestSchema.safeParse({ entries }).success).toBe(false); }); + test('accepts an optional droppedSinceLastFlush count; rejects a negative one', () => { + const entries = [{ level: 'info' as const, message: 'x' }]; + expect(ClientLogsRequestSchema.safeParse({ entries, droppedSinceLastFlush: 3 }).success).toBe( + true, + ); + expect(ClientLogsRequestSchema.safeParse({ entries }).success).toBe(true); + expect(ClientLogsRequestSchema.safeParse({ entries, droppedSinceLastFlush: -1 }).success).toBe( + false, + ); + }); + test('rejects an oversized message', () => { const result = ClientLogsRequestSchema.safeParse({ entries: [{ level: 'error', message: 'a'.repeat(RENDERER_LOG_MAX_MESSAGE_BYTES + 1) }], diff --git a/packages/core/src/schemas/api/client-logs.ts b/packages/core/src/schemas/api/client-logs.ts index 26d168548..73a978c6a 100644 --- a/packages/core/src/schemas/api/client-logs.ts +++ b/packages/core/src/schemas/api/client-logs.ts @@ -33,10 +33,17 @@ export const ClientLogEntrySchema = z .loose() satisfies StandardSchemaV1; export type ClientLogEntry = z.infer; -/** Request body for `POST /api/client-logs` — a bounded batch of entries. */ +/** + * Request body for `POST /api/client-logs` — a bounded batch of entries. + * `droppedSinceLastFlush` is the number of entries the client forwarder lost + * since its last delivered batch (buffer overflow or failed POSTs — reconnect + * storms are the typical cause); the server records it so the persisted log + * carries an explicit gap marker instead of silently missing entries. + */ export const ClientLogsRequestSchema = z .object({ entries: z.array(ClientLogEntrySchema).max(RENDERER_LOG_MAX_ENTRIES), + droppedSinceLastFlush: z.number().int().min(0).optional(), }) .loose() satisfies StandardSchemaV1; export type ClientLogsRequest = z.infer; diff --git a/packages/core/src/schemas/api/metrics.test.ts b/packages/core/src/schemas/api/metrics.test.ts index 7cbd36a21..b0e7d067e 100644 --- a/packages/core/src/schemas/api/metrics.test.ts +++ b/packages/core/src/schemas/api/metrics.test.ts @@ -7,6 +7,7 @@ import { AgentBurstDiffSuccessSchema, AgentPresenceEntrySchema, InstalledAgentsSuccessSchema, + MetricsAgentEffectsSuccessSchema, MetricsAgentPresenceSuccessSchema, MetricsParseHealthSuccessSchema, MetricsReconciliationSuccessSchema, @@ -326,6 +327,49 @@ describe('MetricsAgentPresenceSuccessSchema', () => { }); }); +describe('MetricsAgentEffectsSuccessSchema', () => { + test('parses an empty effects list', () => { + expect(MetricsAgentEffectsSuccessSchema.safeParse({ effects: [] }).success).toBe(true); + }); + test('parses a populated per-doc block with summarized entries', () => { + expect( + MetricsAgentEffectsSuccessSchema.safeParse({ + effects: [ + { + 'doc.name': 'meetings/standup', + entries: [ + { + sessionId: 'agent-a1', + agentType: 'claude', + ts: 1, + insertedChars: 12, + deletedChars: 0, + }, + ], + }, + ], + }).success, + ).toBe(true); + }); + test('rejects body missing effects field', () => { + expect(MetricsAgentEffectsSuccessSchema.safeParse({}).success).toBe(false); + }); + test('rejects a negative character count', () => { + expect( + MetricsAgentEffectsSuccessSchema.safeParse({ + effects: [ + { + 'doc.name': 'a', + entries: [ + { sessionId: 's', agentType: 't', ts: 1, insertedChars: -1, deletedChars: 0 }, + ], + }, + ], + }).success, + ).toBe(false); + }); +}); + describe('InstalledAgentsSuccessSchema', () => { test('parses a populated boolean record', () => { expect( diff --git a/packages/core/src/schemas/api/metrics.ts b/packages/core/src/schemas/api/metrics.ts index a836262f7..c9076e4ea 100644 --- a/packages/core/src/schemas/api/metrics.ts +++ b/packages/core/src/schemas/api/metrics.ts @@ -1,10 +1,11 @@ /** * metrics + agent activity + test handlers. * - * Eight handlers — `handleAgentActivity`, `handleAgentBurstDiff`, + * Nine handlers — `handleAgentActivity`, `handleAgentBurstDiff`, * `handleTestReset`, `handleTestRescanBacklinks`, * `handleMetricsReconciliation`, `handleMetricsParseHealth`, - * `handleMetricsAgentPresence`, `handleInstalledAgentsRoute`. No new URN + * `handleMetricsAgentPresence`, `handleMetricsAgentEffects`, + * `handleInstalledAgentsRoute`. No new URN * tokens — every error path reuses existing tokens (`invalid-request`, * `reserved-doc-name`, `no-active-session`, `not-found`, `loopback-required`, * `host-not-allowed`, `invalid-origin`, `method-not-allowed`, @@ -166,6 +167,51 @@ export const MetricsAgentPresenceSuccessSchema = z .loose() satisfies StandardSchemaV1; export type MetricsAgentPresenceSuccess = z.infer; +/** + * One summarized agent-effects ring-buffer entry on + * `MetricsAgentEffectsSuccessSchema.effects[].entries`. Deltas are reduced to + * character counts — the diagnostic signal is who wrote how much to which doc + * and when; the raw delta text (user content) never leaves the live doc via + * this endpoint. + */ +export const AgentEffectEntrySchema = z + .object({ + sessionId: z.string().min(1), + agentType: z.string().min(1), + ts: z.number().int().min(0), + insertedChars: z.number().int().min(0), + deletedChars: z.number().int().min(0), + }) + .loose() satisfies StandardSchemaV1; +export type AgentEffectEntryWire = z.infer; + +/** + * One per-doc block on `MetricsAgentEffectsSuccessSchema.effects`. The doc + * name is carried under the literal key `doc.name` — the same key the + * diagnostics-bundle redactor hashes — so a staged copy of this response is + * anonymized by the existing pass without a redactor change. + */ +export const AgentEffectsDocSchema = z + .object({ + 'doc.name': z.string().min(1), + entries: z.array(AgentEffectEntrySchema), + }) + .loose() satisfies StandardSchemaV1; +export type AgentEffectsDocWire = z.infer; + +/** + * Success response for `GET /api/metrics/agent-effects`. Aggregates the + * bounded per-doc `agent-effects` ring buffers across currently-loaded docs + * (never force-loads unloaded ones). Loopback + Host-allowlist gated like + * `/api/metrics/agent-presence`. + */ +export const MetricsAgentEffectsSuccessSchema = z + .object({ + effects: z.array(AgentEffectsDocSchema), + }) + .loose() satisfies StandardSchemaV1; +export type MetricsAgentEffectsSuccess = z.infer; + /** * Success response for `GET /api/installed-agents`. Returns a flat boolean * record keyed by agent scheme name (`claude` / `codex` / `cursor`). The diff --git a/packages/server/src/api-extension.ts b/packages/server/src/api-extension.ts index 4cefaaec8..6a70a0226 100644 --- a/packages/server/src/api-extension.ts +++ b/packages/server/src/api-extension.ts @@ -128,6 +128,8 @@ import { MANAGED_ARTIFACT_PREFIX_SKILL, MANAGED_ARTIFACT_PREFIX_TEMPLATE, MarkdownlintRuleWriteRequestSchema, + type MetricsAgentEffectsSuccess, + MetricsAgentEffectsSuccessSchema, MetricsAgentPresenceSuccessSchema, MetricsParseHealthSuccessSchema, MetricsReconciliationSuccessSchema, @@ -251,7 +253,7 @@ import busboy from 'busboy'; import { fileTypeFromBuffer } from 'file-type'; import { parse as parseYaml, stringify as stringifyYaml } from 'yaml'; import { z } from 'zod'; -import { captureEffect } from './activity-log.ts'; +import { captureEffect, type EffectValue } from './activity-log.ts'; import { listAgentActivity, synthesizeVersionDiff } from './agent-activity.ts'; import type { AgentFocusBroadcaster } from './agent-focus.ts'; import { type AgentPresenceBroadcaster, BROADCASTER_EVICTION_MS } from './agent-presence.ts'; @@ -8784,6 +8786,92 @@ export function createApiExtension(options: ApiExtensionOptions): Extension { } } + async function handleMetricsAgentEffects( + req: IncomingMessage, + res: ServerResponse, + ): Promise { + // Diagnostic view of the per-doc `agent-effects` ring buffers, which + // otherwise live only inside live Y.Docs and are invisible to bundles. + // Loopback + Host-header gated with auth-before-method-dispatch ordering + // — same pattern + rationale as `handleMetricsAgentPresence` (per-agent + // identity plus per-doc write timing are local-editing-only signals). + if (!isLoopbackAddress(req.socket.remoteAddress)) { + errorResponse(res, 403, 'urn:ok:error:loopback-required', 'Loopback required.', { + handler: 'metrics-agent-effects', + }); + return; + } + if (!isAllowedWorkspaceHostHeader(req.headers.host)) { + errorResponse(res, 403, 'urn:ok:error:host-not-allowed', 'Host header not allowed.', { + handler: 'metrics-agent-effects', + }); + return; + } + if (req.method !== 'GET') { + errorResponse(res, 405, 'urn:ok:error:method-not-allowed', 'Method not allowed.', { + handler: 'metrics-agent-effects', + extraHeaders: { Allow: 'GET' }, + }); + return; + } + // Tracks the doc being summarized so a throw in the per-doc reduction (e.g. + // a malformed `EffectValue` from an older schema) names its source in the + // catch log. Rides the `doc.name` key the bundle redactor hashes. + let failingDocName: string | undefined; + try { + // Currently-loaded docs only — iterating `hocuspocus.documents` never + // materializes an unloaded doc. The `share.has` probe avoids even + // creating the lazy Y.Map placeholder on docs no agent ever wrote to. + const effects: MetricsAgentEffectsSuccess['effects'] = []; + for (const [effectsDocName, document] of hocuspocus.documents) { + if (isSystemDoc(effectsDocName) || isConfigDoc(effectsDocName)) continue; + if (!document.share.has('agent-effects')) continue; + failingDocName = effectsDocName; + const effectsMap = document.getMap('agent-effects'); + if (effectsMap.size === 0) continue; + // Deltas reduce to character counts: the diagnostic signal is who + // wrote how much to which doc and when. Raw delta text is user + // content and stays in the live doc. + const entries = [...effectsMap.values()] + .map((effect) => { + let insertedChars = 0; + let deletedChars = 0; + for (const op of effect.delta) { + if (typeof op.insert === 'string') insertedChars += op.insert.length; + else if (op.insert !== undefined) insertedChars += 1; + if (typeof op.delete === 'number') deletedChars += op.delete; + } + return { + sessionId: effect.sessionId, + agentType: effect.agent_type, + ts: effect.timestamp, + insertedChars, + deletedChars, + }; + }) + .sort((a, b) => a.ts - b.ts); + // The doc name rides under the literal `doc.name` key — the key the + // diagnostics-bundle redactor hashes — so a staged copy of this + // response is anonymized by the existing pass. + effects.push({ 'doc.name': effectsDocName, entries }); + } + effects.sort((a, b) => a['doc.name'].localeCompare(b['doc.name'])); + successResponse( + res, + 200, + MetricsAgentEffectsSuccessSchema, + { effects }, + { handler: 'metrics-agent-effects' }, + ); + } catch (e) { + log.error({ err: e, 'doc.name': failingDocName }, '[metrics-agent-effects] handler failed'); + errorResponse(res, 500, 'urn:ok:error:internal-server-error', 'Internal server error.', { + handler: 'metrics-agent-effects', + cause: e, + }); + } + } + async function handleEmbedDetect(req: IncomingMessage, res: ServerResponse): Promise { // Diagnostic endpoint for the Cursor / Codex / Claude Code embedded-viewer // detection spikes. Reads from the in-process ring buffer populated in @@ -17707,6 +17795,21 @@ export function createApiExtension(options: ApiExtensionOptions): Extension { async (_req, res, body) => { try { const logger = getLogger('renderer'); + if (body.droppedSinceLastFlush !== undefined && body.droppedSinceLastFlush > 0) { + // Gap marker: the forwarder lost entries (buffer overflow / failed + // POSTs) between the previous delivered batch and this one. Persist + // it as its own line so a log reader knows the silence was loss, + // not inactivity. + logger.warn( + { + source: 'renderer-console', + transport: 'web', + event: 'client-log-entries-dropped', + droppedSinceLastFlush: body.droppedSinceLastFlush, + }, + 'client-log-entries-dropped', + ); + } for (const entry of body.entries) { // Per-entry guard: one entry that trips a pino serialization fault // must not drop the rest of the batch (the response still reports the @@ -18633,6 +18736,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension { '/api/metrics/reconciliation': handleMetricsReconciliation, '/api/metrics/parse-health': handleMetricsParseHealth, '/api/metrics/agent-presence': handleMetricsAgentPresence, + '/api/metrics/agent-effects': handleMetricsAgentEffects, '/api/__embed-detect': handleEmbedDetect, '/api/server-info': handleServerInfo, '/api/share/construct-url': handleShareConstructUrl, diff --git a/packages/server/src/server-observers.ts b/packages/server/src/server-observers.ts index 85e2bb9be..d7081b90c 100644 --- a/packages/server/src/server-observers.ts +++ b/packages/server/src/server-observers.ts @@ -60,6 +60,7 @@ import { } from './bridge-watchdog.ts'; import { isConfigDoc, isSystemDoc } from './cc1-broadcast.ts'; import { recordFrontmatterEditSurface } from './frontmatter-telemetry.ts'; +import { getLogger } from './logger.ts'; import { computeMapDrivenBodySplice } from './map-driven-splice.ts'; import { incrementBridgeMergeCheckpointCreated, @@ -81,6 +82,14 @@ import { import { type ShadowHandle, saveInMemoryCheckpoint } from './shadow-repo.ts'; import { setActiveSpanAttributes, withSpanSync } from './telemetry.ts'; +/** + * Pino line for checkpoint-write failures, alongside the existing + * console.warn: a failed checkpoint means the recovery snapshot for a + * detected content-loss event was NOT persisted, and the pino sink is what + * reaches the local log files (and, through them, diagnostics bundles). + */ +const checkpointLog = getLogger('server-observers'); + // ───────────────────────────────────────────────────────────── // Origin constant // ───────────────────────────────────────────────────────────── @@ -675,6 +684,10 @@ export function setupServerObservers(opts: SetupServerObserversOpts): () => void message: err.message, stack: err.stack?.split('\n').slice(0, 4).join('\n'), }); + checkpointLog.warn( + { err, 'doc.name': opts.docName ?? null, branch, kind: 'bridge-merge-loss' }, + 'checkpoint write failed', + ); }); }); }; @@ -759,6 +772,10 @@ export function setupServerObservers(opts: SetupServerObserversOpts): () => void message: e.message, stack: e.stack?.split('\n').slice(0, 4).join('\n'), }); + checkpointLog.warn( + { err: e, 'doc.name': docName, branch, kind: 'observer-a-duplication' }, + 'checkpoint write failed', + ); }); }); }; @@ -1100,6 +1117,10 @@ export function setupServerObservers(opts: SetupServerObserversOpts): () => void message: e.message, stack: e.stack?.split('\n').slice(0, 4).join('\n'), }); + checkpointLog.warn( + { err: e, 'doc.name': docName, branch, kind: 'producer-guard-loss' }, + 'checkpoint write failed', + ); }); }); };