diff --git a/.changeset/bundle-checkpoint-effects-drop-counts.md b/.changeset/bundle-checkpoint-effects-drop-counts.md new file mode 100644 index 000000000..cb9ee7930 --- /dev/null +++ b/.changeset/bundle-checkpoint-effects-drop-counts.md @@ -0,0 +1,5 @@ +--- +"@inkeep/open-knowledge": patch +--- + +Diagnostics bundles now capture more of what matters for content-loss triage. The full bundle stages the shadow repo's recovery-checkpoint refs (`state/checkpoint-refs.txt` — ref name, date, and content-free subject per checkpoint) and a new `state/agent-effects.json` snapshot of the per-document agent activity ring buffers, served by a new loopback-only `GET /api/metrics/agent-effects` endpoint that summarizes agent writes as character counts (never raw text). Doc names in the new artifacts are anonymized by the existing `--redact` pass, which now also hashes the agent-presence bundle's `currentDoc` path so a redacted `state/agent-presence.json` no longer leaks the document a person was editing. The web client's log forwarder also stops losing entries silently: drops from buffer overflow or failed uploads are counted and recorded as an explicit `droppedSinceLastFlush` gap marker in the server log on the next delivered batch, and a failed recovery-checkpoint write now also emits a structured server-log line with doc and branch context. diff --git a/packages/app/src/components/FileTree.tsx b/packages/app/src/components/FileTree.tsx index a5a401d1a..9ef72868b 100644 --- a/packages/app/src/components/FileTree.tsx +++ b/packages/app/src/components/FileTree.tsx @@ -43,8 +43,6 @@ import { TriangleAlert, UnfoldVertical, } from 'lucide-react'; -import { __iconNode as botIcon } from 'lucide-react/dist/esm/icons/bot'; -import { __iconNode as link2Icon } from 'lucide-react/dist/esm/icons/link-2'; import { useTheme } from 'next-themes'; import { type DragEvent as ReactDragEvent, @@ -61,10 +59,7 @@ import { import { toast } from 'sonner'; import { DeleteConfirmationDialog } from '@/components/DeleteConfirmationDialog'; import { FileTreeFilteredToZeroNotice } from '@/components/FileTreeFilteredToZeroNotice'; -import { - MARKDOWN_FILE_ICON_PATH_D, - MARKDOWN_FILE_ICON_VIEWBOX, -} from '@/components/file-entry-icon'; +import { MARKDOWN_FILE_ICON_VIEWBOX } from '@/components/file-entry-icon'; import { appendSidebarUploadFields, collectTreeFolderPathsFromDocuments, @@ -101,6 +96,12 @@ import { applyExtensionBadges, FILE_TREE_EXT_BADGE_CSS, } from '@/components/file-tree-extension-badge'; +import { + AGENT_DECORATION_ICON_ID, + FILE_TREE_DECORATION_SPRITE_SHEET, + LINK_DECORATION_ICON_ID, + MARKDOWN_FILE_ICON_ID, +} from '@/components/file-tree-icon-sprite'; import { buildOkignorePatternFromTarget } from '@/components/file-tree-okignore'; import { applyDeleteToDocuments, @@ -116,6 +117,20 @@ import { type RenamedFolderMapping, remapActiveDocName, } from '@/components/file-tree-operations'; +import { + alternateMarkdownTreePath, + buildRowDecorationIndex, + collectTabsToCloseForDelete, + deleteTargetCoversPendingCreate, + hasSameStemMarkdownSiblingTreePath, + isAgentTreePath, + isEditableKeyboardTarget, + markdownTreeExtension, + parseAlreadyExistsRenamePath, + resolveDuplicableKeyboardTarget, + resolveKeyboardDeleteTargets, + selectedTreePathsToDeleteTargets, +} from '@/components/file-tree-path-helpers'; import { applyRenameInputAffordance, FILE_TREE_RENAME_INPUT_CSS, @@ -135,7 +150,6 @@ import { classifyEmptyTree, type DocumentEntry, type FileEntry, - type FolderEntry, filterVisibleEntries, hasOkPathSegment, isAssetEntry, @@ -228,18 +242,6 @@ import { useInstalledAgents } from './handoff/useInstalledAgents'; import { cancelHoverPrewarm, scheduleHoverPrewarm } from './sidebar-hover-prewarm'; import { useSidebar } from './ui/sidebar'; -const MARKDOWN_TREE_EXTENSION_PATTERN = /\.(md|mdx)$/i; - -function parseAlreadyExistsRenamePath(message: string): string | null { - const match = message.match(/^"(.+)" already exists\.$/); - return match ? match[1] : null; -} - -function markdownTreeExtension(path: string): string | null { - const match = path.match(MARKDOWN_TREE_EXTENSION_PATTERN); - return match ? match[0] : null; -} - function focusEditorAfterRename(docName: string): void { window.requestAnimationFrame(() => { const editor = getEditorForDoc(docName); @@ -300,43 +302,6 @@ async function copyToClipboard(text: string, kind: 'full' | 'relative'): Promise } } -const AGENT_FILE_NAMES = new Set(['agents', 'agent', 'claude', 'skill']); -const LINK_DECORATION_ICON_ID = 'ok-file-tree-link-decoration'; -const AGENT_DECORATION_ICON_ID = 'ok-file-tree-agent-decoration'; -const MARKDOWN_FILE_ICON_ID = 'ok-file-tree-markdown'; -// Custom Markdown file glyph (document with an "MD" label) overriding Pierre's -// built-in `complete`-set markdown glyph. `fill="currentColor"` lets -// `--trees-file-icon-color-markdown` (set in createFileTreeStyle, see -// file-tree-density.ts) color it. -const MARKDOWN_FILE_ICON_SYMBOL = ``; - -type IconNode = [string, Record][]; - -function iconNodeToSvg(iconNode: IconNode): string { - return ( - iconNode - // remove React key - .map(([tag, { key: _, ...attrs }]) => { - const attrString = Object.entries(attrs) - .map(([k, v]) => `${k}="${v}"`) - .join(' '); - return `<${tag} ${attrString} />`; - }) - .join('') - ); -} - -function createLucideSpriteSymbol(id: string, iconNode: IconNode): string { - const symbolContent = iconNodeToSvg(iconNode); - return `${symbolContent}`; -} - -const FILE_TREE_DECORATION_SPRITE_SHEET = ``; - // Drop-to-root affordance. The patched `@pierre/trees` sets // `data-file-tree-root-drag-target="true"` on the virtualized root while an // in-tree drag hovers empty content area (or a top-level file) — i.e. when the @@ -432,11 +397,6 @@ const FILE_TREE_CREATION_CLEARED_CSS = ` // CSS + DOM-mutation contract stays in one place. const FILE_TREE_UNSAFE_CSS = `${FILE_TREE_EXT_BADGE_CSS}\n${FILE_TREE_RENAME_INPUT_CSS}\n${FILE_TREE_ROOT_DROP_CSS}\n${FILE_TREE_EXTERNAL_FILE_DROP_CSS}\n${FILE_TREE_CREATION_CLEARED_CSS}\n${FILE_TREE_INDENT_GUIDE_CSS}\n${FILE_TREE_STICKY_HEADER_CSS}`; -function isAgentTreePath(treePath: string): boolean { - const name = treePath.split('/').pop()?.replace(/\.md$/i, '').toLowerCase(); - return !!name && AGENT_FILE_NAMES.has(name); -} - interface PendingCreate { kind: 'file' | 'folder'; renamePath: string; @@ -592,161 +552,6 @@ interface FileTreeMenuProps { documents: readonly FileEntry[]; } -function treePathToTarget(treePath: string, documents: readonly FileEntry[]): FileTreeTarget { - return treeItemToTarget( - { - kind: treePath.endsWith('/') ? 'directory' : 'file', - name: treePath, - path: treePath, - }, - documents, - ); -} - -function alternateMarkdownTreePath(treePath: string): string | null { - const match = treePath.match(/\.(md|mdx)$/i); - if (!match) return null; - const ext = match[0].toLowerCase(); - const alternateExt = ext === '.md' ? '.mdx' : '.md'; - return `${treePath.slice(0, -match[0].length)}${alternateExt}`; -} - -function hasSameStemMarkdownSiblingTreePath( - treePath: string, - treePaths: readonly string[], -): boolean { - const alternate = alternateMarkdownTreePath(treePath); - if (!alternate) return false; - return treePaths.includes(alternate); -} - -function isTreePathInsideFolder(treePath: string, folderTreePath: string): boolean { - return treePath !== folderTreePath && treePath.startsWith(folderTreePath); -} - -function selectedTreePathsToDeleteTargets( - selectedTreePaths: readonly string[], - documents: readonly FileEntry[], -): FileTreeTarget[] { - // Revealed `.ok` rows are read-only OK-managed state — they never become - // delete targets, even when swept into a multi-selection beside deletable - // rows (this also keeps the confirm dialog's item count honest). - const uniqueDeletablePaths = [...new Set(selectedTreePaths)].filter( - (treePath) => !hasOkPathSegment(treePath), - ); - const selectedFolderPaths = uniqueDeletablePaths.filter((treePath) => treePath.endsWith('/')); - return uniqueDeletablePaths - .filter( - (treePath) => - !selectedFolderPaths.some((folderPath) => isTreePathInsideFolder(treePath, folderPath)), - ) - .map((treePath) => treePathToTarget(treePath, documents)); -} - -function normalizeTreePathFromModel(model: PierreFileTreeModel, treePath: string): string { - const selectedItem = - model.getItem(treePath) ?? model.getItem(folderPathToTreeDirectoryPath(treePath)); - return selectedItem?.isDirectory() - ? folderPathToTreeDirectoryPath(treeDirectoryPathToFolderPath(selectedItem.getPath())) - : treePath; -} - -function focusedOrFirstSelectedTreePath(model: PierreFileTreeModel): string | null { - const selectedPath = model.getFocusedPath() ?? model.getSelectedPaths()[0] ?? null; - return selectedPath ? normalizeTreePathFromModel(model, selectedPath) : null; -} - -function resolveDuplicableKeyboardTarget( - model: PierreFileTreeModel, - documents: readonly FileEntry[], - assetTreePaths: ReadonlySet, -): FileTreeTarget | null { - const selectedPath = focusedOrFirstSelectedTreePath(model); - // Revealed `.ok` rows are read-only OK-managed state — no keyboard - // copy/paste/duplicate, matching their menu's suppressed affordances. - if (!selectedPath || assetTreePaths.has(selectedPath) || hasOkPathSegment(selectedPath)) { - return null; - } - return treePathToTarget(selectedPath, documents); -} - -function resolveKeyboardDeleteTargets( - model: PierreFileTreeModel, - documents: readonly FileEntry[], -): FileTreeTarget[] { - const selectedPaths = model.getSelectedPaths(); - const focusedPath = focusedOrFirstSelectedTreePath(model); - const paths = - selectedPaths.length > 0 - ? selectedPaths.map((treePath) => normalizeTreePathFromModel(model, treePath)) - : focusedPath - ? [focusedPath] - : []; - return selectedTreePathsToDeleteTargets(paths, documents); -} - -function isPathAtOrInsideFolder(path: string, folderPath: string): boolean { - return path === folderPath || path.startsWith(`${folderPath}/`); -} - -function isEditableKeyboardTarget(target: EventTarget | null): boolean { - if (!(target instanceof HTMLElement)) return false; - const tagName = target.tagName; - return ( - tagName === 'INPUT' || - tagName === 'TEXTAREA' || - tagName === 'SELECT' || - target.isContentEditable || - target.closest('[contenteditable="true"]') !== null - ); -} - -function collectTabsToCloseForDelete( - targets: readonly FileTreeTarget[], - documents: readonly FileEntry[], - folderTreePaths: readonly string[], -): { docNames: Set; folderPaths: Set; assetPaths: Set } { - const docNames = new Set(); - const folderPaths = new Set(); - const assetPaths = new Set(); - - for (const target of targets) { - if (target.kind === 'file') { - docNames.add(target.path); - continue; - } - if (target.kind === 'asset') { - assetPaths.add(target.path); - continue; - } - - folderPaths.add(target.path); - for (const entry of documents) { - if (isDocumentEntry(entry) && entry.docName.startsWith(`${target.path}/`)) { - docNames.add(entry.docName); - } else if (isAssetEntry(entry) && entry.path.startsWith(`${target.path}/`)) { - assetPaths.add(entry.path); - } - } - for (const treePath of folderTreePaths) { - const folderPath = treeDirectoryPathToFolderPath(treePath); - if (isPathAtOrInsideFolder(folderPath, target.path)) { - folderPaths.add(folderPath); - } - } - } - - return { docNames, folderPaths, assetPaths }; -} - -function deleteTargetCoversPendingCreate(target: FileTreeTarget, pending: PendingCreate): boolean { - if (target.kind === 'file') { - return pending.kind === 'file' && target.path === pending.createdPath; - } - if (target.kind === 'asset') return false; - return isPathAtOrInsideFolder(pending.createdPath, target.path); -} - function FileTreeMenu({ item, context, @@ -1386,25 +1191,6 @@ export function FileTree({ } ); } - function navigateToWithPulse( - targetPath: string, - size?: number, - options?: { registerPage?: boolean }, - ) { - if (options?.registerPage) addPage(targetPath); - openTarget(navigationTargetForDocument(targetPath, size), { tabBehavior: 'replace-active' }); - replaceHashWithoutNavigation(hashFromDocName(targetPath)); - notifySidebarFileSelected(); - } - function navigateToFolderWithPulse(folderPath: string) { - const nextHash = hashFromFolderPath(folderPath); - openTarget( - { kind: 'folder', target: folderPath, folderPath }, - { tabBehavior: 'replace-active' }, - ); - replaceHashWithoutNavigation(nextHash); - notifySidebarFileSelected(); - } const [documents, setDocuments] = useState([]); const [loading, setLoading] = useState(true); const [error, setError] = useState(null); @@ -1467,6 +1253,45 @@ export function FileTree({ const documentsRef = useRef(documents); const pageMetaRef = useRef(pageMeta); const pendingExactFileSelectionRef = useRef(null); + // Single navigation path for tree-initiated opens: resolve the target per + // kind, open it in the active tab, sync the hash, and pulse the sidebar. + function navigateWithPulse( + target: + | { kind: 'doc'; docName: string; size?: number; registerPage?: boolean } + | { kind: 'folder'; folderPath: string } + | { kind: 'asset'; assetPath: string; entries?: readonly FileEntry[] }, + ) { + if (target.kind === 'doc') { + if (target.registerPage) addPage(target.docName); + openTarget(navigationTargetForDocument(target.docName, target.size), { + tabBehavior: 'replace-active', + }); + replaceHashWithoutNavigation(hashFromDocName(target.docName)); + } else if (target.kind === 'folder') { + openTarget( + { kind: 'folder', target: target.folderPath, folderPath: target.folderPath }, + { tabBehavior: 'replace-active' }, + ); + replaceHashWithoutNavigation(hashFromFolderPath(target.folderPath)); + } else { + const currentEntries = target.entries ?? documentsRef.current; + const entry = currentEntries.find( + (item): item is Extract => + isAssetEntry(item) && item.path === target.assetPath, + ); + openTarget( + { + kind: 'asset', + target: target.assetPath, + assetPath: target.assetPath, + mediaKind: entry?.mediaKind ?? null, + }, + { tabBehavior: 'replace-active' }, + ); + replaceHashWithoutNavigation(hashFromAssetPath(target.assetPath)); + } + notifySidebarFileSelected(); + } function activateTreePath(treePath: string, entries: readonly FileEntry[] = documents) { const action = resolveFileTreeSelectionAction(treePath, entries); if (action.kind === 'none') { @@ -1491,7 +1316,7 @@ export function FileTree({ return; } if (action.kind === 'folder') { - navigateToFolderWithPulse(action.path); + navigateWithPulse({ kind: 'folder', folderPath: action.path }); return; } const docEntry = entries.find( @@ -1513,36 +1338,23 @@ export function FileTree({ return; } if (okTarget?.kind === 'doc') { - navigateToWithPulse(okTarget.docName); + navigateWithPulse({ kind: 'doc', docName: okTarget.docName }); return; } - navigateToWithPulse(action.path, docEntry?.size, { + navigateWithPulse({ + kind: 'doc', + docName: action.path, + size: docEntry?.size, registerPage: hasSupportedDocumentExtension(action.path), }); } - function navigateToAssetWithPulse(assetPath: string, entries?: readonly FileEntry[]) { - const currentEntries = entries ?? documentsRef.current; - const entry = currentEntries.find( - (item): item is Extract => - isAssetEntry(item) && item.path === assetPath, - ); - openTarget( - { - kind: 'asset', - target: assetPath, - assetPath, - mediaKind: entry?.mediaKind ?? null, - }, - { tabBehavior: 'replace-active' }, - ); - replaceHashWithoutNavigation(hashFromAssetPath(assetPath)); - notifySidebarFileSelected(); - } const activeDocNameRef = useRef(activeDocName); const assetTreePaths = new Set( documents.filter(isAssetEntry).map((entry) => fileEntryToTreePath(entry)), ); const assetTreePathsRef = useRef(assetTreePaths); + const rowDecorationIndex = buildRowDecorationIndex(documents); + const rowDecorationIndexRef = useRef(rowDecorationIndex); const activeAncestorTreePathsRef = useRef([]); const pendingCreateRef = useRef(null); const cleanupPendingCreateRef = useRef< @@ -1854,10 +1666,7 @@ export function FileTree({ onSelectionChange: (selectedPaths) => handleSelectionChangeRef.current(selectedPaths), renderRowDecoration: ({ item }) => { if (item.kind === 'file') { - const doc = documentsRef.current.find( - (entry): entry is DocumentEntry => - isDocumentEntry(entry) && docNameToTreePath(entry.docName, entry.docExt) === item.path, - ); + const doc = rowDecorationIndexRef.current.docsByTreePath.get(item.path); if (doc?.isSymlink) { const targetPath = doc.targetPath; return { @@ -1876,10 +1685,8 @@ export function FileTree({ // Symlinked directories carry isSymlink on their FolderEntry. Badge the // alias folder itself (Finder-style — its contents are not separately // marked, since they live behind the one symlink). - const folder = documentsRef.current.find( - (entry): entry is FolderEntry => - isFolderEntry(entry) && - folderPathToTreeDirectoryPath(entry.path) === folderPathToTreeDirectoryPath(item.path), + const folder = rowDecorationIndexRef.current.foldersByTreeDirectoryPath.get( + folderPathToTreeDirectoryPath(item.path), ); if (folder?.isSymlink) { const targetPath = folder.targetPath; @@ -2162,9 +1969,9 @@ export function FileTree({ if (success.path !== target.path) { if (success.kind === 'folder') { - navigateToFolderWithPulse(success.path); + navigateWithPulse({ kind: 'folder', folderPath: success.path }); } else { - navigateToWithPulse(success.path); + navigateWithPulse({ kind: 'doc', docName: success.path }); } } toast.success(success.kind === 'folder' ? t`Folder duplicated` : t`File duplicated`, { @@ -2830,15 +2637,19 @@ export function FileTree({ nextActiveFolderPath && nextActiveFolderPath !== currentActiveFolderPath ) { - navigateToFolderWithPulse(nextActiveFolderPath); + navigateWithPulse({ kind: 'folder', folderPath: nextActiveFolderPath }); } else if (nextActiveDocName && nextActiveDocName !== currentActiveDocName) { - navigateToWithPulse(nextActiveDocName); + navigateWithPulse({ kind: 'doc', docName: nextActiveDocName }); focusEditorAfterRename(nextActiveDocName); } else if ( nextActiveAssetPath && (activeDocToAssetPath || nextActiveAssetPath !== currentActiveAssetPath) ) { - navigateToAssetWithPulse(nextActiveAssetPath, nextDocumentsForRename ?? documentsRef.current); + navigateWithPulse({ + kind: 'asset', + assetPath: nextActiveAssetPath, + entries: nextDocumentsForRename ?? documentsRef.current, + }); } emitDocumentsChanged(['files', 'backlinks', 'graph']); }; @@ -3279,7 +3090,7 @@ export function FileTree({ return next; }); emitDocumentsChanged(['files', 'backlinks', 'graph']); - navigateToWithPulse(docName); + navigateWithPulse({ kind: 'doc', docName }); } else { const folderPath = treeDirectoryPathToFolderPath(placeholder.addPath); const res = await fetch('/api/create-folder', { @@ -3319,7 +3130,7 @@ export function FileTree({ return next; }); emitDocumentsChanged(['files']); - navigateToFolderWithPulse(createdPath); + navigateWithPulse({ kind: 'folder', folderPath: createdPath }); } let disposed = false; @@ -3403,6 +3214,7 @@ export function FileTree({ useLayoutEffect(() => { documentsRef.current = documents; + rowDecorationIndexRef.current = rowDecorationIndex; pageMetaRef.current = pageMeta; activeDocNameRef.current = activeDocName; activeTargetRef.current = activeTarget; @@ -4350,7 +4162,7 @@ export function FileTree({ event.preventDefault(); event.stopPropagation(); if (folderItem && !folderItem.isExpanded()) folderItem.expand(); - queueMicrotask(() => navigateToFolderWithPulse(folderPath)); + queueMicrotask(() => navigateWithPulse({ kind: 'folder', folderPath })); return; } if (model.getSelectedPaths().length !== 1) return; @@ -4358,7 +4170,7 @@ export function FileTree({ event.preventDefault(); event.stopPropagation(); if (folderItem && !folderItem.isExpanded()) folderItem.expand(); - queueMicrotask(() => navigateToFolderWithPulse(folderPath)); + queueMicrotask(() => navigateWithPulse({ kind: 'folder', folderPath })); return; } @@ -4372,7 +4184,7 @@ export function FileTree({ pendingExactFileSelectionRef.current = path; // Let handleSelectionChange's microtask consume the exact file selection // before navigation commits the extension-qualified URL. - setTimeout(() => navigateToWithPulse(path, undefined, { registerPage: true }), 0); + setTimeout(() => navigateWithPulse({ kind: 'doc', docName: path, registerPage: true }), 0); return; } queueMicrotask(() => activateTreePath(path)); diff --git a/packages/app/src/components/file-tree-icon-sprite.ts b/packages/app/src/components/file-tree-icon-sprite.ts new file mode 100644 index 000000000..d3f72644d --- /dev/null +++ b/packages/app/src/components/file-tree-icon-sprite.ts @@ -0,0 +1,50 @@ +/** + * Inline SVG sprite sheet fed to the @pierre/trees `icons.spriteSheet` option: + * row-decoration glyphs (symlink, agent file) plus the custom Markdown file + * icon. Lucide icons are imported as raw `__iconNode` data (not React + * components) so they can be serialized into `` markup once at module + * load. + */ + +import { __iconNode as botIcon } from 'lucide-react/dist/esm/icons/bot'; +import { __iconNode as link2Icon } from 'lucide-react/dist/esm/icons/link-2'; +import { + MARKDOWN_FILE_ICON_PATH_D, + MARKDOWN_FILE_ICON_VIEWBOX, +} from '@/components/file-entry-icon'; + +export const LINK_DECORATION_ICON_ID = 'ok-file-tree-link-decoration'; +export const AGENT_DECORATION_ICON_ID = 'ok-file-tree-agent-decoration'; +export const MARKDOWN_FILE_ICON_ID = 'ok-file-tree-markdown'; +// Custom Markdown file glyph (document with an "MD" label) overriding Pierre's +// built-in `complete`-set markdown glyph. `fill="currentColor"` lets +// `--trees-file-icon-color-markdown` (set in createFileTreeStyle, see +// file-tree-density.ts) color it. +const MARKDOWN_FILE_ICON_SYMBOL = ``; + +type IconNode = [string, Record][]; + +function iconNodeToSvg(iconNode: IconNode): string { + return ( + iconNode + // remove React key + .map(([tag, { key: _, ...attrs }]) => { + const attrString = Object.entries(attrs) + .map(([k, v]) => `${k}="${v}"`) + .join(' '); + return `<${tag} ${attrString} />`; + }) + .join('') + ); +} + +function createLucideSpriteSymbol(id: string, iconNode: IconNode): string { + const symbolContent = iconNodeToSvg(iconNode); + return `${symbolContent}`; +} + +export const FILE_TREE_DECORATION_SPRITE_SHEET = ``; diff --git a/packages/app/src/components/file-tree-path-helpers.test.ts b/packages/app/src/components/file-tree-path-helpers.test.ts new file mode 100644 index 000000000..65714cab0 --- /dev/null +++ b/packages/app/src/components/file-tree-path-helpers.test.ts @@ -0,0 +1,189 @@ +import { describe, expect, test } from 'bun:test'; +import type { FileTreeTarget } from './file-tree-operations'; +import { + alternateMarkdownTreePath, + buildRowDecorationIndex, + deleteTargetCoversPendingCreate, + hasSameStemMarkdownSiblingTreePath, + isAgentTreePath, + markdownTreeExtension, + parseAlreadyExistsRenamePath, + selectedTreePathsToDeleteTargets, +} from './file-tree-path-helpers'; +import type { FileEntry } from './file-tree-utils'; + +const doc = (docName: string, docExt?: string): FileEntry => ({ + kind: 'document', + docName, + docExt, + size: 1, + modified: '2026-04-13T00:00:00.000Z', +}); + +const folder = (path: string): FileEntry => ({ + kind: 'folder', + path, + size: 0, + modified: '2026-04-13T00:00:00.000Z', +}); + +const asset = (path: string): FileEntry => ({ + kind: 'asset', + path, + assetExt: '.png', + mediaKind: 'image', + size: 1, + modified: '2026-04-13T00:00:00.000Z', +}); + +describe('file-tree-path-helpers', () => { + test('parseAlreadyExistsRenamePath extracts the quoted path', () => { + expect(parseAlreadyExistsRenamePath('"docs/notes.md" already exists.')).toBe('docs/notes.md'); + expect(parseAlreadyExistsRenamePath('some other error')).toBeNull(); + }); + + test('markdownTreeExtension returns the matched extension or null', () => { + expect(markdownTreeExtension('docs/notes.md')).toBe('.md'); + expect(markdownTreeExtension('docs/notes.MDX')).toBe('.MDX'); + expect(markdownTreeExtension('docs/image.png')).toBeNull(); + }); + + test('isAgentTreePath flags agent/skill filenames case-insensitively', () => { + expect(isAgentTreePath('AGENTS.md')).toBe(true); + expect(isAgentTreePath('nested/claude.md')).toBe(true); + expect(isAgentTreePath('skill')).toBe(true); + expect(isAgentTreePath('docs/notes.md')).toBe(false); + }); + + test('alternateMarkdownTreePath swaps .md <-> .mdx and preserves the stem', () => { + expect(alternateMarkdownTreePath('docs/notes.md')).toBe('docs/notes.mdx'); + expect(alternateMarkdownTreePath('docs/notes.mdx')).toBe('docs/notes.md'); + expect(alternateMarkdownTreePath('docs/image.png')).toBeNull(); + }); + + test('hasSameStemMarkdownSiblingTreePath detects the alternate-extension sibling', () => { + const paths = ['docs/notes.mdx', 'docs/other.md']; + expect(hasSameStemMarkdownSiblingTreePath('docs/notes.md', paths)).toBe(true); + expect(hasSameStemMarkdownSiblingTreePath('docs/other.md', paths)).toBe(false); + }); + + describe('selectedTreePathsToDeleteTargets', () => { + const documents: FileEntry[] = [doc('docs/notes'), folder('docs'), doc('docs/nested/page')]; + + test('deduplicates repeated selections', () => { + const targets = selectedTreePathsToDeleteTargets( + ['docs/notes.md', 'docs/notes.md'], + documents, + ); + expect(targets).toHaveLength(1); + expect(targets[0].path).toBe('docs/notes'); + }); + + test('excludes read-only .ok rows', () => { + const targets = selectedTreePathsToDeleteTargets( + ['docs/notes.md', 'docs/.ok/templates/trip.md'], + documents, + ); + expect(targets.map((t) => t.path)).toEqual(['docs/notes']); + }); + + test('drops paths already covered by a selected ancestor folder', () => { + const targets = selectedTreePathsToDeleteTargets(['docs/', 'docs/notes.md'], documents); + // The folder subsumes the file inside it — only the folder survives. + expect(targets).toHaveLength(1); + expect(targets[0].kind).toBe('folder'); + expect(targets[0].path).toBe('docs'); + }); + }); + + describe('deleteTargetCoversPendingCreate', () => { + const fileTarget = (path: string): FileTreeTarget => ({ kind: 'file', name: path, path }); + const folderTarget = (path: string): FileTreeTarget => ({ kind: 'folder', name: path, path }); + const assetTarget = (path: string): FileTreeTarget => ({ kind: 'asset', name: path, path }); + + test('file target covers a pending file at the same path only', () => { + expect( + deleteTargetCoversPendingCreate(fileTarget('docs/new'), { + kind: 'file', + createdPath: 'docs/new', + }), + ).toBe(true); + expect( + deleteTargetCoversPendingCreate(fileTarget('docs/new'), { + kind: 'file', + createdPath: 'docs/other', + }), + ).toBe(false); + // A file target never covers a pending folder. + expect( + deleteTargetCoversPendingCreate(fileTarget('docs/new'), { + kind: 'folder', + createdPath: 'docs/new', + }), + ).toBe(false); + }); + + test('asset target never covers a pending create', () => { + expect( + deleteTargetCoversPendingCreate(assetTarget('docs/img.png'), { + kind: 'file', + createdPath: 'docs/img.png', + }), + ).toBe(false); + }); + + test('folder target covers a pending create at or inside the folder', () => { + expect( + deleteTargetCoversPendingCreate(folderTarget('docs'), { + kind: 'file', + createdPath: 'docs/new', + }), + ).toBe(true); + expect( + deleteTargetCoversPendingCreate(folderTarget('docs'), { + kind: 'folder', + createdPath: 'docs', + }), + ).toBe(true); + // A sibling folder that only shares a name prefix is not inside it. + expect( + deleteTargetCoversPendingCreate(folderTarget('docs'), { + kind: 'folder', + createdPath: 'docs-archive/new', + }), + ).toBe(false); + }); + }); + + describe('buildRowDecorationIndex', () => { + test('keys documents by tree path and folders by directory path; skips assets', () => { + const index = buildRowDecorationIndex([ + doc('docs/notes', '.md'), + doc('README'), + folder('docs/empty'), + asset('docs/image.png'), + ]); + + expect(index.docsByTreePath.get('docs/notes.md')?.kind).toBe('document'); + expect(index.docsByTreePath.get('README.md')?.kind).toBe('document'); + expect(index.foldersByTreeDirectoryPath.get('docs/empty/')?.kind).toBe('folder'); + // Assets are not indexed by either map. + expect(index.docsByTreePath.has('docs/image.png')).toBe(false); + expect(index.foldersByTreeDirectoryPath.has('docs/image.png')).toBe(false); + }); + + test('first entry wins on document key collision, matching Array.find', () => { + const first = doc('dup', '.md'); + const second = doc('dup', '.md'); + const index = buildRowDecorationIndex([first, second]); + expect(index.docsByTreePath.get('dup.md')).toBe(first); + }); + + test('first entry wins on folder key collision, matching Array.find', () => { + const first = folder('dup'); + const second = folder('dup'); + const index = buildRowDecorationIndex([first, second]); + expect(index.foldersByTreeDirectoryPath.get('dup/')).toBe(first); + }); + }); +}); diff --git a/packages/app/src/components/file-tree-path-helpers.ts b/packages/app/src/components/file-tree-path-helpers.ts new file mode 100644 index 000000000..560e4f985 --- /dev/null +++ b/packages/app/src/components/file-tree-path-helpers.ts @@ -0,0 +1,239 @@ +/** + * Pure tree-path helpers for the file sidebar: keyboard/selection target + * resolution, delete-target planning, markdown-sibling checks, and the + * row-decoration lookup index. Everything here is a pure function of its + * inputs — no React, no DOM mutation, no network — except + * `isEditableKeyboardTarget`, which performs a DOM read. + */ + +import type { FileTree as PierreFileTreeModel } from '@pierre/trees'; +import { + docNameToTreePath, + folderPathToTreeDirectoryPath, + treeDirectoryPathToFolderPath, + treeItemToTarget, +} from '@/components/file-tree-adapter'; +import type { FileTreeTarget } from '@/components/file-tree-operations'; +import { + type DocumentEntry, + type FileEntry, + type FolderEntry, + hasOkPathSegment, + isAssetEntry, + isDocumentEntry, + isFolderEntry, +} from '@/components/file-tree-utils'; + +const MARKDOWN_TREE_EXTENSION_PATTERN = /\.(md|mdx)$/i; + +export function parseAlreadyExistsRenamePath(message: string): string | null { + const match = message.match(/^"(.+)" already exists\.$/); + return match ? match[1] : null; +} + +export function markdownTreeExtension(path: string): string | null { + const match = path.match(MARKDOWN_TREE_EXTENSION_PATTERN); + return match ? match[0] : null; +} + +const AGENT_FILE_NAMES = new Set(['agents', 'agent', 'claude', 'skill']); + +export function isAgentTreePath(treePath: string): boolean { + const name = treePath.split('/').pop()?.replace(/\.md$/i, '').toLowerCase(); + return !!name && AGENT_FILE_NAMES.has(name); +} + +function treePathToTarget(treePath: string, documents: readonly FileEntry[]): FileTreeTarget { + return treeItemToTarget( + { + kind: treePath.endsWith('/') ? 'directory' : 'file', + name: treePath, + path: treePath, + }, + documents, + ); +} + +export function alternateMarkdownTreePath(treePath: string): string | null { + const match = treePath.match(/\.(md|mdx)$/i); + if (!match) return null; + const ext = match[0].toLowerCase(); + const alternateExt = ext === '.md' ? '.mdx' : '.md'; + return `${treePath.slice(0, -match[0].length)}${alternateExt}`; +} + +export function hasSameStemMarkdownSiblingTreePath( + treePath: string, + treePaths: readonly string[], +): boolean { + const alternate = alternateMarkdownTreePath(treePath); + if (!alternate) return false; + return treePaths.includes(alternate); +} + +function isTreePathInsideFolder(treePath: string, folderTreePath: string): boolean { + return treePath !== folderTreePath && treePath.startsWith(folderTreePath); +} + +export function selectedTreePathsToDeleteTargets( + selectedTreePaths: readonly string[], + documents: readonly FileEntry[], +): FileTreeTarget[] { + // Revealed `.ok` rows are read-only OK-managed state — they never become + // delete targets, even when swept into a multi-selection beside deletable + // rows (this also keeps the confirm dialog's item count honest). + const uniqueDeletablePaths = [...new Set(selectedTreePaths)].filter( + (treePath) => !hasOkPathSegment(treePath), + ); + const selectedFolderPaths = uniqueDeletablePaths.filter((treePath) => treePath.endsWith('/')); + return uniqueDeletablePaths + .filter( + (treePath) => + !selectedFolderPaths.some((folderPath) => isTreePathInsideFolder(treePath, folderPath)), + ) + .map((treePath) => treePathToTarget(treePath, documents)); +} + +function normalizeTreePathFromModel(model: PierreFileTreeModel, treePath: string): string { + const selectedItem = + model.getItem(treePath) ?? model.getItem(folderPathToTreeDirectoryPath(treePath)); + return selectedItem?.isDirectory() + ? folderPathToTreeDirectoryPath(treeDirectoryPathToFolderPath(selectedItem.getPath())) + : treePath; +} + +function focusedOrFirstSelectedTreePath(model: PierreFileTreeModel): string | null { + const selectedPath = model.getFocusedPath() ?? model.getSelectedPaths()[0] ?? null; + return selectedPath ? normalizeTreePathFromModel(model, selectedPath) : null; +} + +export function resolveDuplicableKeyboardTarget( + model: PierreFileTreeModel, + documents: readonly FileEntry[], + assetTreePaths: ReadonlySet, +): FileTreeTarget | null { + const selectedPath = focusedOrFirstSelectedTreePath(model); + // Revealed `.ok` rows are read-only OK-managed state — no keyboard + // copy/paste/duplicate, matching their menu's suppressed affordances. + if (!selectedPath || assetTreePaths.has(selectedPath) || hasOkPathSegment(selectedPath)) { + return null; + } + return treePathToTarget(selectedPath, documents); +} + +export function resolveKeyboardDeleteTargets( + model: PierreFileTreeModel, + documents: readonly FileEntry[], +): FileTreeTarget[] { + const selectedPaths = model.getSelectedPaths(); + const focusedPath = focusedOrFirstSelectedTreePath(model); + const paths = + selectedPaths.length > 0 + ? selectedPaths.map((treePath) => normalizeTreePathFromModel(model, treePath)) + : focusedPath + ? [focusedPath] + : []; + return selectedTreePathsToDeleteTargets(paths, documents); +} + +function isPathAtOrInsideFolder(path: string, folderPath: string): boolean { + return path === folderPath || path.startsWith(`${folderPath}/`); +} + +export function isEditableKeyboardTarget(target: EventTarget | null): boolean { + if (!(target instanceof HTMLElement)) return false; + const tagName = target.tagName; + return ( + tagName === 'INPUT' || + tagName === 'TEXTAREA' || + tagName === 'SELECT' || + target.isContentEditable || + target.closest('[contenteditable="true"]') !== null + ); +} + +export function collectTabsToCloseForDelete( + targets: readonly FileTreeTarget[], + documents: readonly FileEntry[], + folderTreePaths: readonly string[], +): { docNames: Set; folderPaths: Set; assetPaths: Set } { + const docNames = new Set(); + const folderPaths = new Set(); + const assetPaths = new Set(); + + for (const target of targets) { + if (target.kind === 'file') { + docNames.add(target.path); + continue; + } + if (target.kind === 'asset') { + assetPaths.add(target.path); + continue; + } + + folderPaths.add(target.path); + for (const entry of documents) { + if (isDocumentEntry(entry) && entry.docName.startsWith(`${target.path}/`)) { + docNames.add(entry.docName); + } else if (isAssetEntry(entry) && entry.path.startsWith(`${target.path}/`)) { + assetPaths.add(entry.path); + } + } + for (const treePath of folderTreePaths) { + const folderPath = treeDirectoryPathToFolderPath(treePath); + if (isPathAtOrInsideFolder(folderPath, target.path)) { + folderPaths.add(folderPath); + } + } + } + + return { docNames, folderPaths, assetPaths }; +} + +/** + * The slice of FileTree's pending-create state this module needs — kept + * structural so the component's richer lifecycle record stays local to it. + */ +export interface PendingCreateTarget { + kind: 'file' | 'folder'; + createdPath: string; +} + +export function deleteTargetCoversPendingCreate( + target: FileTreeTarget, + pending: PendingCreateTarget, +): boolean { + if (target.kind === 'file') { + return pending.kind === 'file' && target.path === pending.createdPath; + } + if (target.kind === 'asset') return false; + return isPathAtOrInsideFolder(pending.createdPath, target.path); +} + +export interface RowDecorationIndex { + docsByTreePath: ReadonlyMap; + foldersByTreeDirectoryPath: ReadonlyMap; +} + +/** + * O(1) lookup index for `renderRowDecoration`, which the tree invokes per + * visible row on every redraw — a linear scan of the document list there is + * O(rows × entries). First entry wins on key collision, matching the + * front-to-back `Array.find` this replaces. + */ +export function buildRowDecorationIndex(entries: readonly FileEntry[]): RowDecorationIndex { + const docsByTreePath = new Map(); + const foldersByTreeDirectoryPath = new Map(); + for (const entry of entries) { + if (isDocumentEntry(entry)) { + const treePath = docNameToTreePath(entry.docName, entry.docExt); + if (!docsByTreePath.has(treePath)) docsByTreePath.set(treePath, entry); + } else if (isFolderEntry(entry)) { + const directoryPath = folderPathToTreeDirectoryPath(entry.path); + if (!foldersByTreeDirectoryPath.has(directoryPath)) { + foldersByTreeDirectoryPath.set(directoryPath, entry); + } + } + } + return { docsByTreePath, foldersByTreeDirectoryPath }; +} diff --git a/packages/app/src/lib/install-client-log-forwarder.test.ts b/packages/app/src/lib/install-client-log-forwarder.test.ts index a39695e58..d43d85e55 100644 --- a/packages/app/src/lib/install-client-log-forwarder.test.ts +++ b/packages/app/src/lib/install-client-log-forwarder.test.ts @@ -228,6 +228,109 @@ describe('installClientLogForwarder', () => { expect((entry?.message as string).length).toBeLessThanOrEqual(8192); }); + test('a failed POST is counted and carried as droppedSinceLastFlush on the next batch', async () => { + let failNext = true; + const fetchSpy = mock((_url: string, _init?: RequestInit) => + failNext + ? Promise.reject(new Error('network down')) + : Promise.resolve(new Response(null, { status: 200 })), + ); + const { con } = install(fetchSpy); + + con.warn('lost-1'); + con.warn('lost-2'); + handle?.flushNow(); + await new Promise((r) => setTimeout(r, 0)); // let the rejection handler run + + failNext = false; + con.warn('delivered'); + handle?.flushNow(); + expect(fetchSpy).toHaveBeenCalledTimes(2); + const body = bodyOf(fetchSpy) as { entries: unknown[]; droppedSinceLastFlush?: number }; + expect(body.entries).toHaveLength(1); + expect(body.droppedSinceLastFlush).toBe(2); + }); + + test('a non-2xx response counts the batch as dropped', async () => { + let reject = true; + const fetchSpy = mock((_url: string, _init?: RequestInit) => + Promise.resolve(new Response(null, { status: reject ? 500 : 200 })), + ); + const { con } = install(fetchSpy); + + con.warn('rejected'); + handle?.flushNow(); + await new Promise((r) => setTimeout(r, 0)); + + reject = false; + con.warn('delivered'); + handle?.flushNow(); + const body = bodyOf(fetchSpy) as { droppedSinceLastFlush?: number }; + expect(body.droppedSinceLastFlush).toBe(1); + }); + + test('a sustained failing burst past the entry cap preserves the accumulated drop count on recovery', async () => { + // The reconnect-storm scenario the feature targets: entries arrive faster + // than they can be delivered while the server is unreachable. Each auto-flush + // at the entry cap drains the queue and fails its POST — the failed-batch + // accounting must accumulate across cycles and ride the first delivered batch + // once the server recovers. (The ring-overflow `shift` in `enqueue` is not + // exercised here: the flush-at-cap drains the queue at exactly the cap, so it + // never grows past it — the failed-POST path is the real bound.) + let failNext = true; + const fetchSpy = mock((_url: string, _init?: RequestInit) => + failNext + ? Promise.reject(new Error('network down')) + : Promise.resolve(new Response(null, { status: 200 })), + ); + const { con } = install(fetchSpy); + + // Two full cap-sized batches, all failing → two auto-flush cycles. + for (let i = 0; i < RENDERER_LOG_MAX_ENTRIES * 2; i++) con.warn(`storm-${i}`); + await new Promise((r) => setTimeout(r, 0)); // let both rejection handlers run + + failNext = false; + con.warn('delivered'); + handle?.flushNow(); + const body = bodyOf(fetchSpy) as { entries: unknown[]; droppedSinceLastFlush?: number }; + expect(body.entries).toHaveLength(1); + expect(body.droppedSinceLastFlush).toBe(RENDERER_LOG_MAX_ENTRIES * 2); + }); + + test('drop count resets after a delivered batch (field absent when zero)', async () => { + let failNext = true; + const fetchSpy = mock((_url: string, _init?: RequestInit) => + failNext + ? Promise.reject(new Error('network down')) + : Promise.resolve(new Response(null, { status: 200 })), + ); + const { con } = install(fetchSpy); + + con.warn('lost'); + handle?.flushNow(); + await new Promise((r) => setTimeout(r, 0)); + + failNext = false; + con.warn('carries the count'); + handle?.flushNow(); + await new Promise((r) => setTimeout(r, 0)); // let the success handler subtract + + con.warn('clean batch'); + handle?.flushNow(); + const body = bodyOf(fetchSpy) as { entries: unknown[]; droppedSinceLastFlush?: number }; + expect(body.entries).toHaveLength(1); + expect(body.droppedSinceLastFlush).toBeUndefined(); + }); + + test('no droppedSinceLastFlush field when nothing was dropped', () => { + const fetchSpy = makeFetchSpy(); + const { con } = install(fetchSpy); + con.warn('all good'); + handle?.flushNow(); + const body = bodyOf(fetchSpy) as { droppedSinceLastFlush?: number }; + expect(body.droppedSinceLastFlush).toBeUndefined(); + }); + test('uninstall restores console and clears the marker (fresh install works)', () => { const fetchSpy = makeFetchSpy(); const con = makeFakeConsole(); diff --git a/packages/app/src/lib/install-client-log-forwarder.ts b/packages/app/src/lib/install-client-log-forwarder.ts index 5237b5db6..8a14be77d 100644 --- a/packages/app/src/lib/install-client-log-forwarder.ts +++ b/packages/app/src/lib/install-client-log-forwarder.ts @@ -145,6 +145,12 @@ export function installClientLogForwarder( // Set while flushing so neither the flush path nor any transitive console // call it triggers gets re-captured (recursion guard). let inForward = false; + // Entries lost since the last delivered batch (ring overflow + failed + // POSTs). Carried as `droppedSinceLastFlush` on the next successful batch so + // the persisted log records that a gap exists — the drops cluster exactly + // when diagnostics matter most (reconnect storms), and a silent gap reads + // as "nothing happened". + let droppedSinceLastFlush = 0; const original: ConsoleLike = { log: con.log.bind(con), @@ -161,19 +167,43 @@ export function installClientLogForwarder( if (queue.length === 0) return; const entries = queue.splice(0, RENDERER_LOG_MAX_ENTRIES); pendingBytes = 0; + // Snapshot the drop count at send time: on delivery only this snapshot is + // subtracted (not a reset to 0), so drops that accrue while the POST is in + // flight survive to the next batch. + const droppedAtSend = droppedSinceLastFlush; inForward = true; try { void doFetch('/api/client-logs', { method: 'POST', headers: { 'content-type': 'application/json' }, keepalive: true, - body: JSON.stringify({ entries }), - }).catch(() => { - // Network failure — drop. Never surface; never console.* here (would - // re-capture). The entries are already removed from the queue. - }); + body: JSON.stringify( + droppedAtSend > 0 ? { entries, droppedSinceLastFlush: droppedAtSend } : { entries }, + ), + }).then( + (res) => { + if (res.ok) { + // Clamp at zero: `inForward` is released in the `finally` before this + // promise settles, so a second flush can fire and snapshot the same + // non-zero `droppedAtSend`. Two overlapping 2xx deliveries would then + // each subtract it and drive the counter negative, suppressing the + // next genuine gap marker. The clamp keeps double-reporting from + // going below zero. + droppedSinceLastFlush = Math.max(0, droppedSinceLastFlush - droppedAtSend); + } else { + // Non-2xx: the server rejected the batch; these entries are gone. + droppedSinceLastFlush += entries.length; + } + }, + () => { + // Network failure — drop. Never surface; never console.* here (would + // re-capture). The entries are already removed from the queue. + droppedSinceLastFlush += entries.length; + }, + ); } catch { // Synchronous failure (e.g. serialization) — drop the batch. + droppedSinceLastFlush += entries.length; } finally { inForward = false; } @@ -185,7 +215,10 @@ export function installClientLogForwarder( // Bounded ring — drop oldest under sustained backpressure. if (queue.length > RENDERER_LOG_MAX_ENTRIES) { const dropped = queue.shift(); - if (dropped) pendingBytes -= estimateEntryBytes(dropped); + if (dropped) { + pendingBytes -= estimateEntryBytes(dropped); + droppedSinceLastFlush += 1; + } } // Flush on entry count OR byte budget — the byte cap keeps each POST under // the browser's ~64KB `keepalive` limit so unload-time flushes aren't diff --git a/packages/app/tests/integration/api-error-envelope/metrics-agent-effects.test.ts b/packages/app/tests/integration/api-error-envelope/metrics-agent-effects.test.ts new file mode 100644 index 000000000..c44e16182 --- /dev/null +++ b/packages/app/tests/integration/api-error-envelope/metrics-agent-effects.test.ts @@ -0,0 +1,107 @@ +/** + * Per-handler narrow-integration smoke test for `handleMetricsAgentEffects`. + * + * Asserts the canonical RFC 9457 wire shape for + * `GET /api/metrics/agent-effects`. This handler shares the same + * auth-before-method-dispatch ordering as `handleMetricsAgentPresence` + * (loopback gate → host-allowlist gate → method check) so a bad Host never + * leaks "verb the endpoint expects" via 405. + * + * Coverage: + * - happy path: 200 + `application/json` + body parses against + * `MetricsAgentEffectsSuccessSchema`, no `ok` discriminator. + * - populated path: an agent write is captured into the doc's ring buffer + * and surfaces as a summarized per-doc block (character counts only — + * never the raw delta text). + * - DNS-rebinding Host → 403 `urn:ok:error:host-not-allowed` (must + * emit BEFORE the method check). + * - method-not-allowed on POST → 405 `urn:ok:error:method-not-allowed` + * with `Allow: GET`. + */ + +import { afterAll, beforeAll, describe, expect, test } from 'bun:test'; +import { + MetricsAgentEffectsSuccessSchema, + ProblemDetailsSchema, +} from '@inkeep/open-knowledge-core'; +import { HARNESS_BOOT_TIMEOUT_MS } from '../harness-boot-timeout'; +import { fetchWithHostHeader } from '../host-header-request.test-helper'; +import { createTestServer, type TestServer } from '../test-harness'; + +let server: TestServer; + +beforeAll(async () => { + server = await createTestServer(); +}, HARNESS_BOOT_TIMEOUT_MS); + +afterAll(async () => { + await server.cleanup(); +}); + +describe('metrics-agent-effects envelope (RFC 9457)', () => { + test('happy path emits flat success body with application/json', async () => { + const res = await fetch(`http://127.0.0.1:${server.port}/api/metrics/agent-effects`); + expect(res.status).toBe(200); + expect(res.headers.get('content-type')).toBe('application/json'); + + const body = await res.json(); + expect(MetricsAgentEffectsSuccessSchema.safeParse(body).success).toBe(true); + expect((body as Record).ok).toBeUndefined(); + }); + + test('an agent write surfaces as a summarized per-doc block (no raw delta text)', async () => { + const docName = `agent-effects-populated-${crypto.randomUUID().slice(0, 8)}`; + const written = '# Agent effects probe\n'; + const writeRes = await fetch(`http://127.0.0.1:${server.port}/api/agent-write-md`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ markdown: written, position: 'replace', docName }), + }); + expect(writeRes.status).toBe(200); + + const res = await fetch(`http://127.0.0.1:${server.port}/api/metrics/agent-effects`); + expect(res.status).toBe(200); + const parsed = MetricsAgentEffectsSuccessSchema.safeParse(await res.json()); + expect(parsed.success).toBe(true); + if (!parsed.success) return; + + const block = parsed.data.effects.find((b) => b['doc.name'] === docName); + expect(block).toBeDefined(); + expect(block?.entries.length).toBeGreaterThan(0); + const entry = block?.entries[0]; + expect(entry?.insertedChars).toBeGreaterThan(0); + // Summaries only: the response must never carry the written text. + expect(JSON.stringify(parsed.data)).not.toContain('Agent effects probe'); + }); + + test('DNS-rebinding Host emits 403 urn:ok:error:host-not-allowed BEFORE method check', async () => { + const res = await fetchWithHostHeader( + `http://127.0.0.1:${server.port}/api/metrics/agent-effects`, + 'evil.example.com', + { method: 'POST' }, + ); + // Auth-before-method-dispatch ordering: bad Host → 403, NOT 405. + expect(res.status).toBe(403); + expect(res.headers.get('content-type')).toBe('application/problem+json'); + + const parsed = ProblemDetailsSchema.safeParse(await res.json()); + expect(parsed.success).toBe(true); + if (parsed.success) { + expect(parsed.data.type).toBe('urn:ok:error:host-not-allowed'); + } + }); + + test('method-not-allowed on POST (with valid Host) emits problem+json with Allow: GET', async () => { + const res = await fetch(`http://127.0.0.1:${server.port}/api/metrics/agent-effects`, { + method: 'POST', + }); + expect(res.status).toBe(405); + expect(res.headers.get('allow')).toBe('GET'); + + const parsed = ProblemDetailsSchema.safeParse(await res.json()); + expect(parsed.success).toBe(true); + if (parsed.success) { + expect(parsed.data.type).toBe('urn:ok:error:method-not-allowed'); + } + }); +}); diff --git a/packages/app/tests/integration/attribution-sweep-coverage.test.ts b/packages/app/tests/integration/attribution-sweep-coverage.test.ts index da42fb579..eaa92a3a7 100644 --- a/packages/app/tests/integration/attribution-sweep-coverage.test.ts +++ b/packages/app/tests/integration/attribution-sweep-coverage.test.ts @@ -188,6 +188,10 @@ const EXEMPT_HANDLERS = new Set([ 'handleMetricsReconciliation', 'handleMetricsParseHealth', 'handleMetricsAgentPresence', + // `/api/metrics/agent-effects` — GET-only loopback + host-gated diagnostic + // (per-doc agent-effects ring-buffer summaries). Read path, no writes, + // nothing to attribute — same posture as `handleMetricsAgentPresence`. + 'handleMetricsAgentEffects', // `/api/client-logs` — web/browser renderer console-log ingest. Writes only // to the `renderer` pino log (diagnostics), no Y.Docs / agent content; gated // by `checkLocalOpSecurity` like the local-op handlers. No identity needed. diff --git a/packages/app/tests/integration/client-logs-endpoint.test.ts b/packages/app/tests/integration/client-logs-endpoint.test.ts index 9d7b04777..902cb33db 100644 --- a/packages/app/tests/integration/client-logs-endpoint.test.ts +++ b/packages/app/tests/integration/client-logs-endpoint.test.ts @@ -57,6 +57,23 @@ describe('POST /api/client-logs', () => { expect(body.accepted).toBe(2); }); + test('accepts a batch carrying a droppedSinceLastFlush gap marker', async () => { + const res = await postLogs({ + entries: [{ level: 'warn', message: 'after a reconnect storm' }], + droppedSinceLastFlush: 7, + }); + expect(res.status).toBe(200); + expect(((await res.json()) as { accepted: number }).accepted).toBe(1); + }); + + test('rejects a negative droppedSinceLastFlush with 400', async () => { + const res = await postLogs({ + entries: [{ level: 'warn', message: 'x' }], + droppedSinceLastFlush: -1, + }); + expect(res.status).toBe(400); + }); + test('accepts an empty batch (accepted: 0)', async () => { const res = await postLogs({ entries: [] }); expect(res.status).toBe(200); diff --git a/packages/app/tests/integration/conflict-gate-coverage.test.ts b/packages/app/tests/integration/conflict-gate-coverage.test.ts index 935b0ac2d..8f5fa0023 100644 --- a/packages/app/tests/integration/conflict-gate-coverage.test.ts +++ b/packages/app/tests/integration/conflict-gate-coverage.test.ts @@ -125,6 +125,11 @@ const EXEMPT_HANDLERS = new Set([ 'handleMetricsReconciliation', 'handleMetricsParseHealth', 'handleMetricsAgentPresence', + // `/api/metrics/agent-effects` — GET-only loopback + host-gated diagnostic + // summarizing the per-doc `Y.Map('agent-effects')` ring buffers. Reads only; + // targets no Y.Doc write, so the per-doc conflict gate does not apply — same + // posture as its `handleMetricsAgentPresence` sibling. + 'handleMetricsAgentEffects', // `/api/__embed-detect` — read-only loopback + host-gated diagnostic for the // embedded-viewer detection spikes; reads the in-process UA ring buffer and // returns boolean signals, targets no Y.Doc, so the per-doc conflict gate diff --git a/packages/cli/src/diagnose/bundle-redact.test.ts b/packages/cli/src/diagnose/bundle-redact.test.ts index 948553238..2fe6333b7 100644 --- a/packages/cli/src/diagnose/bundle-redact.test.ts +++ b/packages/cli/src/diagnose/bundle-redact.test.ts @@ -358,6 +358,78 @@ describe('redactStagedBundle — contentDir substitution', () => { expect(Object.values(result.docNameMap)).toContain('live-doc'); }); + test('hashes the real `currentDoc` field the presence endpoint emits', () => { + // The live `/api/metrics/agent-presence` wire body carries the doc path + // under `currentDoc`, not `doc.name`. The walker must hash that exact key + // so a staged presence artifact never leaks the doc a human is editing. + const stagingDir = makeStagingDir(); + const contentDir = '/Users/test/notes'; + writeStaged( + stagingDir, + 'state/agent-presence.json', + `${JSON.stringify({ + presence: { + 'agent-a1': { + displayName: 'Claude', + icon: 'claude', + color: '#abc', + currentDoc: 'meetings/secret-standup', + mode: 'writing', + ts: 1, + }, + }, + })}`, + ); + + const result = redactStagedBundle({ stagingDir, contentDir }); + + const after = JSON.parse(readStaged(stagingDir, 'state/agent-presence.json')); + expect(after.presence['agent-a1'].currentDoc).toMatch(/^doc:[a-f0-9]{8}$/); + expect(JSON.stringify(after)).not.toContain('meetings/secret-standup'); + expect(Object.values(result.docNameMap)).toContain('meetings/secret-standup'); + }); + + test('leaves a null `currentDoc` untouched (only strings hash)', () => { + const stagingDir = makeStagingDir(); + const contentDir = '/Users/test/notes'; + writeStaged( + stagingDir, + 'state/agent-presence.json', + `${JSON.stringify({ + presence: { 'agent-a1': { currentDoc: null, mode: 'idle', ts: 1 } }, + })}`, + ); + + redactStagedBundle({ stagingDir, contentDir }); + + const after = JSON.parse(readStaged(stagingDir, 'state/agent-presence.json')); + expect(after.presence['agent-a1'].currentDoc).toBeNull(); + }); + + test('hashes doc.name values in state/agent-effects.json via the walker pass', () => { + const stagingDir = makeStagingDir(); + const contentDir = '/Users/test/notes'; + writeStaged( + stagingDir, + 'state/agent-effects.json', + `${JSON.stringify({ + effects: [ + { + 'doc.name': 'meetings/standup', + entries: [{ sessionId: 'agent-a1', agentType: 'claude', ts: 1, insertedChars: 4 }], + }, + ], + })}`, + ); + + const result = redactStagedBundle({ stagingDir, contentDir }); + + const after = JSON.parse(readStaged(stagingDir, 'state/agent-effects.json')); + expect(after.effects[0]['doc.name']).toMatch(/^doc:[a-f0-9]{8}$/); + expect(after.effects[0].entries[0].sessionId).toBe('agent-a1'); + expect(Object.values(result.docNameMap)).toContain('meetings/standup'); + }); + test('strings that do not include contentDir are passed through verbatim', () => { const stagingDir = makeStagingDir(); const contentDir = '/Users/test/notes'; diff --git a/packages/cli/src/diagnose/bundle-redact.ts b/packages/cli/src/diagnose/bundle-redact.ts index 045ad5d32..79e95bdc7 100644 --- a/packages/cli/src/diagnose/bundle-redact.ts +++ b/packages/cli/src/diagnose/bundle-redact.ts @@ -31,8 +31,12 @@ const HASH_HEX_LEN = 8; // Keys whose values are treated as doc-name-shaped. The OTLP attribute pair // form sets `key` to one of these and stores the value under `value.stringValue`; // the Pino flat form sets the key directly on the log record. Single source -// for both shapes — extending DOC_NAME_KEYS covers both at once. -const DOC_NAME_KEYS = new Set(['doc.name']); +// for both shapes — extending DOC_NAME_KEYS covers both at once. `currentDoc` +// is the workspace-relative doc path carried by `agent-presence.json` entries; +// hashing it keeps a redacted presence artifact from leaking the doc a human +// was editing (same anonymization guarantee the `doc.name`-keyed effects +// artifact already gets). +const DOC_NAME_KEYS = new Set(['doc.name', 'currentDoc']); export interface RedactStagedBundleOpts { /** Absolute path to the staging dir (contains telemetry/, logs/, state/). */ @@ -263,10 +267,10 @@ function walkDirFiles(dir: string): string[] { } } -// State files that are JSON-shaped get a full walker pass (agent-presence may -// carry per-agent doc.name fields; runtime.json may carry the contentDir). -// Other state files get substring-only substitution. -const STATE_JSON_FILES = new Set(['agent-presence.json', 'runtime.json']); +// State files that are JSON-shaped get a full walker pass (agent-presence and +// agent-effects may carry per-doc `doc.name` fields; runtime.json may carry +// the contentDir). Other state files get substring-only substitution. +const STATE_JSON_FILES = new Set(['agent-presence.json', 'agent-effects.json', 'runtime.json']); export function redactStagedBundle(opts: RedactStagedBundleOpts): RedactStagedBundleResult { const ctx: RedactCtx = { diff --git a/packages/cli/src/diagnose/bundle.test.ts b/packages/cli/src/diagnose/bundle.test.ts index 7f8b60c76..37385c8db 100644 --- a/packages/cli/src/diagnose/bundle.test.ts +++ b/packages/cli/src/diagnose/bundle.test.ts @@ -44,7 +44,9 @@ afterEach(() => { function makeDeterministicDeps(over: Partial = {}): CollectBundleDeps { return { fetchAgentPresence: async () => null, + fetchAgentEffects: async () => null, readShadowHead: () => null, + readCheckpointRefs: () => null, now: () => new Date('2026-05-28T14:22:01.000Z'), okVersion: () => '0.7.99', readDesktopEnv: () => null, @@ -310,6 +312,48 @@ describe('collectBundle — server status', () => { collected.cleanup(); }); + test('lock present + agent-effects 2xx → agent-effects.json staged', async () => { + const contentDir = makeTmpDir(); + writeAt( + contentDir, + '.ok/local/server.lock', + JSON.stringify({ pid: 1, port: 4711, hostname: 'h', startedAt: 't', worktreeRoot: '/' }), + ); + let queriedPort = -1; + const deps = makeDeterministicDeps({ + fetchAgentPresence: async () => JSON.stringify({ presence: {} }), + fetchAgentEffects: async (port) => { + queriedPort = port; + return JSON.stringify({ effects: [] }); + }, + }); + const collected = await collectBundle({ contentDir, deps }); + + expect(queriedPort).toBe(4711); + const effectsPath = join(collected.stagingDir, 'state', 'agent-effects.json'); + expect(existsSync(effectsPath)).toBe(true); + expect(JSON.parse(readFileSync(effectsPath, 'utf-8'))).toEqual({ effects: [] }); + collected.cleanup(); + }); + + test('agent-effects failure never affects serverStatus (presence is the probe)', async () => { + const contentDir = makeTmpDir(); + writeAt( + contentDir, + '.ok/local/server.lock', + JSON.stringify({ pid: 1, port: 4711, hostname: 'h', startedAt: 't', worktreeRoot: '/' }), + ); + const deps = makeDeterministicDeps({ + fetchAgentPresence: async () => JSON.stringify({ presence: {} }), + fetchAgentEffects: async () => null, + }); + const collected = await collectBundle({ contentDir, deps }); + + expect(collected.manifest.serverStatus).toBe('running'); + expect(existsSync(join(collected.stagingDir, 'state', 'agent-effects.json'))).toBe(false); + collected.cleanup(); + }); + test('corrupt lock → not-running, lock still staged for forensics', async () => { const contentDir = makeTmpDir(); writeAt(contentDir, '.ok/local/server.lock', 'not json {'); @@ -345,6 +389,28 @@ describe('collectBundle — state files', () => { collected.cleanup(); }); + test('checkpoint-refs.txt is written when readCheckpointRefs returns content', async () => { + const contentDir = makeTmpDir(); + const listing = + 'refs/checkpoints/main/deadbee\t2026-05-28T14:00:00+00:00\tcheckpoint: Before concurrent merge @ 2026-05-28T14:00:00.000Z\n'; + const deps = makeDeterministicDeps({ + readCheckpointRefs: () => listing, + }); + const collected = await collectBundle({ contentDir, deps }); + expect(readFileSync(join(collected.stagingDir, 'state', 'checkpoint-refs.txt'), 'utf-8')).toBe( + listing, + ); + expect(collected.manifest.files.map((f) => f.path)).toContain('state/checkpoint-refs.txt'); + collected.cleanup(); + }); + + test('checkpoint-refs.txt is omitted when readCheckpointRefs returns null', async () => { + const contentDir = makeTmpDir(); + const collected = await collectBundle({ contentDir, deps: makeDeterministicDeps() }); + expect(existsSync(join(collected.stagingDir, 'state', 'checkpoint-refs.txt'))).toBe(false); + collected.cleanup(); + }); + test('last-server-exit.json is staged when the desktop host wrote one', async () => { const contentDir = makeTmpDir(); const body = `${JSON.stringify( diff --git a/packages/cli/src/diagnose/bundle.ts b/packages/cli/src/diagnose/bundle.ts index ba63ba839..26f5e037e 100644 --- a/packages/cli/src/diagnose/bundle.ts +++ b/packages/cli/src/diagnose/bundle.ts @@ -202,12 +202,28 @@ export interface CollectBundleDeps { * on any failure (network, timeout, non-2xx). */ fetchAgentPresence?: (port: number) => Promise; + /** + * Fetch the running server's `GET /api/metrics/agent-effects` response + * (per-doc agent activity ring-buffer summaries), within the 1s budget. + * Returns the response body string on 2xx, or `null` on any failure. + * Best-effort like agent-presence; a `null` never affects `serverStatus`. + */ + fetchAgentEffects?: (port: number) => Promise; /** * Read the last 50 entries of the shadow repo log at * `/.git/ok/`. Returns the stdout string or `null` if the * shadow repo is missing or git fails. */ readShadowHead?: (contentDir: string) => string | null; + /** + * List the most recent `refs/checkpoints/*` refs of the shadow repo at + * `/.git/ok/` — the content-loss / rescue recovery snapshots. + * One tab-separated line per ref: ref name, creation date, commit subject + * (the subject is a content-free label; the commit BODY, which can carry + * doc names and lost content, is deliberately not read). Returns the + * listing string or `null` if the shadow repo is missing or git fails. + */ + readCheckpointRefs?: (contentDir: string) => string | null; /** Returns the current timestamp. Override in tests for determinism. */ now?: () => Date; /** Returns the OK CLI's package version. */ @@ -322,6 +338,7 @@ export const _pathHelpersForTests = { const AGENT_PRESENCE_TIMEOUT_MS = 1000; const SHADOW_GIT_LOG_LIMIT = 50; +const CHECKPOINT_REF_LIMIT = 50; async function defaultFetchAgentPresence(port: number): Promise { try { @@ -338,6 +355,21 @@ async function defaultFetchAgentPresence(port: number): Promise { } } +async function defaultFetchAgentEffects(port: number): Promise { + try { + const response = await fetch(`http://127.0.0.1:${port}/api/metrics/agent-effects`, { + signal: AbortSignal.timeout(AGENT_PRESENCE_TIMEOUT_MS), + }); + if (!response.ok) return null; + return await response.text(); + } catch { + // Same all-failures-collapse-to-null posture as agent-presence — a + // server without the endpoint (older version) or not running must not + // fail the bundle. + return null; + } +} + function defaultReadShadowHead(contentDir: string): string | null { const shadowDir = join(contentDir, '.git', 'ok'); if (!existsSync(shadowDir)) return null; @@ -350,6 +382,31 @@ function defaultReadShadowHead(contentDir: string): string | null { return result.stdout ?? ''; } +function defaultReadCheckpointRefs(contentDir: string): string | null { + const shadowDir = join(contentDir, '.git', 'ok'); + if (!existsSync(shadowDir)) return null; + // `%(contents:subject)` only: checkpoint commit bodies carry an + // `ok-checkpoint-v1` JSON line with the doc name and (for merge-loss + // checkpoints) lost-content substrings — the subject is a content-free + // label, so restricting the format keeps the artifact safe to stage as + // plain text under the same redaction rules as shadow-head.txt. + const result = spawnSync( + 'git', + [ + '-C', + shadowDir, + 'for-each-ref', + '--sort=-creatordate', + `--count=${CHECKPOINT_REF_LIMIT}`, + '--format=%(refname)%09%(creatordate:iso-strict)%09%(contents:subject)', + 'refs/checkpoints/', + ], + withHiddenWindowsConsole({ encoding: 'utf-8', timeout: 2000 }), + ); + if (result.error || result.status !== 0) return null; + return result.stdout ?? ''; +} + export function defaultReadDesktopEnv(): DesktopMetadata | null { const electronVersion = process.env.OK_DESKTOP_VERSION; const packagedRaw = process.env.OK_DESKTOP_PACKAGED; @@ -569,7 +626,9 @@ export async function collectBundle(opts: CollectBundleOpts): Promise new Date()); const okVersion = deps.okVersion ?? (() => PACKAGE_VERSION); const readDesktopEnv = deps.readDesktopEnv ?? defaultReadDesktopEnv; @@ -621,6 +680,13 @@ export async function collectBundle(opts: CollectBundleOpts): Promise { expect(ClientLogsRequestSchema.safeParse({ entries }).success).toBe(false); }); + test('accepts an optional droppedSinceLastFlush count; rejects a negative one', () => { + const entries = [{ level: 'info' as const, message: 'x' }]; + expect(ClientLogsRequestSchema.safeParse({ entries, droppedSinceLastFlush: 3 }).success).toBe( + true, + ); + expect(ClientLogsRequestSchema.safeParse({ entries }).success).toBe(true); + expect(ClientLogsRequestSchema.safeParse({ entries, droppedSinceLastFlush: -1 }).success).toBe( + false, + ); + }); + test('rejects an oversized message', () => { const result = ClientLogsRequestSchema.safeParse({ entries: [{ level: 'error', message: 'a'.repeat(RENDERER_LOG_MAX_MESSAGE_BYTES + 1) }], diff --git a/packages/core/src/schemas/api/client-logs.ts b/packages/core/src/schemas/api/client-logs.ts index 26d168548..73a978c6a 100644 --- a/packages/core/src/schemas/api/client-logs.ts +++ b/packages/core/src/schemas/api/client-logs.ts @@ -33,10 +33,17 @@ export const ClientLogEntrySchema = z .loose() satisfies StandardSchemaV1; export type ClientLogEntry = z.infer; -/** Request body for `POST /api/client-logs` — a bounded batch of entries. */ +/** + * Request body for `POST /api/client-logs` — a bounded batch of entries. + * `droppedSinceLastFlush` is the number of entries the client forwarder lost + * since its last delivered batch (buffer overflow or failed POSTs — reconnect + * storms are the typical cause); the server records it so the persisted log + * carries an explicit gap marker instead of silently missing entries. + */ export const ClientLogsRequestSchema = z .object({ entries: z.array(ClientLogEntrySchema).max(RENDERER_LOG_MAX_ENTRIES), + droppedSinceLastFlush: z.number().int().min(0).optional(), }) .loose() satisfies StandardSchemaV1; export type ClientLogsRequest = z.infer; diff --git a/packages/core/src/schemas/api/metrics.test.ts b/packages/core/src/schemas/api/metrics.test.ts index 7cbd36a21..b0e7d067e 100644 --- a/packages/core/src/schemas/api/metrics.test.ts +++ b/packages/core/src/schemas/api/metrics.test.ts @@ -7,6 +7,7 @@ import { AgentBurstDiffSuccessSchema, AgentPresenceEntrySchema, InstalledAgentsSuccessSchema, + MetricsAgentEffectsSuccessSchema, MetricsAgentPresenceSuccessSchema, MetricsParseHealthSuccessSchema, MetricsReconciliationSuccessSchema, @@ -326,6 +327,49 @@ describe('MetricsAgentPresenceSuccessSchema', () => { }); }); +describe('MetricsAgentEffectsSuccessSchema', () => { + test('parses an empty effects list', () => { + expect(MetricsAgentEffectsSuccessSchema.safeParse({ effects: [] }).success).toBe(true); + }); + test('parses a populated per-doc block with summarized entries', () => { + expect( + MetricsAgentEffectsSuccessSchema.safeParse({ + effects: [ + { + 'doc.name': 'meetings/standup', + entries: [ + { + sessionId: 'agent-a1', + agentType: 'claude', + ts: 1, + insertedChars: 12, + deletedChars: 0, + }, + ], + }, + ], + }).success, + ).toBe(true); + }); + test('rejects body missing effects field', () => { + expect(MetricsAgentEffectsSuccessSchema.safeParse({}).success).toBe(false); + }); + test('rejects a negative character count', () => { + expect( + MetricsAgentEffectsSuccessSchema.safeParse({ + effects: [ + { + 'doc.name': 'a', + entries: [ + { sessionId: 's', agentType: 't', ts: 1, insertedChars: -1, deletedChars: 0 }, + ], + }, + ], + }).success, + ).toBe(false); + }); +}); + describe('InstalledAgentsSuccessSchema', () => { test('parses a populated boolean record', () => { expect( diff --git a/packages/core/src/schemas/api/metrics.ts b/packages/core/src/schemas/api/metrics.ts index a836262f7..c9076e4ea 100644 --- a/packages/core/src/schemas/api/metrics.ts +++ b/packages/core/src/schemas/api/metrics.ts @@ -1,10 +1,11 @@ /** * metrics + agent activity + test handlers. * - * Eight handlers — `handleAgentActivity`, `handleAgentBurstDiff`, + * Nine handlers — `handleAgentActivity`, `handleAgentBurstDiff`, * `handleTestReset`, `handleTestRescanBacklinks`, * `handleMetricsReconciliation`, `handleMetricsParseHealth`, - * `handleMetricsAgentPresence`, `handleInstalledAgentsRoute`. No new URN + * `handleMetricsAgentPresence`, `handleMetricsAgentEffects`, + * `handleInstalledAgentsRoute`. No new URN * tokens — every error path reuses existing tokens (`invalid-request`, * `reserved-doc-name`, `no-active-session`, `not-found`, `loopback-required`, * `host-not-allowed`, `invalid-origin`, `method-not-allowed`, @@ -166,6 +167,51 @@ export const MetricsAgentPresenceSuccessSchema = z .loose() satisfies StandardSchemaV1; export type MetricsAgentPresenceSuccess = z.infer; +/** + * One summarized agent-effects ring-buffer entry on + * `MetricsAgentEffectsSuccessSchema.effects[].entries`. Deltas are reduced to + * character counts — the diagnostic signal is who wrote how much to which doc + * and when; the raw delta text (user content) never leaves the live doc via + * this endpoint. + */ +export const AgentEffectEntrySchema = z + .object({ + sessionId: z.string().min(1), + agentType: z.string().min(1), + ts: z.number().int().min(0), + insertedChars: z.number().int().min(0), + deletedChars: z.number().int().min(0), + }) + .loose() satisfies StandardSchemaV1; +export type AgentEffectEntryWire = z.infer; + +/** + * One per-doc block on `MetricsAgentEffectsSuccessSchema.effects`. The doc + * name is carried under the literal key `doc.name` — the same key the + * diagnostics-bundle redactor hashes — so a staged copy of this response is + * anonymized by the existing pass without a redactor change. + */ +export const AgentEffectsDocSchema = z + .object({ + 'doc.name': z.string().min(1), + entries: z.array(AgentEffectEntrySchema), + }) + .loose() satisfies StandardSchemaV1; +export type AgentEffectsDocWire = z.infer; + +/** + * Success response for `GET /api/metrics/agent-effects`. Aggregates the + * bounded per-doc `agent-effects` ring buffers across currently-loaded docs + * (never force-loads unloaded ones). Loopback + Host-allowlist gated like + * `/api/metrics/agent-presence`. + */ +export const MetricsAgentEffectsSuccessSchema = z + .object({ + effects: z.array(AgentEffectsDocSchema), + }) + .loose() satisfies StandardSchemaV1; +export type MetricsAgentEffectsSuccess = z.infer; + /** * Success response for `GET /api/installed-agents`. Returns a flat boolean * record keyed by agent scheme name (`claude` / `codex` / `cursor`). The diff --git a/packages/server/src/api-extension.ts b/packages/server/src/api-extension.ts index 4cefaaec8..6a70a0226 100644 --- a/packages/server/src/api-extension.ts +++ b/packages/server/src/api-extension.ts @@ -128,6 +128,8 @@ import { MANAGED_ARTIFACT_PREFIX_SKILL, MANAGED_ARTIFACT_PREFIX_TEMPLATE, MarkdownlintRuleWriteRequestSchema, + type MetricsAgentEffectsSuccess, + MetricsAgentEffectsSuccessSchema, MetricsAgentPresenceSuccessSchema, MetricsParseHealthSuccessSchema, MetricsReconciliationSuccessSchema, @@ -251,7 +253,7 @@ import busboy from 'busboy'; import { fileTypeFromBuffer } from 'file-type'; import { parse as parseYaml, stringify as stringifyYaml } from 'yaml'; import { z } from 'zod'; -import { captureEffect } from './activity-log.ts'; +import { captureEffect, type EffectValue } from './activity-log.ts'; import { listAgentActivity, synthesizeVersionDiff } from './agent-activity.ts'; import type { AgentFocusBroadcaster } from './agent-focus.ts'; import { type AgentPresenceBroadcaster, BROADCASTER_EVICTION_MS } from './agent-presence.ts'; @@ -8784,6 +8786,92 @@ export function createApiExtension(options: ApiExtensionOptions): Extension { } } + async function handleMetricsAgentEffects( + req: IncomingMessage, + res: ServerResponse, + ): Promise { + // Diagnostic view of the per-doc `agent-effects` ring buffers, which + // otherwise live only inside live Y.Docs and are invisible to bundles. + // Loopback + Host-header gated with auth-before-method-dispatch ordering + // — same pattern + rationale as `handleMetricsAgentPresence` (per-agent + // identity plus per-doc write timing are local-editing-only signals). + if (!isLoopbackAddress(req.socket.remoteAddress)) { + errorResponse(res, 403, 'urn:ok:error:loopback-required', 'Loopback required.', { + handler: 'metrics-agent-effects', + }); + return; + } + if (!isAllowedWorkspaceHostHeader(req.headers.host)) { + errorResponse(res, 403, 'urn:ok:error:host-not-allowed', 'Host header not allowed.', { + handler: 'metrics-agent-effects', + }); + return; + } + if (req.method !== 'GET') { + errorResponse(res, 405, 'urn:ok:error:method-not-allowed', 'Method not allowed.', { + handler: 'metrics-agent-effects', + extraHeaders: { Allow: 'GET' }, + }); + return; + } + // Tracks the doc being summarized so a throw in the per-doc reduction (e.g. + // a malformed `EffectValue` from an older schema) names its source in the + // catch log. Rides the `doc.name` key the bundle redactor hashes. + let failingDocName: string | undefined; + try { + // Currently-loaded docs only — iterating `hocuspocus.documents` never + // materializes an unloaded doc. The `share.has` probe avoids even + // creating the lazy Y.Map placeholder on docs no agent ever wrote to. + const effects: MetricsAgentEffectsSuccess['effects'] = []; + for (const [effectsDocName, document] of hocuspocus.documents) { + if (isSystemDoc(effectsDocName) || isConfigDoc(effectsDocName)) continue; + if (!document.share.has('agent-effects')) continue; + failingDocName = effectsDocName; + const effectsMap = document.getMap('agent-effects'); + if (effectsMap.size === 0) continue; + // Deltas reduce to character counts: the diagnostic signal is who + // wrote how much to which doc and when. Raw delta text is user + // content and stays in the live doc. + const entries = [...effectsMap.values()] + .map((effect) => { + let insertedChars = 0; + let deletedChars = 0; + for (const op of effect.delta) { + if (typeof op.insert === 'string') insertedChars += op.insert.length; + else if (op.insert !== undefined) insertedChars += 1; + if (typeof op.delete === 'number') deletedChars += op.delete; + } + return { + sessionId: effect.sessionId, + agentType: effect.agent_type, + ts: effect.timestamp, + insertedChars, + deletedChars, + }; + }) + .sort((a, b) => a.ts - b.ts); + // The doc name rides under the literal `doc.name` key — the key the + // diagnostics-bundle redactor hashes — so a staged copy of this + // response is anonymized by the existing pass. + effects.push({ 'doc.name': effectsDocName, entries }); + } + effects.sort((a, b) => a['doc.name'].localeCompare(b['doc.name'])); + successResponse( + res, + 200, + MetricsAgentEffectsSuccessSchema, + { effects }, + { handler: 'metrics-agent-effects' }, + ); + } catch (e) { + log.error({ err: e, 'doc.name': failingDocName }, '[metrics-agent-effects] handler failed'); + errorResponse(res, 500, 'urn:ok:error:internal-server-error', 'Internal server error.', { + handler: 'metrics-agent-effects', + cause: e, + }); + } + } + async function handleEmbedDetect(req: IncomingMessage, res: ServerResponse): Promise { // Diagnostic endpoint for the Cursor / Codex / Claude Code embedded-viewer // detection spikes. Reads from the in-process ring buffer populated in @@ -17707,6 +17795,21 @@ export function createApiExtension(options: ApiExtensionOptions): Extension { async (_req, res, body) => { try { const logger = getLogger('renderer'); + if (body.droppedSinceLastFlush !== undefined && body.droppedSinceLastFlush > 0) { + // Gap marker: the forwarder lost entries (buffer overflow / failed + // POSTs) between the previous delivered batch and this one. Persist + // it as its own line so a log reader knows the silence was loss, + // not inactivity. + logger.warn( + { + source: 'renderer-console', + transport: 'web', + event: 'client-log-entries-dropped', + droppedSinceLastFlush: body.droppedSinceLastFlush, + }, + 'client-log-entries-dropped', + ); + } for (const entry of body.entries) { // Per-entry guard: one entry that trips a pino serialization fault // must not drop the rest of the batch (the response still reports the @@ -18633,6 +18736,7 @@ export function createApiExtension(options: ApiExtensionOptions): Extension { '/api/metrics/reconciliation': handleMetricsReconciliation, '/api/metrics/parse-health': handleMetricsParseHealth, '/api/metrics/agent-presence': handleMetricsAgentPresence, + '/api/metrics/agent-effects': handleMetricsAgentEffects, '/api/__embed-detect': handleEmbedDetect, '/api/server-info': handleServerInfo, '/api/share/construct-url': handleShareConstructUrl, diff --git a/packages/server/src/server-observers.ts b/packages/server/src/server-observers.ts index 85e2bb9be..d7081b90c 100644 --- a/packages/server/src/server-observers.ts +++ b/packages/server/src/server-observers.ts @@ -60,6 +60,7 @@ import { } from './bridge-watchdog.ts'; import { isConfigDoc, isSystemDoc } from './cc1-broadcast.ts'; import { recordFrontmatterEditSurface } from './frontmatter-telemetry.ts'; +import { getLogger } from './logger.ts'; import { computeMapDrivenBodySplice } from './map-driven-splice.ts'; import { incrementBridgeMergeCheckpointCreated, @@ -81,6 +82,14 @@ import { import { type ShadowHandle, saveInMemoryCheckpoint } from './shadow-repo.ts'; import { setActiveSpanAttributes, withSpanSync } from './telemetry.ts'; +/** + * Pino line for checkpoint-write failures, alongside the existing + * console.warn: a failed checkpoint means the recovery snapshot for a + * detected content-loss event was NOT persisted, and the pino sink is what + * reaches the local log files (and, through them, diagnostics bundles). + */ +const checkpointLog = getLogger('server-observers'); + // ───────────────────────────────────────────────────────────── // Origin constant // ───────────────────────────────────────────────────────────── @@ -675,6 +684,10 @@ export function setupServerObservers(opts: SetupServerObserversOpts): () => void message: err.message, stack: err.stack?.split('\n').slice(0, 4).join('\n'), }); + checkpointLog.warn( + { err, 'doc.name': opts.docName ?? null, branch, kind: 'bridge-merge-loss' }, + 'checkpoint write failed', + ); }); }); }; @@ -759,6 +772,10 @@ export function setupServerObservers(opts: SetupServerObserversOpts): () => void message: e.message, stack: e.stack?.split('\n').slice(0, 4).join('\n'), }); + checkpointLog.warn( + { err: e, 'doc.name': docName, branch, kind: 'observer-a-duplication' }, + 'checkpoint write failed', + ); }); }); }; @@ -1100,6 +1117,10 @@ export function setupServerObservers(opts: SetupServerObserversOpts): () => void message: e.message, stack: e.stack?.split('\n').slice(0, 4).join('\n'), }); + checkpointLog.warn( + { err: e, 'doc.name': docName, branch, kind: 'producer-guard-loss' }, + 'checkpoint write failed', + ); }); }); };