From cb3e1e9c2db64c83c877f4f4eb075db50a0afbf9 Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Fri, 16 Oct 2026 14:05:07 -0400 Subject: [PATCH 01/34] fix(search): index vault changes that land while the server starts The file watcher now starts before the startup rebuild and holds every change it reports while the rebuild runs. One bounded batch then checks each held path against the disk, removing missing paths and re-reading present ones, and events that arrive during the batch are replayed in arrival order once the watcher goes live. Startup waits at most 10 s for the batch before the server listens. Co-Authored-By: Claude Opus 5.5 --- .../search/__tests__/file-watcher.test.ts | 588 +++++++++++++++++- src/vault-mcp/search/file-watcher.ts | 355 ++++++++++- src/vault-mcp/search/search-index.ts | 6 +- src/vault-mcp/server.ts | 9 +- 4 files changed, 898 insertions(+), 60 deletions(-) diff --git a/src/vault-mcp/search/__tests__/file-watcher.test.ts b/src/vault-mcp/search/__tests__/file-watcher.test.ts index 1e92865df..618fdb7c0 100644 --- a/src/vault-mcp/search/__tests__/file-watcher.test.ts +++ b/src/vault-mcp/search/__tests__/file-watcher.test.ts @@ -20,7 +20,7 @@ import { createSearchIndex } from "../search-index.js" import { extractPdfText } from "../../obsidian-markdown/pdf.js" import { readdirOrNull, statOrNull } from "../../../utils/fs.js" import type { SearchIndex } from "../search-index.js" -import { startFileWatcher } from "../file-watcher.js" +import { watchVaultAndBuildIndex } from "../file-watcher.js" import { logger } from "../../../logger.js" // Auto-spy chokidar: spy: true keeps the real implementation, so the @@ -38,6 +38,21 @@ vi.mock("../../obsidian-markdown/pdf.js", { spy: true }) let vault: string let index: SearchIndex +/** Starts the watcher with the startup rebuild stubbed to find nothing, so the + * only indexing a test sees comes from the watcher's own events and rescans, + * including for files the test wrote before starting it. */ +const startWatcherWithoutRebuild = async ( + vaultPath: string, + search: SearchIndex, + options?: Parameters[2], +): Promise => { + vi.spyOn(search, "rebuildFromVault").mockResolvedValue({ + count: 0, + embedding: Promise.resolve(), + }) + await watchVaultAndBuildIndex(vaultPath, search, options) +} + /** Poll until a condition is met, with timeout. */ const waitFor = async (check: () => boolean, timeoutMs = 8000, intervalMs = 100): Promise => { const deadline = Date.now() + timeoutMs @@ -75,7 +90,7 @@ describe("file-watcher", REAL_WATCHER_RETRY, () => { registerSharedVaultHooks() it("indexes a new .md file", { timeout: 15000 }, async () => { - await startFileWatcher(vault, index, { + await startWatcherWithoutRebuild(vault, index, { stabilityThreshold: 200, pollInterval: 50, }) @@ -89,7 +104,7 @@ describe("file-watcher", REAL_WATCHER_RETRY, () => { }) it("re-indexes a modified file", { timeout: 15000 }, async () => { - await startFileWatcher(vault, index, { + await startWatcherWithoutRebuild(vault, index, { stabilityThreshold: 200, pollInterval: 50, }) @@ -105,7 +120,7 @@ describe("file-watcher", REAL_WATCHER_RETRY, () => { }) it("removes a deleted file from index", { timeout: 15000 }, async () => { - await startFileWatcher(vault, index, { + await startWatcherWithoutRebuild(vault, index, { stabilityThreshold: 200, pollInterval: 50, }) @@ -121,7 +136,7 @@ describe("file-watcher", REAL_WATCHER_RETRY, () => { }) it("ignores non-.md files", { timeout: 15000 }, async () => { - await startFileWatcher(vault, index, { + await startWatcherWithoutRebuild(vault, index, { stabilityThreshold: 200, pollInterval: 50, }) @@ -148,7 +163,7 @@ describe("file-watcher", REAL_WATCHER_RETRY, () => { vi.mocked(statOrNull).mockRestore() }) - await startFileWatcher(vault, index, { + await startWatcherWithoutRebuild(vault, index, { stabilityThreshold: 200, pollInterval: 50, }) @@ -173,7 +188,7 @@ describe("file-watcher", REAL_WATCHER_RETRY, () => { it("ignores hidden directories", { timeout: 15000 }, async () => { await mkdir(join(vault, ".obsidian"), { recursive: true }) - await startFileWatcher(vault, index, { + await startWatcherWithoutRebuild(vault, index, { stabilityThreshold: 200, pollInterval: 50, }) @@ -190,7 +205,7 @@ describe("file-watcher", REAL_WATCHER_RETRY, () => { it("indexes a symlinked .md file", { timeout: 15000 }, async () => { await writeFile(join(vault, "real.md"), "symlink watcher target\n", "utf8") - await startFileWatcher(vault, index, { + await startWatcherWithoutRebuild(vault, index, { stabilityThreshold: 200, pollInterval: 50, }) @@ -217,7 +232,7 @@ describe("file-watcher", REAL_WATCHER_RETRY, () => { capturedVersion.resolve(sourceVersion) return sourceVersion }) - await startFileWatcher(vault, index, { + await startWatcherWithoutRebuild(vault, index, { stabilityThreshold: 200, pollInterval: 50, }) @@ -244,7 +259,7 @@ describe("file-watcher", REAL_WATCHER_RETRY, () => { "indexes a note atomically written into a brand-new directory", { timeout: 15000 }, async () => { - await startFileWatcher(vault, index, { + await startWatcherWithoutRebuild(vault, index, { stabilityThreshold: 200, pollInterval: 50, newDirectoryRescanDelay: 500, @@ -269,7 +284,7 @@ describe("file-watcher", REAL_WATCHER_RETRY, () => { // WSL2 bridge). It works on any filesystem, so this verifies the usePolling // option is wired through and indexing still happens under polling. it("indexes a new .md file when polling", { timeout: 15000 }, async () => { - await startFileWatcher(vault, index, { + await startWatcherWithoutRebuild(vault, index, { stabilityThreshold: 200, pollInterval: 50, usePolling: true, @@ -293,7 +308,7 @@ describe("file-watcher — file content indexing", REAL_WATCHER_RETRY, () => { }) const upsertSpy = vi.spyOn(fileIndex, "upsertFileContent") - await startFileWatcher(vault, fileIndex, { + await startWatcherWithoutRebuild(vault, fileIndex, { stabilityThreshold: 200, pollInterval: 50, }) @@ -315,7 +330,7 @@ describe("file-watcher — file content indexing", REAL_WATCHER_RETRY, () => { const upsertSpy = vi.spyOn(fileIndex, "upsertFileContent") const removeSpy = vi.spyOn(fileIndex, "removeFileContent") - await startFileWatcher(vault, fileIndex, { + await startWatcherWithoutRebuild(vault, fileIndex, { stabilityThreshold: 200, pollInterval: 50, }) @@ -337,7 +352,7 @@ describe("file-watcher — file content indexing", REAL_WATCHER_RETRY, () => { }) const upsertSpy = vi.spyOn(fileIndex, "upsertFileContent") - await startFileWatcher(vault, fileIndex, { + await startWatcherWithoutRebuild(vault, fileIndex, { stabilityThreshold: 200, pollInterval: 50, }) @@ -364,7 +379,7 @@ describe("file-watcher — file content indexing", REAL_WATCHER_RETRY, () => { return sourceVersion }) - await startFileWatcher(vault, fileIndex, { + await startWatcherWithoutRebuild(vault, fileIndex, { stabilityThreshold: 200, pollInterval: 50, }) @@ -380,7 +395,7 @@ describe("file-watcher — file content indexing", REAL_WATCHER_RETRY, () => { }) }) -describe("startFileWatcher — obsolete events and embedding queues", () => { +describe("file watcher — obsolete events and embedding queues", () => { const createControlledWatcher = async (embedder?: Parameters[1]) => { const testVault = await mkdtemp(join(tmpdir(), "watcher-events-")) onTestFinished(() => rm(testVault, { recursive: true })) @@ -397,7 +412,7 @@ describe("startFileWatcher — obsolete events and embedding queues", () => { watchMock.mockRestore() await watcher.close() }) - const starting = startFileWatcher(testVault, search) + const starting = startWatcherWithoutRebuild(testVault, search) watcher.emit("ready") await starting @@ -1382,14 +1397,539 @@ describe("startFileWatcher — obsolete events and embedding queues", () => { }) }) -describe("startFileWatcher — chokidar watch options", () => { +/** Pauses the first readFile of one path. "before-disk" waits before reading, + * so the read sees the disk as it is on release; "after-disk" reads first and + * waits before returning what it read. */ +const gateFirstRead = async (filePath: string, placement: "before-disk" | "after-disk") => { + const actualFs = await vi.importActual("node:fs/promises") + const reached = Promise.withResolvers() + const release = Promise.withResolvers() + const gatedPaths = new Set() + vi.mocked(readFile).mockImplementation(async (requestedPath, options) => { + const isGatedRead = requestedPath === filePath && !gatedPaths.has(filePath) + + if (!isGatedRead) return actualFs.readFile(requestedPath, options) + gatedPaths.add(filePath) + if (placement === "before-disk") { + reached.resolve(undefined) + await release.promise + return actualFs.readFile(requestedPath, options) + } + const content = await actualFs.readFile(requestedPath, options) + reached.resolve(undefined) + await release.promise + return content + }) + onTestFinished(() => vi.mocked(readFile).mockRestore()) + return { reached: reached.promise, release: () => release.resolve(undefined) } +} + +describe("watchVaultAndBuildIndex — changes during startup", () => { + /** A controlled watcher whose startup rebuild waits for finishRebuild, unless + * realRebuild is set. The database lives outside the vault, so its own files + * never appear among the held changes. */ + const createControlledStartup = async (setup?: { + embedder?: Parameters[1] + heldChangesDeadline?: number + rebuild?: "stubbed" | "real" + seedNotes?: Readonly> + /** Runs against the empty vault before the watcher starts, e.g. to write + * files the real rebuild reads or to gate one of those reads. */ + beforeStart?: (testVault: string) => Promise + }) => { + const testVault = await mkdtemp(join(tmpdir(), "watcher-startup-vault-")) + const dataDirectory = await mkdtemp(join(tmpdir(), "watcher-startup-data-")) + onTestFinished(async () => { + await rm(testVault, { recursive: true }) + await rm(dataDirectory, { recursive: true }) + }) + const databasePath = join(dataDirectory, "index.db") + const search = createSearchIndex(databasePath, setup?.embedder, undefined, { + fileToolsEnabled: true, + }) + const database = new Database(databasePath, { readonly: true }) + sqliteVec.load(database) + onTestFinished(() => { + database.close() + }) + for (const [fileName, content] of Object.entries(setup?.seedNotes ?? {})) { + search.upsertNote( + { filePath: fileName, rawContent: content, fileStat: { mtimeMs: 0, size: content.length } }, + logger, + ) + } + const rebuildGate = Promise.withResolvers() + + if (setup?.rebuild !== "real") { + vi.spyOn(search, "rebuildFromVault").mockImplementation(async () => { + await rebuildGate.promise + return { count: 0, embedding: Promise.resolve() } + }) + } + await setup?.beforeStart?.(testVault) + const watcher = new FSWatcher() + const watchMock = vi.mocked(watch).mockReturnValue(watcher) + onTestFinished(async () => { + watchMock.mockRestore() + await watcher.close() + }) + const heldChangesDeadline = setup?.heldChangesDeadline + const startup = watchVaultAndBuildIndex( + testVault, + search, + heldChangesDeadline ? { heldChangesDeadline } : {}, + ) + watcher.emit("ready") + + const fire = async (event: "add" | "change" | "unlink", fileName: string): Promise => { + const handler = watcher.listeners(event)[0] + + if (!handler) throw new Error(`${event} handler was not registered`) + await handler(join(testVault, fileName)) + } + const notePaths = (): unknown[] => { + return database.prepare("SELECT path FROM notes ORDER BY path").all() + } + return { + testVault, + search, + database, + fire, + notePaths, + startup, + finishRebuild: () => rebuildGate.resolve(undefined), + } + } + + const spyOnLogger = (level: "debug" | "info" | "error") => { + const levelSpy = vi.spyOn(logger, level) + onTestFinished(() => levelSpy.mockRestore()) + return levelSpy + } + + it("indexes a change held during the rebuild once the rebuild returns", async () => { + const { testVault, search, fire, startup, finishRebuild } = await createControlledStartup() + const debugSpy = spyOnLogger("debug") + const infoSpy = spyOnLogger("info") + await writeFile(join(testVault, "note.md"), "heldamber text\n") + + await fire("add", "note.md") + + expect(debugSpy).toHaveBeenCalledWith("held a change until startup indexing ends", { + path: "note.md", + }) + expect(search.fullTextSearch({ query: "heldamber" }, logger)).toEqual([]) + finishRebuild() + await startup + expect(search.fullTextSearch({ query: "heldamber" }, logger).map((hit) => hit.path)).toEqual([ + "note.md", + ]) + expect(infoSpy).toHaveBeenCalledWith("applied the changes held during startup indexing", { + batchPaths: 1, + replayedEvents: 0, + }) + }) + + it("removes a held deletion's rows even when a change for the path arrives after it", async () => { + const { search, database, fire, startup, finishRebuild } = await createControlledStartup({ + seedNotes: { "gone.md": "ghostquartz text links [[other]]\n\n- [ ] ghost task\n" }, + }) + const debugSpy = spyOnLogger("debug") + const linkRows = () => database.prepare("SELECT source, target FROM links").all() + const taskRows = () => { + return database.prepare("SELECT note_path FROM tasks").all() + } + expect(search.fullTextSearch({ query: "ghostquartz" }, logger).map((hit) => hit.path)).toEqual([ + "gone.md", + ]) + expect(linkRows()).toEqual([{ source: "gone.md", target: "other" }]) + expect(taskRows()).toEqual([{ note_path: "gone.md" }]) + + await fire("unlink", "gone.md") + await fire("change", "gone.md") + + expect(debugSpy).toHaveBeenCalledWith("held a deletion until startup indexing ends", { + path: "gone.md", + }) + expect(search.fullTextSearch({ query: "ghostquartz" }, logger).map((hit) => hit.path)).toEqual([ + "gone.md", + ]) + finishRebuild() + await startup + expect(search.fullTextSearch({ query: "ghostquartz" }, logger)).toEqual([]) + expect(database.prepare("SELECT path FROM notes").all()).toEqual([]) + expect(linkRows()).toEqual([]) + expect(taskRows()).toEqual([]) + }) + + it("reads a held path whose last event was an unlink but which exists again", async () => { + const { testVault, search, fire, startup, finishRebuild } = await createControlledStartup({ + seedNotes: { "back.md": "oldtext\n" }, + }) + await writeFile(join(testVault, "back.md"), "returnedopal\n") + + await fire("change", "back.md") + await fire("unlink", "back.md") + finishRebuild() + await startup + + expect(search.fullTextSearch({ query: "returnedopal" }, logger).map((hit) => hit.path)).toEqual( + ["back.md"], + ) + expect(search.fullTextSearch({ query: "oldtext" }, logger)).toEqual([]) + }) + + it("removes a held path deleted between its check and its read", async () => { + const { testVault, notePaths, fire, startup, finishRebuild } = await createControlledStartup({ + seedNotes: { "late.md": "oldtext\n" }, + }) + const notePath = join(testVault, "late.md") + await writeFile(notePath, "newer text\n") + const readGate = await gateFirstRead(notePath, "before-disk") + expect(notePaths()).toEqual([{ path: "late.md" }]) + + await fire("change", "late.md") + finishRebuild() + await readGate.reached + await unlink(notePath) + readGate.release() + await startup + + expect(notePaths()).toEqual([]) + }) + + it("removes a renamed note before reading its new path, so incoming links follow", async () => { + const { testVault, search, fire, startup, finishRebuild } = await createControlledStartup({ + seedNotes: { "B.md": "old home\n", "source.md": "points at [[B]]\n" }, + }) + await mkdir(join(testVault, "folder")) + await writeFile(join(testVault, "folder", "B.md"), "new home\n") + await writeFile(join(testVault, "source.md"), "points at [[B]]\n") + const linkFromSource = () => { + const outgoingLinks = search.getOutgoingLinks({ path: "source.md" }, logger) + return outgoingLinks.map((link) => ({ path: link.path, exists: link.exists })) + } + expect(linkFromSource()).toEqual([{ path: "B.md", exists: true }]) + + await fire("add", "folder/B.md") + await fire("unlink", "B.md") + finishRebuild() + await startup + + expect(linkFromSource()).toEqual([{ path: "folder/B.md", exists: true }]) + }) + + it("replays events from during the batch in arrival order, so the later state wins", async () => { + const { testVault, search, fire, startup, finishRebuild } = await createControlledStartup({ + seedNotes: { "P.md": "oldamber\n", "Q.md": "quartzrow\n" }, + }) + const infoSpy = spyOnLogger("info") + const pPath = join(testVault, "P.md") + await writeFile(pPath, "oldamber\n") + await writeFile(join(testVault, "Q.md"), "quartzrow\n") + const readGate = await gateFirstRead(pPath, "after-disk") + const pathsFor = (query: string) => { + return search.fullTextSearch({ query }, logger).map((hit) => hit.path) + } + + await fire("change", "P.md") + finishRebuild() + await readGate.reached + await writeFile(pPath, "newopal\n") + await fire("change", "P.md") + await unlink(join(testVault, "Q.md")) + await fire("unlink", "Q.md") + await fire("change", "Q.md") + await writeFile(join(testVault, "R.md"), "replayjade\n") + await fire("change", "R.md") + readGate.release() + await startup + + expect(infoSpy).toHaveBeenCalledWith("applied the changes held during startup indexing", { + batchPaths: 1, + replayedEvents: 4, + }) + await vi.waitFor(() => { + expect(pathsFor("newopal")).toEqual(["P.md"]) + expect(pathsFor("oldamber")).toEqual([]) + expect(pathsFor("quartzrow")).toEqual([]) + expect(pathsFor("replayjade")).toEqual(["R.md"]) + }) + }) + + it("contains a failed read and keeps applying the other held paths", async () => { + const { testVault, search, fire, startup, finishRebuild } = await createControlledStartup() + const errorSpy = spyOnLogger("error") + const failingPath = join(testVault, "locked.md") + await writeFile(failingPath, "lockedtext\n") + await writeFile(join(testVault, "fine.md"), "finetext\n") + const accessFailure = Object.assign(new Error("controlled access failure"), { code: "EACCES" }) + const actualFs = await vi.importActual("node:fs/promises") + vi.mocked(readFile).mockImplementation(async (requestedPath, options) => { + if (requestedPath === failingPath) throw accessFailure + return actualFs.readFile(requestedPath, options) + }) + onTestFinished(() => vi.mocked(readFile).mockRestore()) + + await fire("change", "locked.md") + await fire("change", "fine.md") + finishRebuild() + await expect(startup).resolves.toEqual({ count: 0, embedding: expect.any(Promise) }) + + expect(errorSpy).toHaveBeenCalledExactlyOnceWith("failed to process file change", { + path: "locked.md", + error: "[Error]: controlled access failure", + }) + expect(search.fullTextSearch({ query: "finetext" }, logger).map((hit) => hit.path)).toEqual([ + "fine.md", + ]) + }) + + it("leaves a path whose check fails to its read, which reports the failure", async () => { + const { testVault, search, fire, startup, finishRebuild } = await createControlledStartup() + const errorSpy = spyOnLogger("error") + const failingPath = join(testVault, "locked.md") + await writeFile(failingPath, "lockedtext\n") + await writeFile(join(testVault, "fine.md"), "finetext\n") + const accessFailure = Object.assign(new Error("controlled stat failure"), { code: "EACCES" }) + const actualFs = await vi.importActual("node:fs/promises") + vi.mocked(stat).mockImplementation(async (requestedPath, options) => { + if (requestedPath === failingPath) throw accessFailure + return actualFs.stat(requestedPath, options) + }) + onTestFinished(() => vi.mocked(stat).mockRestore()) + + await fire("change", "locked.md") + await fire("change", "fine.md") + finishRebuild() + await startup + + expect(errorSpy).toHaveBeenCalledExactlyOnceWith("failed to process file change", { + path: "locked.md", + error: "[Error]: controlled stat failure", + }) + expect(search.fullTextSearch({ query: "finetext" }, logger).map((hit) => hit.path)).toEqual([ + "fine.md", + ]) + }) + + it("reports a note embedding that fails during the batch once, with no unhandled rejection", async () => { + const vector = new Float32Array(384).fill(0.1) + const embedder = { + embedText: vi.fn(async (text: string) => { + if (text === "failing\n\nfailingtext") throw new Error("controlled model failure") + return vector + }), + embedBatch: vi.fn(async (texts: readonly string[]) => texts.map(() => vector)), + } + const { testVault, search, fire, startup, finishRebuild } = await createControlledStartup({ + embedder, + }) + const errorLogged = Promise.withResolvers() + const errorSpy = vi.spyOn(logger, "error").mockImplementation((message) => { + if (message === "failed to process file change") errorLogged.resolve(undefined) + }) + onTestFinished(() => errorSpy.mockRestore()) + const slowPath = join(testVault, "slow.md") + await writeFile(join(testVault, "failing.md"), "failingtext\n") + await writeFile(slowPath, "slowtext\n") + const readGate = await gateFirstRead(slowPath, "before-disk") + + await fire("change", "failing.md") + await fire("change", "slow.md") + finishRebuild() + await readGate.reached + await errorLogged.promise + readGate.release() + await startup + + expect(errorSpy).toHaveBeenCalledExactlyOnceWith("failed to process file change", { + path: "failing.md", + error: "[Error]: controlled model failure", + }) + expect(search.fullTextSearch({ query: "slowtext" }, logger).map((hit) => hit.path)).toEqual([ + "slow.md", + ]) + }) + + it("lets startup finish while held paths' embeddings are still running", async () => { + const vector = new Float32Array(384).fill(0.1) + const modelReady = Promise.withResolvers() + const embedder = { + embedText: vi.fn(async () => { + await modelReady.promise + return vector + }), + embedBatch: vi.fn(async (texts: readonly string[]) => { + await modelReady.promise + return texts.map(() => vector) + }), + } + const { testVault, search, database, fire, startup, finishRebuild } = + await createControlledStartup({ embedder }) + await writeFile(join(testVault, "note.md"), "pendingopal\n") + await writeFile(join(testVault, "content.txt"), "pendingjade") + + await fire("change", "note.md") + await fire("change", "content.txt") + finishRebuild() + const waitForStartup = async (): Promise => { + await startup + return "resolved" + } + const startupOutcome = await Promise.race([ + waitForStartup(), + new Promise((resolve) => setTimeout(() => resolve("still waiting"), 1000)), + ]) + + expect(startupOutcome).toBe("resolved") + expect(search.fullTextSearch({ query: "pendingopal" }, logger).map((hit) => hit.path)).toEqual([ + "note.md", + ]) + expect(database.prepare("SELECT path FROM file_content").all()).toEqual([ + { path: "content.txt" }, + ]) + expect(database.prepare("SELECT note_path FROM note_chunks").all()).toEqual([]) + modelReady.resolve(undefined) + await vi.waitFor(() => { + expect(database.prepare("SELECT note_path, chunk_text FROM note_chunks").all()).toEqual([ + { note_path: "note.md", chunk_text: "note\n\npendingopal" }, + ]) + }) + }) + + it("reads at most 16 held files at once", async () => { + const { testVault, search, fire, startup, finishRebuild } = await createControlledStartup() + const fileNames = Array.from({ length: 40 }, (_unused, noteIndex) => `bulk-${noteIndex}.md`) + for (const fileName of fileNames) { + await writeFile(join(testVault, fileName), "bulktopaz\n") + } + const actualFs = await vi.importActual("node:fs/promises") + const readCounts = { inFlight: 0, peak: 0 } + vi.mocked(readFile).mockImplementation(async (requestedPath, options) => { + readCounts.inFlight += 1 + readCounts.peak = Math.max(readCounts.peak, readCounts.inFlight) + await new Promise((resolve) => setTimeout(resolve, 5)) + try { + return await actualFs.readFile(requestedPath, options) + } finally { + readCounts.inFlight -= 1 + } + }) + onTestFinished(() => vi.mocked(readFile).mockRestore()) + + for (const fileName of fileNames) { + await fire("add", fileName) + } + finishRebuild() + await startup + + expect(readCounts.peak).toBe(16) + expect(search.fullTextSearch({ query: "bulktopaz", limit: 50 }, logger)).toHaveLength(40) + }) + + it("returns at the deadline and finishes the held changes in the background", async () => { + const { testVault, search, fire, startup, finishRebuild } = await createControlledStartup({ + heldChangesDeadline: 50, + }) + const infoSpy = spyOnLogger("info") + const debugSpy = spyOnLogger("debug") + const slowPath = join(testVault, "slow.md") + await writeFile(slowPath, "slowtext\n") + const readGate = await gateFirstRead(slowPath, "before-disk") + const pathsFor = (query: string) => { + return search.fullTextSearch({ query }, logger).map((hit) => hit.path) + } + + await fire("change", "slow.md") + finishRebuild() + await startup + + expect(infoSpy).toHaveBeenCalledWith( + "startup continues while the changes held during it are still applied", + { heldPaths: 1, heldChangesDeadline: 50 }, + ) + expect(pathsFor("slowtext")).toEqual([]) + await writeFile(join(testVault, "S.md"), "aftertext\n") + await fire("change", "S.md") + expect(debugSpy).toHaveBeenCalledWith("held a change until startup indexing ends", { + path: "S.md", + }) + expect(pathsFor("aftertext")).toEqual([]) + readGate.release() + await vi.waitFor(() => { + expect(pathsFor("slowtext")).toEqual(["slow.md"]) + expect(pathsFor("aftertext")).toEqual(["S.md"]) + }) + }) + + it("does not index the old name again after a case-only rename", async () => { + const { testVault, notePaths, fire, startup, finishRebuild } = await createControlledStartup({ + seedNotes: { "note.md": "caseopal\n" }, + }) + const renamedPath = join(testVault, "Note.md") + const oldPath = join(testVault, "note.md") + await writeFile(renamedPath, "caseopal\n") + const actualFs = await vi.importActual("node:fs/promises") + // A case-insensitive filesystem answers for the old spelling too; the + // folder listing still holds only the new one. + const asFoundOnDisk = (requestedPath: RequestedPath): RequestedPath | string => { + return requestedPath === oldPath ? renamedPath : requestedPath + } + vi.mocked(stat).mockImplementation(async (requestedPath, options) => { + return actualFs.stat(asFoundOnDisk(requestedPath), options) + }) + vi.mocked(readFile).mockImplementation(async (requestedPath, options) => { + return actualFs.readFile(asFoundOnDisk(requestedPath), options) + }) + onTestFinished(() => { + vi.mocked(stat).mockRestore() + vi.mocked(readFile).mockRestore() + }) + + await fire("unlink", "note.md") + await fire("add", "Note.md") + finishRebuild() + await startup + + expect(notePaths()).toEqual([{ path: "Note.md" }]) + }) + + it("lets a held change win over the rebuild's older read of the same note", async () => { + const readGates: Array>> = [] + const { testVault, search, fire, startup } = await createControlledStartup({ + rebuild: "real", + beforeStart: async (emptyVault) => { + await writeFile(join(emptyVault, "note.md"), "oldquartz\n") + readGates.push(await gateFirstRead(join(emptyVault, "note.md"), "after-disk")) + }, + }) + const [readGate] = readGates + const notePath = join(testVault, "note.md") + const pathsFor = (query: string) => { + return search.fullTextSearch({ query }, logger).map((hit) => hit.path) + } + + if (!readGate) throw new Error("the rebuild's read gate was not installed") + await readGate.reached + await writeFile(notePath, "newopal\n") + await fire("change", "note.md") + readGate.release() + await startup + + expect(pathsFor("newopal")).toEqual(["note.md"]) + expect(pathsFor("oldquartz")).toEqual([]) + }) +}) + +describe("file watcher — chokidar watch options", () => { registerSharedVaultHooks() type FakeWatcher = { on: (event: string, handler: (...args: unknown[]) => void) => FakeWatcher } - /** Chainable watcher stub that fires "ready" so startFileWatcher resolves. */ + /** Chainable watcher stub that fires "ready" so the watcher becomes ready. */ const createFakeWatcher = (): FakeWatcher => { const watcher: FakeWatcher = { on(event, handler) { @@ -1405,13 +1945,13 @@ describe("startFileWatcher — chokidar watch options", () => { * with. mockRestore (via onTestFinished) hands the real, spied watch back to * the integration tests. */ const chokidarOptionsFrom = async ( - watcherOptions?: Parameters[2], + watcherOptions?: Parameters[2], ): Promise> => { const watchMock = vi.mocked(watch) watchMock.mockReset() watchMock.mockImplementation(() => createFakeWatcher() as unknown as ReturnType) onTestFinished(() => watchMock.mockRestore()) - await startFileWatcher("/vault", index, watcherOptions) + await startWatcherWithoutRebuild("/vault", index, watcherOptions) expect(watchMock).toHaveBeenCalledTimes(1) return watchMock.mock.calls[0]?.[1] as Record } @@ -1448,7 +1988,7 @@ describe("startFileWatcher — chokidar watch options", () => { // outside, so these tests drive the reconciliation directly: a fake watcher // captures the addDir handler, reports test-controlled tracking via // getWatched(), and records add() calls — against a real temp vault and index. -describe("startFileWatcher — new-directory rescan", REAL_WATCHER_RETRY, () => { +describe("file watcher — new-directory rescan", REAL_WATCHER_RETRY, () => { registerSharedVaultHooks() const RESCAN_TEST_OPTIONS = { @@ -1467,7 +2007,7 @@ describe("startFileWatcher — new-directory rescan", REAL_WATCHER_RETRY, () => * hands the real, spied watch back to the integration tests. */ const startWatcherWithFakeChokidar = async ( watchedChildren: Record, - watcherOptions: Parameters[2], + watcherOptions: Parameters[2], ): Promise => { const handlers = new Map void>() const addedPaths: string[] = [] @@ -1486,7 +2026,7 @@ describe("startFileWatcher — new-directory rescan", REAL_WATCHER_RETRY, () => watchMock.mockReset() watchMock.mockImplementation(() => fakeWatcher as unknown as ReturnType) onTestFinished(() => watchMock.mockRestore()) - await startFileWatcher(vault, index, watcherOptions) + await startWatcherWithoutRebuild(vault, index, watcherOptions) const fireAddDir = (dirPath: string): void => { const addDirHandler = handlers.get("addDir") diff --git a/src/vault-mcp/search/file-watcher.ts b/src/vault-mcp/search/file-watcher.ts index a7ec2b182..9d33eb095 100644 --- a/src/vault-mcp/search/file-watcher.ts +++ b/src/vault-mcp/search/file-watcher.ts @@ -1,20 +1,22 @@ -/** File watcher — keeps the SQLite FTS5 index and vector embeddings current - * via chokidar events, plus a rescan safety net for the files chokidar's - * new-directory handling loses (see rescanNewDirectory). */ +/** File watcher — builds the search index at startup and keeps it and the + * vector embeddings current via chokidar events, plus a rescan safety net for + * the files chokidar's new-directory handling loses (see rescanNewDirectory). + * Startup order is watchVaultAndBuildIndex's job. */ import { watch } from "chokidar" import { DateTime } from "luxon" -import { readFile, stat } from "node:fs/promises" +import { readFile, readdir, stat } from "node:fs/promises" import type { Stats } from "node:fs" -import { extname, join, relative, resolve as resolvePath } from "node:path" +import { basename, dirname, extname, join, relative, resolve as resolvePath } from "node:path" import { INDEXABLE_TEXT_EXTENSIONS } from "./search-index.js" import type { SearchIndex } from "./search-index.js" import { extractPdfText } from "../obsidian-markdown/pdf.js" import { logger } from "../../logger.js" import { describeError, describeErrorForLog } from "../../utils/describe-error.js" import { isErrnoException } from "../../utils/is-errno-exception.js" -import { readdirOrNull, realpathOrNull, statOrNull } from "../../utils/fs.js" +import { isMissingPathError, readdirOrNull, realpathOrNull, statOrNull } from "../../utils/fs.js" import { hasHiddenPathSegment } from "../../utils/has-hidden-path-segment.js" +import { mapWithConcurrency } from "../../utils/map-with-concurrency.js" /** ms between filesystem polls when usePolling is on. chokidar's raw default is * 100ms, which stat()s the whole tree 10×/sec; 300ms meaningfully cuts CPU, and @@ -25,6 +27,16 @@ const POLLING_INTERVAL_MS = 300 /** Default for FileWatcherOptions.stabilityThreshold (see its doc). */ const DEFAULT_STABILITY_THRESHOLD_MS = 2000 +/** Default for FileWatcherOptions.heldChangesDeadline. It sits inside the local + * compose health check's 20 s start period. A batch that runs longer means + * Sync is delivering a backlog during startup, which search lags anyway, and + * the CLI stops waiting for health after 120 s. */ +const DEFAULT_HELD_CHANGES_DEADLINE_MS = 10_000 + +/** Paths the startup batch checks and reads at once: the rebuild's own file + * concurrency, so a large batch cannot open every file at once. */ +const HELD_CHANGES_CONCURRENCY = 16 + type FileWatcherOptions = Readonly<{ /** ms a file's size must stay unchanged before we index it (default 2000). * Prevents reading partial writes from Obsidian Sync. */ @@ -40,19 +52,51 @@ type FileWatcherOptions = Readonly<{ * past the awaitWriteFinish window lets in-flight writes settle before the * rescan reads them. */ newDirectoryRescanDelay?: number + /** ms startup waits for the changes held during the rebuild before returning + * anyway (default 10000); the rest are applied in the background. */ + heldChangesDeadline?: number +}> + +/** A change chokidar reported while the watcher held it. `add` and `change` + * both record "change", since either means "read this path again". */ +type HeldEvent = Readonly<{ kind: "change" | "unlink"; filePath: string }> + +/** What indexing one changed file did. A note's embedding is still running when + * this returns, so whoever receives `note-indexed` reports its failure. */ +type ChangeIndexResult = + | { status: "note-indexed"; embedding: Promise } + | { status: "file-indexed" } + | { status: "vanished" } + | { status: "superseded" } + | { status: "failed" } + +/** Whether a held path names a file on disk. "unknown" leaves the path to its + * read, which reports why it cannot be checked. */ +type HeldPathPresence = "present" | "missing" | "unknown" + +type FolderListing = + { status: "listed"; names: ReadonlySet } | { status: "missing" } | { status: "failed" } + +type HoldingWatcher = Readonly<{ + /** Resolves once chokidar's initial scan is done and every folder is watched. */ + ready: Promise + /** Starts applying what chokidar reported while holding, and returns how many + * distinct paths that is. `applied` resolves once their index writes commit + * and the watcher is live. */ + applyHeldChanges: () => { heldPathCount: number; applied: Promise } }> /** * Watches the vault and mirrors every file add/change/delete into the search - * index, so search stays current without rebuilds. Resolves once the watcher - * is ready (initial scan done); the watcher then runs for the process - * lifetime — there is no shutdown path. + * index, holding every change from the moment chokidar starts until + * applyHeldChanges runs. The watcher then runs for the process lifetime — + * there is no shutdown path. */ -export const startFileWatcher = ( +const startHoldingWatcher = ( vaultPath: string, search: SearchIndex, options?: FileWatcherOptions, -): Promise => { +): HoldingWatcher => { // Serializing per path limits model concurrency; source versions reject // obsolete results independently of the order jobs finish. const pendingEmbeds = new Map>() @@ -61,33 +105,45 @@ export const startFileWatcher = ( * Source versions belong to the index and protect model work after the handler returns. */ const currentEvents = new Map() + /** Each held event's path and kind in arrival order, or undefined once the + * watcher is live. Reassigned twice: the startup batch takes the list and + * starts a new one, and going live ends holding. */ + let heldEvents: HeldEvent[] | undefined = [] + /** Indexes an added or modified file: non-md files land in the asset table; - * notes are read from disk, upserted into the FTS index, and re-embedded - * (embeds serialized per path via pendingEmbeds). */ - const handleChange = async (filePath: string): Promise => { + * notes are read from disk and upserted into the FTS index, and their + * re-embedding is queued (serialized per path via pendingEmbeds). */ + const indexChangedFile = async (filePath: string): Promise => { const relativePath = relative(vaultPath, filePath) const eventToken = Symbol() currentEvents.set(relativePath, eventToken) try { if (!filePath.endsWith(".md")) { - const readNonMdFileStat = async (): Promise => { + const readNonMdFileStat = async (): Promise< + { status: "found"; fileStat: Stats } | { status: "missing" } | { status: "failed" } + > => { try { - return await statOrNull(filePath) + const fileStat = await statOrNull(filePath) + return fileStat ? { status: "found", fileStat } : { status: "missing" } } catch (error) { logger.error("failed to stat non-md file", { path: relativePath, error: describeError(error), }) - return null + return { status: "failed" } } } - const fileStat = await readNonMdFileStat() + const nonMdStat = await readNonMdFileStat() - if (currentEvents.get(relativePath) !== eventToken) return + if (currentEvents.get(relativePath) !== eventToken) return { status: "superseded" } // Vanished between the watcher event and the stat — the unlink event // that follows will remove any existing row. - if (!fileStat) return + if (nonMdStat.status === "missing") return { status: "vanished" } + if (nonMdStat.status === "failed") return { status: "failed" } + + const { fileStat } = nonMdStat + try { search.upsertNonMdFile(relativePath, fileStat.size) } catch (error) { @@ -95,7 +151,7 @@ export const startFileWatcher = ( path: relativePath, error: describeError(error), }) - return + return { status: "failed" } } // Canvas files are always read — link extraction is unconditional. @@ -117,7 +173,7 @@ export const startFileWatcher = ( } const contentToIndex = await readContentToIndex() - if (currentEvents.get(relativePath) !== eventToken) return + if (currentEvents.get(relativePath) !== eventToken) return { status: "superseded" } const sourceVersion = search.upsertFileContent( { @@ -168,7 +224,7 @@ export const startFileWatcher = ( } logger.debug("indexed non-md file", { path: relativePath }) - return + return { status: "file-indexed" } } try { @@ -188,8 +244,8 @@ export const startFileWatcher = ( } const noteSource = await readNoteSource() - if (currentEvents.get(relativePath) !== eventToken) return - if (!noteSource) return + if (currentEvents.get(relativePath) !== eventToken) return { status: "superseded" } + if (!noteSource) return { status: "vanished" } const { content, fileStat } = noteSource @@ -219,18 +275,20 @@ export const startFileWatcher = ( }) pendingEmbeds.set(relativePath, currentEmbed) - /** This awaited job reports failures through the handler's catch. Await the - * cleanup promise too, so its rejection is handled and newer queued jobs remain registered. */ - await currentEmbed.finally(() => { + /** The receiver of this result reports the job's failures. Hand over the + * cleanup promise, so its rejection is the one handled and newer queued jobs remain registered. */ + const embedding = currentEmbed.finally(() => { if (pendingEmbeds.get(relativePath) === currentEmbed) { pendingEmbeds.delete(relativePath) } }) + return { status: "note-indexed", embedding } } catch (err) { logger.error("failed to process file change", { path: relativePath, error: describeError(err), }) + return { status: "failed" } } } finally { // A newer handler may already have finished and removed its entry. @@ -241,9 +299,38 @@ export const startFileWatcher = ( } } + /** Logs a note embedding's failure under the same message as a failed read, + * so each change reports one error whichever step failed. */ + const reportEmbeddingFailure = (filePath: string, error: unknown): void => { + logger.error("failed to process file change", { + path: relative(vaultPath, filePath), + error: describeError(error), + }) + } + + /** chokidar's add and change listener. While the watcher holds, it records + * the path for applyHeldChanges instead of indexing it. */ + const handleChange = async (filePath: string): Promise => { + if (heldEvents) { + heldEvents.push({ kind: "change", filePath }) + logger.debug("held a change until startup indexing ends", { + path: relative(vaultPath, filePath), + }) + return + } + const indexResult = await indexChangedFile(filePath) + + if (indexResult.status !== "note-indexed") return + try { + await indexResult.embedding + } catch (error) { + reportEmbeddingFailure(filePath, error) + } + } + /** Removes a deleted file from the index — the asset table for non-md * files, the note tables (FTS, links, tasks, vectors) for notes. */ - const handleDelete = (filePath: string): void => { + const removeFromIndex = (filePath: string): void => { const relativePath = relative(vaultPath, filePath) currentEvents.delete(relativePath) @@ -288,6 +375,19 @@ export const startFileWatcher = ( logger.debug("removed from index", { path: relativePath }) } + /** chokidar's unlink listener. While the watcher holds, it records the path + * for applyHeldChanges instead of removing it. */ + const handleDelete = (filePath: string): void => { + if (heldEvents) { + heldEvents.push({ kind: "unlink", filePath }) + logger.debug("held a deletion until startup indexing ends", { + path: relative(vaultPath, filePath), + }) + return + } + removeFromIndex(filePath) + } + const stabilityThreshold = options?.stabilityThreshold ?? DEFAULT_STABILITY_THRESHOLD_MS const newDirectoryRescanDelay = options?.newDirectoryRescanDelay ?? 2 * stabilityThreshold @@ -492,6 +592,155 @@ export const startFileWatcher = ( rescanTimer.unref() } + /** Lists a folder's entry names for the startup batch's presence checks. */ + const listFolderNames = async (folderPath: string): Promise => { + try { + const names = await readdir(folderPath) + return { status: "listed", names: new Set(names) } + } catch (error) { + if (isMissingPathError(error)) return { status: "missing" } + + logger.warn("could not list a folder of held changes, so its files are read instead", { + path: relative(vaultPath, folderPath), + error: describeError(error), + }) + return { status: "failed" } + } + } + + /** + * Decides whether a held path still names a file. The parent folder's listing + * must hold the exact name: on a case-insensitive filesystem, `stat` of + * `note.md` still succeeds after a rename to `Note.md`, and reading it would + * index the old name again. + */ + const checkHeldPathOnDisk = async ( + filePath: string, + listFolderOnce: (folderPath: string) => Promise, + ): Promise => { + const folderListing = await listFolderOnce(dirname(filePath)) + const missingFromListing = + folderListing.status === "listed" && !folderListing.names.has(basename(filePath)) + + if (folderListing.status === "missing" || missingFromListing) return "missing" + try { + const fileStat = await statOrNull(filePath) + return fileStat?.isFile() ? "present" : "missing" + } catch (error) { + // The path's read runs next and reports the failure at error level. + logger.debug("could not stat a held path, so its read reports why", { + path: relative(vaultPath, filePath), + error: describeError(error), + }) + return "unknown" + } + } + + /** Removes a held path the startup batch found missing on disk. */ + const removeHeldPath = (filePath: string): void => { + logger.debug("removing a held path that is gone from disk", { + path: relative(vaultPath, filePath), + }) + removeFromIndex(filePath) + } + + /** Re-reads one held path for the startup batch, without waiting for its embedding. */ + const indexHeldPath = async (filePath: string): Promise => { + const indexResult = await indexChangedFile(filePath) + + if (indexResult.status === "vanished") { + removeHeldPath(filePath) + return + } + if (indexResult.status !== "note-indexed") return + // Caught here, inside the batch step, so a fast rejection is never unhandled. + indexResult.embedding.catch((error) => { + reportEmbeddingFailure(filePath, error) + }) + } + + /** Ends holding and replays the events held while the batch ran, in arrival order. */ + const goLiveAndReplay = (batchPathCount: number): void => { + const eventsDuringBatch = heldEvents ?? [] + heldEvents = undefined + // Every replayed event starts in this one tick, before any newer live + // event for the same path, so that newer event takes the path's token and + // wins. chokidar delivers a burst the same way. + for (const heldEvent of eventsDuringBatch) { + if (heldEvent.kind === "unlink") { + handleDelete(heldEvent.filePath) + continue + } + // As with chokidar's own emit, nothing awaits it; it logs its own failures. + void handleChange(heldEvent.filePath) + } + logger.info("applied the changes held during startup indexing", { + batchPaths: batchPathCount, + replayedEvents: eventsDuringBatch.length, + }) + } + + /** + * Brings the index up to date with the held paths, in one batch, then goes live: + * + * 1. Each path is checked against the disk; missing ones are removed first, + * so a rename seen whole in the batch resolves incoming links to the new path. + * 2. The rest are read and indexed, 16 at a time; one that vanished by its read is removed. + * 3. Events held while steps 1 and 2 ran are replayed live. + * + * Removing a path because its check found it missing is safe only while + * holding: once live, the removal could clear the event token of an `add` + * that started during the check, and the added note would stay unindexed. + */ + const applyHeldBatch = async (batchPaths: ReadonlyArray): Promise => { + const folderListings = new Map>() + + /** Lists each folder once per batch, however many held paths sit in it. */ + const listFolderOnce = (folderPath: string): Promise => { + const knownListing = folderListings.get(folderPath) + + if (knownListing) return knownListing + const folderListing = listFolderNames(folderPath) + folderListings.set(folderPath, folderListing) + return folderListing + } + + const presenceChecks = await mapWithConcurrency({ + items: batchPaths, + concurrency: HELD_CHANGES_CONCURRENCY, + mapper: async (filePath) => { + return { filePath, presence: await checkHeldPathOnDisk(filePath, listFolderOnce) } + }, + }) + const missingPaths = presenceChecks + .filter((check) => check.presence === "missing") + .map((check) => check.filePath) + const pathsToRead = presenceChecks + .filter((check) => check.presence !== "missing") + .map((check) => check.filePath) + + for (const filePath of missingPaths) { + removeHeldPath(filePath) + } + await mapWithConcurrency({ + items: pathsToRead, + concurrency: HELD_CHANGES_CONCURRENCY, + mapper: indexHeldPath, + }) + goLiveAndReplay(batchPaths.length) + } + + const applyHeldChanges = (): { heldPathCount: number; applied: Promise } => { + if (!heldEvents) return { heldPathCount: 0, applied: Promise.resolve() } + + const batchEvents = heldEvents + // Events arriving while the batch runs collect here and are replayed after it. + heldEvents = [] + const batchPaths = [...new Set(batchEvents.map((heldEvent) => heldEvent.filePath))] + + return { heldPathCount: batchPaths.length, applied: applyHeldBatch(batchPaths) } + } + watcher .on("add", handleChange) .on("change", handleChange) @@ -503,10 +752,56 @@ export const startFileWatcher = ( }) }) - return new Promise((resolve) => { + const ready = new Promise((resolve) => { watcher.on("ready", () => { logger.info("file watcher started", { vaultPath }) resolve() }) }) + + return { ready, applyHeldChanges } +} + +/** + * Starts the watcher, builds the search index from disk, then applies the + * changes the watcher saw meanwhile. The watcher is ready before the rebuild + * reads anything and holds every change until the rebuild returns, so a change + * landing mid-rebuild is applied after it instead of being missed or replaced + * by the rebuild's older read. Resolves with the rebuild's result once the + * held changes are indexed, or at heldChangesDeadline if they take longer; + * the rest then apply in the background. + */ +export const watchVaultAndBuildIndex = async ( + vaultPath: string, + search: SearchIndex, + options?: FileWatcherOptions, +): ReturnType => { + const holdingWatcher = startHoldingWatcher(vaultPath, search, options) + await holdingWatcher.ready + + const rebuildResult = await search.rebuildFromVault({ vaultPath }, logger) + const { heldPathCount, applied } = holdingWatcher.applyHeldChanges() + const heldChangesDeadline = options?.heldChangesDeadline ?? DEFAULT_HELD_CHANGES_DEADLINE_MS + const { promise: deadlinePassed, resolve: passDeadline } = Promise.withResolvers<"deadline">() + const deadlineTimer = setTimeout(() => passDeadline("deadline"), heldChangesDeadline) + const waitForHeldChanges = async (): Promise<"applied"> => { + await applied + return "applied" + } + const outcome = await Promise.race([waitForHeldChanges(), deadlinePassed]) + clearTimeout(deadlineTimer) + + if (outcome === "deadline") { + logger.info("startup continues while the changes held during it are still applied", { + heldPaths: heldPathCount, + heldChangesDeadline, + }) + // Nothing awaits the batch from here, so its last-resort failure is logged. + applied.catch((error) => { + logger.error("failed to apply the changes held during startup", { + error: describeError(error), + }) + }) + } + return rebuildResult } diff --git a/src/vault-mcp/search/search-index.ts b/src/vault-mcp/search/search-index.ts index 5f1ed3977..9dc65df65 100644 --- a/src/vault-mcp/search/search-index.ts +++ b/src/vault-mcp/search/search-index.ts @@ -2077,7 +2077,11 @@ export const createSearchIndex = ( } /** Rebuilds note, task, file and graph indexes; reconciles retained memory rows and vectors. - * Returns the note count and a background embedding promise. */ + * Returns the note count and a background embedding promise. + * - No source write may run while it does: a write that lands during its reads is + * replaced by the older content it read. + * - The file watcher is the only runtime source writer, and watchVaultAndBuildIndex + * holds it until this returns; a new runtime writer must go through that hold. */ const rebuildFromVault = async ( params: { vaultPath: string }, logger: Logger, diff --git a/src/vault-mcp/server.ts b/src/vault-mcp/server.ts index 4735eb9cd..0a3e974fe 100644 --- a/src/vault-mcp/server.ts +++ b/src/vault-mcp/server.ts @@ -10,7 +10,7 @@ import { createReranker } from "./search/reranker.js" import { createMemoryStore } from "./vault-operations/memory-store.js" import { readTaskFormatConfig } from "./vault-operations/task-format-config.js" import { trashSweeper } from "./vault-operations/trash-sweeper.js" -import { startFileWatcher } from "./search/file-watcher.js" +import { watchVaultAndBuildIndex } from "./search/file-watcher.js" import { createOAuthProvider } from "./oauth/oauth-provider.js" import { createOAuthRoutes } from "./oauth/oauth-routes.js" import { createMcpRouter } from "./mcp-core/mcp-router.js" @@ -146,13 +146,12 @@ const startServer = async (): Promise => { fileToolsEnabled: config.fileToolsEnabled, statusRegistry, }) + // Before the watcher starts, so its template writes are not held and re-read. await bootstrapMemoryIfEnabled(config, vaultPath) - const { count } = await search.rebuildFromVault({ vaultPath }, logger) - logger.info("initial index built", { count }) - - await startFileWatcher(vaultPath, search, { + const { count } = await watchVaultAndBuildIndex(vaultPath, search, { usePolling: config.windowsBindMount, }) + logger.info("initial index built", { count }) const oauthProvider = createOAuthProvider({ authToken, From dbbff20d3b877cbcffffa4e542f6780b10820499 Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Fri, 16 Oct 2026 14:08:14 -0400 Subject: [PATCH 02/34] test(search): cover the held rescan, real chokidar and server wiring; document startup order Co-Authored-By: Claude Opus 5.5 --- ARCHITECTURE.md | 18 +- .../integration/server-integration.test.ts | 35 +++ .../search/__tests__/file-watcher.test.ts | 215 +++++++++++++++++- 3 files changed, 257 insertions(+), 11 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 1e3a16c6f..bc638b20f 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -492,12 +492,23 @@ When no embedder is configured (`EMBEDDING_ENABLED=false`), no vectors are index ### Indexing -**Indexing flow:** `rebuildFromVault` runs three passes, then returns so the server can start accepting requests: +**Indexing flow:** `rebuildFromVault` runs three passes, then returns: 1. **Pass 1** — index notes (FTS5 + metadata) 2. **Pass 2** — extract links (with the complete path list for resolution), then index file content (canvas, PDF, text → FTS5) 3. **Pass 3 (background)** — embed notes, then file content. Search works with FTS-only until vectors are ready +**Startup order:** `watchVaultAndBuildIndex` starts the file watcher before the rebuild, so a change that lands while the rebuild reads the vault is neither missed nor replaced by the rebuild's older read: + +1. chokidar finishes its initial scan, and the watcher holds every change it reports from then on. +2. `rebuildFromVault` runs. The watcher is the only other writer of the index, so nothing else writes while it reads. +3. One batch checks each held path against the disk, 16 at a time. Paths gone from disk are removed first, then the rest are read again. +4. The watcher goes live and replays, in arrival order, the changes reported while the batch ran. + +The server starts accepting requests once the step 3 batch commits, or after 10 s if a Sync backlog makes the batch longer; the batch then finishes in the background. + +A file that is created after step 1 and deleted within chokidar's 2 s write-stability window emits no event, so if the rebuild read it, it stays searchable until the next restart. + **Startup cleanup:** before resetting the source tables, the rebuild removes vectors whose parent chunk or memory-entry row is missing and retains vectors with surviving parents. Vector tables persist across restarts and rebuilds (only FTS, notes, links, tasks, non-md, and file content tables are cleared). Pass 3 cleans up vectors for deleted notes and files, then embeds only new or modified chunks via content-hash gating. @@ -928,8 +939,9 @@ graph LR both processes read/write the shared `/vault` volume. On startup: bootstraps memory templates if the memory folder doesn't exist, `MEMORY_ENABLED` is not `false`, and the server is not in - `READONLY_MODE`, builds the FTS5 search index including those templates, - then starts the file watcher. + `READONLY_MODE`, starts the file watcher, builds the FTS5 search index + including those templates, then applies the changes the watcher saw while + the index was built (see [Startup order](#indexing)). `svc-vault-mcp` declares `svc-obsidian-sync` in its `dependencies.d`, so the MCP server starts only after the full init chain has finished and the sync diff --git a/src/__tests__/integration/server-integration.test.ts b/src/__tests__/integration/server-integration.test.ts index 8b25a4ff7..2932d39e6 100644 --- a/src/__tests__/integration/server-integration.test.ts +++ b/src/__tests__/integration/server-integration.test.ts @@ -77,6 +77,41 @@ it("indexes newly bootstrapped memory templates before accepting requests", asyn ) }) +// Startup holds file-watcher events until its index batch ends, so a server +// that never stopped holding would never index this note. +it("indexes a note written after startup once the file watcher reports it", async () => { + const server = await startServer(await freePort()) + onTestFinished(server.cleanup) + const client = await createTestClient(server.port) + onTestFinished(() => client.close()) + + const writeResult = await callTool({ + client, + name: "vault_write_note", + args: { path: "watcher-smoke.md", body: "watchersmokeopal\n" }, + }) + expect(writeResult.isError).not.toBe(true) + + // Polls without throwing, so a server that never indexes the note fails on + // the exact assertion below rather than on a timeout. + const pollDeadline = DateTime.now().plus({ seconds: 10 }) + const searchUntilFound = async (): Promise => { + const searchResult = await callTool({ + client, + name: "vault_search", + args: { query: "watchersmokeopal" }, + }) + const foundPaths = JSON.parse(textContent(searchResult)).results.map( + (entry: { path: string }) => entry.path, + ) + + if (foundPaths.length > 0 || DateTime.now() > pollDeadline) return foundPaths + await new Promise((resolve) => setTimeout(resolve, 250)) + return searchUntilFound() + } + expect(await searchUntilFound()).toEqual(["watcher-smoke.md"]) +}, 30_000) + /** Extract joined text from a prompt result's messages. */ const promptText = (result: Awaited>): string => { return result.messages diff --git a/src/vault-mcp/search/__tests__/file-watcher.test.ts b/src/vault-mcp/search/__tests__/file-watcher.test.ts index 618fdb7c0..23dc54ea7 100644 --- a/src/vault-mcp/search/__tests__/file-watcher.test.ts +++ b/src/vault-mcp/search/__tests__/file-watcher.test.ts @@ -1922,6 +1922,142 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { }) }) +describe("watchVaultAndBuildIndex — startup through real chokidar", REAL_WATCHER_RETRY, () => { + it( + "indexes edits, creations and deletions that land while the rebuild reads the vault", + { timeout: 20000 }, + async () => { + const testVault = await mkdtemp(join(tmpdir(), "watcher-startup-real-")) + const dataDirectory = await mkdtemp(join(tmpdir(), "watcher-startup-real-data-")) + onTestFinished(async () => { + await rm(testVault, { recursive: true }) + await rm(dataDirectory, { recursive: true }) + }) + const databasePath = join(dataDirectory, "index.db") + const search = createSearchIndex(databasePath, undefined, undefined, { + fileToolsEnabled: true, + }) + const database = new Database(databasePath, { readonly: true }) + onTestFinished(() => { + database.close() + }) + const modifiedPath = join(testVault, "modified.md") + const deletedPath = join(testVault, "deleted.md") + const deletedTextPath = join(testVault, "deleted.txt") + await writeFile(deletedPath, "ghostquartz links [[modified]]\n") + await writeFile(modifiedPath, "oldtext\n") + await writeFile(deletedTextPath, "ghostquartz") + const { mtime: modifiedTimeTheRebuildSaw } = await stat(modifiedPath) + + // The rebuild's read of modified.md is held after it read "oldtext"; + // deleted.md's read is only observed, so the test deletes the file + // after the rebuild has read it. + const actualFs = await vi.importActual("node:fs/promises") + const modifiedReadReached = Promise.withResolvers() + const releaseModifiedRead = Promise.withResolvers() + const deletedNoteRead = Promise.withResolvers() + const gatedPaths = new Set() + vi.mocked(readFile).mockImplementation(async (requestedPath, options) => { + const content = await actualFs.readFile(requestedPath, options) + + if (requestedPath === deletedPath) deletedNoteRead.resolve(undefined) + if (requestedPath === modifiedPath && !gatedPaths.has(modifiedPath)) { + gatedPaths.add(modifiedPath) + modifiedReadReached.resolve(undefined) + await releaseModifiedRead.promise + } + return content + }) + onTestFinished(() => vi.mocked(readFile).mockRestore()) + + // A path counts as delivered once the watcher either holds its event + // (the intended path) or writes it to the index at once (a broken hold), + // so the state assertions below decide which happened. + const deliveredPaths = new Set() + const debugSpy = vi.spyOn(logger, "debug").mockImplementation((message, fields) => { + const isHeldLine = message.startsWith("held a ") + const heldPath = fields?.path + + if (isHeldLine && typeof heldPath === "string") deliveredPaths.add(heldPath) + }) + const infoSpy = vi.spyOn(logger, "info") + onTestFinished(() => { + debugSpy.mockRestore() + infoSpy.mockRestore() + }) + const recordDeliveredPath = ( + operation: (...operationArguments: Arguments) => Result, + pathOf: (...operationArguments: Arguments) => string, + ) => { + return (...operationArguments: Arguments): Result => { + deliveredPaths.add(pathOf(...operationArguments)) + return operation(...operationArguments) + } + } + // The real functions are captured before spying, because inside + // mockImplementation's argument the property already holds the spy, + // and wrapping the spy would recurse. + const { upsertNote, upsertNonMdFile, removeNote, removeNonMdFile } = search + vi.spyOn(search, "upsertNote").mockImplementation( + recordDeliveredPath(upsertNote, (params) => params.filePath), + ) + vi.spyOn(search, "upsertNonMdFile").mockImplementation( + recordDeliveredPath(upsertNonMdFile, (filePath) => filePath), + ) + vi.spyOn(search, "removeNote").mockImplementation( + recordDeliveredPath(removeNote, (filePath) => filePath), + ) + vi.spyOn(search, "removeNonMdFile").mockImplementation( + recordDeliveredPath(removeNonMdFile, (filePath) => filePath), + ) + + const startup = watchVaultAndBuildIndex(testVault, search, { + stabilityThreshold: 100, + pollInterval: 25, + }) + await modifiedReadReached.promise + await deletedNoteRead.promise + await writeFile(modifiedPath, "newopal is longer than the old text\n") + // Sync writes with equal access and modification times; the old + // modification time shows the outcome does not depend on it changing. + await utimes(modifiedPath, modifiedTimeTheRebuildSaw, modifiedTimeTheRebuildSaw) + await unlink(deletedPath) + await unlink(deletedTextPath) + await writeFile(join(testVault, "created.md"), "newjade\n") + await vi.waitFor( + () => { + expect([...deliveredPaths].toSorted()).toEqual([ + "created.md", + "deleted.md", + "deleted.txt", + "modified.md", + ]) + }, + { timeout: 8000 }, + ) + releaseModifiedRead.resolve(undefined) + await startup + + const pathsFor = (query: string) => { + return search.fullTextSearch({ query }, logger).map((hit) => hit.path) + } + expect(pathsFor("newopal")).toEqual(["modified.md"]) + expect(pathsFor("oldtext")).toEqual([]) + expect(pathsFor("newjade")).toEqual(["created.md"]) + expect(pathsFor("ghostquartz")).toEqual([]) + expect( + database.prepare("SELECT source FROM links WHERE source = 'deleted.md'").all(), + ).toEqual([]) + expect(database.prepare("SELECT path FROM non_md_files").all()).toEqual([]) + expect(database.prepare("SELECT path FROM file_content").all()).toEqual([]) + expect(infoSpy).toHaveBeenCalledWith("applied the changes held during startup indexing", { + batchPaths: 4, + replayedEvents: 0, + }) + }, + ) +}) + describe("file watcher — chokidar watch options", () => { registerSharedVaultHooks() @@ -2002,13 +2138,10 @@ describe("file watcher — new-directory rescan", REAL_WATCHER_RETRY, () => { addedPaths: string[] } - /** Stubs chokidar.watch with a fake exposing what the rescan consumes, then - * starts the file watcher against it. mockRestore (via onTestFinished) - * hands the real, spied watch back to the integration tests. */ - const startWatcherWithFakeChokidar = async ( - watchedChildren: Record, - watcherOptions: Parameters[2], - ): Promise => { + /** Stubs chokidar.watch with a fake exposing what the rescan consumes. + * mockRestore (via onTestFinished) hands the real, spied watch back to the + * integration tests. */ + const installFakeChokidar = (watchedChildren: Record): RescanFakeWatcher => { const handlers = new Map void>() const addedPaths: string[] = [] const fakeWatcher = { @@ -2026,7 +2159,6 @@ describe("file watcher — new-directory rescan", REAL_WATCHER_RETRY, () => { watchMock.mockReset() watchMock.mockImplementation(() => fakeWatcher as unknown as ReturnType) onTestFinished(() => watchMock.mockRestore()) - await startWatcherWithoutRebuild(vault, index, watcherOptions) const fireAddDir = (dirPath: string): void => { const addDirHandler = handlers.get("addDir") @@ -2039,6 +2171,16 @@ describe("file watcher — new-directory rescan", REAL_WATCHER_RETRY, () => { return { fireAddDir, addedPaths } } + /** Starts the file watcher against a fake chokidar (see installFakeChokidar). */ + const startWatcherWithFakeChokidar = async ( + watchedChildren: Record, + watcherOptions: Parameters[2], + ): Promise => { + const fakeChokidar = installFakeChokidar(watchedChildren) + await startWatcherWithoutRebuild(vault, index, watcherOptions) + return fakeChokidar + } + /** Backdates a file's mtime (10 minutes) so the rescan's "still being * written" guard sees it as settled. */ const backdateMtime = async (filePath: string): Promise => { @@ -2065,6 +2207,63 @@ describe("file watcher — new-directory rescan", REAL_WATCHER_RETRY, () => { expect(addedPaths).toEqual([missedPath]) }) + it( + "holds a rescan's indexing until the startup rebuild returns", + { timeout: 15000 }, + async () => { + const newDirectory = join(vault, "new-folder") + await mkdir(newDirectory) + const missedPath = join(newDirectory, "missed.md") + await writeFile(missedPath, "missed during startup\n", "utf8") + await backdateMtime(missedPath) + const rebuildGate = Promise.withResolvers() + vi.spyOn(index, "rebuildFromVault").mockImplementation(async () => { + await rebuildGate.promise + return { count: 0, embedding: Promise.resolve() } + }) + // Resolves when the rescan reaches the missed file, whether it holds the + // change (the intended path) or indexes it at once (a broken hold), so the + // state assertion below decides which happened. + const missedFileReached = Promise.withResolvers() + const debugSpy = vi.spyOn(logger, "debug").mockImplementation((message, fields) => { + const isMissedFileHeld = + message === "held a change until startup indexing ends" && + fields?.path === "new-folder/missed.md" + + if (isMissedFileHeld) missedFileReached.resolve(undefined) + }) + const realUpsertNote = index.upsertNote + vi.spyOn(index, "upsertNote").mockImplementation((params, requestLogger) => { + missedFileReached.resolve(undefined) + return realUpsertNote(params, requestLogger) + }) + const infoSpy = vi.spyOn(logger, "info") + onTestFinished(() => { + debugSpy.mockRestore() + infoSpy.mockRestore() + }) + const { fireAddDir } = installFakeChokidar({}) + const startup = watchVaultAndBuildIndex(vault, index, RESCAN_TEST_OPTIONS) + await vi.waitFor(() => { + expect(index.rebuildFromVault).toHaveBeenCalledTimes(1) + }) + + fireAddDir(newDirectory) + await missedFileReached.promise + + expect(index.fullTextSearch({ query: "missed" }, logger)).toEqual([]) + rebuildGate.resolve(undefined) + await startup + expect(index.fullTextSearch({ query: "missed" }, logger).map((hit) => hit.path)).toEqual([ + "new-folder/missed.md", + ]) + expect(infoSpy).toHaveBeenCalledWith("applied the changes held during startup indexing", { + batchPaths: 1, + replayedEvents: 0, + }) + }, + ) + it("does not re-index a file chokidar already tracks", { timeout: 15000 }, async () => { const newDirectory = join(vault, "new-folder") await mkdir(newDirectory) From 9b1bed53524bcdf228f4c1439ec2c45ab07d0d45 Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Fri, 16 Oct 2026 14:12:33 -0400 Subject: [PATCH 03/34] test(search): make the deadline and embedding-failure tests fail on state, not on a wait Co-Authored-By: Claude Opus 5.5 --- .../search/__tests__/file-watcher.test.ts | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/vault-mcp/search/__tests__/file-watcher.test.ts b/src/vault-mcp/search/__tests__/file-watcher.test.ts index 23dc54ea7..dfff725a3 100644 --- a/src/vault-mcp/search/__tests__/file-watcher.test.ts +++ b/src/vault-mcp/search/__tests__/file-watcher.test.ts @@ -1725,11 +1725,7 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { const { testVault, search, fire, startup, finishRebuild } = await createControlledStartup({ embedder, }) - const errorLogged = Promise.withResolvers() - const errorSpy = vi.spyOn(logger, "error").mockImplementation((message) => { - if (message === "failed to process file change") errorLogged.resolve(undefined) - }) - onTestFinished(() => errorSpy.mockRestore()) + const errorSpy = spyOnLogger("error") const slowPath = join(testVault, "slow.md") await writeFile(join(testVault, "failing.md"), "failingtext\n") await writeFile(slowPath, "slowtext\n") @@ -1739,7 +1735,12 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { await fire("change", "slow.md") finishRebuild() await readGate.reached - await errorLogged.promise + await vi.waitFor(() => { + expect(embedder.embedText).toHaveBeenCalledWith("failing\n\nfailingtext") + }) + // One macrotask lets Node report the rejection as unhandled if nothing + // caught it while the other path's read is still held open. + await new Promise((resolve) => setImmediate(resolve)) readGate.release() await startup @@ -1843,8 +1844,16 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { await fire("change", "slow.md") finishRebuild() - await startup + const waitForStartup = async (): Promise => { + await startup + return "resolved" + } + const startupOutcome = await Promise.race([ + waitForStartup(), + new Promise((resolve) => setTimeout(() => resolve("still waiting"), 1000)), + ]) + expect(startupOutcome).toBe("resolved") expect(infoSpy).toHaveBeenCalledWith( "startup continues while the changes held during it are still applied", { heldPaths: 1, heldChangesDeadline: 50 }, From b10d8beb7278169f7e62576802e1c75d5e592724 Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Fri, 16 Oct 2026 14:14:38 -0400 Subject: [PATCH 04/34] test(search): assert exact results instead of lengths and call-log reads in the watcher tests Co-Authored-By: Claude Opus 5.5 --- .../search/__tests__/file-watcher.test.ts | 60 +++++++++++-------- 1 file changed, 36 insertions(+), 24 deletions(-) diff --git a/src/vault-mcp/search/__tests__/file-watcher.test.ts b/src/vault-mcp/search/__tests__/file-watcher.test.ts index dfff725a3..5cc2e9340 100644 --- a/src/vault-mcp/search/__tests__/file-watcher.test.ts +++ b/src/vault-mcp/search/__tests__/file-watcher.test.ts @@ -99,8 +99,7 @@ describe("file-watcher", REAL_WATCHER_RETRY, () => { await waitFor(() => index.fullTextSearch({ query: "watcher" }, logger).length > 0) const results = index.fullTextSearch({ query: "watcher" }, logger) - expect(results).toHaveLength(1) - expect(results[0]?.path).toBe("test.md") + expect(results.map((result) => result.path)).toEqual(["test.md"]) }) it("re-indexes a modified file", { timeout: 15000 }, async () => { @@ -116,7 +115,7 @@ describe("file-watcher", REAL_WATCHER_RETRY, () => { await waitFor(() => index.fullTextSearch({ query: "updated" }, logger).length > 0) const results = index.fullTextSearch({ query: "updated" }, logger) - expect(results).toHaveLength(1) + expect(results.map((result) => result.path)).toEqual(["modify.md"]) }) it("removes a deleted file from index", { timeout: 15000 }, async () => { @@ -220,6 +219,8 @@ describe("file-watcher", REAL_WATCHER_RETRY, () => { const paths = index .fullTextSearch({ query: "symlink watcher" }, logger) .map((result) => result.path) + // On macOS a late event for real.md, written just before the watcher + // started, can index it too, so only the link's presence is asserted. expect(paths).toContain("linked.md") }) @@ -275,8 +276,7 @@ describe("file-watcher", REAL_WATCHER_RETRY, () => { await waitFor(() => index.fullTextSearch({ query: "raced" }, logger).length > 0) const results = index.fullTextSearch({ query: "raced" }, logger) - expect(results).toHaveLength(1) - expect(results[0]?.path).toBe("brand-new/nested/note.md") + expect(results.map((result) => result.path)).toEqual(["brand-new/nested/note.md"]) }, ) @@ -294,8 +294,7 @@ describe("file-watcher", REAL_WATCHER_RETRY, () => { await waitFor(() => index.fullTextSearch({ query: "polled" }, logger).length > 0) const results = index.fullTextSearch({ query: "polled" }, logger) - expect(results).toHaveLength(1) - expect(results[0]?.path).toBe("polled.md") + expect(results.map((result) => result.path)).toEqual(["polled.md"]) }) }) @@ -2209,8 +2208,7 @@ describe("file watcher — new-directory rescan", REAL_WATCHER_RETRY, () => { await waitFor(() => index.fullTextSearch({ query: "missed" }, logger).length > 0) const results = index.fullTextSearch({ query: "missed" }, logger) - expect(results).toHaveLength(1) - expect(results[0]?.path).toBe("new-folder/missed.md") + expect(results.map((result) => result.path)).toEqual(["new-folder/missed.md"]) // The file must also be registered with chokidar — indexed but // untracked, its later deletion would emit no unlink (ghost entry). expect(addedPaths).toEqual([missedPath]) @@ -2293,8 +2291,14 @@ describe("file watcher — new-directory rescan", REAL_WATCHER_RETRY, () => { // The untracked sibling getting indexed proves the rescan ran — the // tracked file being skipped can't be a silent no-op. await waitFor(() => index.fullTextSearch({ query: "sibling" }, logger).length > 0) - expect(upsertNoteSpy).toHaveBeenCalledTimes(1) - expect(upsertNoteSpy.mock.calls[0]?.[0]?.filePath).toBe("new-folder/missed.md") + expect(upsertNoteSpy).toHaveBeenCalledExactlyOnceWith( + { + filePath: "new-folder/missed.md", + rawContent: "missed sibling note\n", + fileStat: { mtimeMs: expect.any(Number), size: 20 }, + }, + logger, + ) // Only the missed file is registered — the tracked one is not re-added. expect(addedPaths).toEqual([missedPath]) }) @@ -2343,8 +2347,7 @@ describe("file watcher — new-directory rescan", REAL_WATCHER_RETRY, () => { await waitFor(() => index.fullTextSearch({ query: "nested" }, logger).length > 0) const results = index.fullTextSearch({ query: "nested" }, logger) - expect(results).toHaveLength(1) - expect(results[0]?.path).toBe("new-folder/missed-subdir/nested.md") + expect(results.map((result) => result.path)).toEqual(["new-folder/missed-subdir/nested.md"]) // The subdirectory and the missed file are both handed back to chokidar // so they gain watches and future events (unlink included) fire. expect(addedPaths).toEqual([missedSubdirectory, nestedPath]) @@ -2390,9 +2393,8 @@ describe("file watcher — new-directory rescan", REAL_WATCHER_RETRY, () => { await waitFor(() => index.fullTextSearch({ query: "symlinked" }, logger).length > 0) const results = index.fullTextSearch({ query: "symlinked" }, logger) - expect(results).toHaveLength(1) // Indexed under the link path, matching chokidar's followSymlinks view. - expect(results[0]?.path).toBe("new-folder/linked-dir/inside.md") + expect(results.map((result) => result.path)).toEqual(["new-folder/linked-dir/inside.md"]) // The inner file registers during the recursion, then the link itself. expect(addedPaths).toEqual([join(linkedDirectory, "inside.md"), linkedDirectory]) }) @@ -2428,8 +2430,7 @@ describe("file watcher — new-directory rescan", REAL_WATCHER_RETRY, () => { await waitFor(() => index.fullTextSearch({ query: "symlinked" }, logger).length > 0) const results = index.fullTextSearch({ query: "symlinked" }, logger) - expect(results).toHaveLength(1) - expect(results[0]?.path).toBe("new-folder/linked-dir/inside.md") + expect(results.map((result) => result.path)).toEqual(["new-folder/linked-dir/inside.md"]) // Registration order proves the retry ran: the symlinked directory // registered during the rescan; the file only after its retry settled. expect(addedPaths).toEqual([linkedDirectory, join(linkedDirectory, "inside.md")]) @@ -2459,8 +2460,14 @@ describe("file watcher — new-directory rescan", REAL_WATCHER_RETRY, () => { // Give a (broken) unbounded recursion time to produce extra upserts // before asserting the exact count. await new Promise((finished) => setTimeout(finished, 300)) - expect(upsertNoteSpy).toHaveBeenCalledTimes(1) - expect(upsertNoteSpy.mock.calls[0]?.[0]?.filePath).toBe("new-folder/settled.md") + expect(upsertNoteSpy).toHaveBeenCalledExactlyOnceWith( + { + filePath: "new-folder/settled.md", + rawContent: "settled beside a cycle\n", + fileStat: { mtimeMs: expect.any(Number), size: 23 }, + }, + logger, + ) // readdir order isn't guaranteed, so compare sorted ("loop" < "settled.md"). expect([...addedPaths].sort()).toEqual([loopLink, settledPath]) }, @@ -2489,10 +2496,16 @@ describe("file watcher — new-directory rescan", REAL_WATCHER_RETRY, () => { // swallowed — not merely absent from the listing. expect(debugSpy).toHaveBeenCalledWith("rescan skipped unreadable entry", { path: "new-folder/dangling.md", - error: expect.stringContaining("ENOENT"), + error: `[Error]: ENOENT: no such file or directory, stat '${danglingLink}'`, }) - expect(upsertNoteSpy).toHaveBeenCalledTimes(1) - expect(upsertNoteSpy.mock.calls[0]?.[0]?.filePath).toBe("new-folder/settled.md") + expect(upsertNoteSpy).toHaveBeenCalledExactlyOnceWith( + { + filePath: "new-folder/settled.md", + rawContent: "settled beside a broken link\n", + fileStat: { mtimeMs: expect.any(Number), size: 29 }, + }, + logger, + ) expect(addedPaths).toEqual([settledPath]) }) @@ -2617,8 +2630,7 @@ describe("file watcher — new-directory rescan", REAL_WATCHER_RETRY, () => { await waitFor(() => index.fullTextSearch({ query: "skew" }, logger).length > 0) const results = index.fullTextSearch({ query: "skew" }, logger) - expect(results).toHaveLength(1) - expect(results[0]?.path).toBe("new-folder/skewed.md") + expect(results.map((result) => result.path)).toEqual(["new-folder/skewed.md"]) expect(addedPaths).toEqual([skewedPath]) }, ) From 5169beacbd714fa69be831086a552cb4bdc7d65c Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Fri, 16 Oct 2026 14:20:30 -0400 Subject: [PATCH 05/34] fix(review): go live and resolve startup when the held-change batch throws MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The batch's last-resort catch ran only after the deadline, so a throw before it failed startup, and either way the watcher never went live and dropped every later change. The batch now logs the failure and goes live in all cases. Ship-Check: pr-review · claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- .../search/__tests__/file-watcher.test.ts | 27 +++++++++++ src/vault-mcp/search/file-watcher.ts | 45 +++++++++++-------- 2 files changed, 53 insertions(+), 19 deletions(-) diff --git a/src/vault-mcp/search/__tests__/file-watcher.test.ts b/src/vault-mcp/search/__tests__/file-watcher.test.ts index 5cc2e9340..7fd902868 100644 --- a/src/vault-mcp/search/__tests__/file-watcher.test.ts +++ b/src/vault-mcp/search/__tests__/file-watcher.test.ts @@ -1712,6 +1712,33 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { ]) }) + it("goes live and resolves startup when the batch throws unexpectedly", async () => { + const { testVault, search, fire, startup, finishRebuild } = await createControlledStartup({ + seedNotes: { "gone.md": "ghostquartz text\n" }, + }) + const errorSpy = spyOnLogger("error") + // No batch step catches its own debug line, so a throw here escapes the batch. + spyOnLogger("debug").mockImplementation((message) => { + if (message === "removing a held path that is gone from disk") { + throw new Error("controlled batch failure") + } + }) + + await fire("unlink", "gone.md") + finishRebuild() + await expect(startup).resolves.toEqual({ count: 0, embedding: expect.any(Promise) }) + expect(errorSpy).toHaveBeenCalledExactlyOnceWith( + "failed to apply the changes held during startup", + { error: "[Error]: controlled batch failure" }, + ) + + await writeFile(join(testVault, "live.md"), "livetopaz\n") + await fire("add", "live.md") + expect(search.fullTextSearch({ query: "livetopaz" }, logger).map((hit) => hit.path)).toEqual([ + "live.md", + ]) + }) + it("reports a note embedding that fails during the batch once, with no unhandled rejection", async () => { const vector = new Float32Array(384).fill(0.1) const embedder = { diff --git a/src/vault-mcp/search/file-watcher.ts b/src/vault-mcp/search/file-watcher.ts index 9d33eb095..7d4ecbf8e 100644 --- a/src/vault-mcp/search/file-watcher.ts +++ b/src/vault-mcp/search/file-watcher.ts @@ -680,19 +680,11 @@ const startHoldingWatcher = ( }) } - /** - * Brings the index up to date with the held paths, in one batch, then goes live: - * - * 1. Each path is checked against the disk; missing ones are removed first, - * so a rename seen whole in the batch resolves incoming links to the new path. - * 2. The rest are read and indexed, 16 at a time; one that vanished by its read is removed. - * 3. Events held while steps 1 and 2 ran are replayed live. - * - * Removing a path because its check found it missing is safe only while - * holding: once live, the removal could clear the event token of an `add` - * that started during the check, and the added note would stay unindexed. - */ - const applyHeldBatch = async (batchPaths: ReadonlyArray): Promise => { + /** Steps 1 and 2 of applyHeldBatch. Removing a path because its check found + * it missing is safe only while holding: once live, the removal could clear + * the event token of an `add` that started during the check, and the added + * note would stay unindexed. */ + const reconcileHeldPaths = async (batchPaths: ReadonlyArray): Promise => { const folderListings = new Map>() /** Lists each folder once per batch, however many held paths sit in it. */ @@ -727,6 +719,27 @@ const startHoldingWatcher = ( concurrency: HELD_CHANGES_CONCURRENCY, mapper: indexHeldPath, }) + } + + /** + * Brings the index up to date with the held paths, in one batch, then goes live: + * + * 1. Each path is checked against the disk; missing ones are removed first, + * so a rename seen whole in the batch resolves incoming links to the new path. + * 2. The rest are read and indexed, 16 at a time; one that vanished by its read is removed. + * 3. Events held while steps 1 and 2 ran are replayed live. + * + * Never rejects, and step 3 runs even if an earlier step threw: a watcher + * left holding would drop every later change. + */ + const applyHeldBatch = async (batchPaths: ReadonlyArray): Promise => { + try { + await reconcileHeldPaths(batchPaths) + } catch (error) { + logger.error("failed to apply the changes held during startup", { + error: describeError(error), + }) + } goLiveAndReplay(batchPaths.length) } @@ -796,12 +809,6 @@ export const watchVaultAndBuildIndex = async ( heldPaths: heldPathCount, heldChangesDeadline, }) - // Nothing awaits the batch from here, so its last-resort failure is logged. - applied.catch((error) => { - logger.error("failed to apply the changes held during startup", { - error: describeError(error), - }) - }) } return rebuildResult } From d1f7fd5c46eb4b57c4c0823a32ca61e4b9927bf6 Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Fri, 16 Oct 2026 14:32:31 -0400 Subject: [PATCH 06/34] style(search): split indexChangedFile, model the hold as a state union, and explain the startup batch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - indexChangedFile keeps the event token and delegates to indexChangedNote and indexChangedNonMdFile. - The watcher's holding state is a WatcherState union instead of an array-or-undefined. - Comments define event tokens and source versions, and say why file embeddings detach, why the content-indexing failure warns, why removals run before reads, and when the replay's token order matters. - watchVaultAndBuildIndex races the held changes directly against the deadline and states its return shape. - The rebuildFromVault docstring defines a source write; ARCHITECTURE.md fixes the pass-3 return, the listen point, and the Startup order link. Ship-Check: code-quality · claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- ARCHITECTURE.md | 6 +- src/vault-mcp/search/file-watcher.ts | 483 ++++++++++++++++----------- src/vault-mcp/search/search-index.ts | 10 +- src/vault-mcp/server.ts | 3 + 4 files changed, 298 insertions(+), 204 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index bc638b20f..0b17c24e0 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -492,7 +492,7 @@ When no embedder is configured (`EMBEDDING_ENABLED=false`), no vectors are index ### Indexing -**Indexing flow:** `rebuildFromVault` runs three passes, then returns: +**Indexing flow:** `rebuildFromVault` runs passes 1 and 2, then returns while pass 3 runs in the background: 1. **Pass 1** — index notes (FTS5 + metadata) 2. **Pass 2** — extract links (with the complete path list for resolution), then index file content (canvas, PDF, text → FTS5) @@ -505,7 +505,7 @@ When no embedder is configured (`EMBEDDING_ENABLED=false`), no vectors are index 3. One batch checks each held path against the disk, 16 at a time. Paths gone from disk are removed first, then the rest are read again. 4. The watcher goes live and replays, in arrival order, the changes reported while the batch ran. -The server starts accepting requests once the step 3 batch commits, or after 10 s if a Sync backlog makes the batch longer; the batch then finishes in the background. +The server starts accepting requests once the step 3 batch commits and step 4 starts its replay, or after 10 s if a Sync backlog makes the batch longer; the batch then finishes in the background. A file that is created after step 1 and deleted within chokidar's 2 s write-stability window emits no event, so if the rebuild read it, it stays searchable until the next restart. @@ -941,7 +941,7 @@ graph LR doesn't exist, `MEMORY_ENABLED` is not `false`, and the server is not in `READONLY_MODE`, starts the file watcher, builds the FTS5 search index including those templates, then applies the changes the watcher saw while - the index was built (see [Startup order](#indexing)). + the index was built (see **Startup order** under [Indexing](#indexing)). `svc-vault-mcp` declares `svc-obsidian-sync` in its `dependencies.d`, so the MCP server starts only after the full init chain has finished and the sync diff --git a/src/vault-mcp/search/file-watcher.ts b/src/vault-mcp/search/file-watcher.ts index 7d4ecbf8e..7ab3c08f6 100644 --- a/src/vault-mcp/search/file-watcher.ts +++ b/src/vault-mcp/search/file-watcher.ts @@ -27,14 +27,15 @@ const POLLING_INTERVAL_MS = 300 /** Default for FileWatcherOptions.stabilityThreshold (see its doc). */ const DEFAULT_STABILITY_THRESHOLD_MS = 2000 -/** Default for FileWatcherOptions.heldChangesDeadline. It sits inside the local - * compose health check's 20 s start period. A batch that runs longer means - * Sync is delivering a backlog during startup, which search lags anyway, and - * the CLI stops waiting for health after 120 s. */ +/** Default for FileWatcherOptions.heldChangesDeadline. The local compose health + * check's 20 s start period sets the bound: 10 s leaves the server time to + * listen inside it. Without a deadline, a Sync backlog arriving during startup + * could keep the server from listening past the CLI's 120 s health wait. */ const DEFAULT_HELD_CHANGES_DEADLINE_MS = 10_000 -/** Paths the startup batch checks and reads at once: the rebuild's own file - * concurrency, so a large batch cannot open every file at once. */ +/** Paths the startup batch checks and reads at once. It matches the rebuild's + * REBUILD_IO_CONCURRENCY in search-index.ts, so the batch opens no more files + * at once than the rebuild did. */ const HELD_CHANGES_CONCURRENCY = 16 type FileWatcherOptions = Readonly<{ @@ -58,16 +59,26 @@ type FileWatcherOptions = Readonly<{ }> /** A change chokidar reported while the watcher held it. `add` and `change` - * both record "change", since either means "read this path again". */ + * both record "change", since either means "read this path again". Only the + * replay of events that arrive during the startup batch uses `kind`; the batch + * itself checks each path against the disk instead. */ type HeldEvent = Readonly<{ kind: "change" | "unlink"; filePath: string }> +/** While holding, chokidar's events are recorded in arrival order instead of + * applied; once live, each event is applied as it arrives. */ +type WatcherState = { mode: "holding"; heldEvents: HeldEvent[] } | { mode: "live" } + /** What indexing one changed file did. A note's embedding is still running when * this returns, so whoever receives `note-indexed` reports its failure. */ type ChangeIndexResult = | { status: "note-indexed"; embedding: Promise } | { status: "file-indexed" } + // The file was gone when read. | { status: "vanished" } + // A newer event or a removal for the same path arrived during the read, so + // this read's content was dropped. | { status: "superseded" } + // A read, stat, extraction or index write failed; already logged. | { status: "failed" } /** Whether a held path names a file on disk. "unknown" leaves the path to its @@ -97,199 +108,239 @@ const startHoldingWatcher = ( search: SearchIndex, options?: FileWatcherOptions, ): HoldingWatcher => { - // Serializing per path limits model concurrency; source versions reject - // obsolete results independently of the order jobs finish. + // Each path's latest embedding job. A new job for the path waits for this one, + // so a note edited many times runs one model job at a time. A job that still + // finishes after a newer edit writes nothing: the index checks the source + // version the job carries (the token upsertNote or upsertFileContent returned + // for the content it embeds) against the path's current one. const pendingEmbeds = new Map>() - /** Event tokens reject superseded filesystem reads until their handler finishes. - * Source versions belong to the index and protect model work after the handler returns. */ + /** Each path's event token. indexChangedFile stores a fresh token before it + * reads the file and drops what it read if the token changed meanwhile, + * because a newer event or a removal replaced it. The token guards only the + * read; source versions guard the embedding that runs after it. */ const currentEvents = new Map() - /** Each held event's path and kind in arrival order, or undefined once the - * watcher is live. Reassigned twice: the startup batch takes the list and - * starts a new one, and going live ends holding. */ - let heldEvents: HeldEvent[] | undefined = [] - - /** Indexes an added or modified file: non-md files land in the asset table; - * notes are read from disk and upserted into the FTS index, and their - * re-embedding is queued (serialized per path via pendingEmbeds). */ - const indexChangedFile = async (filePath: string): Promise => { - const relativePath = relative(vaultPath, filePath) - const eventToken = Symbol() - currentEvents.set(relativePath, eventToken) - - try { - if (!filePath.endsWith(".md")) { - const readNonMdFileStat = async (): Promise< - { status: "found"; fileStat: Stats } | { status: "missing" } | { status: "failed" } - > => { - try { - const fileStat = await statOrNull(filePath) - return fileStat ? { status: "found", fileStat } : { status: "missing" } - } catch (error) { - logger.error("failed to stat non-md file", { - path: relativePath, - error: describeError(error), - }) - return { status: "failed" } - } - } - const nonMdStat = await readNonMdFileStat() - - if (currentEvents.get(relativePath) !== eventToken) return { status: "superseded" } - // Vanished between the watcher event and the stat — the unlink event - // that follows will remove any existing row. - if (nonMdStat.status === "missing") return { status: "vanished" } - if (nonMdStat.status === "failed") return { status: "failed" } + // Reassigned twice, so `let`: the startup batch swaps in a new held list for + // events that arrive while it runs, and going live ends holding. + let watcherState: WatcherState = { mode: "holding", heldEvents: [] } + + /** Indexes an added or modified non-md file: its asset-table row, plus its + * content when it is a canvas or an indexable text or PDF file. */ + const indexChangedNonMdFile = async (params: { + filePath: string + relativePath: string + eventToken: symbol + }): Promise => { + const { filePath, relativePath, eventToken } = params + + const readNonMdFileStat = async (): Promise< + { status: "found"; fileStat: Stats } | { status: "missing" } | { status: "failed" } + > => { + try { + const fileStat = await statOrNull(filePath) + return fileStat ? { status: "found", fileStat } : { status: "missing" } + } catch (error) { + logger.error("failed to stat non-md file", { + path: relativePath, + error: describeError(error), + }) + return { status: "failed" } + } + } + const nonMdStat = await readNonMdFileStat() - const { fileStat } = nonMdStat + if (currentEvents.get(relativePath) !== eventToken) return { status: "superseded" } + // Vanished between the event and the stat. Live, the unlink event that + // follows removes any existing row; the startup batch removes it itself. + if (nonMdStat.status === "missing") return { status: "vanished" } + if (nonMdStat.status === "failed") return { status: "failed" } - try { - search.upsertNonMdFile(relativePath, fileStat.size) - } catch (error) { - logger.error("failed to index non-md file metadata", { - path: relativePath, - error: describeError(error), - }) - return { status: "failed" } - } + const { fileStat } = nonMdStat - // Canvas files are always read — link extraction is unconditional. - // PDF and text files are only read when file content FTS is enabled. - const extension = extname(filePath) - const isCanvas = extension === ".canvas" - const isIndexableNonCanvas = - search.fileContentIndexingEnabled && INDEXABLE_TEXT_EXTENSIONS.has(extension) - - if (isCanvas || isIndexableNonCanvas) { - try { - const readContentToIndex = async (): Promise => { - if (extension !== ".pdf") return await readFile(filePath, "utf8") - - const buffer = await readFile(filePath) - const pdfData = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength) - const pdfResult = await extractPdfText(pdfData) - return pdfResult.text - } - const contentToIndex = await readContentToIndex() - - if (currentEvents.get(relativePath) !== eventToken) return { status: "superseded" } - - const sourceVersion = search.upsertFileContent( - { - filePath: relativePath, - rawContent: contentToIndex, - fileStat: { mtimeMs: fileStat.mtimeMs, size: fileStat.size }, - }, - logger, - ) - - // Embed file content vectors — serialized per path via the same - // pendingEmbeds map (note paths end in .md, file paths don't). - // Reads the processed content from the file_content table. - const previousEmbed = pendingEmbeds.get(relativePath) ?? Promise.resolve() - const currentEmbed = previousEmbed - .catch((previousError) => { - logger.debug("previous file embed failed, proceeding with current", { - path: relativePath, - error: describeError(previousError), - }) - }) - .then(() => { - return search.embedFileContent({ filePath: relativePath, sourceVersion }, logger) - }) - pendingEmbeds.set(relativePath, currentEmbed) - - /** This detached job owns its error reporting and queue cleanup after the - * file handler returns; an older job must leave a newer queued job registered. */ - currentEmbed - .catch((embedError) => { - logger.error("file content embedding failed", { - path: relativePath, - error: describeError(embedError), - }) - }) - .finally(() => { - if (pendingEmbeds.get(relativePath) === currentEmbed) { - pendingEmbeds.delete(relativePath) - } - }) - } catch (error) { - // A PDF reader that failed to start arrives wrapped; the cause holds why - logger.warn("file content indexing failed", { - path: relativePath, - ...describeErrorForLog(error), - }) - } - } + // Nothing is awaited between the token check above and this write, so it + // cannot be superseded; only the content read below needs a second check. + try { + search.upsertNonMdFile(relativePath, fileStat.size) + } catch (error) { + logger.error("failed to index non-md file metadata", { + path: relativePath, + error: describeError(error), + }) + return { status: "failed" } + } - logger.debug("indexed non-md file", { path: relativePath }) - return { status: "file-indexed" } - } + // Canvas files are always read — link extraction is unconditional. + // PDF and text files are only read when file content FTS is enabled. + const extension = extname(filePath) + const isCanvas = extension === ".canvas" + const isIndexableNonCanvas = + search.fileContentIndexingEnabled && INDEXABLE_TEXT_EXTENSIONS.has(extension) + if (isCanvas || isIndexableNonCanvas) { try { - const readNoteSource = async (): Promise<{ content: string; fileStat: Stats } | null> => { - try { - const [content, fileStat] = await Promise.all([ - readFile(filePath, "utf8"), - stat(filePath), - ]) - return { content, fileStat } - } catch (error) { - if (!isErrnoException(error, "ENOENT")) throw error - - logger.debug("change event skipped, file vanished", { path: relativePath }) - return null - } + const readContentToIndex = async (): Promise => { + if (extension !== ".pdf") return await readFile(filePath, "utf8") + + const buffer = await readFile(filePath) + const pdfData = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength) + const pdfResult = await extractPdfText(pdfData) + return pdfResult.text } - const noteSource = await readNoteSource() + const contentToIndex = await readContentToIndex() if (currentEvents.get(relativePath) !== eventToken) return { status: "superseded" } - if (!noteSource) return { status: "vanished" } - - const { content, fileStat } = noteSource - const sourceVersion = search.upsertNote( + const sourceVersion = search.upsertFileContent( { filePath: relativePath, - rawContent: content, + rawContent: contentToIndex, fileStat: { mtimeMs: fileStat.mtimeMs, size: fileStat.size }, }, logger, ) - // Each path queues model work; recovering a rejected predecessor - // allows later jobs to run after a transient failure. + + // Embed file content vectors — serialized per path via the same + // pendingEmbeds map (note paths end in .md, file paths don't). + // Reads the processed content from the file_content table. const previousEmbed = pendingEmbeds.get(relativePath) ?? Promise.resolve() const currentEmbed = previousEmbed .catch((previousError) => { - logger.debug("previous embed failed, proceeding with current", { + logger.debug("previous file embed failed, proceeding with current", { path: relativePath, error: describeError(previousError), }) }) .then(() => { - return search.embedNote( - { notePath: relativePath, rawContent: content, sourceVersion }, - logger, - ) + return search.embedFileContent({ filePath: relativePath, sourceVersion }, logger) }) pendingEmbeds.set(relativePath, currentEmbed) - /** The receiver of this result reports the job's failures. Hand over the - * cleanup promise, so its rejection is the one handled and newer queued jobs remain registered. */ - const embedding = currentEmbed.finally(() => { - if (pendingEmbeds.get(relativePath) === currentEmbed) { - pendingEmbeds.delete(relativePath) - } - }) - return { status: "note-indexed", embedding } - } catch (err) { - logger.error("failed to process file change", { + // Unlike a note's, this embedding is not returned: neither the live + // handler nor the startup batch waits on file-content model work. So + // the job reports its own failure and removes its own queue entry, + // leaving a newer queued job registered. + currentEmbed + .catch((embedError) => { + logger.error("file content embedding failed", { + path: relativePath, + error: describeError(embedError), + }) + }) + .finally(() => { + if (pendingEmbeds.get(relativePath) === currentEmbed) { + pendingEmbeds.delete(relativePath) + } + }) + } catch (error) { + // Warn, as the rebuild does for an unreadable file: the asset-table row + // above is indexed, so only the file's content search is missing. + // describeErrorForLog keeps the cause, because a PDF reader that + // failed to start arrives wrapped. + logger.warn("file content indexing failed", { path: relativePath, - error: describeError(err), + ...describeErrorForLog(error), }) - return { status: "failed" } } + } + + logger.debug("indexed non-md file", { path: relativePath }) + return { status: "file-indexed" } + } + + /** Indexes an added or modified note and queues its re-embedding. The + * embedding is returned unawaited, so the caller decides whether to wait. */ + const indexChangedNote = async (params: { + filePath: string + relativePath: string + eventToken: symbol + }): Promise => { + const { filePath, relativePath, eventToken } = params + + try { + const readNoteSource = async (): Promise<{ content: string; fileStat: Stats } | null> => { + try { + const [content, fileStat] = await Promise.all([ + readFile(filePath, "utf8"), + stat(filePath), + ]) + return { content, fileStat } + } catch (error) { + if (!isErrnoException(error, "ENOENT")) throw error + + logger.debug("change event skipped, file vanished", { path: relativePath }) + return null + } + } + const noteSource = await readNoteSource() + + if (currentEvents.get(relativePath) !== eventToken) return { status: "superseded" } + if (!noteSource) return { status: "vanished" } + + const { content, fileStat } = noteSource + + const sourceVersion = search.upsertNote( + { + filePath: relativePath, + rawContent: content, + fileStat: { mtimeMs: fileStat.mtimeMs, size: fileStat.size }, + }, + logger, + ) + + // Each path queues model work; recovering a rejected predecessor + // allows later jobs to run after a transient failure. + const previousEmbed = pendingEmbeds.get(relativePath) ?? Promise.resolve() + const currentEmbed = previousEmbed + .catch((previousError) => { + logger.debug("previous embed failed, proceeding with current", { + path: relativePath, + error: describeError(previousError), + }) + }) + .then(() => { + return search.embedNote( + { notePath: relativePath, rawContent: content, sourceVersion }, + logger, + ) + }) + pendingEmbeds.set(relativePath, currentEmbed) + + // The caller reports the job's failure, so it receives the promise with + // the queue cleanup attached rather than currentEmbed: `.finally` passes + // a rejection through, and a cleanup promise nobody received would reject + // unhandled. The cleanup leaves a newer queued job registered. + const embedding = currentEmbed.finally(() => { + if (pendingEmbeds.get(relativePath) === currentEmbed) { + pendingEmbeds.delete(relativePath) + } + }) + return { status: "note-indexed", embedding } + } catch (err) { + logger.error("failed to process file change", { + path: relativePath, + error: describeError(err), + }) + return { status: "failed" } + } + } + + /** Indexes an added or modified file and reports what it did. */ + const indexChangedFile = async (filePath: string): Promise => { + const relativePath = relative(vaultPath, filePath) + + // Taken before the first await, so events take their tokens in the order + // their handlers start. goLiveAndReplay relies on this: a replayed event + // must take its token before a newer live event for the same path does. + const eventToken = Symbol() + currentEvents.set(relativePath, eventToken) + + const indexParams = { filePath, relativePath, eventToken } + + // Awaited inside the try, so the token is released only once indexing ends. + try { + if (filePath.endsWith(".md")) return await indexChangedNote(indexParams) + return await indexChangedNonMdFile(indexParams) } finally { // A newer handler may already have finished and removed its entry. // Only the current event owns cleanup; absence still invalidates old reads. @@ -311,16 +362,18 @@ const startHoldingWatcher = ( /** chokidar's add and change listener. While the watcher holds, it records * the path for applyHeldChanges instead of indexing it. */ const handleChange = async (filePath: string): Promise => { - if (heldEvents) { - heldEvents.push({ kind: "change", filePath }) + if (watcherState.mode === "holding") { + watcherState.heldEvents.push({ kind: "change", filePath }) logger.debug("held a change until startup indexing ends", { path: relative(vaultPath, filePath), }) return } + const indexResult = await indexChangedFile(filePath) if (indexResult.status !== "note-indexed") return + try { await indexResult.embedding } catch (error) { @@ -332,6 +385,9 @@ const startHoldingWatcher = ( * files, the note tables (FTS, links, tasks, vectors) for notes. */ const removeFromIndex = (filePath: string): void => { const relativePath = relative(vaultPath, filePath) + + // Dropping the path's token makes a read still in flight for it return + // "superseded", so that read cannot re-add the file removed here. currentEvents.delete(relativePath) if (!filePath.endsWith(".md")) { @@ -343,6 +399,9 @@ const startHoldingWatcher = ( error: describeError(error), }) } + + // The content rows live in their own tables, so they are removed even + // when the asset-table removal above failed. const deletedExtension = extname(filePath) const isDeletedCanvas = deletedExtension === ".canvas" const isDeletedIndexable = @@ -378,8 +437,8 @@ const startHoldingWatcher = ( /** chokidar's unlink listener. While the watcher holds, it records the path * for applyHeldChanges instead of removing it. */ const handleDelete = (filePath: string): void => { - if (heldEvents) { - heldEvents.push({ kind: "unlink", filePath }) + if (watcherState.mode === "holding") { + watcherState.heldEvents.push({ kind: "unlink", filePath }) logger.debug("held a deletion until startup indexing ends", { path: relative(vaultPath, filePath), }) @@ -440,6 +499,9 @@ const startHoldingWatcher = ( const indexIfSettled = async (): Promise => { const fileStat = await stat(filePath) + // A file still being written is checked again after another window. + // The retries have no limit, since a file that keeps changing is still + // arriving; a deleted file ends them when the stat above rejects. if (isWithinStabilityWindow(fileStat.mtimeMs)) { scheduleUnstableFileRetry(filePath) return @@ -477,7 +539,9 @@ const startHoldingWatcher = ( dirPath: string, visitedRealPaths: Set, ): Promise => { - /** The wrapper lists descendants recursively; symlinked directories are followed below. */ + // readdirOrNull lists every descendant recursively, or returns null when the + // directory is gone. It does not enter symlinked directories; the loop + // below rescans those. const entries = await readdirOrNull(dirPath) if (!entries) { @@ -595,6 +659,8 @@ const startHoldingWatcher = ( /** Lists a folder's entry names for the startup batch's presence checks. */ const listFolderNames = async (folderPath: string): Promise => { try { + // A plain readdir, not readdirOrNull: the check needs only this folder's + // own names, and readdirOrNull lists every descendant. const names = await readdir(folderPath) return { status: "listed", names: new Set(names) } } catch (error) { @@ -612,7 +678,8 @@ const startHoldingWatcher = ( * Decides whether a held path still names a file. The parent folder's listing * must hold the exact name: on a case-insensitive filesystem, `stat` of * `note.md` still succeeds after a rename to `Note.md`, and reading it would - * index the old name again. + * index the old name again. `listFolderOnce` returns the batch's shared + * listing of a folder, so each folder is read once per batch. */ const checkHeldPathOnDisk = async ( filePath: string, @@ -623,8 +690,14 @@ const startHoldingWatcher = ( folderListing.status === "listed" && !folderListing.names.has(basename(filePath)) if (folderListing.status === "missing" || missingFromListing) return "missing" + + // A folder that could not be listed proves nothing either way, so the stat + // below decides. try { const fileStat = await statOrNull(filePath) + + // A folder now at the path also counts as missing. Removing it is + // harmless: it clears only rows for a file that is no longer there. return fileStat?.isFile() ? "present" : "missing" } catch (error) { // The path's read runs next and reports the failure at error level. @@ -653,7 +726,10 @@ const startHoldingWatcher = ( return } if (indexResult.status !== "note-indexed") return - // Caught here, inside the batch step, so a fast rejection is never unhandled. + + // The batch does not wait for embeddings, so the failure handler is + // attached now. Attached after the batch, an embedding that rejected first + // would be reported as an unhandled rejection. indexResult.embedding.catch((error) => { reportEmbeddingFailure(filePath, error) }) @@ -661,11 +737,15 @@ const startHoldingWatcher = ( /** Ends holding and replays the events held while the batch ran, in arrival order. */ const goLiveAndReplay = (batchPathCount: number): void => { - const eventsDuringBatch = heldEvents ?? [] - heldEvents = undefined + // Already live only when applyHeldChanges ran twice and the first batch + // replayed everything. + const eventsDuringBatch = watcherState.mode === "holding" ? watcherState.heldEvents : [] + watcherState = { mode: "live" } + // Every replayed event starts in this one tick, before any newer live // event for the same path, so that newer event takes the path's token and - // wins. chokidar delivers a burst the same way. + // wins. indexChangedFile takes the token before its first await, and + // chokidar delivers a burst the same way. for (const heldEvent of eventsDuringBatch) { if (heldEvent.kind === "unlink") { handleDelete(heldEvent.filePath) @@ -680,10 +760,11 @@ const startHoldingWatcher = ( }) } - /** Steps 1 and 2 of applyHeldBatch. Removing a path because its check found - * it missing is safe only while holding: once live, the removal could clear - * the event token of an `add` that started during the check, and the added - * note would stay unindexed. */ + /** Checks each held path against the disk, removes the missing ones, then + * re-reads the rest. It must run while the watcher holds: removeFromIndex + * drops the path's event token, so once live, a removal could cancel the + * read of an `add` that started during the check, and the added file would + * stay unindexed. */ const reconcileHeldPaths = async (batchPaths: ReadonlyArray): Promise => { const folderListings = new Map>() @@ -711,9 +792,13 @@ const startHoldingWatcher = ( .filter((check) => check.presence !== "missing") .map((check) => check.filePath) + // Removals run before reads. upsertNote re-resolves only links whose target + // is not indexed, so when a renamed note's old path is removed first, the + // new path claims the links that pointed at the old one. for (const filePath of missingPaths) { removeHeldPath(filePath) } + await mapWithConcurrency({ items: pathsToRead, concurrency: HELD_CHANGES_CONCURRENCY, @@ -724,9 +809,9 @@ const startHoldingWatcher = ( /** * Brings the index up to date with the held paths, in one batch, then goes live: * - * 1. Each path is checked against the disk; missing ones are removed first, - * so a rename seen whole in the batch resolves incoming links to the new path. - * 2. The rest are read and indexed, 16 at a time; one that vanished by its read is removed. + * 1. Each path is checked against the disk, and missing ones are removed. + * 2. The rest are read and indexed, HELD_CHANGES_CONCURRENCY at a time; one + * that vanished by its read is removed. * 3. Events held while steps 1 and 2 ran are replayed live. * * Never rejects, and step 3 runs even if an earlier step threw: a watcher @@ -743,12 +828,15 @@ const startHoldingWatcher = ( goLiveAndReplay(batchPaths.length) } + /** Starts the startup batch over the distinct paths held so far. The batch + * ignores each event's kind, because it checks every path against the disk. */ const applyHeldChanges = (): { heldPathCount: number; applied: Promise } => { - if (!heldEvents) return { heldPathCount: 0, applied: Promise.resolve() } + if (watcherState.mode === "live") return { heldPathCount: 0, applied: Promise.resolve() } - const batchEvents = heldEvents - // Events arriving while the batch runs collect here and are replayed after it. - heldEvents = [] + const batchEvents = watcherState.heldEvents + // Events arriving while the batch runs collect in a new list and are + // replayed after it. + watcherState = { mode: "holding", heldEvents: [] } const batchPaths = [...new Set(batchEvents.map((heldEvent) => heldEvent.filePath))] return { heldPathCount: batchPaths.length, applied: applyHeldBatch(batchPaths) } @@ -780,31 +868,32 @@ const startHoldingWatcher = ( * changes the watcher saw meanwhile. The watcher is ready before the rebuild * reads anything and holds every change until the rebuild returns, so a change * landing mid-rebuild is applied after it instead of being missed or replaced - * by the rebuild's older read. Resolves with the rebuild's result once the - * held changes are indexed, or at heldChangesDeadline if they take longer; - * the rest then apply in the background. + * by the rebuild's older read. Resolves with the rebuild's result (the note + * count and its background embedding promise) once the held changes are + * indexed, or at heldChangesDeadline if they take longer; the rest then apply + * in the background. */ export const watchVaultAndBuildIndex = async ( vaultPath: string, search: SearchIndex, options?: FileWatcherOptions, -): ReturnType => { +): Promise<{ count: number; embedding: Promise }> => { const holdingWatcher = startHoldingWatcher(vaultPath, search, options) await holdingWatcher.ready const rebuildResult = await search.rebuildFromVault({ vaultPath }, logger) + + // `applied` never rejects, so past the deadline it can finish unawaited. const { heldPathCount, applied } = holdingWatcher.applyHeldChanges() const heldChangesDeadline = options?.heldChangesDeadline ?? DEFAULT_HELD_CHANGES_DEADLINE_MS const { promise: deadlinePassed, resolve: passDeadline } = Promise.withResolvers<"deadline">() const deadlineTimer = setTimeout(() => passDeadline("deadline"), heldChangesDeadline) - const waitForHeldChanges = async (): Promise<"applied"> => { - await applied - return "applied" - } - const outcome = await Promise.race([waitForHeldChanges(), deadlinePassed]) + + // `applied` resolves with nothing, so only the deadline makes this "deadline". + const firstSettled = await Promise.race([applied, deadlinePassed]) clearTimeout(deadlineTimer) - if (outcome === "deadline") { + if (firstSettled === "deadline") { logger.info("startup continues while the changes held during it are still applied", { heldPaths: heldPathCount, heldChangesDeadline, diff --git a/src/vault-mcp/search/search-index.ts b/src/vault-mcp/search/search-index.ts index 9dc65df65..4d34c4887 100644 --- a/src/vault-mcp/search/search-index.ts +++ b/src/vault-mcp/search/search-index.ts @@ -2078,10 +2078,12 @@ export const createSearchIndex = ( /** Rebuilds note, task, file and graph indexes; reconciles retained memory rows and vectors. * Returns the note count and a background embedding promise. - * - No source write may run while it does: a write that lands during its reads is - * replaced by the older content it read. - * - The file watcher is the only runtime source writer, and watchVaultAndBuildIndex - * holds it until this returns; a new runtime writer must go through that hold. */ + * - No source write may run while it does. A source write is an upsert or remove call + * that changes the tables this clears; one that lands during its reads is replaced by + * the older content it read. + * - The file watcher is the only caller that makes source writes at runtime, and + * watchVaultAndBuildIndex holds it until this returns; a new runtime caller must go + * through that hold. */ const rebuildFromVault = async ( params: { vaultPath: string }, logger: Logger, diff --git a/src/vault-mcp/server.ts b/src/vault-mcp/server.ts index 0a3e974fe..367757127 100644 --- a/src/vault-mcp/server.ts +++ b/src/vault-mcp/server.ts @@ -146,8 +146,11 @@ const startServer = async (): Promise => { fileToolsEnabled: config.fileToolsEnabled, statusRegistry, }) + // Before the watcher starts, so its template writes are not held and re-read. await bootstrapMemoryIfEnabled(config, vaultPath) + + // The returned embedding promise is not awaited: the rebuild logs its failure itself. const { count } = await watchVaultAndBuildIndex(vaultPath, search, { usePolling: config.windowsBindMount, }) From 57efaa91e4722ef385e09c2218e348b866f30fec Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Fri, 16 Oct 2026 14:38:41 -0400 Subject: [PATCH 07/34] test(search): cover the startup batch's folder, stat and rebuild-failure branches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A held path now a folder, or in a folder that is gone, is removed (the latter without a listing warning). - A folder that cannot be listed leaves its held notes to their reads. - A held file whose stat fails keeps its rows; one deleted between its check and its read loses them. - watchVaultAndBuildIndex rejects with the rebuild's error. - Seeded rows are asserted before acting in the unlink-then-exists and replay tests, and the 16-at-once test asserts the exact indexed paths. Ship-Check: test-audit · claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- .../search/__tests__/file-watcher.test.ts | 158 +++++++++++++++++- 1 file changed, 156 insertions(+), 2 deletions(-) diff --git a/src/vault-mcp/search/__tests__/file-watcher.test.ts b/src/vault-mcp/search/__tests__/file-watcher.test.ts index 7fd902868..14c9e30ab 100644 --- a/src/vault-mcp/search/__tests__/file-watcher.test.ts +++ b/src/vault-mcp/search/__tests__/file-watcher.test.ts @@ -10,6 +10,7 @@ import { symlink, utimes, readFile, + readdir, } from "node:fs/promises" import { join, resolve } from "node:path" import { tmpdir } from "node:os" @@ -1497,10 +1498,11 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { notePaths, startup, finishRebuild: () => rebuildGate.resolve(undefined), + failRebuild: (error: Error) => rebuildGate.reject(error), } } - const spyOnLogger = (level: "debug" | "info" | "error") => { + const spyOnLogger = (level: "debug" | "info" | "warn" | "error") => { const levelSpy = vi.spyOn(logger, level) onTestFinished(() => levelSpy.mockRestore()) return levelSpy @@ -1566,6 +1568,9 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { seedNotes: { "back.md": "oldtext\n" }, }) await writeFile(join(testVault, "back.md"), "returnedopal\n") + expect(search.fullTextSearch({ query: "oldtext" }, logger).map((hit) => hit.path)).toEqual([ + "back.md", + ]) await fire("change", "back.md") await fire("unlink", "back.md") @@ -1630,6 +1635,8 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { const pathsFor = (query: string) => { return search.fullTextSearch({ query }, logger).map((hit) => hit.path) } + expect(pathsFor("oldamber")).toEqual(["P.md"]) + expect(pathsFor("quartzrow")).toEqual(["Q.md"]) await fire("change", "P.md") finishRebuild() @@ -1852,7 +1859,10 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { await startup expect(readCounts.peak).toBe(16) - expect(search.fullTextSearch({ query: "bulktopaz", limit: 50 }, logger)).toHaveLength(40) + const indexedPaths = search + .fullTextSearch({ query: "bulktopaz", limit: 50 }, logger) + .map((hit) => hit.path) + expect(indexedPaths.toSorted()).toEqual(fileNames.toSorted()) }) it("returns at the deadline and finishes the held changes in the background", async () => { @@ -1930,6 +1940,150 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { expect(notePaths()).toEqual([{ path: "Note.md" }]) }) + it("removes a held note whose path is now a folder", async () => { + const { testVault, notePaths, fire, startup, finishRebuild } = await createControlledStartup({ + seedNotes: { "shape.md": "shapeopal\n" }, + }) + const errorSpy = spyOnLogger("error") + await mkdir(join(testVault, "shape.md")) + expect(notePaths()).toEqual([{ path: "shape.md" }]) + + await fire("change", "shape.md") + finishRebuild() + await startup + + expect(notePaths()).toEqual([]) + expect(errorSpy).not.toHaveBeenCalled() + }) + + it("removes a held note whose folder is gone, with no listing warning", async () => { + const { notePaths, fire, startup, finishRebuild } = await createControlledStartup({ + seedNotes: { "gone-folder/note.md": "folderopal\n" }, + }) + const warnSpy = spyOnLogger("warn") + expect(notePaths()).toEqual([{ path: "gone-folder/note.md" }]) + + await fire("unlink", "gone-folder/note.md") + finishRebuild() + await startup + + expect(notePaths()).toEqual([]) + expect(warnSpy).not.toHaveBeenCalled() + }) + + it("reads a held note whose folder cannot be listed", async () => { + const { testVault, search, fire, startup, finishRebuild } = await createControlledStartup() + const warnSpy = spyOnLogger("warn") + const lockedFolder = join(testVault, "locked") + await mkdir(lockedFolder) + await writeFile(join(lockedFolder, "note.md"), "listedopal\n") + const actualFs = await vi.importActual("node:fs/promises") + const listingFailure = Object.assign(new Error("controlled listing failure"), { + code: "EACCES", + }) + vi.mocked(readdir).mockImplementation(async (requestedPath, options) => { + if (requestedPath === lockedFolder) throw listingFailure + return actualFs.readdir(requestedPath, options) + }) + onTestFinished(() => vi.mocked(readdir).mockRestore()) + + await fire("change", "locked/note.md") + finishRebuild() + await startup + + expect(warnSpy).toHaveBeenCalledExactlyOnceWith( + "could not list a folder of held changes, so its files are read instead", + { path: "locked", error: "[Error]: controlled listing failure" }, + ) + expect(search.fullTextSearch({ query: "listedopal" }, logger).map((hit) => hit.path)).toEqual([ + "locked/note.md", + ]) + }) + + it("keeps a held file's rows when its stat fails during the batch", async () => { + const { testVault, search, database, fire, startup, finishRebuild } = + await createControlledStartup() + const errorSpy = spyOnLogger("error") + const imagePath = join(testVault, "image.png") + await writeFile(imagePath, "image data") + await writeFile(join(testVault, "fine.md"), "finetext\n") + search.upsertNonMdFile("image.png", 10) + const assetPaths = () => database.prepare("SELECT path FROM non_md_files").all() + expect(assetPaths()).toEqual([{ path: "image.png" }]) + const actualFsUtils = + await vi.importActual("../../../utils/fs.js") + const statFailure = Object.assign(new Error("controlled stat failure"), { code: "EACCES" }) + vi.mocked(statOrNull).mockImplementation(async (requestedPath) => { + if (requestedPath === imagePath) throw statFailure + return actualFsUtils.statOrNull(requestedPath) + }) + onTestFinished(() => vi.mocked(statOrNull).mockRestore()) + + await fire("change", "image.png") + await fire("change", "fine.md") + finishRebuild() + await startup + + expect(errorSpy).toHaveBeenCalledExactlyOnceWith("failed to stat non-md file", { + path: "image.png", + error: "[Error]: controlled stat failure", + }) + expect(assetPaths()).toEqual([{ path: "image.png" }]) + expect(search.fullTextSearch({ query: "finetext" }, logger).map((hit) => hit.path)).toEqual([ + "fine.md", + ]) + }) + + it("removes a held file's rows when it is deleted between its check and its read", async () => { + const { testVault, search, database, fire, startup, finishRebuild } = + await createControlledStartup() + const contentPath = join(testVault, "content.txt") + await writeFile(contentPath, "vanishjade") + search.upsertNonMdFile("content.txt", 10) + search.upsertFileContent( + { filePath: "content.txt", rawContent: "vanishjade", fileStat: { mtimeMs: 0, size: 10 } }, + logger, + ) + const fileRows = () => ({ + assets: database.prepare("SELECT path FROM non_md_files").all(), + contents: database.prepare("SELECT path FROM file_content").all(), + }) + expect(fileRows()).toEqual({ + assets: [{ path: "content.txt" }], + contents: [{ path: "content.txt" }], + }) + // The batch's check sees the file, which is deleted before the check + // returns, so only the read that follows can find it gone. + const actualFsUtils = + await vi.importActual("../../../utils/fs.js") + const checkedPaths = new Set() + vi.mocked(statOrNull).mockImplementation(async (requestedPath) => { + const fileStat = await actualFsUtils.statOrNull(requestedPath) + + if (requestedPath === contentPath && !checkedPaths.has(contentPath)) { + checkedPaths.add(contentPath) + await unlink(contentPath) + } + return fileStat + }) + onTestFinished(() => vi.mocked(statOrNull).mockRestore()) + + await fire("change", "content.txt") + finishRebuild() + await startup + + expect(checkedPaths).toEqual(new Set([contentPath])) + expect(fileRows()).toEqual({ assets: [], contents: [] }) + }) + + it("rejects with the rebuild's error when the rebuild fails", async () => { + const { startup, failRebuild } = await createControlledStartup() + + failRebuild(new Error("controlled rebuild failure")) + + await expect(startup).rejects.toThrow(new Error("controlled rebuild failure")) + }) + it("lets a held change win over the rebuild's older read of the same note", async () => { const readGates: Array>> = [] const { testVault, search, fire, startup } = await createControlledStartup({ From a820697c4606526d3911801da7d199a8626e8a28 Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Fri, 16 Oct 2026 14:43:12 -0400 Subject: [PATCH 08/34] refactor(search): share one file-read bound between the rebuild and the startup batch The startup batch and the rebuild bound the same resource, open vault files, so they read one exported constant. A non-md removal whose asset-table row failed to delete no longer also logs that it was processed. Co-Authored-By: Claude Opus 5.5 --- src/vault-mcp/search/file-watcher.ts | 35 ++++++++++++++-------------- src/vault-mcp/search/search-index.ts | 15 +++++++----- 2 files changed, 27 insertions(+), 23 deletions(-) diff --git a/src/vault-mcp/search/file-watcher.ts b/src/vault-mcp/search/file-watcher.ts index 7ab3c08f6..adbe4d11f 100644 --- a/src/vault-mcp/search/file-watcher.ts +++ b/src/vault-mcp/search/file-watcher.ts @@ -8,7 +8,7 @@ import { DateTime } from "luxon" import { readFile, readdir, stat } from "node:fs/promises" import type { Stats } from "node:fs" import { basename, dirname, extname, join, relative, resolve as resolvePath } from "node:path" -import { INDEXABLE_TEXT_EXTENSIONS } from "./search-index.js" +import { INDEXABLE_TEXT_EXTENSIONS, INDEXING_FILE_READ_CONCURRENCY } from "./search-index.js" import type { SearchIndex } from "./search-index.js" import { extractPdfText } from "../obsidian-markdown/pdf.js" import { logger } from "../../logger.js" @@ -33,11 +33,6 @@ const DEFAULT_STABILITY_THRESHOLD_MS = 2000 * could keep the server from listening past the CLI's 120 s health wait. */ const DEFAULT_HELD_CHANGES_DEADLINE_MS = 10_000 -/** Paths the startup batch checks and reads at once. It matches the rebuild's - * REBUILD_IO_CONCURRENCY in search-index.ts, so the batch opens no more files - * at once than the rebuild did. */ -const HELD_CHANGES_CONCURRENCY = 16 - type FileWatcherOptions = Readonly<{ /** ms a file's size must stay unchanged before we index it (default 2000). * Prevents reading partial writes from Obsidian Sync. */ @@ -391,14 +386,19 @@ const startHoldingWatcher = ( currentEvents.delete(relativePath) if (!filePath.endsWith(".md")) { - try { - search.removeNonMdFile(relativePath) - } catch (error) { - logger.error("failed to remove non-md file metadata", { - path: relativePath, - error: describeError(error), - }) + const removeAssetRow = (): boolean => { + try { + search.removeNonMdFile(relativePath) + return true + } catch (error) { + logger.error("failed to remove non-md file metadata", { + path: relativePath, + error: describeError(error), + }) + return false + } } + const assetRowRemoved = removeAssetRow() // The content rows live in their own tables, so they are removed even // when the asset-table removal above failed. @@ -418,7 +418,8 @@ const startHoldingWatcher = ( return } } - logger.debug("processed non-md file removal", { path: relativePath }) + // Reported only when every removal succeeded; a failure logged its own error. + if (assetRowRemoved) logger.debug("processed non-md file removal", { path: relativePath }) return } @@ -780,7 +781,7 @@ const startHoldingWatcher = ( const presenceChecks = await mapWithConcurrency({ items: batchPaths, - concurrency: HELD_CHANGES_CONCURRENCY, + concurrency: INDEXING_FILE_READ_CONCURRENCY, mapper: async (filePath) => { return { filePath, presence: await checkHeldPathOnDisk(filePath, listFolderOnce) } }, @@ -801,7 +802,7 @@ const startHoldingWatcher = ( await mapWithConcurrency({ items: pathsToRead, - concurrency: HELD_CHANGES_CONCURRENCY, + concurrency: INDEXING_FILE_READ_CONCURRENCY, mapper: indexHeldPath, }) } @@ -810,7 +811,7 @@ const startHoldingWatcher = ( * Brings the index up to date with the held paths, in one batch, then goes live: * * 1. Each path is checked against the disk, and missing ones are removed. - * 2. The rest are read and indexed, HELD_CHANGES_CONCURRENCY at a time; one + * 2. The rest are read and indexed, INDEXING_FILE_READ_CONCURRENCY at a time; one * that vanished by its read is removed. * 3. Events held while steps 1 and 2 ran are replayed live. * diff --git a/src/vault-mcp/search/search-index.ts b/src/vault-mcp/search/search-index.ts index 4d34c4887..7691fb1fd 100644 --- a/src/vault-mcp/search/search-index.ts +++ b/src/vault-mcp/search/search-index.ts @@ -39,6 +39,11 @@ import { hybridSearch } from "./hybrid-search.js" // ── File content indexing constants ───────────────────────────── +/** Files read at once while indexing from disk, by the startup rebuild and by + * the file watcher's startup batch, so a large vault cannot open every file + * at once. */ +export const INDEXING_FILE_READ_CONCURRENCY = 16 + /** Extensions whose content is indexed into file_content_fts for vault_search. * Excludes .canvas (own path with link extraction) and .md (notes_fts). */ export const INDEXABLE_TEXT_EXTENSIONS = new Set([ @@ -2183,11 +2188,9 @@ export const createSearchIndex = ( return null } } - /** Bound filesystem work so a large vault cannot open every file at once. */ - const REBUILD_IO_CONCURRENCY = 16 const nonMarkdownFileSizeResults = await mapWithConcurrency({ items: allNonMdFiles, - concurrency: REBUILD_IO_CONCURRENCY, + concurrency: INDEXING_FILE_READ_CONCURRENCY, mapper: readFileSize, }) const nonMarkdownFileSizes = nonMarkdownFileSizeResults.filter((entry) => entry !== null) @@ -2232,7 +2235,7 @@ export const createSearchIndex = ( } const canvasContentResults = await mapWithConcurrency({ items: canvasFiles, - concurrency: REBUILD_IO_CONCURRENCY, + concurrency: INDEXING_FILE_READ_CONCURRENCY, mapper: (file) => readRebuildFileContent({ file, sourceKind: "canvas file" }), }) const canvasContents = canvasContentResults.filter((entry) => entry !== null) @@ -2279,14 +2282,14 @@ export const createSearchIndex = ( // Read text files for content indexing (raw UTF-8). const textFileContentResults = await mapWithConcurrency({ items: textFiles, - concurrency: REBUILD_IO_CONCURRENCY, + concurrency: INDEXING_FILE_READ_CONCURRENCY, mapper: (file) => readRebuildFileContent({ file, sourceKind: "text file" }), }) const textFileContents = textFileContentResults.filter((entry) => entry !== null) const noteContentResults = await mapWithConcurrency({ items: markdownFiles, - concurrency: REBUILD_IO_CONCURRENCY, + concurrency: INDEXING_FILE_READ_CONCURRENCY, mapper: (file) => readRebuildFileContent({ file, sourceKind: "note" }), }) const noteContents = noteContentResults.filter((entry) => entry !== null) From ea411a9dc89c81711665d95aaefb3d387ab677be Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Fri, 16 Oct 2026 14:53:05 -0400 Subject: [PATCH 09/34] fix(search): check every folder on a held path so a folder case-rename does not re-add the old path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On a case-insensitive filesystem, a held `notes/x.md` still lists and stats after its folder becomes `Notes/`, so the startup batch re-indexed the old path beside the new one. The presence check now requires each folder from the vault root down to list the next name exactly, reusing the batch's cached listings. Also corrects three comments: which failures return "failed" (a non-md file's content failure stays "file-indexed"), what content search keeps when a file's content cannot be re-read, and how long the watcher holds. Ship-Check: bug-check · claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- .../search/__tests__/file-watcher.test.ts | 42 +++++++++++++++++++ src/vault-mcp/search/file-watcher.ts | 41 +++++++++++------- 2 files changed, 68 insertions(+), 15 deletions(-) diff --git a/src/vault-mcp/search/__tests__/file-watcher.test.ts b/src/vault-mcp/search/__tests__/file-watcher.test.ts index 14c9e30ab..442426653 100644 --- a/src/vault-mcp/search/__tests__/file-watcher.test.ts +++ b/src/vault-mcp/search/__tests__/file-watcher.test.ts @@ -1940,6 +1940,48 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { expect(notePaths()).toEqual([{ path: "Note.md" }]) }) + it("does not index the old path again after a case-only rename of its folder", async () => { + const { testVault, notePaths, fire, startup, finishRebuild } = await createControlledStartup({ + seedNotes: { "notes/x.md": "folderjade\n" }, + }) + const oldFolder = join(testVault, "notes") + const renamedFolder = join(testVault, "Notes") + await mkdir(renamedFolder) + await writeFile(join(renamedFolder, "x.md"), "folderjade\n") + const actualFs = await vi.importActual("node:fs/promises") + // A case-insensitive filesystem answers for the old folder spelling too; + // only the vault root's listing holds the new one. + const asFoundOnDisk = (requestedPath: RequestedPath): RequestedPath | string => { + if (typeof requestedPath !== "string") return requestedPath + if (requestedPath !== oldFolder && !requestedPath.startsWith(`${oldFolder}/`)) { + return requestedPath + } + return `${renamedFolder}${requestedPath.slice(oldFolder.length)}` + } + vi.mocked(readdir).mockImplementation(async (requestedPath, options) => { + return actualFs.readdir(asFoundOnDisk(requestedPath), options) + }) + vi.mocked(stat).mockImplementation(async (requestedPath, options) => { + return actualFs.stat(asFoundOnDisk(requestedPath), options) + }) + vi.mocked(readFile).mockImplementation(async (requestedPath, options) => { + return actualFs.readFile(asFoundOnDisk(requestedPath), options) + }) + onTestFinished(() => { + vi.mocked(readdir).mockRestore() + vi.mocked(stat).mockRestore() + vi.mocked(readFile).mockRestore() + }) + expect(notePaths()).toEqual([{ path: "notes/x.md" }]) + + await fire("unlink", "notes/x.md") + await fire("add", "Notes/x.md") + finishRebuild() + await startup + + expect(notePaths()).toEqual([{ path: "Notes/x.md" }]) + }) + it("removes a held note whose path is now a folder", async () => { const { testVault, notePaths, fire, startup, finishRebuild } = await createControlledStartup({ seedNotes: { "shape.md": "shapeopal\n" }, diff --git a/src/vault-mcp/search/file-watcher.ts b/src/vault-mcp/search/file-watcher.ts index adbe4d11f..7078ce8a3 100644 --- a/src/vault-mcp/search/file-watcher.ts +++ b/src/vault-mcp/search/file-watcher.ts @@ -7,7 +7,7 @@ import { watch } from "chokidar" import { DateTime } from "luxon" import { readFile, readdir, stat } from "node:fs/promises" import type { Stats } from "node:fs" -import { basename, dirname, extname, join, relative, resolve as resolvePath } from "node:path" +import { extname, join, relative, resolve as resolvePath, sep } from "node:path" import { INDEXABLE_TEXT_EXTENSIONS, INDEXING_FILE_READ_CONCURRENCY } from "./search-index.js" import type { SearchIndex } from "./search-index.js" import { extractPdfText } from "../obsidian-markdown/pdf.js" @@ -73,7 +73,9 @@ type ChangeIndexResult = // A newer event or a removal for the same path arrived during the read, so // this read's content was dropped. | { status: "superseded" } - // A read, stat, extraction or index write failed; already logged. + // A note's read, stat or index write failed, or a non-md file's stat or + // asset-table write did; already logged. A non-md file whose content alone + // could not be read, extracted or indexed is still "file-indexed". | { status: "failed" } /** Whether a held path names a file on disk. "unknown" leaves the path to its @@ -94,9 +96,9 @@ type HoldingWatcher = Readonly<{ /** * Watches the vault and mirrors every file add/change/delete into the search - * index, holding every change from the moment chokidar starts until - * applyHeldChanges runs. The watcher then runs for the process lifetime — - * there is no shutdown path. + * index, holding every change from the moment chokidar starts until the batch + * applyHeldChanges starts has finished. The watcher then runs for the process + * lifetime — there is no shutdown path. */ const startHoldingWatcher = ( vaultPath: string, @@ -229,7 +231,7 @@ const startHoldingWatcher = ( }) } catch (error) { // Warn, as the rebuild does for an unreadable file: the asset-table row - // above is indexed, so only the file's content search is missing. + // above is indexed, and content search keeps what it had for the file. // describeErrorForLog keeps the cause, because a PDF reader that // failed to start arrives wrapped. logger.warn("file content indexing failed", { @@ -676,21 +678,30 @@ const startHoldingWatcher = ( } /** - * Decides whether a held path still names a file. The parent folder's listing - * must hold the exact name: on a case-insensitive filesystem, `stat` of - * `note.md` still succeeds after a rename to `Note.md`, and reading it would - * index the old name again. `listFolderOnce` returns the batch's shared - * listing of a folder, so each folder is read once per batch. + * Decides whether a held path still names a file. + * - Each folder from the vault root down must list the next name in the path + * exactly. On a case-insensitive filesystem, `stat` still succeeds for the + * old path after a case-only rename of the file (`note.md` to `Note.md`) or + * of a folder above it (`notes/` to `Notes/`), and reading it would index + * the old path again. + * - `listFolderOnce` returns the batch's shared listing of a folder, so each + * folder is read once per batch. */ const checkHeldPathOnDisk = async ( filePath: string, listFolderOnce: (folderPath: string) => Promise, ): Promise => { - const folderListing = await listFolderOnce(dirname(filePath)) - const missingFromListing = - folderListing.status === "listed" && !folderListing.names.has(basename(filePath)) + const pathSegments = relative(vaultPath, filePath).split(sep) - if (folderListing.status === "missing" || missingFromListing) return "missing" + // Sequential, so the first folder that lacks the next name ends the check. + for (const [segmentIndex, segmentName] of pathSegments.entries()) { + const folderPath = join(vaultPath, ...pathSegments.slice(0, segmentIndex)) + const folderListing = await listFolderOnce(folderPath) + const missingFromListing = + folderListing.status === "listed" && !folderListing.names.has(segmentName) + + if (folderListing.status === "missing" || missingFromListing) return "missing" + } // A folder that could not be listed proves nothing either way, so the stat // below decides. From cdd42497fb4464c013fca54e4bf7cf362ba02ec8 Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Fri, 16 Oct 2026 14:55:41 -0400 Subject: [PATCH 10/34] fix(search): extract held PDFs at the rebuild's lower bound during the startup batch PDF text extraction is CPU-heavy, so the rebuild extracts 4 at a time; the startup batch now reads PDFs last at the same exported bound instead of 16 at a time with the other files. Co-Authored-By: Claude Opus 5.5 --- .../search/__tests__/file-watcher.test.ts | 29 +++++++++++++++++++ src/vault-mcp/search/file-watcher.ts | 22 +++++++++++--- src/vault-mcp/search/search-index.ts | 6 +++- 3 files changed, 52 insertions(+), 5 deletions(-) diff --git a/src/vault-mcp/search/__tests__/file-watcher.test.ts b/src/vault-mcp/search/__tests__/file-watcher.test.ts index 442426653..7bf5bdbbb 100644 --- a/src/vault-mcp/search/__tests__/file-watcher.test.ts +++ b/src/vault-mcp/search/__tests__/file-watcher.test.ts @@ -1865,6 +1865,35 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { expect(indexedPaths.toSorted()).toEqual(fileNames.toSorted()) }) + it("extracts at most 4 held PDFs at once", async () => { + const { testVault, database, fire, startup, finishRebuild } = await createControlledStartup() + const fileNames = Array.from({ length: 10 }, (_unused, pdfIndex) => `doc-${pdfIndex}.pdf`) + // Extraction is stubbed, so the bytes only need to exist. + for (const fileName of fileNames) { + await writeFile(join(testVault, fileName), "controlled PDF bytes") + } + const extractionCounts = { inFlight: 0, peak: 0 } + const extractionSpy = vi.mocked(extractPdfText).mockImplementation(async () => { + extractionCounts.inFlight += 1 + extractionCounts.peak = Math.max(extractionCounts.peak, extractionCounts.inFlight) + await new Promise((resolve) => setTimeout(resolve, 5)) + extractionCounts.inFlight -= 1 + return { text: "pdftopaz", totalPages: 1 } + }) + onTestFinished(() => extractionSpy.mockRestore()) + + for (const fileName of fileNames) { + await fire("add", fileName) + } + finishRebuild() + await startup + + expect(extractionCounts.peak).toBe(4) + expect(database.prepare("SELECT path FROM file_content ORDER BY path").all()).toEqual( + fileNames.toSorted().map((path) => ({ path })), + ) + }) + it("returns at the deadline and finishes the held changes in the background", async () => { const { testVault, search, fire, startup, finishRebuild } = await createControlledStartup({ heldChangesDeadline: 50, diff --git a/src/vault-mcp/search/file-watcher.ts b/src/vault-mcp/search/file-watcher.ts index 7078ce8a3..efe302645 100644 --- a/src/vault-mcp/search/file-watcher.ts +++ b/src/vault-mcp/search/file-watcher.ts @@ -8,7 +8,11 @@ import { DateTime } from "luxon" import { readFile, readdir, stat } from "node:fs/promises" import type { Stats } from "node:fs" import { extname, join, relative, resolve as resolvePath, sep } from "node:path" -import { INDEXABLE_TEXT_EXTENSIONS, INDEXING_FILE_READ_CONCURRENCY } from "./search-index.js" +import { + INDEXABLE_TEXT_EXTENSIONS, + INDEXING_FILE_READ_CONCURRENCY, + PDF_EXTRACTION_CONCURRENCY, +} from "./search-index.js" import type { SearchIndex } from "./search-index.js" import { extractPdfText } from "../obsidian-markdown/pdf.js" import { logger } from "../../logger.js" @@ -803,6 +807,9 @@ const startHoldingWatcher = ( const pathsToRead = presenceChecks .filter((check) => check.presence !== "missing") .map((check) => check.filePath) + const isPdf = (filePath: string): boolean => extname(filePath) === ".pdf" + const pdfPathsToRead = pathsToRead.filter(isPdf) + const otherPathsToRead = pathsToRead.filter((filePath) => !isPdf(filePath)) // Removals run before reads. upsertNote re-resolves only links whose target // is not indexed, so when a renamed note's old path is removed first, the @@ -812,18 +819,25 @@ const startHoldingWatcher = ( } await mapWithConcurrency({ - items: pathsToRead, + items: otherPathsToRead, concurrency: INDEXING_FILE_READ_CONCURRENCY, mapper: indexHeldPath, }) + // PDFs go last at the rebuild's lower bound, since each extraction is CPU-heavy. + await mapWithConcurrency({ + items: pdfPathsToRead, + concurrency: PDF_EXTRACTION_CONCURRENCY, + mapper: indexHeldPath, + }) } /** * Brings the index up to date with the held paths, in one batch, then goes live: * * 1. Each path is checked against the disk, and missing ones are removed. - * 2. The rest are read and indexed, INDEXING_FILE_READ_CONCURRENCY at a time; one - * that vanished by its read is removed. + * 2. The rest are read and indexed, INDEXING_FILE_READ_CONCURRENCY at a time + * (PDFs PDF_EXTRACTION_CONCURRENCY at a time, after the others); one that + * vanished by its read is removed. * 3. Events held while steps 1 and 2 ran are replayed live. * * Never rejects, and step 3 runs even if an earlier step threw: a watcher diff --git a/src/vault-mcp/search/search-index.ts b/src/vault-mcp/search/search-index.ts index 7691fb1fd..9013d22e3 100644 --- a/src/vault-mcp/search/search-index.ts +++ b/src/vault-mcp/search/search-index.ts @@ -44,6 +44,10 @@ import { hybridSearch } from "./hybrid-search.js" * at once. */ export const INDEXING_FILE_READ_CONCURRENCY = 16 +/** PDFs whose text is extracted at once while indexing. Extraction is + * CPU-heavy, so fewer run together than other file reads. */ +export const PDF_EXTRACTION_CONCURRENCY = 4 + /** Extensions whose content is indexed into file_content_fts for vault_search. * Excludes .canvas (own path with link extraction) and .md (notes_fts). */ export const INDEXABLE_TEXT_EXTENSIONS = new Set([ @@ -2274,7 +2278,7 @@ export const createSearchIndex = ( } const pdfResults = await mapWithConcurrency({ items: pdfFiles, - concurrency: 4, + concurrency: PDF_EXTRACTION_CONCURRENCY, mapper: extractPdfContent, }) const pdfContents = pdfResults.filter((entry) => entry !== null) From 00081dda38914aded2b0d6916483f41196511ba9 Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Fri, 16 Oct 2026 14:59:50 -0400 Subject: [PATCH 11/34] fix(search): hold back only the held PDFs whose text is extracted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Without file content indexing a PDF is only statted, as in the rebuild, so it keeps the general bound instead of the extraction bound. Ship-Check: bug-check · claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- src/vault-mcp/search/file-watcher.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/vault-mcp/search/file-watcher.ts b/src/vault-mcp/search/file-watcher.ts index efe302645..7db6e0166 100644 --- a/src/vault-mcp/search/file-watcher.ts +++ b/src/vault-mcp/search/file-watcher.ts @@ -807,9 +807,13 @@ const startHoldingWatcher = ( const pathsToRead = presenceChecks .filter((check) => check.presence !== "missing") .map((check) => check.filePath) - const isPdf = (filePath: string): boolean => extname(filePath) === ".pdf" - const pdfPathsToRead = pathsToRead.filter(isPdf) - const otherPathsToRead = pathsToRead.filter((filePath) => !isPdf(filePath)) + // indexChangedNonMdFile extracts a PDF's text only when file content + // indexing is on; otherwise a PDF is only statted and keeps the general bound. + const needsPdfExtraction = (filePath: string): boolean => { + return search.fileContentIndexingEnabled && extname(filePath) === ".pdf" + } + const pdfPathsToRead = pathsToRead.filter(needsPdfExtraction) + const otherPathsToRead = pathsToRead.filter((filePath) => !needsPdfExtraction(filePath)) // Removals run before reads. upsertNote re-resolves only links whose target // is not indexed, so when a renamed note's old path is removed first, the @@ -836,8 +840,8 @@ const startHoldingWatcher = ( * * 1. Each path is checked against the disk, and missing ones are removed. * 2. The rest are read and indexed, INDEXING_FILE_READ_CONCURRENCY at a time - * (PDFs PDF_EXTRACTION_CONCURRENCY at a time, after the others); one that - * vanished by its read is removed. + * (PDFs whose text is extracted PDF_EXTRACTION_CONCURRENCY at a time, after + * the others); one that vanished by its read is removed. * 3. Events held while steps 1 and 2 ran are replayed live. * * Never rejects, and step 3 runs even if an earlier step threw: a watcher From f0ae4b47982d0e13e04bfb0efb94f51c773689a3 Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Fri, 16 Oct 2026 15:16:46 -0400 Subject: [PATCH 12/34] fix(search): bound the startup replay's reads and close the watcher when the rebuild fails The changes held while the startup batch runs are now replayed with the batch's limits: deletions at once, then each changed path read 16 at a time (PDFs whose text is extracted 4 at a time). A failed startup rebuild now closes the watcher instead of leaving it holding every later change. Co-Authored-By: Claude Opus 5.5 --- ARCHITECTURE.md | 2 +- .../search/__tests__/file-watcher.test.ts | 56 ++++++++- src/vault-mcp/search/file-watcher.ts | 118 ++++++++++++------ 3 files changed, 135 insertions(+), 41 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 0b17c24e0..215d10294 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -503,7 +503,7 @@ When no embedder is configured (`EMBEDDING_ENABLED=false`), no vectors are index 1. chokidar finishes its initial scan, and the watcher holds every change it reports from then on. 2. `rebuildFromVault` runs. The watcher is the only other writer of the index, so nothing else writes while it reads. 3. One batch checks each held path against the disk, 16 at a time. Paths gone from disk are removed first, then the rest are read again. -4. The watcher goes live and replays, in arrival order, the changes reported while the batch ran. +4. The watcher goes live and replays the changes reported while the batch ran: deletions at once, then the changed paths read again with the batch's limits. The server starts accepting requests once the step 3 batch commits and step 4 starts its replay, or after 10 s if a Sync backlog makes the batch longer; the batch then finishes in the background. diff --git a/src/vault-mcp/search/__tests__/file-watcher.test.ts b/src/vault-mcp/search/__tests__/file-watcher.test.ts index 7bf5bdbbb..ca8b9c324 100644 --- a/src/vault-mcp/search/__tests__/file-watcher.test.ts +++ b/src/vault-mcp/search/__tests__/file-watcher.test.ts @@ -1494,6 +1494,7 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { testVault, search, database, + watcher, fire, notePaths, startup, @@ -1623,7 +1624,7 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { expect(linkFromSource()).toEqual([{ path: "folder/B.md", exists: true }]) }) - it("replays events from during the batch in arrival order, so the later state wins", async () => { + it("replays events from during the batch, so the latest state on disk wins", async () => { const { testVault, search, fire, startup, finishRebuild } = await createControlledStartup({ seedNotes: { "P.md": "oldamber\n", "Q.md": "quartzrow\n" }, }) @@ -1865,6 +1866,53 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { expect(indexedPaths.toSorted()).toEqual(fileNames.toSorted()) }) + it("reads at most 16 files at once when replaying the changes from during the batch", async () => { + const { testVault, search, fire, startup, finishRebuild } = await createControlledStartup() + const infoSpy = spyOnLogger("info") + const batchPath = join(testVault, "batch.md") + await writeFile(batchPath, "batchonyx\n") + const fileNames = Array.from({ length: 40 }, (_unused, noteIndex) => `late-${noteIndex}.md`) + for (const fileName of fileNames) { + await writeFile(join(testVault, fileName), "latetopaz\n") + } + const readGate = await gateFirstRead(batchPath, "before-disk") + const pathsFor = (query: string) => { + return search.fullTextSearch({ query, limit: 50 }, logger).map((hit) => hit.path) + } + + await fire("add", "batch.md") + finishRebuild() + await readGate.reached + // Installed while the batch's one read waits, so only the replay's reads count. + const actualFs = await vi.importActual("node:fs/promises") + const readCounts = { inFlight: 0, peak: 0 } + vi.mocked(readFile).mockImplementation(async (requestedPath, options) => { + readCounts.inFlight += 1 + readCounts.peak = Math.max(readCounts.peak, readCounts.inFlight) + await new Promise((resolve) => setTimeout(resolve, 5)) + try { + return await actualFs.readFile(requestedPath, options) + } finally { + readCounts.inFlight -= 1 + } + }) + for (const fileName of fileNames) { + await fire("add", fileName) + } + readGate.release() + await startup + + expect(infoSpy).toHaveBeenCalledWith("applied the changes held during startup indexing", { + batchPaths: 1, + replayedEvents: 40, + }) + await vi.waitFor(() => { + expect(pathsFor("latetopaz").toSorted()).toEqual(fileNames.toSorted()) + }) + expect(pathsFor("batchonyx")).toEqual(["batch.md"]) + expect(readCounts.peak).toBe(16) + }) + it("extracts at most 4 held PDFs at once", async () => { const { testVault, database, fire, startup, finishRebuild } = await createControlledStartup() const fileNames = Array.from({ length: 10 }, (_unused, pdfIndex) => `doc-${pdfIndex}.pdf`) @@ -2147,12 +2195,14 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { expect(fileRows()).toEqual({ assets: [], contents: [] }) }) - it("rejects with the rebuild's error when the rebuild fails", async () => { - const { startup, failRebuild } = await createControlledStartup() + it("closes the watcher and rejects with the rebuild's error when the rebuild fails", async () => { + const { watcher, startup, failRebuild } = await createControlledStartup() + expect(watcher.closed).toBe(false) failRebuild(new Error("controlled rebuild failure")) await expect(startup).rejects.toThrow(new Error("controlled rebuild failure")) + expect(watcher.closed).toBe(true) }) it("lets a held change win over the rebuild's older read of the same note", async () => { diff --git a/src/vault-mcp/search/file-watcher.ts b/src/vault-mcp/search/file-watcher.ts index 7db6e0166..f73297a37 100644 --- a/src/vault-mcp/search/file-watcher.ts +++ b/src/vault-mcp/search/file-watcher.ts @@ -96,13 +96,15 @@ type HoldingWatcher = Readonly<{ * distinct paths that is. `applied` resolves once their index writes commit * and the watcher is live. */ applyHeldChanges: () => { heldPathCount: number; applied: Promise } + /** Stops watching. Used only when startup fails before the held changes apply. */ + close: () => Promise }> /** * Watches the vault and mirrors every file add/change/delete into the search - * index, holding every change from the moment chokidar starts until the batch - * applyHeldChanges starts has finished. The watcher then runs for the process - * lifetime — there is no shutdown path. + * index. Every change is held from the moment chokidar starts until the + * startup batch that applyHeldChanges runs has finished. After a successful + * startup the watcher runs for the process lifetime. */ const startHoldingWatcher = ( vaultPath: string, @@ -331,8 +333,7 @@ const startHoldingWatcher = ( const relativePath = relative(vaultPath, filePath) // Taken before the first await, so events take their tokens in the order - // their handlers start. goLiveAndReplay relies on this: a replayed event - // must take its token before a newer live event for the same path does. + // their handlers start, and the read that starts last is the one indexed. const eventToken = Symbol() currentEvents.set(relativePath, eventToken) @@ -733,7 +734,10 @@ const startHoldingWatcher = ( removeFromIndex(filePath) } - /** Re-reads one held path for the startup batch, without waiting for its embedding. */ + /** Re-reads one held path for the startup batch or its replay, without + * waiting for its embedding. A path that vanished by its read is removed: + * a live event for it that arrived during the read would have taken the + * token and made the read "superseded" instead, so nothing newer is lost. */ const indexHeldPath = async (filePath: string): Promise => { const indexResult = await indexChangedFile(filePath) @@ -751,25 +755,64 @@ const startHoldingWatcher = ( }) } - /** Ends holding and replays the events held while the batch ran, in arrival order. */ + /** Reads and indexes paths at the rebuild's bounds: PDFs whose text is + * extracted PDF_EXTRACTION_CONCURRENCY at a time, after the others, which go + * INDEXING_FILE_READ_CONCURRENCY at a time. */ + const indexPathsWithinBounds = async (filePaths: ReadonlyArray): Promise => { + // indexChangedNonMdFile extracts a PDF's text only when file content + // indexing is on; otherwise a PDF is only statted and keeps the general bound. + const needsPdfExtraction = (filePath: string): boolean => { + return search.fileContentIndexingEnabled && extname(filePath) === ".pdf" + } + const pdfPaths = filePaths.filter(needsPdfExtraction) + const otherPaths = filePaths.filter((filePath) => !needsPdfExtraction(filePath)) + + await mapWithConcurrency({ + items: otherPaths, + concurrency: INDEXING_FILE_READ_CONCURRENCY, + mapper: indexHeldPath, + }) + await mapWithConcurrency({ + items: pdfPaths, + concurrency: PDF_EXTRACTION_CONCURRENCY, + mapper: indexHeldPath, + }) + } + + /** Reads the paths changed while the startup batch ran. Nothing awaits it, + * so it logs its own failure. */ + const replayChangedPaths = async (changedPaths: ReadonlyArray): Promise => { + try { + await indexPathsWithinBounds(changedPaths) + } catch (error) { + logger.error("failed to replay the changes held during the startup batch", { + error: describeError(error), + }) + } + } + + /** Ends holding and replays the events held while the batch ran: deletions + * at once, then each changed path read again at the batch's bounds. Startup + * does not wait for those reads, as it does not wait for later live events. */ const goLiveAndReplay = (batchPathCount: number): void => { // Already live only when applyHeldChanges ran twice and the first batch // replayed everything. const eventsDuringBatch = watcherState.mode === "holding" ? watcherState.heldEvents : [] watcherState = { mode: "live" } - // Every replayed event starts in this one tick, before any newer live - // event for the same path, so that newer event takes the path's token and - // wins. indexChangedFile takes the token before its first await, and - // chokidar delivers a burst the same way. + // Deletions apply in this tick, before any live event arrives, so none can + // remove what a newer event for the same path indexes. The reads index each + // file as it is on disk; when a live event reads the same path, the read + // that starts last wins, and it sees the newer content. + const changedPaths = new Set() for (const heldEvent of eventsDuringBatch) { if (heldEvent.kind === "unlink") { handleDelete(heldEvent.filePath) continue } - // As with chokidar's own emit, nothing awaits it; it logs its own failures. - void handleChange(heldEvent.filePath) + changedPaths.add(heldEvent.filePath) } + void replayChangedPaths([...changedPaths]) logger.info("applied the changes held during startup indexing", { batchPaths: batchPathCount, replayedEvents: eventsDuringBatch.length, @@ -807,13 +850,6 @@ const startHoldingWatcher = ( const pathsToRead = presenceChecks .filter((check) => check.presence !== "missing") .map((check) => check.filePath) - // indexChangedNonMdFile extracts a PDF's text only when file content - // indexing is on; otherwise a PDF is only statted and keeps the general bound. - const needsPdfExtraction = (filePath: string): boolean => { - return search.fileContentIndexingEnabled && extname(filePath) === ".pdf" - } - const pdfPathsToRead = pathsToRead.filter(needsPdfExtraction) - const otherPathsToRead = pathsToRead.filter((filePath) => !needsPdfExtraction(filePath)) // Removals run before reads. upsertNote re-resolves only links whose target // is not indexed, so when a renamed note's old path is removed first, the @@ -822,27 +858,18 @@ const startHoldingWatcher = ( removeHeldPath(filePath) } - await mapWithConcurrency({ - items: otherPathsToRead, - concurrency: INDEXING_FILE_READ_CONCURRENCY, - mapper: indexHeldPath, - }) - // PDFs go last at the rebuild's lower bound, since each extraction is CPU-heavy. - await mapWithConcurrency({ - items: pdfPathsToRead, - concurrency: PDF_EXTRACTION_CONCURRENCY, - mapper: indexHeldPath, - }) + await indexPathsWithinBounds(pathsToRead) } /** * Brings the index up to date with the held paths, in one batch, then goes live: * * 1. Each path is checked against the disk, and missing ones are removed. - * 2. The rest are read and indexed, INDEXING_FILE_READ_CONCURRENCY at a time - * (PDFs whose text is extracted PDF_EXTRACTION_CONCURRENCY at a time, after - * the others); one that vanished by its read is removed. - * 3. Events held while steps 1 and 2 ran are replayed live. + * 2. The rest are read and indexed within the rebuild's bounds; one that + * vanished by its read is removed. + * 3. The watcher goes live, and the events held while steps 1 and 2 ran are + * replayed: deletions at once, then the changed paths read again in the + * background. * * Never rejects, and step 3 runs even if an earlier step threw: a watcher * left holding would drop every later change. @@ -890,7 +917,11 @@ const startHoldingWatcher = ( }) }) - return { ready, applyHeldChanges } + const close = async (): Promise => { + await watcher.close() + } + + return { ready, applyHeldChanges, close } } /** @@ -911,7 +942,20 @@ export const watchVaultAndBuildIndex = async ( const holdingWatcher = startHoldingWatcher(vaultPath, search, options) await holdingWatcher.ready - const rebuildResult = await search.rebuildFromVault({ vaultPath }, logger) + // A failed rebuild closes the watcher rather than leave it holding every + // later change; the caller sees the rebuild's error. + const rebuildIndexOrCloseWatcher = async (): Promise<{ + count: number + embedding: Promise + }> => { + try { + return await search.rebuildFromVault({ vaultPath }, logger) + } catch (error) { + await holdingWatcher.close() + throw error + } + } + const rebuildResult = await rebuildIndexOrCloseWatcher() // `applied` never rejects, so past the deadline it can finish unawaited. const { heldPathCount, applied } = holdingWatcher.applyHeldChanges() From b458e62e4db1424a63013c6be1dd5030dd032bda Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Fri, 16 Oct 2026 15:16:51 -0400 Subject: [PATCH 13/34] docs(deploy): list the startup tasks in their new order Co-Authored-By: Claude Opus 5.5 --- deploy/local/README.md | 6 ++++-- deploy/remote/README.md | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/deploy/local/README.md b/deploy/local/README.md index e26a8ad2b..e856e802e 100644 --- a/deploy/local/README.md +++ b/deploy/local/README.md @@ -192,8 +192,10 @@ The server runs these startup tasks on every boot: 1. Create memory template files if the memory folder doesn't exist (skipped when `MEMORY_ENABLED=false` or `READONLY_MODE=true`). -2. Rebuild the search index, including any new memory template files. -3. Start the file watcher. +2. Start the file watcher, which holds the changes it sees until step 4. +3. Rebuild the search index, including any new memory template files. +4. Apply the changes the watcher held, so search reflects notes added, edited + or deleted while the server started. Restarting the container re-runs this flow (useful when testing bootstrap behavior). The command is the same for both setup methods, diff --git a/deploy/remote/README.md b/deploy/remote/README.md index 2e433e087..cc0214c4f 100644 --- a/deploy/remote/README.md +++ b/deploy/remote/README.md @@ -437,8 +437,10 @@ starts. The server then runs these startup tasks: 1. Create memory template files if the memory folder doesn't exist (skipped when `MEMORY_ENABLED=false` or `READONLY_MODE=true`). -2. Rebuild the search index, including any new memory template files. -3. Start the file watcher. +2. Start the file watcher, which holds the changes it sees until step 4. +3. Rebuild the search index, including any new memory template files. +4. Apply the changes the watcher held, so search reflects notes added, edited + or deleted while the server started. Restarting the container re-runs this flow (useful when testing bootstrap behavior). The command is the same for every setup From 676874643669814ba7c3e4a937c1b2b41516d8f6 Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Fri, 16 Oct 2026 15:19:45 -0400 Subject: [PATCH 14/34] fix(search): keep the rebuild's error when closing the watcher fails Co-Authored-By: Claude Opus 5.5 --- src/vault-mcp/search/__tests__/file-watcher.test.ts | 13 +++++++++++++ src/vault-mcp/search/file-watcher.ts | 9 +++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/vault-mcp/search/__tests__/file-watcher.test.ts b/src/vault-mcp/search/__tests__/file-watcher.test.ts index ca8b9c324..91ee56c07 100644 --- a/src/vault-mcp/search/__tests__/file-watcher.test.ts +++ b/src/vault-mcp/search/__tests__/file-watcher.test.ts @@ -2205,6 +2205,19 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { expect(watcher.closed).toBe(true) }) + it("keeps the rebuild's error when closing the watcher also fails", async () => { + const { watcher, startup, failRebuild } = await createControlledStartup() + const errorSpy = spyOnLogger("error") + vi.spyOn(watcher, "close").mockRejectedValueOnce(new Error("controlled close failure")) + + failRebuild(new Error("controlled rebuild failure")) + + await expect(startup).rejects.toThrow(new Error("controlled rebuild failure")) + expect(errorSpy).toHaveBeenCalledWith("failed to close the file watcher", { + error: "[Error]: controlled close failure", + }) + }) + it("lets a held change win over the rebuild's older read of the same note", async () => { const readGates: Array>> = [] const { testVault, search, fire, startup } = await createControlledStartup({ diff --git a/src/vault-mcp/search/file-watcher.ts b/src/vault-mcp/search/file-watcher.ts index f73297a37..58578d58b 100644 --- a/src/vault-mcp/search/file-watcher.ts +++ b/src/vault-mcp/search/file-watcher.ts @@ -96,7 +96,8 @@ type HoldingWatcher = Readonly<{ * distinct paths that is. `applied` resolves once their index writes commit * and the watcher is live. */ applyHeldChanges: () => { heldPathCount: number; applied: Promise } - /** Stops watching. Used only when startup fails before the held changes apply. */ + /** Stops watching. Used only when startup fails before the held changes + * apply. Never rejects, so the startup failure stays the error reported. */ close: () => Promise }> @@ -918,7 +919,11 @@ const startHoldingWatcher = ( }) const close = async (): Promise => { - await watcher.close() + try { + await watcher.close() + } catch (error) { + logger.error("failed to close the file watcher", { error: describeError(error) }) + } } return { ready, applyHeldChanges, close } From d177e518c31dd5a04b09e9296248e103c4c75325 Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Fri, 16 Oct 2026 15:23:06 -0400 Subject: [PATCH 15/34] refactor(search): explain the rebuild's orphan sweep, second link clear and embedding contract Comments for the orphan-vector sweep, the second links clear, the re-check before memory entries, the returned embedding promise's rejection and oauth.db's folder; named booleans for the two source-version checks; the PDF reader reuses RebuildFileContent. No behaviour change. Co-Authored-By: Claude Opus 5.5 --- src/vault-mcp/search/search-index.ts | 47 ++++++++++++++-------------- src/vault-mcp/server.ts | 1 + 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/src/vault-mcp/search/search-index.ts b/src/vault-mcp/search/search-index.ts index 9013d22e3..f35c8c670 100644 --- a/src/vault-mcp/search/search-index.ts +++ b/src/vault-mcp/search/search-index.ts @@ -2092,12 +2092,17 @@ export const createSearchIndex = ( * the older content it read. * - The file watcher is the only caller that makes source writes at runtime, and * watchVaultAndBuildIndex holds it until this returns; a new runtime caller must go - * through that hold. */ + * through that hold. + * - The returned embedding promise rejects if the background pass fails. This function + * already logs that failure, so a caller may leave the promise unawaited. */ const rebuildFromVault = async ( params: { vaultPath: string }, logger: Logger, ): Promise<{ count: number; embedding: Promise }> => { const { vaultPath } = params + // Embedding work that outlived its source can leave vectors whose chunk or + // memory entry is gone. They would take nearest-neighbour slots from live + // vectors, so they are dropped before anything else. const orphanNoteVectorsRemoved = deleteOrphanNoteVectorsStmt?.run().changes ?? 0 const orphanFileVectorsRemoved = deleteOrphanFileVectorsStmt?.run().changes ?? 0 const orphanMemoryVectorsRemoved = deleteOrphanMemoryVectorsStmt?.run().changes ?? 0 @@ -2211,7 +2216,6 @@ export const createSearchIndex = ( }) : [] - // Read canvas files for content indexing + link extraction. const readRebuildFileContent = async (params: { file: RebuildFilePaths sourceKind: "canvas file" | "text file" | "note" @@ -2237,6 +2241,7 @@ export const createSearchIndex = ( return null } } + // Read canvas files for content indexing + link extraction. const canvasContentResults = await mapWithConcurrency({ items: canvasFiles, concurrency: INDEXING_FILE_READ_CONCURRENCY, @@ -2245,15 +2250,9 @@ export const createSearchIndex = ( const canvasContents = canvasContentResults.filter((entry) => entry !== null) // Extract PDF text with bounded concurrency (CPU-intensive pdfjs work). - const extractPdfContent = async (file: { - absolutePath: string - relativePath: string - }): Promise<{ - relativePath: string - content: string - modifiedAtMs: number - sizeBytes: number - } | null> => { + const extractPdfContent = async ( + file: RebuildFilePaths, + ): Promise => { try { const [buffer, fileStat] = await Promise.all([ readFile(file.absolutePath), @@ -2384,6 +2383,8 @@ export const createSearchIndex = ( const pathList = allPaths.map((row) => row.path) cachedCanvasNotePaths = pathList + // Cleared again because the first clear ran before the file reads + // above, so every link left in the table is written by this transaction. db.exec("DELETE FROM links") for (const note of noteContents) { if (skippedNotePaths.has(note.relativePath)) continue @@ -2510,13 +2511,12 @@ export const createSearchIndex = ( let entriesEmbedded = 0 let embedErrors = 0 for (const note of notesForEmbedding) { - if ( - !isCurrentSourceVersion( - { sourcePath: note.relativePath, sourceVersion: note.sourceVersion }, - logger, - ) + const noteVersionIsCurrent = isCurrentSourceVersion( + { sourcePath: note.relativePath, sourceVersion: note.sourceVersion }, + logger, ) - continue + + if (!noteVersionIsCurrent) continue try { chunksEmbedded += await embedAndStoreChunks( @@ -2529,6 +2529,8 @@ export const createSearchIndex = ( ) const memoryFile = memoryFileNameFromPath(note.relativePath) + // Checked again after the await above: a newer write to the + // note during the chunk embedding makes this entry pass obsolete. if ( memoryFile && isCurrentSourceVersion( @@ -2593,13 +2595,12 @@ export const createSearchIndex = ( let fileChunksEmbedded = 0 let fileEmbedErrors = 0 for (const file of filesForEmbedding) { - if ( - !isCurrentSourceVersion( - { sourcePath: file.path, sourceVersion: file.sourceVersion }, - logger, - ) + const fileVersionIsCurrent = isCurrentSourceVersion( + { sourcePath: file.path, sourceVersion: file.sourceVersion }, + logger, ) - continue + + if (!fileVersionIsCurrent) continue try { fileChunksEmbedded += await embedAndStoreFileChunks( diff --git a/src/vault-mcp/server.ts b/src/vault-mcp/server.ts index 367757127..0a17bb164 100644 --- a/src/vault-mcp/server.ts +++ b/src/vault-mcp/server.ts @@ -117,6 +117,7 @@ const startServer = async (): Promise => { // /data is the Docker volume where the container persists indexes and logs const dataDir = indexDbPath ? dirname(indexDbPath) : "/data" const searchDbPath = indexDbPath ?? `${dataDir}/search.db` + // oauth.db sits beside the search index, so INDEX_DB_PATH moves both. const oauthDbPath = `${dataDir}/oauth.db` const port = env.get("PORT").default("8000").asPortNumber() const host = env.get("HOST").default("0.0.0.0").asString() From 6c1746f763951fc590127cf29c973a1be319f217 Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Fri, 16 Oct 2026 15:38:32 -0400 Subject: [PATCH 16/34] fix(search): skip replay reads that a later deletion or event replaced, and keep a closed watcher closed Each change replayed after the startup batch now reserves its path's event token in arrival order, and a held or live deletion or a newer event cancels the read. On a case-insensitive filesystem the old spelling of a renamed note still reads, so an uncancelled read indexed the old path again. A watcher closed after a failed startup no longer re-adds paths or indexes from a rescan or retry that was already scheduled. ARCHITECTURE.md states the batch's read limits. Co-Authored-By: Claude Opus 5.5 --- ARCHITECTURE.md | 4 +- .../search/__tests__/file-watcher.test.ts | 153 +++++++++++++++++- src/vault-mcp/search/file-watcher.ts | 110 +++++++++---- 3 files changed, 226 insertions(+), 41 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 215d10294..9f79f9448 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -502,8 +502,8 @@ When no embedder is configured (`EMBEDDING_ENABLED=false`), no vectors are index 1. chokidar finishes its initial scan, and the watcher holds every change it reports from then on. 2. `rebuildFromVault` runs. The watcher is the only other writer of the index, so nothing else writes while it reads. -3. One batch checks each held path against the disk, 16 at a time. Paths gone from disk are removed first, then the rest are read again. -4. The watcher goes live and replays the changes reported while the batch ran: deletions at once, then the changed paths read again with the batch's limits. +3. One batch checks each held path against the disk, 16 at a time. Paths gone from disk are removed first, then the rest are read again, 16 at a time (PDFs whose text is extracted, 4 at a time). +4. The watcher goes live and replays the changes reported while the batch ran: deletions at once, then the changed paths read again with the same limits. A path deleted or changed again before its read starts is left to that later event. The server starts accepting requests once the step 3 batch commits and step 4 starts its replay, or after 10 s if a Sync backlog makes the batch longer; the batch then finishes in the background. diff --git a/src/vault-mcp/search/__tests__/file-watcher.test.ts b/src/vault-mcp/search/__tests__/file-watcher.test.ts index 91ee56c07..42c7627c6 100644 --- a/src/vault-mcp/search/__tests__/file-watcher.test.ts +++ b/src/vault-mcp/search/__tests__/file-watcher.test.ts @@ -16,6 +16,7 @@ import { join, resolve } from "node:path" import { tmpdir } from "node:os" import { watch, FSWatcher } from "chokidar" import Database from "better-sqlite3" +import { DateTime } from "luxon" import * as sqliteVec from "sqlite-vec" import { createSearchIndex } from "../search-index.js" import { extractPdfText } from "../../obsidian-markdown/pdf.js" @@ -1431,6 +1432,7 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { const createControlledStartup = async (setup?: { embedder?: Parameters[1] heldChangesDeadline?: number + newDirectoryRescanDelay?: number rebuild?: "stubbed" | "real" seedNotes?: Readonly> /** Runs against the empty vault before the watcher starts, e.g. to write @@ -1474,14 +1476,17 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { await watcher.close() }) const heldChangesDeadline = setup?.heldChangesDeadline - const startup = watchVaultAndBuildIndex( - testVault, - search, - heldChangesDeadline ? { heldChangesDeadline } : {}, - ) + const newDirectoryRescanDelay = setup?.newDirectoryRescanDelay + const startup = watchVaultAndBuildIndex(testVault, search, { + ...(heldChangesDeadline ? { heldChangesDeadline } : {}), + ...(newDirectoryRescanDelay ? { newDirectoryRescanDelay } : {}), + }) watcher.emit("ready") - const fire = async (event: "add" | "change" | "unlink", fileName: string): Promise => { + const fire = async ( + event: "add" | "change" | "unlink" | "addDir", + fileName: string, + ): Promise => { const handler = watcher.listeners(event)[0] if (!handler) throw new Error(`${event} handler was not registered`) @@ -1913,6 +1918,142 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { expect(readCounts.peak).toBe(16) }) + it("does not replay a change that a later held deletion of the path replaced", async () => { + const { testVault, notePaths, fire, startup, finishRebuild } = await createControlledStartup({ + seedNotes: { "note.md": "caseopal\n" }, + }) + const batchPath = join(testVault, "batch.md") + const oldPath = join(testVault, "note.md") + const renamedPath = join(testVault, "Note.md") + await writeFile(batchPath, "batchonyx\n") + await writeFile(renamedPath, "caseopal\n") + const readGate = await gateFirstRead(batchPath, "before-disk") + + await fire("add", "batch.md") + finishRebuild() + await readGate.reached + // Installed while the batch's one read waits. A case-insensitive filesystem + // answers for the old spelling too, so reading it would index note.md again. + const actualFs = await vi.importActual("node:fs/promises") + const asFoundOnDisk = (requestedPath: RequestedPath): RequestedPath | string => { + return requestedPath === oldPath ? renamedPath : requestedPath + } + vi.mocked(stat).mockImplementation(async (requestedPath, options) => { + return actualFs.stat(asFoundOnDisk(requestedPath), options) + }) + vi.mocked(readFile).mockImplementation(async (requestedPath, options) => { + return actualFs.readFile(asFoundOnDisk(requestedPath), options) + }) + onTestFinished(() => vi.mocked(stat).mockRestore()) + await fire("change", "note.md") + await fire("unlink", "note.md") + await fire("add", "Note.md") + readGate.release() + await startup + + await vi.waitFor(() => { + expect(notePaths()).toEqual([{ path: "Note.md" }, { path: "batch.md" }]) + }) + // The replay starts note.md's read before Note.md's, so it would have run by now. + expect(vi.mocked(readFile)).not.toHaveBeenCalledWith(oldPath, "utf8") + }) + + it("does not replay a change whose path a live deletion reached first", async () => { + const { testVault, notePaths, fire, startup, finishRebuild } = await createControlledStartup({ + seedNotes: { "note.md": "caseopal\n" }, + }) + const batchPath = join(testVault, "batch.md") + const oldPath = join(testVault, "note.md") + const renamedPath = join(testVault, "Note.md") + await writeFile(batchPath, "batchonyx\n") + await writeFile(renamedPath, "caseopal\n") + await writeFile(join(testVault, "z-last.md"), "lastjade\n") + const blockerNames = Array.from( + { length: 16 }, + (_unused, blockerIndex) => `blocker-${blockerIndex}.md`, + ) + for (const blockerName of blockerNames) { + await writeFile(join(testVault, blockerName), "blockertext\n") + } + const readGate = await gateFirstRead(batchPath, "before-disk") + + await fire("add", "batch.md") + finishRebuild() + await readGate.reached + // Installed while the batch's one read waits. The 16 blocker reads fill the + // replay's first group and wait for blockerGate, so note.md's read stays + // queued. A case-insensitive filesystem answers for the old spelling too. + const actualFs = await vi.importActual("node:fs/promises") + const blockerGate = Promise.withResolvers() + const blockerReads = { started: 0 } + const blockerPathPrefix = join(testVault, "blocker-") + const asFoundOnDisk = (requestedPath: RequestedPath): RequestedPath | string => { + return requestedPath === oldPath ? renamedPath : requestedPath + } + vi.mocked(stat).mockImplementation(async (requestedPath, options) => { + return actualFs.stat(asFoundOnDisk(requestedPath), options) + }) + vi.mocked(readFile).mockImplementation(async (requestedPath, options) => { + const isBlockerRead = + typeof requestedPath === "string" && requestedPath.startsWith(blockerPathPrefix) + + if (isBlockerRead) { + blockerReads.started += 1 + await blockerGate.promise + } + return actualFs.readFile(asFoundOnDisk(requestedPath), options) + }) + onTestFinished(() => vi.mocked(stat).mockRestore()) + for (const blockerName of blockerNames) { + await fire("change", blockerName) + } + await fire("change", "note.md") + await fire("change", "z-last.md") + readGate.release() + await startup + await vi.waitFor(() => { + expect(blockerReads.started).toBe(16) + }) + + await fire("unlink", "note.md") + await fire("add", "Note.md") + blockerGate.resolve(undefined) + + await vi.waitFor(() => { + expect(notePaths()).toContainEqual({ path: "z-last.md" }) + }) + // The replay starts note.md's read before z-last.md's, so it would have run by now. + expect(vi.mocked(readFile)).not.toHaveBeenCalledWith(oldPath, "utf8") + const expectedPaths = ["Note.md", "batch.md", "z-last.md", ...blockerNames].toSorted() + expect(notePaths()).toEqual(expectedPaths.map((path) => ({ path }))) + }) + + it("keeps the watcher closed when a folder rescan scheduled before a failed rebuild runs", async () => { + const { testVault, watcher, fire, startup, failRebuild } = await createControlledStartup({ + newDirectoryRescanDelay: 50, + }) + const debugSpy = spyOnLogger("debug") + const freshNote = join(testVault, "fresh", "note.md") + await mkdir(join(testVault, "fresh")) + await writeFile(freshNote, "freshtext\n") + // Older than the stability window, so the rescan watches and indexes it at once. + const settledSeconds = DateTime.now().minus({ minutes: 10 }).toUnixInteger() + await utimes(freshNote, settledSeconds, settledSeconds) + const addSpy = vi.spyOn(watcher, "add") + + await fire("addDir", "fresh") + failRebuild(new Error("controlled rebuild failure")) + await expect(startup).rejects.toThrow(new Error("controlled rebuild failure")) + await vi.waitFor(() => { + expect(debugSpy).toHaveBeenCalledWith("rescan indexing missed file", { + path: join("fresh", "note.md"), + }) + }) + + expect(addSpy).not.toHaveBeenCalled() + expect(watcher.closed).toBe(true) + }) + it("extracts at most 4 held PDFs at once", async () => { const { testVault, database, fire, startup, finishRebuild } = await createControlledStartup() const fileNames = Array.from({ length: 10 }, (_unused, pdfIndex) => `doc-${pdfIndex}.pdf`) diff --git a/src/vault-mcp/search/file-watcher.ts b/src/vault-mcp/search/file-watcher.ts index 58578d58b..12dd99fa6 100644 --- a/src/vault-mcp/search/file-watcher.ts +++ b/src/vault-mcp/search/file-watcher.ts @@ -64,8 +64,10 @@ type FileWatcherOptions = Readonly<{ type HeldEvent = Readonly<{ kind: "change" | "unlink"; filePath: string }> /** While holding, chokidar's events are recorded in arrival order instead of - * applied; once live, each event is applied as it arrives. */ -type WatcherState = { mode: "holding"; heldEvents: HeldEvent[] } | { mode: "live" } + * applied; once live, each event is applied as it arrives; once closed, they + * are ignored. */ +type WatcherState = + { mode: "holding"; heldEvents: HeldEvent[] } | { mode: "live" } | { mode: "closed" } /** What indexing one changed file did. A note's embedding is still running when * this returns, so whoever receives `note-indexed` reports its failure. */ @@ -96,8 +98,9 @@ type HoldingWatcher = Readonly<{ * distinct paths that is. `applied` resolves once their index writes commit * and the watcher is live. */ applyHeldChanges: () => { heldPathCount: number; applied: Promise } - /** Stops watching. Used only when startup fails before the held changes - * apply. Never rejects, so the startup failure stays the error reported. */ + /** Stops watching, including the rescans and retries still scheduled. Used + * only when startup fails before the held changes apply. Never rejects, so + * the startup failure stays the error reported. */ close: () => Promise }> @@ -119,14 +122,19 @@ const startHoldingWatcher = ( // for the content it embeds) against the path's current one. const pendingEmbeds = new Map>() - /** Each path's event token. indexChangedFile stores a fresh token before it - * reads the file and drops what it read if the token changed meanwhile, - * because a newer event or a removal replaced it. The token guards only the - * read; source versions guard the embedding that runs after it. */ + /** + * Each path's event token, which guards the path's read; source versions + * guard the embedding that runs after it. + * - indexChangedFile stores a fresh token before it reads the file, and drops + * what it read if a newer event or a removal replaced the token meanwhile. + * - goLiveAndReplay stores one for each path it will read again, so a later + * deletion or a newer event for the path cancels that read. + */ const currentEvents = new Map() - // Reassigned twice, so `let`: the startup batch swaps in a new held list for - // events that arrive while it runs, and going live ends holding. + // Reassigned, so `let`: the startup batch swaps in a new held list for + // events that arrive while it runs, going live ends holding, and a failed + // startup closes the watcher. let watcherState: WatcherState = { mode: "holding", heldEvents: [] } /** Indexes an added or modified non-md file: its asset-table row, plus its @@ -365,6 +373,7 @@ const startHoldingWatcher = ( /** chokidar's add and change listener. While the watcher holds, it records * the path for applyHeldChanges instead of indexing it. */ const handleChange = async (filePath: string): Promise => { + if (watcherState.mode === "closed") return if (watcherState.mode === "holding") { watcherState.heldEvents.push({ kind: "change", filePath }) logger.debug("held a change until startup indexing ends", { @@ -446,6 +455,7 @@ const startHoldingWatcher = ( /** chokidar's unlink listener. While the watcher holds, it records the path * for applyHeldChanges instead of removing it. */ const handleDelete = (filePath: string): void => { + if (watcherState.mode === "closed") return if (watcherState.mode === "holding") { watcherState.heldEvents.push({ kind: "unlink", filePath }) logger.debug("held a deletion until startup indexing ends", { @@ -493,6 +503,13 @@ const startHoldingWatcher = ( }, }) + /** Adds a path to chokidar unless the watcher is closed. A rescan or retry + * timer can fire after close, and chokidar's add reopens a closed watcher. */ + const watchPathUnlessClosed = (path: string): void => { + if (watcherState.mode === "closed") return + watcher.add(path) + } + /** * Polls a mid-write file until its writes settle, then indexes it. Serves * the new-directory rescan (rescanNewDirectory, below) for one case: a file @@ -506,6 +523,9 @@ const startHoldingWatcher = ( const scheduleUnstableFileRetry = (filePath: string): void => { const retryTimer = setTimeout(() => { const indexIfSettled = async (): Promise => { + // Ends the retries, which would otherwise poll a closed watcher's file forever. + if (watcherState.mode === "closed") return + const fileStat = await stat(filePath) // A file still being written is checked again after another window. @@ -515,12 +535,13 @@ const startHoldingWatcher = ( scheduleUnstableFileRetry(filePath) return } - watcher.add(filePath) + watchPathUnlessClosed(filePath) logger.debug("rescan indexing settled file after retry", { path: relative(vaultPath, filePath), }) await handleChange(filePath) } + // A timer callback cannot await, so the retry runs detached and logs its own failure. indexIfSettled().catch((err) => { logger.debug("rescan retry skipped unreadable file", { path: relative(vaultPath, filePath), @@ -595,7 +616,7 @@ const startHoldingWatcher = ( // A subdirectory chokidar never saw would stay unwatched forever — // watcher.add() registers its watches. Its contents are already covered // by this recursive listing, so the add's suppressed events don't matter. - watcher.add(fullPath) + watchPathUnlessClosed(fullPath) logger.debug("rescan registered missed directory", { path: relativePath, }) @@ -614,7 +635,7 @@ const startHoldingWatcher = ( // emitting events, which would make a later pass skip them as // already tracked. await rescanNewDirectory(fullPath, visitedRealPaths) - watcher.add(fullPath) + watchPathUnlessClosed(fullPath) logger.debug("rescan registered missed symlinked directory", { path: relativePath, }) @@ -637,7 +658,7 @@ const startHoldingWatcher = ( // Register the file with chokidar too — indexed but untracked, its // later deletion would emit no unlink, leaving a ghost index entry. - watcher.add(fullPath) + watchPathUnlessClosed(fullPath) logger.debug("rescan indexing missed file", { path: relativePath }) await handleChange(fullPath) } catch (err) { @@ -654,6 +675,7 @@ const startHoldingWatcher = ( * reads them. Wired to chokidar's addDir event below. */ const scheduleNewDirectoryRescan = (dirPath: string): void => { const rescanTimer = setTimeout(() => { + // A timer callback cannot await, so the rescan runs detached and logs its own failure. rescanNewDirectory(dirPath, new Set()).catch((err) => { logger.error("failed to rescan new directory", { path: relative(vaultPath, dirPath), @@ -759,7 +781,11 @@ const startHoldingWatcher = ( /** Reads and indexes paths at the rebuild's bounds: PDFs whose text is * extracted PDF_EXTRACTION_CONCURRENCY at a time, after the others, which go * INDEXING_FILE_READ_CONCURRENCY at a time. */ - const indexPathsWithinBounds = async (filePaths: ReadonlyArray): Promise => { + const indexPathsWithinBounds = async (params: { + filePaths: ReadonlyArray + indexPath: (filePath: string) => Promise + }): Promise => { + const { filePaths, indexPath } = params // indexChangedNonMdFile extracts a PDF's text only when file content // indexing is on; otherwise a PDF is only statted and keeps the general bound. const needsPdfExtraction = (filePath: string): boolean => { @@ -771,20 +797,34 @@ const startHoldingWatcher = ( await mapWithConcurrency({ items: otherPaths, concurrency: INDEXING_FILE_READ_CONCURRENCY, - mapper: indexHeldPath, + mapper: indexPath, }) await mapWithConcurrency({ items: pdfPaths, concurrency: PDF_EXTRACTION_CONCURRENCY, - mapper: indexHeldPath, + mapper: indexPath, }) } - /** Reads the paths changed while the startup batch ran. Nothing awaits it, - * so it logs its own failure. */ - const replayChangedPaths = async (changedPaths: ReadonlyArray): Promise => { + /** Reads again each path changed while the startup batch ran, unless a later + * deletion or a newer event replaced the token goLiveAndReplay reserved for + * it. Nothing awaits it, so it logs its own failure. */ + const replayChangedPaths = async (replayTokens: ReadonlyMap): Promise => { + const indexPathIfStillReserved = async (filePath: string): Promise => { + const reservationIsCurrent = + currentEvents.get(relative(vaultPath, filePath)) === replayTokens.get(filePath) + + // On a case-insensitive filesystem the old spelling of a renamed file + // still reads, so a cancelled read would index the old path again. + if (!reservationIsCurrent) return + await indexHeldPath(filePath) + } + try { - await indexPathsWithinBounds(changedPaths) + await indexPathsWithinBounds({ + filePaths: [...replayTokens.keys()], + indexPath: indexPathIfStillReserved, + }) } catch (error) { logger.error("failed to replay the changes held during the startup batch", { error: describeError(error), @@ -796,24 +836,27 @@ const startHoldingWatcher = ( * at once, then each changed path read again at the batch's bounds. Startup * does not wait for those reads, as it does not wait for later live events. */ const goLiveAndReplay = (batchPathCount: number): void => { - // Already live only when applyHeldChanges ran twice and the first batch - // replayed everything. - const eventsDuringBatch = watcherState.mode === "holding" ? watcherState.heldEvents : [] + // The watcher is already live only when applyHeldChanges ran twice and the + // first batch replayed everything, and it is closed only when startup failed. + if (watcherState.mode !== "holding") return + + const eventsDuringBatch = watcherState.heldEvents watcherState = { mode: "live" } - // Deletions apply in this tick, before any live event arrives, so none can - // remove what a newer event for the same path indexes. The reads index each - // file as it is on disk; when a live event reads the same path, the read - // that starts last wins, and it sees the newer content. - const changedPaths = new Set() + // This loop runs in arrival order, in one tick, before any live event + // arrives. Each change reserves a token for its path's read and each + // deletion drops it, so a later deletion or newer event cancels the read. + const replayTokens = new Map() for (const heldEvent of eventsDuringBatch) { if (heldEvent.kind === "unlink") { handleDelete(heldEvent.filePath) continue } - changedPaths.add(heldEvent.filePath) + const replayToken = Symbol() + currentEvents.set(relative(vaultPath, heldEvent.filePath), replayToken) + replayTokens.set(heldEvent.filePath, replayToken) } - void replayChangedPaths([...changedPaths]) + void replayChangedPaths(replayTokens) logger.info("applied the changes held during startup indexing", { batchPaths: batchPathCount, replayedEvents: eventsDuringBatch.length, @@ -859,7 +902,7 @@ const startHoldingWatcher = ( removeHeldPath(filePath) } - await indexPathsWithinBounds(pathsToRead) + await indexPathsWithinBounds({ filePaths: pathsToRead, indexPath: indexHeldPath }) } /** @@ -889,7 +932,7 @@ const startHoldingWatcher = ( /** Starts the startup batch over the distinct paths held so far. The batch * ignores each event's kind, because it checks every path against the disk. */ const applyHeldChanges = (): { heldPathCount: number; applied: Promise } => { - if (watcherState.mode === "live") return { heldPathCount: 0, applied: Promise.resolve() } + if (watcherState.mode !== "holding") return { heldPathCount: 0, applied: Promise.resolve() } const batchEvents = watcherState.heldEvents // Events arriving while the batch runs collect in a new list and are @@ -919,6 +962,7 @@ const startHoldingWatcher = ( }) const close = async (): Promise => { + watcherState = { mode: "closed" } try { await watcher.close() } catch (error) { From 86aee053844ac452db273d3f9fca96670483be08 Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Fri, 16 Oct 2026 18:24:04 -0400 Subject: [PATCH 17/34] fix(search): skip a closed watcher's scheduled folder rescan and pin the default startup deadline A rescan scheduled before a failed startup closed the watcher now returns at once instead of walking the folder, and a rescan already running indexes and watches nothing. Tests pin the closed-watcher guards and the 10 s default listen deadline, and two comments state that a replayed path is skipped when a later deletion or event replaced it. Co-Authored-By: Claude Opus 5.5 --- .../search/__tests__/file-watcher.test.ts | 71 ++++++++++++++++++- src/vault-mcp/search/file-watcher.ts | 18 +++-- 2 files changed, 84 insertions(+), 5 deletions(-) diff --git a/src/vault-mcp/search/__tests__/file-watcher.test.ts b/src/vault-mcp/search/__tests__/file-watcher.test.ts index 42c7627c6..e8e11087f 100644 --- a/src/vault-mcp/search/__tests__/file-watcher.test.ts +++ b/src/vault-mcp/search/__tests__/file-watcher.test.ts @@ -2028,32 +2028,101 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { expect(notePaths()).toEqual(expectedPaths.map((path) => ({ path }))) }) - it("keeps the watcher closed when a folder rescan scheduled before a failed rebuild runs", async () => { + it("skips a folder rescan that was scheduled before a failed rebuild closed the watcher", async () => { const { testVault, watcher, fire, startup, failRebuild } = await createControlledStartup({ newDirectoryRescanDelay: 50, }) const debugSpy = spyOnLogger("debug") + const freshFolder = join(testVault, "fresh") + await mkdir(freshFolder) + await writeFile(join(freshFolder, "note.md"), "freshtext\n") + const addSpy = vi.spyOn(watcher, "add") + + await fire("addDir", "fresh") + failRebuild(new Error("controlled rebuild failure")) + await expect(startup).rejects.toThrow(new Error("controlled rebuild failure")) + await vi.waitFor(() => { + expect(debugSpy).toHaveBeenCalledWith("rescan skipped, watcher closed", { path: "fresh" }) + }) + + expect(vi.mocked(readdirOrNull)).not.toHaveBeenCalledWith(freshFolder) + expect(addSpy).not.toHaveBeenCalled() + expect(watcher.closed).toBe(true) + }) + + it("indexes and watches nothing from a rescan that was running when the watcher closed", async () => { + const { testVault, watcher, fire, startup, failRebuild } = await createControlledStartup({ + newDirectoryRescanDelay: 20, + }) + const debugSpy = spyOnLogger("debug") const freshNote = join(testVault, "fresh", "note.md") await mkdir(join(testVault, "fresh")) await writeFile(freshNote, "freshtext\n") // Older than the stability window, so the rescan watches and indexes it at once. const settledSeconds = DateTime.now().minus({ minutes: 10 }).toUnixInteger() await utimes(freshNote, settledSeconds, settledSeconds) + const actualFs = await vi.importActual("node:fs/promises") + const statReached = Promise.withResolvers() + const statRelease = Promise.withResolvers() + // Holds the rescan's stat of the note until the watcher has closed. + vi.mocked(stat).mockImplementation(async (requestedPath, options) => { + if (requestedPath === freshNote) { + statReached.resolve(undefined) + await statRelease.promise + } + return actualFs.stat(requestedPath, options) + }) + onTestFinished(() => vi.mocked(stat).mockRestore()) const addSpy = vi.spyOn(watcher, "add") await fire("addDir", "fresh") + await statReached.promise failRebuild(new Error("controlled rebuild failure")) await expect(startup).rejects.toThrow(new Error("controlled rebuild failure")) + statRelease.resolve(undefined) await vi.waitFor(() => { expect(debugSpy).toHaveBeenCalledWith("rescan indexing missed file", { path: join("fresh", "note.md"), }) }) + // The rescan logs just before handleChange, which would read the note + // before its first await. + expect(vi.mocked(readFile)).not.toHaveBeenCalledWith(freshNote, "utf8") expect(addSpy).not.toHaveBeenCalled() expect(watcher.closed).toBe(true) }) + it("returns 10 seconds after the rebuild by default while held changes are still applied", async () => { + const { testVault, fire, startup, finishRebuild } = await createControlledStartup() + const infoSpy = spyOnLogger("info") + const slowPath = join(testVault, "slow.md") + await writeFile(slowPath, "slowtext\n") + const readGate = await gateFirstRead(slowPath, "before-disk") + onTestFinished(() => readGate.release()) + // Fake only setTimeout, so the batch's real filesystem work still runs. + vi.useFakeTimers({ toFake: ["setTimeout"] }) + onTestFinished(() => { + vi.useRealTimers() + }) + + await fire("change", "slow.md") + finishRebuild() + await readGate.reached + await vi.advanceTimersByTimeAsync(9_999) + + expect(infoSpy).not.toHaveBeenCalledWith( + "startup continues while the changes held during it are still applied", + { heldPaths: 1, heldChangesDeadline: 10_000 }, + ) + await vi.advanceTimersByTimeAsync(1) + expect(infoSpy).toHaveBeenCalledWith( + "startup continues while the changes held during it are still applied", + { heldPaths: 1, heldChangesDeadline: 10_000 }, + ) + await startup + }) + it("extracts at most 4 held PDFs at once", async () => { const { testVault, database, fire, startup, finishRebuild } = await createControlledStartup() const fileNames = Array.from({ length: 10 }, (_unused, pdfIndex) => `doc-${pdfIndex}.pdf`) diff --git a/src/vault-mcp/search/file-watcher.ts b/src/vault-mcp/search/file-watcher.ts index 12dd99fa6..b589f6b22 100644 --- a/src/vault-mcp/search/file-watcher.ts +++ b/src/vault-mcp/search/file-watcher.ts @@ -523,7 +523,8 @@ const startHoldingWatcher = ( const scheduleUnstableFileRetry = (filePath: string): void => { const retryTimer = setTimeout(() => { const indexIfSettled = async (): Promise => { - // Ends the retries, which would otherwise poll a closed watcher's file forever. + // Ends the retries, which would otherwise keep polling a closed + // watcher's file until it settles. if (watcherState.mode === "closed") return const fileStat = await stat(filePath) @@ -569,6 +570,14 @@ const startHoldingWatcher = ( dirPath: string, visitedRealPaths: Set, ): Promise => { + // A rescan scheduled before close still fires, but a closed watcher has + // nothing to reconcile, and close empties getWatched, so every entry would + // look missed. + if (watcherState.mode === "closed") { + logger.debug("rescan skipped, watcher closed", { path: relative(vaultPath, dirPath) }) + return + } + // readdirOrNull lists every descendant recursively, or returns null when the // directory is gone. It does not enter symlinked directories; the loop // below rescans those. @@ -833,8 +842,9 @@ const startHoldingWatcher = ( } /** Ends holding and replays the events held while the batch ran: deletions - * at once, then each changed path read again at the batch's bounds. Startup - * does not wait for those reads, as it does not wait for later live events. */ + * at once, then each changed path read again at the batch's bounds unless a + * later deletion or event replaced it. Startup does not wait for those + * reads, as it does not wait for later live events. */ const goLiveAndReplay = (batchPathCount: number): void => { // The watcher is already live only when applyHeldChanges ran twice and the // first batch replayed everything, and it is closed only when startup failed. @@ -913,7 +923,7 @@ const startHoldingWatcher = ( * vanished by its read is removed. * 3. The watcher goes live, and the events held while steps 1 and 2 ran are * replayed: deletions at once, then the changed paths read again in the - * background. + * background, except those a later deletion or event replaced. * * Never rejects, and step 3 runs even if an earlier step threw: a watcher * left holding would drop every later change. From 97a82da417a7e562fd211c7dcf1f04b2749ba704 Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Fri, 16 Oct 2026 18:41:58 -0400 Subject: [PATCH 18/34] test(search): assert the watcher's chokidar options and asset upsert through matchers The chokidar option tests assert the whole options object, so an interval passed without polling fails the match, and the vanished-file test pins the asset row's size. Both replaced readbacks of the mock's call log. Co-Authored-By: Claude Opus 5.5 --- .../search/__tests__/file-watcher.test.ts | 70 +++++++++++++------ 1 file changed, 48 insertions(+), 22 deletions(-) diff --git a/src/vault-mcp/search/__tests__/file-watcher.test.ts b/src/vault-mcp/search/__tests__/file-watcher.test.ts index e8e11087f..76a96061e 100644 --- a/src/vault-mcp/search/__tests__/file-watcher.test.ts +++ b/src/vault-mcp/search/__tests__/file-watcher.test.ts @@ -183,7 +183,10 @@ describe("file-watcher", REAL_WATCHER_RETRY, () => { // must not upsert (without the guard, reading .size off null would throw // inside the watcher callback). await waitFor(() => upsertNonMdFileSpy.mock.calls.length > 0) - expect(upsertNonMdFileSpy.mock.calls.map(([path]) => path)).toEqual(["kept.png"]) + expect(upsertNonMdFileSpy).toHaveBeenCalledExactlyOnceWith( + "kept.png", + Buffer.byteLength("here"), + ) }) it("ignores hidden directories", { timeout: 15000 }, async () => { @@ -1944,6 +1947,7 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { vi.mocked(readFile).mockImplementation(async (requestedPath, options) => { return actualFs.readFile(asFoundOnDisk(requestedPath), options) }) + // gateFirstRead's cleanup restores readFile. onTestFinished(() => vi.mocked(stat).mockRestore()) await fire("change", "note.md") await fire("unlink", "note.md") @@ -2003,6 +2007,7 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { } return actualFs.readFile(asFoundOnDisk(requestedPath), options) }) + // gateFirstRead's cleanup restores readFile. onTestFinished(() => vi.mocked(stat).mockRestore()) for (const blockerName of blockerNames) { await fire("change", blockerName) @@ -2609,45 +2614,66 @@ describe("file watcher — chokidar watch options", () => { return watcher } - /** Starts the watcher with the given FileWatcherOptions (chokidar.watch stubbed - * for this one test) and returns the options object chokidar.watch was called - * with. mockRestore (via onTestFinished) hands the real, spied watch back to - * the integration tests. */ - const chokidarOptionsFrom = async ( + /** Starts the watcher with the given FileWatcherOptions, with chokidar.watch + * stubbed for this one test. mockRestore (via onTestFinished) hands the + * real, spied watch back to the integration tests. */ + const startWatcherWithStubbedChokidar = async ( watcherOptions?: Parameters[2], - ): Promise> => { + ): Promise => { const watchMock = vi.mocked(watch) watchMock.mockReset() watchMock.mockImplementation(() => createFakeWatcher() as unknown as ReturnType) onTestFinished(() => watchMock.mockRestore()) await startWatcherWithoutRebuild("/vault", index, watcherOptions) - expect(watchMock).toHaveBeenCalledTimes(1) - return watchMock.mock.calls[0]?.[1] as Record } + // The options every watcher passes chokidar. The whole object is asserted, + // so an interval key passed without polling fails the match. + const sharedChokidarOptions = { + ignored: expect.any(Function), + persistent: true, + ignoreInitial: true, + } + const defaultWriteFinish = { stabilityThreshold: 2000, pollInterval: 100 } + it("defaults to native events (usePolling false) with no interval when unset", async () => { - const chokidarOptions = await chokidarOptionsFrom() - expect(chokidarOptions.usePolling).toBe(false) - expect("interval" in chokidarOptions).toBe(false) + await startWatcherWithStubbedChokidar() + + expect(vi.mocked(watch)).toHaveBeenCalledExactlyOnceWith("/vault", { + ...sharedChokidarOptions, + usePolling: false, + awaitWriteFinish: defaultWriteFinish, + }) }) it("omits interval when usePolling is false", async () => { - const chokidarOptions = await chokidarOptionsFrom({ usePolling: false }) - expect(chokidarOptions.usePolling).toBe(false) - expect("interval" in chokidarOptions).toBe(false) + await startWatcherWithStubbedChokidar({ usePolling: false }) + + expect(vi.mocked(watch)).toHaveBeenCalledExactlyOnceWith("/vault", { + ...sharedChokidarOptions, + usePolling: false, + awaitWriteFinish: defaultWriteFinish, + }) }) it("passes interval (300ms) only when usePolling is true", async () => { - const chokidarOptions = await chokidarOptionsFrom({ usePolling: true }) - expect(chokidarOptions.usePolling).toBe(true) - expect(chokidarOptions.interval).toBe(300) + await startWatcherWithStubbedChokidar({ usePolling: true }) + + expect(vi.mocked(watch)).toHaveBeenCalledExactlyOnceWith("/vault", { + ...sharedChokidarOptions, + usePolling: true, + interval: 300, + awaitWriteFinish: defaultWriteFinish, + }) }) it("defaults awaitWriteFinish to 2000ms stability and 100ms poll interval", async () => { - const chokidarOptions = await chokidarOptionsFrom() - expect(chokidarOptions.awaitWriteFinish).toEqual({ - stabilityThreshold: 2000, - pollInterval: 100, + await startWatcherWithStubbedChokidar() + + expect(vi.mocked(watch)).toHaveBeenCalledExactlyOnceWith("/vault", { + ...sharedChokidarOptions, + usePolling: false, + awaitWriteFinish: { stabilityThreshold: 2000, pollInterval: 100 }, }) }) }) From 955b8d325bf04b5e96e255a72bef883cf54920aa Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Fri, 16 Oct 2026 19:01:44 -0400 Subject: [PATCH 19/34] fix(search): log a failure to start the startup replay instead of rejecting Going live after the startup batch gets its own catch, because nothing awaits the batch once the listen deadline has passed. The comment on the early return names it as a guard against a second applyHeldChanges call, and the server's startup line names its count as the rebuild's. Co-Authored-By: Claude Opus 5.5 --- .../search/__tests__/file-watcher.test.ts | 25 +++++++++++++++++++ src/vault-mcp/search/file-watcher.ts | 15 ++++++++--- src/vault-mcp/server.ts | 2 +- 3 files changed, 38 insertions(+), 4 deletions(-) diff --git a/src/vault-mcp/search/__tests__/file-watcher.test.ts b/src/vault-mcp/search/__tests__/file-watcher.test.ts index 76a96061e..892dea359 100644 --- a/src/vault-mcp/search/__tests__/file-watcher.test.ts +++ b/src/vault-mcp/search/__tests__/file-watcher.test.ts @@ -1755,6 +1755,31 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { ]) }) + it("logs a failure to start the replay instead of rejecting startup", async () => { + const { testVault, search, fire, startup, finishRebuild } = await createControlledStartup() + const errorSpy = spyOnLogger("error") + // The replay's summary line is the last step of going live, so a throw + // there escapes it after the watcher is live. + spyOnLogger("info").mockImplementation((message) => { + if (message === "applied the changes held during startup indexing") { + throw new Error("controlled replay failure") + } + }) + + finishRebuild() + await expect(startup).resolves.toEqual({ count: 0, embedding: expect.any(Promise) }) + expect(errorSpy).toHaveBeenCalledExactlyOnceWith( + "failed to start replaying the changes held during the startup batch", + { error: "[Error]: controlled replay failure" }, + ) + + await writeFile(join(testVault, "live.md"), "livetopaz\n") + await fire("add", "live.md") + expect(search.fullTextSearch({ query: "livetopaz" }, logger).map((hit) => hit.path)).toEqual([ + "live.md", + ]) + }) + it("reports a note embedding that fails during the batch once, with no unhandled rejection", async () => { const vector = new Float32Array(384).fill(0.1) const embedder = { diff --git a/src/vault-mcp/search/file-watcher.ts b/src/vault-mcp/search/file-watcher.ts index b589f6b22..e96e42254 100644 --- a/src/vault-mcp/search/file-watcher.ts +++ b/src/vault-mcp/search/file-watcher.ts @@ -846,8 +846,9 @@ const startHoldingWatcher = ( * later deletion or event replaced it. Startup does not wait for those * reads, as it does not wait for later live events. */ const goLiveAndReplay = (batchPathCount: number): void => { - // The watcher is already live only when applyHeldChanges ran twice and the - // first batch replayed everything, and it is closed only when startup failed. + // Unreachable today, because applyHeldChanges runs once per startup and the + // watcher closes only when the rebuild fails, before that call. A second + // call would start a second batch, and this keeps it from going live twice. if (watcherState.mode !== "holding") return const eventsDuringBatch = watcherState.heldEvents @@ -936,7 +937,15 @@ const startHoldingWatcher = ( error: describeError(error), }) } - goLiveAndReplay(batchPaths.length) + // Its own catch, because nothing awaits `applied` once the deadline has + // passed, so a throw here would be an unhandled rejection. + try { + goLiveAndReplay(batchPaths.length) + } catch (error) { + logger.error("failed to start replaying the changes held during the startup batch", { + error: describeError(error), + }) + } } /** Starts the startup batch over the distinct paths held so far. The batch diff --git a/src/vault-mcp/server.ts b/src/vault-mcp/server.ts index 0a17bb164..d8e09ce0e 100644 --- a/src/vault-mcp/server.ts +++ b/src/vault-mcp/server.ts @@ -155,7 +155,7 @@ const startServer = async (): Promise => { const { count } = await watchVaultAndBuildIndex(vaultPath, search, { usePolling: config.windowsBindMount, }) - logger.info("initial index built", { count }) + logger.info("initial index built", { notesIndexedByRebuild: count }) const oauthProvider = createOAuthProvider({ authToken, From b009c80bd5c559e354f74edfd5e1fe8490a2d0b5 Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Fri, 16 Oct 2026 19:09:35 -0400 Subject: [PATCH 20/34] docs(search): state why the rebuild clears links a second time Co-Authored-By: Claude Opus 5.5 --- src/vault-mcp/search/search-index.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/vault-mcp/search/search-index.ts b/src/vault-mcp/search/search-index.ts index f35c8c670..1f17af5d3 100644 --- a/src/vault-mcp/search/search-index.ts +++ b/src/vault-mcp/search/search-index.ts @@ -2383,8 +2383,9 @@ export const createSearchIndex = ( const pathList = allPaths.map((row) => row.path) cachedCanvasNotePaths = pathList - // Cleared again because the first clear ran before the file reads - // above, so every link left in the table is written by this transaction. + // No source write may run during a rebuild, so this second clear + // normally finds nothing. It stays so that a link written during the + // file reads above, against that rule, cannot outlive the rebuild. db.exec("DELETE FROM links") for (const note of noteContents) { if (skippedNotePaths.has(note.relativePath)) continue From 98c753a6759f5712ea8673fec9cc9f58cf4af251 Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Fri, 16 Oct 2026 19:22:33 -0400 Subject: [PATCH 21/34] test(search): pin that a held change after a held deletion is replayed Co-Authored-By: Claude Opus 5.5 --- .../search/__tests__/file-watcher.test.ts | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/vault-mcp/search/__tests__/file-watcher.test.ts b/src/vault-mcp/search/__tests__/file-watcher.test.ts index 892dea359..2b59a2a7b 100644 --- a/src/vault-mcp/search/__tests__/file-watcher.test.ts +++ b/src/vault-mcp/search/__tests__/file-watcher.test.ts @@ -1946,6 +1946,32 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { expect(readCounts.peak).toBe(16) }) + it("replays a change that follows a held deletion of the same path", async () => { + const { testVault, search, fire, startup, finishRebuild } = await createControlledStartup({ + seedNotes: { "note.md": "oldtext\n" }, + }) + const batchPath = join(testVault, "batch.md") + await writeFile(batchPath, "batchonyx\n") + const readGate = await gateFirstRead(batchPath, "before-disk") + const pathsFor = (query: string) => { + return search.fullTextSearch({ query }, logger).map((hit) => hit.path) + } + + await fire("add", "batch.md") + finishRebuild() + await readGate.reached + await writeFile(join(testVault, "note.md"), "newopal\n") + await fire("unlink", "note.md") + await fire("change", "note.md") + readGate.release() + await startup + + await vi.waitFor(() => { + expect(pathsFor("newopal")).toEqual(["note.md"]) + }) + expect(pathsFor("oldtext")).toEqual([]) + }) + it("does not replay a change that a later held deletion of the path replaced", async () => { const { testVault, notePaths, fire, startup, finishRebuild } = await createControlledStartup({ seedNotes: { "note.md": "caseopal\n" }, From 14b7072cff5f0639ffce34e91bfe7ac957af9804 Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Fri, 16 Oct 2026 21:21:22 -0400 Subject: [PATCH 22/34] refactor(search): take named params and the caller's logger in watchVaultAndBuildIndex watchVaultAndBuildIndex now follows the data layer's (params, logger) pattern: { vaultPath, search, watcherOptions } plus the logger, which the watcher uses instead of importing the root logger. server.ts passes its logger. AGENTS.md now says background modules take the logger through their entry point, and startup wiring passes the root logger. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 5 +- .../search/__tests__/file-watcher.test.ts | 207 +++++++++++------- src/vault-mcp/search/file-watcher.ts | 37 ++-- src/vault-mcp/server.ts | 7 +- 4 files changed, 160 insertions(+), 96 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 4ba717abd..2074301fd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -343,8 +343,9 @@ root logger (src/logger.ts) - Data-layer functions (`vault-filesystem`, `vault-patcher`, `note-mover`, `memory-store`, `search-index`) take the logger as a **required** second argument (two-arg pattern: `(params, logger)`) -- Background callers (file-watcher, startup) use the root logger - directly — no request context available +- Background modules (`file-watcher`) take the logger the same way, + through their entry point. Startup wiring in `server.ts` has no + request context, so it passes the root logger **Two-arg `(params, logger)` pattern:** diff --git a/src/vault-mcp/search/__tests__/file-watcher.test.ts b/src/vault-mcp/search/__tests__/file-watcher.test.ts index 2b59a2a7b..e31d56f36 100644 --- a/src/vault-mcp/search/__tests__/file-watcher.test.ts +++ b/src/vault-mcp/search/__tests__/file-watcher.test.ts @@ -40,19 +40,23 @@ vi.mock("../../obsidian-markdown/pdf.js", { spy: true }) let vault: string let index: SearchIndex +type WatcherOptions = NonNullable[0]["watcherOptions"]> + /** Starts the watcher with the startup rebuild stubbed to find nothing, so the * only indexing a test sees comes from the watcher's own events and rescans, * including for files the test wrote before starting it. */ const startWatcherWithoutRebuild = async ( - vaultPath: string, - search: SearchIndex, - options?: Parameters[2], + target: Readonly<{ vaultPath: string; search: SearchIndex }>, + watcherOptions?: WatcherOptions, ): Promise => { - vi.spyOn(search, "rebuildFromVault").mockResolvedValue({ + vi.spyOn(target.search, "rebuildFromVault").mockResolvedValue({ count: 0, embedding: Promise.resolve(), }) - await watchVaultAndBuildIndex(vaultPath, search, options) + await watchVaultAndBuildIndex( + { ...target, ...(watcherOptions ? { watcherOptions } : {}) }, + logger, + ) } /** Poll until a condition is met, with timeout. */ @@ -92,10 +96,13 @@ describe("file-watcher", REAL_WATCHER_RETRY, () => { registerSharedVaultHooks() it("indexes a new .md file", { timeout: 15000 }, async () => { - await startWatcherWithoutRebuild(vault, index, { - stabilityThreshold: 200, - pollInterval: 50, - }) + await startWatcherWithoutRebuild( + { vaultPath: vault, search: index }, + { + stabilityThreshold: 200, + pollInterval: 50, + }, + ) await writeFile(join(vault, "test.md"), "---\ntitle: Test\n---\n\nHello watcher\n", "utf8") @@ -105,10 +112,13 @@ describe("file-watcher", REAL_WATCHER_RETRY, () => { }) it("re-indexes a modified file", { timeout: 15000 }, async () => { - await startWatcherWithoutRebuild(vault, index, { - stabilityThreshold: 200, - pollInterval: 50, - }) + await startWatcherWithoutRebuild( + { vaultPath: vault, search: index }, + { + stabilityThreshold: 200, + pollInterval: 50, + }, + ) await writeFile(join(vault, "modify.md"), "original content\n", "utf8") await waitFor(() => index.fullTextSearch({ query: "original" }, logger).length > 0) @@ -121,10 +131,13 @@ describe("file-watcher", REAL_WATCHER_RETRY, () => { }) it("removes a deleted file from index", { timeout: 15000 }, async () => { - await startWatcherWithoutRebuild(vault, index, { - stabilityThreshold: 200, - pollInterval: 50, - }) + await startWatcherWithoutRebuild( + { vaultPath: vault, search: index }, + { + stabilityThreshold: 200, + pollInterval: 50, + }, + ) await writeFile(join(vault, "delete-me.md"), "ephemeral\n", "utf8") await waitFor(() => index.fullTextSearch({ query: "ephemeral" }, logger).length > 0) @@ -137,10 +150,13 @@ describe("file-watcher", REAL_WATCHER_RETRY, () => { }) it("ignores non-.md files", { timeout: 15000 }, async () => { - await startWatcherWithoutRebuild(vault, index, { - stabilityThreshold: 200, - pollInterval: 50, - }) + await startWatcherWithoutRebuild( + { vaultPath: vault, search: index }, + { + stabilityThreshold: 200, + pollInterval: 50, + }, + ) await writeFile(join(vault, "data.json"), '{"key": "value"}', "utf8") await writeFile(join(vault, "check.md"), "check file\n", "utf8") @@ -164,10 +180,13 @@ describe("file-watcher", REAL_WATCHER_RETRY, () => { vi.mocked(statOrNull).mockRestore() }) - await startWatcherWithoutRebuild(vault, index, { - stabilityThreshold: 200, - pollInterval: 50, - }) + await startWatcherWithoutRebuild( + { vaultPath: vault, search: index }, + { + stabilityThreshold: 200, + pollInterval: 50, + }, + ) await writeFile(join(vault, "vanished.png"), "gone", "utf8") await writeFile(join(vault, "kept.png"), "here", "utf8") @@ -192,10 +211,13 @@ describe("file-watcher", REAL_WATCHER_RETRY, () => { it("ignores hidden directories", { timeout: 15000 }, async () => { await mkdir(join(vault, ".obsidian"), { recursive: true }) - await startWatcherWithoutRebuild(vault, index, { - stabilityThreshold: 200, - pollInterval: 50, - }) + await startWatcherWithoutRebuild( + { vaultPath: vault, search: index }, + { + stabilityThreshold: 200, + pollInterval: 50, + }, + ) await writeFile(join(vault, ".obsidian/workspace.md"), "hidden content\n", "utf8") await writeFile(join(vault, "visible.md"), "visible content\n", "utf8") @@ -209,10 +231,13 @@ describe("file-watcher", REAL_WATCHER_RETRY, () => { it("indexes a symlinked .md file", { timeout: 15000 }, async () => { await writeFile(join(vault, "real.md"), "symlink watcher target\n", "utf8") - await startWatcherWithoutRebuild(vault, index, { - stabilityThreshold: 200, - pollInterval: 50, - }) + await startWatcherWithoutRebuild( + { vaultPath: vault, search: index }, + { + stabilityThreshold: 200, + pollInterval: 50, + }, + ) await symlink("real.md", join(vault, "linked.md")) @@ -238,10 +263,13 @@ describe("file-watcher", REAL_WATCHER_RETRY, () => { capturedVersion.resolve(sourceVersion) return sourceVersion }) - await startWatcherWithoutRebuild(vault, index, { - stabilityThreshold: 200, - pollInterval: 50, - }) + await startWatcherWithoutRebuild( + { vaultPath: vault, search: index }, + { + stabilityThreshold: 200, + pollInterval: 50, + }, + ) await writeFile( join(vault, "embed-test.md"), @@ -265,11 +293,14 @@ describe("file-watcher", REAL_WATCHER_RETRY, () => { "indexes a note atomically written into a brand-new directory", { timeout: 15000 }, async () => { - await startWatcherWithoutRebuild(vault, index, { - stabilityThreshold: 200, - pollInterval: 50, - newDirectoryRescanDelay: 500, - }) + await startWatcherWithoutRebuild( + { vaultPath: vault, search: index }, + { + stabilityThreshold: 200, + pollInterval: 50, + newDirectoryRescanDelay: 500, + }, + ) // vault_write_note's sequence — mkdir -p, stage a temp file, rename over // the target — is the pattern that races chokidar's new-directory scan. @@ -289,11 +320,14 @@ describe("file-watcher", REAL_WATCHER_RETRY, () => { // WSL2 bridge). It works on any filesystem, so this verifies the usePolling // option is wired through and indexing still happens under polling. it("indexes a new .md file when polling", { timeout: 15000 }, async () => { - await startWatcherWithoutRebuild(vault, index, { - stabilityThreshold: 200, - pollInterval: 50, - usePolling: true, - }) + await startWatcherWithoutRebuild( + { vaultPath: vault, search: index }, + { + stabilityThreshold: 200, + pollInterval: 50, + usePolling: true, + }, + ) await writeFile(join(vault, "polled.md"), "polled content\n", "utf8") @@ -312,10 +346,13 @@ describe("file-watcher — file content indexing", REAL_WATCHER_RETRY, () => { }) const upsertSpy = vi.spyOn(fileIndex, "upsertFileContent") - await startWatcherWithoutRebuild(vault, fileIndex, { - stabilityThreshold: 200, - pollInterval: 50, - }) + await startWatcherWithoutRebuild( + { vaultPath: vault, search: fileIndex }, + { + stabilityThreshold: 200, + pollInterval: 50, + }, + ) await writeFile(join(vault, "notes.txt"), "deployment checklist for production release", "utf8") @@ -334,10 +371,13 @@ describe("file-watcher — file content indexing", REAL_WATCHER_RETRY, () => { const upsertSpy = vi.spyOn(fileIndex, "upsertFileContent") const removeSpy = vi.spyOn(fileIndex, "removeFileContent") - await startWatcherWithoutRebuild(vault, fileIndex, { - stabilityThreshold: 200, - pollInterval: 50, - }) + await startWatcherWithoutRebuild( + { vaultPath: vault, search: fileIndex }, + { + stabilityThreshold: 200, + pollInterval: 50, + }, + ) await writeFile(join(vault, "temp.csv"), "id,name\n1,ephemeral-data-row", "utf8") await waitFor(() => upsertSpy.mock.calls.length > 0) @@ -356,10 +396,13 @@ describe("file-watcher — file content indexing", REAL_WATCHER_RETRY, () => { }) const upsertSpy = vi.spyOn(fileIndex, "upsertFileContent") - await startWatcherWithoutRebuild(vault, fileIndex, { - stabilityThreshold: 200, - pollInterval: 50, - }) + await startWatcherWithoutRebuild( + { vaultPath: vault, search: fileIndex }, + { + stabilityThreshold: 200, + pollInterval: 50, + }, + ) await writeFile(join(vault, "doc.pdf"), buildMinimalPdf()) await waitFor(() => upsertSpy.mock.calls.length > 0) @@ -383,10 +426,13 @@ describe("file-watcher — file content indexing", REAL_WATCHER_RETRY, () => { return sourceVersion }) - await startWatcherWithoutRebuild(vault, fileIndex, { - stabilityThreshold: 200, - pollInterval: 50, - }) + await startWatcherWithoutRebuild( + { vaultPath: vault, search: fileIndex }, + { + stabilityThreshold: 200, + pollInterval: 50, + }, + ) await writeFile(join(vault, "data.csv"), "id,name,value\n1,deploy,active\n", "utf8") @@ -416,7 +462,7 @@ describe("file watcher — obsolete events and embedding queues", () => { watchMock.mockRestore() await watcher.close() }) - const starting = startWatcherWithoutRebuild(testVault, search) + const starting = startWatcherWithoutRebuild({ vaultPath: testVault, search }) watcher.emit("ready") await starting @@ -1480,10 +1526,14 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { }) const heldChangesDeadline = setup?.heldChangesDeadline const newDirectoryRescanDelay = setup?.newDirectoryRescanDelay - const startup = watchVaultAndBuildIndex(testVault, search, { + const watcherOptions = { ...(heldChangesDeadline ? { heldChangesDeadline } : {}), ...(newDirectoryRescanDelay ? { newDirectoryRescanDelay } : {}), - }) + } + const startup = watchVaultAndBuildIndex( + { vaultPath: testVault, search, watcherOptions }, + logger, + ) watcher.emit("ready") const fire = async ( @@ -2600,10 +2650,14 @@ describe("watchVaultAndBuildIndex — startup through real chokidar", REAL_WATCH recordDeliveredPath(removeNonMdFile, (filePath) => filePath), ) - const startup = watchVaultAndBuildIndex(testVault, search, { - stabilityThreshold: 100, - pollInterval: 25, - }) + const startup = watchVaultAndBuildIndex( + { + vaultPath: testVault, + search, + watcherOptions: { stabilityThreshold: 100, pollInterval: 25 }, + }, + logger, + ) await modifiedReadReached.promise await deletedNoteRead.promise await writeFile(modifiedPath, "newopal is longer than the old text\n") @@ -2669,13 +2723,13 @@ describe("file watcher — chokidar watch options", () => { * stubbed for this one test. mockRestore (via onTestFinished) hands the * real, spied watch back to the integration tests. */ const startWatcherWithStubbedChokidar = async ( - watcherOptions?: Parameters[2], + watcherOptions?: WatcherOptions, ): Promise => { const watchMock = vi.mocked(watch) watchMock.mockReset() watchMock.mockImplementation(() => createFakeWatcher() as unknown as ReturnType) onTestFinished(() => watchMock.mockRestore()) - await startWatcherWithoutRebuild("/vault", index, watcherOptions) + await startWatcherWithoutRebuild({ vaultPath: "/vault", search: index }, watcherOptions) } // The options every watcher passes chokidar. The whole object is asserted, @@ -2784,10 +2838,10 @@ describe("file watcher — new-directory rescan", REAL_WATCHER_RETRY, () => { /** Starts the file watcher against a fake chokidar (see installFakeChokidar). */ const startWatcherWithFakeChokidar = async ( watchedChildren: Record, - watcherOptions: Parameters[2], + watcherOptions: WatcherOptions, ): Promise => { const fakeChokidar = installFakeChokidar(watchedChildren) - await startWatcherWithoutRebuild(vault, index, watcherOptions) + await startWatcherWithoutRebuild({ vaultPath: vault, search: index }, watcherOptions) return fakeChokidar } @@ -2852,7 +2906,10 @@ describe("file watcher — new-directory rescan", REAL_WATCHER_RETRY, () => { infoSpy.mockRestore() }) const { fireAddDir } = installFakeChokidar({}) - const startup = watchVaultAndBuildIndex(vault, index, RESCAN_TEST_OPTIONS) + const startup = watchVaultAndBuildIndex( + { vaultPath: vault, search: index, watcherOptions: RESCAN_TEST_OPTIONS }, + logger, + ) await vi.waitFor(() => { expect(index.rebuildFromVault).toHaveBeenCalledTimes(1) }) diff --git a/src/vault-mcp/search/file-watcher.ts b/src/vault-mcp/search/file-watcher.ts index e96e42254..52e028938 100644 --- a/src/vault-mcp/search/file-watcher.ts +++ b/src/vault-mcp/search/file-watcher.ts @@ -15,7 +15,7 @@ import { } from "./search-index.js" import type { SearchIndex } from "./search-index.js" import { extractPdfText } from "../obsidian-markdown/pdf.js" -import { logger } from "../../logger.js" +import type { Logger } from "../../logger.js" import { describeError, describeErrorForLog } from "../../utils/describe-error.js" import { isErrnoException } from "../../utils/is-errno-exception.js" import { isMissingPathError, readdirOrNull, realpathOrNull, statOrNull } from "../../utils/fs.js" @@ -57,6 +57,12 @@ type FileWatcherOptions = Readonly<{ heldChangesDeadline?: number }> +type WatchVaultParams = Readonly<{ + vaultPath: string + search: SearchIndex + watcherOptions?: FileWatcherOptions +}> + /** A change chokidar reported while the watcher held it. `add` and `change` * both record "change", since either means "read this path again". Only the * replay of events that arrive during the startup batch uses `kind`; the batch @@ -110,11 +116,9 @@ type HoldingWatcher = Readonly<{ * startup batch that applyHeldChanges runs has finished. After a successful * startup the watcher runs for the process lifetime. */ -const startHoldingWatcher = ( - vaultPath: string, - search: SearchIndex, - options?: FileWatcherOptions, -): HoldingWatcher => { +const startHoldingWatcher = (params: WatchVaultParams, logger: Logger): HoldingWatcher => { + const { vaultPath, search, watcherOptions } = params + // Each path's latest embedding job. A new job for the path waits for this one, // so a note edited many times runs one model job at a time. A job that still // finishes after a newer edit writes nothing: the index checks the source @@ -466,8 +470,8 @@ const startHoldingWatcher = ( removeFromIndex(filePath) } - const stabilityThreshold = options?.stabilityThreshold ?? DEFAULT_STABILITY_THRESHOLD_MS - const newDirectoryRescanDelay = options?.newDirectoryRescanDelay ?? 2 * stabilityThreshold + const stabilityThreshold = watcherOptions?.stabilityThreshold ?? DEFAULT_STABILITY_THRESHOLD_MS + const newDirectoryRescanDelay = watcherOptions?.newDirectoryRescanDelay ?? 2 * stabilityThreshold /** True when the file's last write falls inside the stability window — * plausibly still being written. A negative age (mtime in the future — @@ -490,16 +494,16 @@ const startHoldingWatcher = ( persistent: true, ignoreInitial: true, // Poll across the Docker Desktop ↔ WSL2 bridge, where inotify is dropped. - usePolling: options?.usePolling ?? false, + usePolling: watcherOptions?.usePolling ?? false, // interval only drives chokidar's polling backend. Pass it only when // polling, so we never depend on how chokidar treats an interval set // without usePolling (today it's ignored — a future release needn't be). - ...(options?.usePolling ? { interval: POLLING_INTERVAL_MS } : {}), + ...(watcherOptions?.usePolling ? { interval: POLLING_INTERVAL_MS } : {}), // Obsidian Sync writes files in chunks — wait for write stability before // indexing to avoid reading partial content awaitWriteFinish: { stabilityThreshold, - pollInterval: options?.pollInterval ?? 100, + pollInterval: watcherOptions?.pollInterval ?? 100, }, }) @@ -1003,11 +1007,11 @@ const startHoldingWatcher = ( * in the background. */ export const watchVaultAndBuildIndex = async ( - vaultPath: string, - search: SearchIndex, - options?: FileWatcherOptions, + params: WatchVaultParams, + logger: Logger, ): Promise<{ count: number; embedding: Promise }> => { - const holdingWatcher = startHoldingWatcher(vaultPath, search, options) + const { vaultPath, search, watcherOptions } = params + const holdingWatcher = startHoldingWatcher(params, logger) await holdingWatcher.ready // A failed rebuild closes the watcher rather than leave it holding every @@ -1027,7 +1031,8 @@ export const watchVaultAndBuildIndex = async ( // `applied` never rejects, so past the deadline it can finish unawaited. const { heldPathCount, applied } = holdingWatcher.applyHeldChanges() - const heldChangesDeadline = options?.heldChangesDeadline ?? DEFAULT_HELD_CHANGES_DEADLINE_MS + const heldChangesDeadline = + watcherOptions?.heldChangesDeadline ?? DEFAULT_HELD_CHANGES_DEADLINE_MS const { promise: deadlinePassed, resolve: passDeadline } = Promise.withResolvers<"deadline">() const deadlineTimer = setTimeout(() => passDeadline("deadline"), heldChangesDeadline) diff --git a/src/vault-mcp/server.ts b/src/vault-mcp/server.ts index d8e09ce0e..49c7191a1 100644 --- a/src/vault-mcp/server.ts +++ b/src/vault-mcp/server.ts @@ -152,9 +152,10 @@ const startServer = async (): Promise => { await bootstrapMemoryIfEnabled(config, vaultPath) // The returned embedding promise is not awaited: the rebuild logs its failure itself. - const { count } = await watchVaultAndBuildIndex(vaultPath, search, { - usePolling: config.windowsBindMount, - }) + const { count } = await watchVaultAndBuildIndex( + { vaultPath, search, watcherOptions: { usePolling: config.windowsBindMount } }, + logger, + ) logger.info("initial index built", { notesIndexedByRebuild: count }) const oauthProvider = createOAuthProvider({ From eade0a4f91e741def0df6e82a6276c26e38d23e9 Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:19:56 -0400 Subject: [PATCH 23/34] refactor(file-watcher): apply held startup changes in one live batch The watcher goes live before the startup batch instead of after it, so the replay of events that arrived during the batch is gone. In one synchronous step it removes held paths whose latest event was a deletion, then reserves an event token per held path; the batch's removal and read of a path go ahead only while that reservation is current. A read that finds the file gone is left to the unlink chokidar sends for it. Co-Authored-By: Claude Opus 5.5 --- .../search/__tests__/file-watcher.test.ts | 380 +++++++++--------- src/vault-mcp/search/file-watcher.ts | 322 +++++++-------- 2 files changed, 347 insertions(+), 355 deletions(-) diff --git a/src/vault-mcp/search/__tests__/file-watcher.test.ts b/src/vault-mcp/search/__tests__/file-watcher.test.ts index e31d56f36..5dbff0709 100644 --- a/src/vault-mcp/search/__tests__/file-watcher.test.ts +++ b/src/vault-mcp/search/__tests__/file-watcher.test.ts @@ -1474,6 +1474,28 @@ const gateFirstRead = async (filePath: string, placement: "before-disk" | "after return { reached: reached.promise, release: () => release.resolve(undefined) } } +/** Pauses the first readdir of the vault root after it has read the folder, + * so the startup batch's presence checks use names from before whatever the + * test does while the listing waits. */ +const gateRootListing = async (testVault: string) => { + const actualFs = await vi.importActual("node:fs/promises") + const reached = Promise.withResolvers() + const release = Promise.withResolvers() + const gateState = { used: false } + vi.mocked(readdir).mockImplementation(async (requestedPath, options) => { + const isGatedListing = requestedPath === testVault && !gateState.used + + if (!isGatedListing) return actualFs.readdir(requestedPath, options) + gateState.used = true + const names = await actualFs.readdir(requestedPath, options) + reached.resolve(undefined) + await release.promise + return names + }) + onTestFinished(() => vi.mocked(readdir).mockRestore()) + return { reached: reached.promise, release: () => release.resolve(undefined) } +} + describe("watchVaultAndBuildIndex — changes during startup", () => { /** A controlled watcher whose startup rebuild waits for finishRebuild, unless * realRebuild is set. The database lives outside the vault, so its own files @@ -1585,8 +1607,7 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { "note.md", ]) expect(infoSpy).toHaveBeenCalledWith("applied the changes held during startup indexing", { - batchPaths: 1, - replayedEvents: 0, + heldPaths: 1, }) }) @@ -1642,47 +1663,168 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { expect(search.fullTextSearch({ query: "oldtext" }, logger)).toEqual([]) }) - it("removes a held path deleted between its check and its read", async () => { - const { testVault, notePaths, fire, startup, finishRebuild } = await createControlledStartup({ - seedNotes: { "late.md": "oldtext\n" }, + // chokidar sends an unlink for every file it tracks, so the batch leaves a + // path that vanished by its read to that live event. + it.each([{ unlinkArrives: "before the read returns" }, { unlinkArrives: "after startup" }])( + "removes a held path deleted between its check and its read once its unlink arrives $unlinkArrives", + async ({ unlinkArrives }) => { + const { testVault, notePaths, fire, startup, finishRebuild } = await createControlledStartup({ + seedNotes: { "late.md": "oldtext\n" }, + }) + const notePath = join(testVault, "late.md") + await writeFile(notePath, "newer text\n") + const readGate = await gateFirstRead(notePath, "before-disk") + expect(notePaths()).toEqual([{ path: "late.md" }]) + + await fire("change", "late.md") + finishRebuild() + await readGate.reached + await unlink(notePath) + if (unlinkArrives === "before the read returns") await fire("unlink", "late.md") + readGate.release() + await startup + if (unlinkArrives === "after startup") await fire("unlink", "late.md") + + expect(notePaths()).toEqual([]) + }, + ) + + it("removes a renamed note before reading its new path, so incoming links follow", async () => { + const { testVault, search, fire, startup, finishRebuild } = await createControlledStartup({ + seedNotes: { "B.md": "old home\n", "source.md": "points at [[B]]\n" }, }) - const notePath = join(testVault, "late.md") - await writeFile(notePath, "newer text\n") - const readGate = await gateFirstRead(notePath, "before-disk") - expect(notePaths()).toEqual([{ path: "late.md" }]) + await mkdir(join(testVault, "folder")) + await writeFile(join(testVault, "folder", "B.md"), "new home\n") + await writeFile(join(testVault, "source.md"), "points at [[B]]\n") + const linkFromSource = () => { + const outgoingLinks = search.getOutgoingLinks({ path: "source.md" }, logger) + return outgoingLinks.map((link) => ({ path: link.path, exists: link.exists })) + } + expect(linkFromSource()).toEqual([{ path: "B.md", exists: true }]) - await fire("change", "late.md") + await fire("add", "folder/B.md") + await fire("unlink", "B.md") finishRebuild() - await readGate.reached - await unlink(notePath) - readGate.release() await startup - expect(notePaths()).toEqual([]) + expect(linkFromSource()).toEqual([{ path: "folder/B.md", exists: true }]) }) - it("removes a renamed note before reading its new path, so incoming links follow", async () => { + it("points incoming links at a renamed note whose new path arrives live during the batch", async () => { const { testVault, search, fire, startup, finishRebuild } = await createControlledStartup({ seedNotes: { "B.md": "old home\n", "source.md": "points at [[B]]\n" }, }) - await mkdir(join(testVault, "folder")) - await writeFile(join(testVault, "folder", "B.md"), "new home\n") await writeFile(join(testVault, "source.md"), "points at [[B]]\n") + const listingGate = await gateRootListing(testVault) const linkFromSource = () => { const outgoingLinks = search.getOutgoingLinks({ path: "source.md" }, logger) return outgoingLinks.map((link) => ({ path: link.path, exists: link.exists })) } expect(linkFromSource()).toEqual([{ path: "B.md", exists: true }]) - await fire("add", "folder/B.md") await fire("unlink", "B.md") finishRebuild() + await listingGate.reached + await mkdir(join(testVault, "folder")) + await writeFile(join(testVault, "folder", "B.md"), "new home\n") + await fire("add", "folder/B.md") + listingGate.release() await startup expect(linkFromSource()).toEqual([{ path: "folder/B.md", exists: true }]) }) - it("replays events from during the batch, so the latest state on disk wins", async () => { + it("keeps a note that a live add indexed while the batch checked its held deletion", async () => { + const { testVault, search, fire, startup, finishRebuild } = await createControlledStartup({ + seedNotes: { "P.md": "oldtext\n" }, + }) + const listingGate = await gateRootListing(testVault) + const pathsFor = (query: string) => { + return search.fullTextSearch({ query }, logger).map((hit) => hit.path) + } + expect(pathsFor("oldtext")).toEqual(["P.md"]) + + await fire("unlink", "P.md") + finishRebuild() + // The root listing was read before P.md came back, so P.md's check finds it missing. + await listingGate.reached + await writeFile(join(testVault, "P.md"), "liveopal\n") + await fire("add", "P.md") + expect(pathsFor("liveopal")).toEqual(["P.md"]) + listingGate.release() + await startup + + expect(pathsFor("liveopal")).toEqual(["P.md"]) + }) + + it("keeps a live add for a held path whose check waits behind 16 others", async () => { + const { testVault, search, fire, startup, finishRebuild } = await createControlledStartup({ + seedNotes: { "P.md": "oldtext\n" }, + }) + const blockerNames = Array.from( + { length: 16 }, + (_unused, blockerIndex) => `blocker-${blockerIndex}.md`, + ) + for (const blockerName of blockerNames) { + await writeFile(join(testVault, blockerName), "blockertext\n") + } + // The 16 blocker checks fill the batch's first group and wait at their stat, + // so P.md's check starts only after its live add. The root listing those + // checks shared was read before P.md came back. + const actualFsUtils = + await vi.importActual("../../../utils/fs.js") + const statGate = Promise.withResolvers() + const blockerStats = { started: 0 } + const blockerPathPrefix = join(testVault, "blocker-") + vi.mocked(statOrNull).mockImplementation(async (requestedPath) => { + const isBlockerCheck = + requestedPath.startsWith(blockerPathPrefix) && blockerStats.started < 16 + + if (isBlockerCheck) { + blockerStats.started += 1 + await statGate.promise + } + return actualFsUtils.statOrNull(requestedPath) + }) + onTestFinished(() => vi.mocked(statOrNull).mockRestore()) + for (const blockerName of blockerNames) { + await fire("change", blockerName) + } + await fire("unlink", "P.md") + + finishRebuild() + await vi.waitFor(() => { + expect(blockerStats.started).toBe(16) + }) + await writeFile(join(testVault, "P.md"), "liveopal\n") + await fire("add", "P.md") + statGate.resolve(undefined) + await startup + + expect(search.fullTextSearch({ query: "liveopal" }, logger).map((hit) => hit.path)).toEqual([ + "P.md", + ]) + }) + + it("removes a held deletion of a re-created note when going live, then reads it again", async () => { + const { testVault, search, fire, startup, finishRebuild } = await createControlledStartup({ + seedNotes: { "back.md": "oldtext\n" }, + }) + const debugSpy = spyOnLogger("debug") + await writeFile(join(testVault, "back.md"), "returnedopal\n") + + await fire("unlink", "back.md") + finishRebuild() + await startup + + expect(debugSpy).toHaveBeenCalledWith("removed from index", { path: "back.md" }) + expect(search.fullTextSearch({ query: "returnedopal" }, logger).map((hit) => hit.path)).toEqual( + ["back.md"], + ) + expect(search.fullTextSearch({ query: "oldtext" }, logger)).toEqual([]) + }) + + it("applies live events from during the batch, so the latest state on disk wins", async () => { const { testVault, search, fire, startup, finishRebuild } = await createControlledStartup({ seedNotes: { "P.md": "oldamber\n", "Q.md": "quartzrow\n" }, }) @@ -1711,15 +1853,12 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { await startup expect(infoSpy).toHaveBeenCalledWith("applied the changes held during startup indexing", { - batchPaths: 1, - replayedEvents: 4, - }) - await vi.waitFor(() => { - expect(pathsFor("newopal")).toEqual(["P.md"]) - expect(pathsFor("oldamber")).toEqual([]) - expect(pathsFor("quartzrow")).toEqual([]) - expect(pathsFor("replayjade")).toEqual(["R.md"]) + heldPaths: 1, }) + expect(pathsFor("newopal")).toEqual(["P.md"]) + expect(pathsFor("oldamber")).toEqual([]) + expect(pathsFor("quartzrow")).toEqual([]) + expect(pathsFor("replayjade")).toEqual(["R.md"]) }) it("contains a failed read and keeps applying the other held paths", async () => { @@ -1805,31 +1944,6 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { ]) }) - it("logs a failure to start the replay instead of rejecting startup", async () => { - const { testVault, search, fire, startup, finishRebuild } = await createControlledStartup() - const errorSpy = spyOnLogger("error") - // The replay's summary line is the last step of going live, so a throw - // there escapes it after the watcher is live. - spyOnLogger("info").mockImplementation((message) => { - if (message === "applied the changes held during startup indexing") { - throw new Error("controlled replay failure") - } - }) - - finishRebuild() - await expect(startup).resolves.toEqual({ count: 0, embedding: expect.any(Promise) }) - expect(errorSpy).toHaveBeenCalledExactlyOnceWith( - "failed to start replaying the changes held during the startup batch", - { error: "[Error]: controlled replay failure" }, - ) - - await writeFile(join(testVault, "live.md"), "livetopaz\n") - await fire("add", "live.md") - expect(search.fullTextSearch({ query: "livetopaz" }, logger).map((hit) => hit.path)).toEqual([ - "live.md", - ]) - }) - it("reports a note embedding that fails during the batch once, with no unhandled rejection", async () => { const vector = new Float32Array(384).fill(0.1) const embedder = { @@ -1949,54 +2063,7 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { expect(indexedPaths.toSorted()).toEqual(fileNames.toSorted()) }) - it("reads at most 16 files at once when replaying the changes from during the batch", async () => { - const { testVault, search, fire, startup, finishRebuild } = await createControlledStartup() - const infoSpy = spyOnLogger("info") - const batchPath = join(testVault, "batch.md") - await writeFile(batchPath, "batchonyx\n") - const fileNames = Array.from({ length: 40 }, (_unused, noteIndex) => `late-${noteIndex}.md`) - for (const fileName of fileNames) { - await writeFile(join(testVault, fileName), "latetopaz\n") - } - const readGate = await gateFirstRead(batchPath, "before-disk") - const pathsFor = (query: string) => { - return search.fullTextSearch({ query, limit: 50 }, logger).map((hit) => hit.path) - } - - await fire("add", "batch.md") - finishRebuild() - await readGate.reached - // Installed while the batch's one read waits, so only the replay's reads count. - const actualFs = await vi.importActual("node:fs/promises") - const readCounts = { inFlight: 0, peak: 0 } - vi.mocked(readFile).mockImplementation(async (requestedPath, options) => { - readCounts.inFlight += 1 - readCounts.peak = Math.max(readCounts.peak, readCounts.inFlight) - await new Promise((resolve) => setTimeout(resolve, 5)) - try { - return await actualFs.readFile(requestedPath, options) - } finally { - readCounts.inFlight -= 1 - } - }) - for (const fileName of fileNames) { - await fire("add", fileName) - } - readGate.release() - await startup - - expect(infoSpy).toHaveBeenCalledWith("applied the changes held during startup indexing", { - batchPaths: 1, - replayedEvents: 40, - }) - await vi.waitFor(() => { - expect(pathsFor("latetopaz").toSorted()).toEqual(fileNames.toSorted()) - }) - expect(pathsFor("batchonyx")).toEqual(["batch.md"]) - expect(readCounts.peak).toBe(16) - }) - - it("replays a change that follows a held deletion of the same path", async () => { + it("indexes a change that follows a deletion of the same path during the batch", async () => { const { testVault, search, fire, startup, finishRebuild } = await createControlledStartup({ seedNotes: { "note.md": "oldtext\n" }, }) @@ -2016,63 +2083,17 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { readGate.release() await startup - await vi.waitFor(() => { - expect(pathsFor("newopal")).toEqual(["note.md"]) - }) + expect(pathsFor("newopal")).toEqual(["note.md"]) expect(pathsFor("oldtext")).toEqual([]) }) - it("does not replay a change that a later held deletion of the path replaced", async () => { + it("drops a queued read of the old spelling when a case-only rename lands during the batch", async () => { const { testVault, notePaths, fire, startup, finishRebuild } = await createControlledStartup({ seedNotes: { "note.md": "caseopal\n" }, }) - const batchPath = join(testVault, "batch.md") const oldPath = join(testVault, "note.md") const renamedPath = join(testVault, "Note.md") - await writeFile(batchPath, "batchonyx\n") - await writeFile(renamedPath, "caseopal\n") - const readGate = await gateFirstRead(batchPath, "before-disk") - - await fire("add", "batch.md") - finishRebuild() - await readGate.reached - // Installed while the batch's one read waits. A case-insensitive filesystem - // answers for the old spelling too, so reading it would index note.md again. - const actualFs = await vi.importActual("node:fs/promises") - const asFoundOnDisk = (requestedPath: RequestedPath): RequestedPath | string => { - return requestedPath === oldPath ? renamedPath : requestedPath - } - vi.mocked(stat).mockImplementation(async (requestedPath, options) => { - return actualFs.stat(asFoundOnDisk(requestedPath), options) - }) - vi.mocked(readFile).mockImplementation(async (requestedPath, options) => { - return actualFs.readFile(asFoundOnDisk(requestedPath), options) - }) - // gateFirstRead's cleanup restores readFile. - onTestFinished(() => vi.mocked(stat).mockRestore()) - await fire("change", "note.md") - await fire("unlink", "note.md") - await fire("add", "Note.md") - readGate.release() - await startup - - await vi.waitFor(() => { - expect(notePaths()).toEqual([{ path: "Note.md" }, { path: "batch.md" }]) - }) - // The replay starts note.md's read before Note.md's, so it would have run by now. - expect(vi.mocked(readFile)).not.toHaveBeenCalledWith(oldPath, "utf8") - }) - - it("does not replay a change whose path a live deletion reached first", async () => { - const { testVault, notePaths, fire, startup, finishRebuild } = await createControlledStartup({ - seedNotes: { "note.md": "caseopal\n" }, - }) - const batchPath = join(testVault, "batch.md") - const oldPath = join(testVault, "note.md") - const renamedPath = join(testVault, "Note.md") - await writeFile(batchPath, "batchonyx\n") - await writeFile(renamedPath, "caseopal\n") - await writeFile(join(testVault, "z-last.md"), "lastjade\n") + await writeFile(oldPath, "caseopal\n") const blockerNames = Array.from( { length: 16 }, (_unused, blockerIndex) => `blocker-${blockerIndex}.md`, @@ -2080,20 +2101,17 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { for (const blockerName of blockerNames) { await writeFile(join(testVault, blockerName), "blockertext\n") } - const readGate = await gateFirstRead(batchPath, "before-disk") - - await fire("add", "batch.md") - finishRebuild() - await readGate.reached - // Installed while the batch's one read waits. The 16 blocker reads fill the - // replay's first group and wait for blockerGate, so note.md's read stays - // queued. A case-insensitive filesystem answers for the old spelling too. + // The 16 blocker reads fill the batch's first group and wait for + // blockerGate, so note.md's read stays queued after its check saw it on + // disk. Once renamed, a case-insensitive filesystem still answers for the + // old spelling, so reading it would index note.md again. const actualFs = await vi.importActual("node:fs/promises") const blockerGate = Promise.withResolvers() const blockerReads = { started: 0 } + const renameState = { renamed: false } const blockerPathPrefix = join(testVault, "blocker-") const asFoundOnDisk = (requestedPath: RequestedPath): RequestedPath | string => { - return requestedPath === oldPath ? renamedPath : requestedPath + return renameState.renamed && requestedPath === oldPath ? renamedPath : requestedPath } vi.mocked(stat).mockImplementation(async (requestedPath, options) => { return actualFs.stat(asFoundOnDisk(requestedPath), options) @@ -2108,29 +2126,27 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { } return actualFs.readFile(asFoundOnDisk(requestedPath), options) }) - // gateFirstRead's cleanup restores readFile. - onTestFinished(() => vi.mocked(stat).mockRestore()) + onTestFinished(() => { + vi.mocked(stat).mockRestore() + vi.mocked(readFile).mockRestore() + }) for (const blockerName of blockerNames) { await fire("change", blockerName) } await fire("change", "note.md") - await fire("change", "z-last.md") - readGate.release() - await startup + + finishRebuild() await vi.waitFor(() => { expect(blockerReads.started).toBe(16) }) - + await rename(oldPath, renamedPath) + renameState.renamed = true await fire("unlink", "note.md") await fire("add", "Note.md") blockerGate.resolve(undefined) + await startup - await vi.waitFor(() => { - expect(notePaths()).toContainEqual({ path: "z-last.md" }) - }) - // The replay starts note.md's read before z-last.md's, so it would have run by now. - expect(vi.mocked(readFile)).not.toHaveBeenCalledWith(oldPath, "utf8") - const expectedPaths = ["Note.md", "batch.md", "z-last.md", ...blockerNames].toSorted() + const expectedPaths = ["Note.md", ...blockerNames].toSorted() expect(notePaths()).toEqual(expectedPaths.map((path) => ({ path }))) }) @@ -2288,16 +2304,17 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { { heldPaths: 1, heldChangesDeadline: 50 }, ) expect(pathsFor("slowtext")).toEqual([]) + // The watcher went live before the batch started, so a change while the + // batch's read is still gated is indexed at once. await writeFile(join(testVault, "S.md"), "aftertext\n") await fire("change", "S.md") - expect(debugSpy).toHaveBeenCalledWith("held a change until startup indexing ends", { + expect(debugSpy).not.toHaveBeenCalledWith("held a change until startup indexing ends", { path: "S.md", }) - expect(pathsFor("aftertext")).toEqual([]) + expect(pathsFor("aftertext")).toEqual(["S.md"]) readGate.release() await vi.waitFor(() => { expect(pathsFor("slowtext")).toEqual(["slow.md"]) - expect(pathsFor("aftertext")).toEqual(["S.md"]) }) }) @@ -2367,7 +2384,9 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { }) expect(notePaths()).toEqual([{ path: "notes/x.md" }]) - await fire("unlink", "notes/x.md") + // On macOS, chokidar reports a case-only folder rename as a change to the + // old path, never an unlink. + await fire("change", "notes/x.md") await fire("add", "Notes/x.md") finishRebuild() await startup @@ -2469,7 +2488,7 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { ]) }) - it("removes a held file's rows when it is deleted between its check and its read", async () => { + it("removes a held file's rows when it is deleted between its check and its read and its unlink arrives", async () => { const { testVault, search, database, fire, startup, finishRebuild } = await createControlledStartup() const contentPath = join(testVault, "content.txt") @@ -2506,8 +2525,11 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { await fire("change", "content.txt") finishRebuild() await startup - + // The batch's read finds the file gone and leaves it to the unlink + // chokidar sends for it. expect(checkedPaths).toEqual(new Set([contentPath])) + await fire("unlink", "content.txt") + expect(fileRows()).toEqual({ assets: [], contents: [] }) }) @@ -2694,8 +2716,7 @@ describe("watchVaultAndBuildIndex — startup through real chokidar", REAL_WATCH expect(database.prepare("SELECT path FROM non_md_files").all()).toEqual([]) expect(database.prepare("SELECT path FROM file_content").all()).toEqual([]) expect(infoSpy).toHaveBeenCalledWith("applied the changes held during startup indexing", { - batchPaths: 4, - replayedEvents: 0, + heldPaths: 4, }) }, ) @@ -2924,8 +2945,7 @@ describe("file watcher — new-directory rescan", REAL_WATCHER_RETRY, () => { "new-folder/missed.md", ]) expect(infoSpy).toHaveBeenCalledWith("applied the changes held during startup indexing", { - batchPaths: 1, - replayedEvents: 0, + heldPaths: 1, }) }, ) diff --git a/src/vault-mcp/search/file-watcher.ts b/src/vault-mcp/search/file-watcher.ts index 52e028938..e370049c9 100644 --- a/src/vault-mcp/search/file-watcher.ts +++ b/src/vault-mcp/search/file-watcher.ts @@ -63,17 +63,22 @@ type WatchVaultParams = Readonly<{ watcherOptions?: FileWatcherOptions }> -/** A change chokidar reported while the watcher held it. `add` and `change` - * both record "change", since either means "read this path again". Only the - * replay of events that arrive during the startup batch uses `kind`; the batch - * itself checks each path against the disk instead. */ -type HeldEvent = Readonly<{ kind: "change" | "unlink"; filePath: string }> - -/** While holding, chokidar's events are recorded in arrival order instead of - * applied; once live, each event is applied as it arrives; once closed, they - * are ignored. */ +/** The kind of the latest event chokidar reported for a held path. `add` and + * `change` both record "change", since either means "read this path again". + * The startup batch checks every held path against the disk; going live uses + * the kind only to remove a deleted path early. */ +type HeldEventKind = "change" | "unlink" + +/** While holding, the paths chokidar reports are recorded instead of applied; + * once live, each event is applied as it arrives; once closed, they are + * ignored. */ type WatcherState = - { mode: "holding"; heldEvents: HeldEvent[] } | { mode: "live" } | { mode: "closed" } + { mode: "holding"; heldPaths: Map } | { mode: "live" } | { mode: "closed" } + +/** A held path and the event token reserved for it when the watcher went live. + * The startup batch's removal or read of the path goes ahead only while the + * reservation is still the path's token. */ +type ReservedPath = Readonly<{ filePath: string; reservation: symbol }> /** What indexing one changed file did. A note's embedding is still running when * this returns, so whoever receives `note-indexed` reports its failure. */ @@ -100,9 +105,9 @@ type FolderListing = type HoldingWatcher = Readonly<{ /** Resolves once chokidar's initial scan is done and every folder is watched. */ ready: Promise - /** Starts applying what chokidar reported while holding, and returns how many - * distinct paths that is. `applied` resolves once their index writes commit - * and the watcher is live. */ + /** Goes live and starts applying what chokidar reported while holding, and + * returns how many distinct paths that is. `applied` resolves once the + * batch's index writes commit, and never rejects. */ applyHeldChanges: () => { heldPathCount: number; applied: Promise } /** Stops watching, including the rescans and retries still scheduled. Used * only when startup fails before the held changes apply. Never rejects, so @@ -112,9 +117,9 @@ type HoldingWatcher = Readonly<{ /** * Watches the vault and mirrors every file add/change/delete into the search - * index. Every change is held from the moment chokidar starts until the - * startup batch that applyHeldChanges runs has finished. After a successful - * startup the watcher runs for the process lifetime. + * index. Every change is held from the moment chokidar starts until + * applyHeldChanges goes live. After a successful startup the watcher runs for + * the process lifetime. */ const startHoldingWatcher = (params: WatchVaultParams, logger: Logger): HoldingWatcher => { const { vaultPath, search, watcherOptions } = params @@ -131,15 +136,15 @@ const startHoldingWatcher = (params: WatchVaultParams, logger: Logger): HoldingW * guard the embedding that runs after it. * - indexChangedFile stores a fresh token before it reads the file, and drops * what it read if a newer event or a removal replaced the token meanwhile. - * - goLiveAndReplay stores one for each path it will read again, so a later - * deletion or a newer event for the path cancels that read. + * - applyHeldChanges reserves one for each held path as the watcher goes + * live. The startup batch reads the path under it, so a later deletion or a + * newer event for the path cancels the batch's removal or read. */ const currentEvents = new Map() - // Reassigned, so `let`: the startup batch swaps in a new held list for - // events that arrive while it runs, going live ends holding, and a failed - // startup closes the watcher. - let watcherState: WatcherState = { mode: "holding", heldEvents: [] } + // Reassigned, so `let`: going live ends holding, and a failed startup closes + // the watcher. + let watcherState: WatcherState = { mode: "holding", heldPaths: new Map() } /** Indexes an added or modified non-md file: its asset-table row, plus its * content when it is a canvas or an indexable text or PDF file. */ @@ -167,8 +172,8 @@ const startHoldingWatcher = (params: WatchVaultParams, logger: Logger): HoldingW const nonMdStat = await readNonMdFileStat() if (currentEvents.get(relativePath) !== eventToken) return { status: "superseded" } - // Vanished between the event and the stat. Live, the unlink event that - // follows removes any existing row; the startup batch removes it itself. + // Vanished between the event and the stat: the unlink event that follows + // removes any existing row. if (nonMdStat.status === "missing") return { status: "vanished" } if (nonMdStat.status === "failed") return { status: "failed" } @@ -341,6 +346,24 @@ const startHoldingWatcher = (params: WatchVaultParams, logger: Logger): HoldingW } } + /** Indexes an added or modified file under the token its caller stored for + * the path, and reports what it did. */ + const indexFileUnderToken = (params: { + filePath: string + relativePath: string + eventToken: symbol + }): Promise => { + if (params.filePath.endsWith(".md")) return indexChangedNote(params) + return indexChangedNonMdFile(params) + } + + /** Releases a path's event token unless a newer event replaced it. A newer + * handler may already have finished and removed its entry; only the current + * event owns cleanup, and absence still invalidates older reads. */ + const releaseEventToken = (relativePath: string, eventToken: symbol): void => { + if (currentEvents.get(relativePath) === eventToken) currentEvents.delete(relativePath) + } + /** Indexes an added or modified file and reports what it did. */ const indexChangedFile = async (filePath: string): Promise => { const relativePath = relative(vaultPath, filePath) @@ -350,18 +373,11 @@ const startHoldingWatcher = (params: WatchVaultParams, logger: Logger): HoldingW const eventToken = Symbol() currentEvents.set(relativePath, eventToken) - const indexParams = { filePath, relativePath, eventToken } - // Awaited inside the try, so the token is released only once indexing ends. try { - if (filePath.endsWith(".md")) return await indexChangedNote(indexParams) - return await indexChangedNonMdFile(indexParams) + return await indexFileUnderToken({ filePath, relativePath, eventToken }) } finally { - // A newer handler may already have finished and removed its entry. - // Only the current event owns cleanup; absence still invalidates old reads. - if (currentEvents.get(relativePath) === eventToken) { - currentEvents.delete(relativePath) - } + releaseEventToken(relativePath, eventToken) } } @@ -379,7 +395,7 @@ const startHoldingWatcher = (params: WatchVaultParams, logger: Logger): HoldingW const handleChange = async (filePath: string): Promise => { if (watcherState.mode === "closed") return if (watcherState.mode === "holding") { - watcherState.heldEvents.push({ kind: "change", filePath }) + watcherState.heldPaths.set(filePath, "change") logger.debug("held a change until startup indexing ends", { path: relative(vaultPath, filePath), }) @@ -461,7 +477,7 @@ const startHoldingWatcher = (params: WatchVaultParams, logger: Logger): HoldingW const handleDelete = (filePath: string): void => { if (watcherState.mode === "closed") return if (watcherState.mode === "holding") { - watcherState.heldEvents.push({ kind: "unlink", filePath }) + watcherState.heldPaths.set(filePath, "unlink") logger.debug("held a deletion until startup indexing ends", { path: relative(vaultPath, filePath), }) @@ -770,120 +786,68 @@ const startHoldingWatcher = (params: WatchVaultParams, logger: Logger): HoldingW removeFromIndex(filePath) } - /** Re-reads one held path for the startup batch or its replay, without - * waiting for its embedding. A path that vanished by its read is removed: - * a live event for it that arrived during the read would have taken the - * token and made the read "superseded" instead, so nothing newer is lost. */ - const indexHeldPath = async (filePath: string): Promise => { - const indexResult = await indexChangedFile(filePath) + /** + * Reads one held path again for the startup batch, under its reservation, + * without waiting for its embedding. + * - A live event for the path since the switch replaced the reservation, so + * the read drops what it read, as an older live read does. + * - A path that vanished by its read is left to the `unlink` chokidar sends + * for every file it tracks, as a live read leaves it. + */ + const indexHeldPath = async ({ filePath, reservation }: ReservedPath): Promise => { + const relativePath = relative(vaultPath, filePath) - if (indexResult.status === "vanished") { - removeHeldPath(filePath) - return - } - if (indexResult.status !== "note-indexed") return + // Awaited inside the try, so the reservation is released only once indexing ends. + try { + const indexResult = await indexFileUnderToken({ + filePath, + relativePath, + eventToken: reservation, + }) - // The batch does not wait for embeddings, so the failure handler is - // attached now. Attached after the batch, an embedding that rejected first - // would be reported as an unhandled rejection. - indexResult.embedding.catch((error) => { - reportEmbeddingFailure(filePath, error) - }) + if (indexResult.status !== "note-indexed") return + + // The batch does not wait for embeddings, so the failure handler is + // attached now. Attached after the batch, an embedding that rejected + // first would be reported as an unhandled rejection. + indexResult.embedding.catch((error) => { + reportEmbeddingFailure(filePath, error) + }) + } finally { + releaseEventToken(relativePath, reservation) + } } - /** Reads and indexes paths at the rebuild's bounds: PDFs whose text is + /** Reads held paths again at the rebuild's bounds: PDFs whose text is * extracted PDF_EXTRACTION_CONCURRENCY at a time, after the others, which go * INDEXING_FILE_READ_CONCURRENCY at a time. */ - const indexPathsWithinBounds = async (params: { - filePaths: ReadonlyArray - indexPath: (filePath: string) => Promise - }): Promise => { - const { filePaths, indexPath } = params + const indexHeldPathsWithinBounds = async ( + reservedPaths: ReadonlyArray, + ): Promise => { // indexChangedNonMdFile extracts a PDF's text only when file content // indexing is on; otherwise a PDF is only statted and keeps the general bound. - const needsPdfExtraction = (filePath: string): boolean => { + const needsPdfExtraction = ({ filePath }: ReservedPath): boolean => { return search.fileContentIndexingEnabled && extname(filePath) === ".pdf" } - const pdfPaths = filePaths.filter(needsPdfExtraction) - const otherPaths = filePaths.filter((filePath) => !needsPdfExtraction(filePath)) + const pdfReads = reservedPaths.filter(needsPdfExtraction) + const otherReads = reservedPaths.filter((reservedPath) => !needsPdfExtraction(reservedPath)) await mapWithConcurrency({ - items: otherPaths, + items: otherReads, concurrency: INDEXING_FILE_READ_CONCURRENCY, - mapper: indexPath, + mapper: indexHeldPath, }) await mapWithConcurrency({ - items: pdfPaths, + items: pdfReads, concurrency: PDF_EXTRACTION_CONCURRENCY, - mapper: indexPath, - }) - } - - /** Reads again each path changed while the startup batch ran, unless a later - * deletion or a newer event replaced the token goLiveAndReplay reserved for - * it. Nothing awaits it, so it logs its own failure. */ - const replayChangedPaths = async (replayTokens: ReadonlyMap): Promise => { - const indexPathIfStillReserved = async (filePath: string): Promise => { - const reservationIsCurrent = - currentEvents.get(relative(vaultPath, filePath)) === replayTokens.get(filePath) - - // On a case-insensitive filesystem the old spelling of a renamed file - // still reads, so a cancelled read would index the old path again. - if (!reservationIsCurrent) return - await indexHeldPath(filePath) - } - - try { - await indexPathsWithinBounds({ - filePaths: [...replayTokens.keys()], - indexPath: indexPathIfStillReserved, - }) - } catch (error) { - logger.error("failed to replay the changes held during the startup batch", { - error: describeError(error), - }) - } - } - - /** Ends holding and replays the events held while the batch ran: deletions - * at once, then each changed path read again at the batch's bounds unless a - * later deletion or event replaced it. Startup does not wait for those - * reads, as it does not wait for later live events. */ - const goLiveAndReplay = (batchPathCount: number): void => { - // Unreachable today, because applyHeldChanges runs once per startup and the - // watcher closes only when the rebuild fails, before that call. A second - // call would start a second batch, and this keeps it from going live twice. - if (watcherState.mode !== "holding") return - - const eventsDuringBatch = watcherState.heldEvents - watcherState = { mode: "live" } - - // This loop runs in arrival order, in one tick, before any live event - // arrives. Each change reserves a token for its path's read and each - // deletion drops it, so a later deletion or newer event cancels the read. - const replayTokens = new Map() - for (const heldEvent of eventsDuringBatch) { - if (heldEvent.kind === "unlink") { - handleDelete(heldEvent.filePath) - continue - } - const replayToken = Symbol() - currentEvents.set(relative(vaultPath, heldEvent.filePath), replayToken) - replayTokens.set(heldEvent.filePath, replayToken) - } - void replayChangedPaths(replayTokens) - logger.info("applied the changes held during startup indexing", { - batchPaths: batchPathCount, - replayedEvents: eventsDuringBatch.length, + mapper: indexHeldPath, }) } /** Checks each held path against the disk, removes the missing ones, then - * re-reads the rest. It must run while the watcher holds: removeFromIndex - * drops the path's event token, so once live, a removal could cancel the - * read of an `add` that started during the check, and the added file would - * stay unindexed. */ - const reconcileHeldPaths = async (batchPaths: ReadonlyArray): Promise => { + * reads the rest again. The watcher is live meanwhile, so a path whose + * reservation a live event replaced is skipped: that event is newer. */ + const reconcileHeldPaths = async (reservedPaths: ReadonlyArray): Promise => { const folderListings = new Map>() /** Lists each folder once per batch, however many held paths sit in it. */ @@ -897,73 +861,81 @@ const startHoldingWatcher = (params: WatchVaultParams, logger: Logger): HoldingW } const presenceChecks = await mapWithConcurrency({ - items: batchPaths, + items: reservedPaths, concurrency: INDEXING_FILE_READ_CONCURRENCY, - mapper: async (filePath) => { - return { filePath, presence: await checkHeldPathOnDisk(filePath, listFolderOnce) } + mapper: async (reservedPath) => { + const presence = await checkHeldPathOnDisk(reservedPath.filePath, listFolderOnce) + return { reservedPath, presence } }, }) const missingPaths = presenceChecks .filter((check) => check.presence === "missing") - .map((check) => check.filePath) + .map((check) => check.reservedPath) const pathsToRead = presenceChecks .filter((check) => check.presence !== "missing") - .map((check) => check.filePath) + .map((check) => check.reservedPath) // Removals run before reads. upsertNote re-resolves only links whose target // is not indexed, so when a renamed note's old path is removed first, the // new path claims the links that pointed at the old one. - for (const filePath of missingPaths) { + for (const { filePath, reservation } of missingPaths) { + // Skips a path a live event has taken since the switch, so a live `add` + // that indexed it is not removed. No await sits between this check and + // the removal, so no live event can take the path in between. + const reservationIsCurrent = currentEvents.get(relative(vaultPath, filePath)) === reservation + + if (!reservationIsCurrent) continue removeHeldPath(filePath) } - await indexPathsWithinBounds({ filePaths: pathsToRead, indexPath: indexHeldPath }) + await indexHeldPathsWithinBounds(pathsToRead) } /** - * Brings the index up to date with the held paths, in one batch, then goes live: + * Goes live and starts the startup batch over the distinct held paths. These + * three steps run in one synchronous step, before any live event can start: * - * 1. Each path is checked against the disk, and missing ones are removed. - * 2. The rest are read and indexed within the rebuild's bounds; one that - * vanished by its read is removed. - * 3. The watcher goes live, and the events held while steps 1 and 2 ran are - * replayed: deletions at once, then the changed paths read again in the - * background, except those a later deletion or event replaced. + * 1. The watcher goes live. + * 2. Each path whose latest held event is a deletion is removed, so a renamed + * note's new path, arriving live, claims the links that pointed at the old + * one. A file that came back is read again by the batch. + * 3. Each held path gets a reserved event token. It comes after step 2, + * because removeFromIndex deletes the path's token. * - * Never rejects, and step 3 runs even if an earlier step threw: a watcher - * left holding would drop every later change. + * `applied` never rejects: one catch logs a failure of the batch. */ - const applyHeldBatch = async (batchPaths: ReadonlyArray): Promise => { - try { - await reconcileHeldPaths(batchPaths) - } catch (error) { - logger.error("failed to apply the changes held during startup", { - error: describeError(error), - }) - } - // Its own catch, because nothing awaits `applied` once the deadline has - // passed, so a throw here would be an unhandled rejection. - try { - goLiveAndReplay(batchPaths.length) - } catch (error) { - logger.error("failed to start replaying the changes held during the startup batch", { - error: describeError(error), - }) - } - } - - /** Starts the startup batch over the distinct paths held so far. The batch - * ignores each event's kind, because it checks every path against the disk. */ const applyHeldChanges = (): { heldPathCount: number; applied: Promise } => { + // Unreachable today, because applyHeldChanges runs once per startup and the + // watcher closes only when the rebuild fails, before that call. if (watcherState.mode !== "holding") return { heldPathCount: 0, applied: Promise.resolve() } - const batchEvents = watcherState.heldEvents - // Events arriving while the batch runs collect in a new list and are - // replayed after it. - watcherState = { mode: "holding", heldEvents: [] } - const batchPaths = [...new Set(batchEvents.map((heldEvent) => heldEvent.filePath))] + const { heldPaths } = watcherState + watcherState = { mode: "live" } + + for (const [filePath, latestEventKind] of heldPaths) { + if (latestEventKind === "unlink") removeFromIndex(filePath) + } + + const reservedPaths: ReservedPath[] = [] + for (const filePath of heldPaths.keys()) { + const reservation = Symbol() + currentEvents.set(relative(vaultPath, filePath), reservation) + reservedPaths.push({ filePath, reservation }) + } - return { heldPathCount: batchPaths.length, applied: applyHeldBatch(batchPaths) } + const applyBatch = async (): Promise => { + try { + await reconcileHeldPaths(reservedPaths) + logger.info("applied the changes held during startup indexing", { + heldPaths: reservedPaths.length, + }) + } catch (error) { + logger.error("failed to apply the changes held during startup", { + error: describeError(error), + }) + } + } + return { heldPathCount: reservedPaths.length, applied: applyBatch() } } watcher @@ -1002,9 +974,9 @@ const startHoldingWatcher = (params: WatchVaultParams, logger: Logger): HoldingW * reads anything and holds every change until the rebuild returns, so a change * landing mid-rebuild is applied after it instead of being missed or replaced * by the rebuild's older read. Resolves with the rebuild's result (the note - * count and its background embedding promise) once the held changes are - * indexed, or at heldChangesDeadline if they take longer; the rest then apply - * in the background. + * count and its background embedding promise) once the batch over the held + * changes commits its index writes, or at heldChangesDeadline if it takes + * longer; the rest then apply in the background. */ export const watchVaultAndBuildIndex = async ( params: WatchVaultParams, From 71b6910a515e7812fe367ffd63d2356a23791a9b Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:23:21 -0400 Subject: [PATCH 24/34] docs(architecture): describe the live startup batch Startup order now says the watcher goes live before the batch, removes held deletions in that step, and that a live change to a held path takes over from the batch. Co-Authored-By: Claude Opus 5.5 --- ARCHITECTURE.md | 8 ++++---- src/vault-mcp/search/__tests__/file-watcher.test.ts | 2 +- src/vault-mcp/search/file-watcher.ts | 5 +++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 9f79f9448..38466d19e 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -502,12 +502,12 @@ When no embedder is configured (`EMBEDDING_ENABLED=false`), no vectors are index 1. chokidar finishes its initial scan, and the watcher holds every change it reports from then on. 2. `rebuildFromVault` runs. The watcher is the only other writer of the index, so nothing else writes while it reads. -3. One batch checks each held path against the disk, 16 at a time. Paths gone from disk are removed first, then the rest are read again, 16 at a time (PDFs whose text is extracted, 4 at a time). -4. The watcher goes live and replays the changes reported while the batch ran: deletions at once, then the changed paths read again with the same limits. A path deleted or changed again before its read starts is left to that later event. +3. The watcher goes live. In the same step, it removes each held path whose latest event was a deletion, and marks every held path so that a live change to it takes over from the batch. +4. One batch checks each held path against the disk, 16 at a time. Paths gone from disk are removed first, then the rest are read again, 16 at a time (PDFs whose text is extracted, 4 at a time). -The server starts accepting requests once the step 3 batch commits and step 4 starts its replay, or after 10 s if a Sync backlog makes the batch longer; the batch then finishes in the background. +The server starts accepting requests once the step 4 batch commits, or after 10 s if a Sync backlog makes the batch longer; the batch then finishes in the background. A held path that a live change takes over is indexed by that change, which can finish just after the server starts. -A file that is created after step 1 and deleted within chokidar's 2 s write-stability window emits no event, so if the rebuild read it, it stays searchable until the next restart. +A file that is created after step 1 and deleted within chokidar's 2 s write-stability window emits no event, so if the rebuild or the step 4 batch read it, it stays searchable until the next restart. **Startup cleanup:** before resetting the source tables, the rebuild removes vectors whose parent chunk or memory-entry row is missing and retains vectors with surviving parents. diff --git a/src/vault-mcp/search/__tests__/file-watcher.test.ts b/src/vault-mcp/search/__tests__/file-watcher.test.ts index 5dbff0709..d17e4050d 100644 --- a/src/vault-mcp/search/__tests__/file-watcher.test.ts +++ b/src/vault-mcp/search/__tests__/file-watcher.test.ts @@ -2308,10 +2308,10 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { // batch's read is still gated is indexed at once. await writeFile(join(testVault, "S.md"), "aftertext\n") await fire("change", "S.md") + expect(pathsFor("aftertext")).toEqual(["S.md"]) expect(debugSpy).not.toHaveBeenCalledWith("held a change until startup indexing ends", { path: "S.md", }) - expect(pathsFor("aftertext")).toEqual(["S.md"]) readGate.release() await vi.waitFor(() => { expect(pathsFor("slowtext")).toEqual(["slow.md"]) diff --git a/src/vault-mcp/search/file-watcher.ts b/src/vault-mcp/search/file-watcher.ts index e370049c9..dc7281ca6 100644 --- a/src/vault-mcp/search/file-watcher.ts +++ b/src/vault-mcp/search/file-watcher.ts @@ -892,8 +892,9 @@ const startHoldingWatcher = (params: WatchVaultParams, logger: Logger): HoldingW } /** - * Goes live and starts the startup batch over the distinct held paths. These - * three steps run in one synchronous step, before any live event can start: + * Goes live and starts the startup batch over the distinct held paths. The + * first three steps run with no await between them, before any live event + * can start: * * 1. The watcher goes live. * 2. Each path whose latest held event is a deletion is removed, so a renamed From 5339e2f7e353009b83ba8685b5425ed474bde199 Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:52:00 -0400 Subject: [PATCH 25/34] style(file-watcher): define held paths and event tokens where first used, and clarify the startup batch's comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Name the poll-interval default, state why pdfjs needs a Uint8Array view, correct close's doc (scheduled rescans fire and do nothing), and say which rename case each removal ordering covers. In ARCHITECTURE.md's Startup order, say when holding starts, why nothing else writes the index during the rebuild, and name the concurrency limits. Ship-Check: code-quality · claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- ARCHITECTURE.md | 12 +-- src/vault-mcp/search/file-watcher.ts | 106 +++++++++++++++++---------- 2 files changed, 72 insertions(+), 46 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 38466d19e..b97a3f876 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -500,14 +500,14 @@ When no embedder is configured (`EMBEDDING_ENABLED=false`), no vectors are index **Startup order:** `watchVaultAndBuildIndex` starts the file watcher before the rebuild, so a change that lands while the rebuild reads the vault is neither missed nor replaced by the rebuild's older read: -1. chokidar finishes its initial scan, and the watcher holds every change it reports from then on. -2. `rebuildFromVault` runs. The watcher is the only other writer of the index, so nothing else writes while it reads. -3. The watcher goes live. In the same step, it removes each held path whose latest event was a deletion, and marks every held path so that a live change to it takes over from the batch. -4. One batch checks each held path against the disk, 16 at a time. Paths gone from disk are removed first, then the rest are read again, 16 at a time (PDFs whose text is extracted, 4 at a time). +1. The watcher starts holding: from then on, it records each path chokidar reports as changed instead of indexing it. Startup waits for chokidar's initial scan to finish. +2. `rebuildFromVault` runs. The watcher is the index's only other writer, because a tool's vault write reaches the index as a watcher event. The watcher is holding, so nothing else writes while the rebuild reads. +3. The watcher goes live. In the same step, it removes each held path whose latest event was a deletion, and marks every held path so that a live change to it takes over from the step 4 batch. +4. One batch checks each held path against the disk, `INDEXING_FILE_READ_CONCURRENCY` (16) at a time. Paths gone from disk are removed first. The rest are read again within the rebuild's limits: other files 16 at a time, then the PDFs whose text is extracted, `PDF_EXTRACTION_CONCURRENCY` (4) at a time. -The server starts accepting requests once the step 4 batch commits, or after 10 s if a Sync backlog makes the batch longer; the batch then finishes in the background. A held path that a live change takes over is indexed by that change, which can finish just after the server starts. +The server starts accepting requests once the step 4 batch has finished writing to the index, or after 10 s if a Sync backlog makes the batch longer; the batch then finishes in the background. A held path that a live change takes over is indexed by that change, which can finish just after the server starts. -A file that is created after step 1 and deleted within chokidar's 2 s write-stability window emits no event, so if the rebuild or the step 4 batch read it, it stays searchable until the next restart. +A file that is created after step 1 and deleted within chokidar's 2 s write-stability window emits no event, so if the rebuild read it, it stays searchable until the next restart. **Startup cleanup:** before resetting the source tables, the rebuild removes vectors whose parent chunk or memory-entry row is missing and retains vectors with surviving parents. diff --git a/src/vault-mcp/search/file-watcher.ts b/src/vault-mcp/search/file-watcher.ts index dc7281ca6..044539c3d 100644 --- a/src/vault-mcp/search/file-watcher.ts +++ b/src/vault-mcp/search/file-watcher.ts @@ -31,10 +31,18 @@ const POLLING_INTERVAL_MS = 300 /** Default for FileWatcherOptions.stabilityThreshold (see its doc). */ const DEFAULT_STABILITY_THRESHOLD_MS = 2000 -/** Default for FileWatcherOptions.heldChangesDeadline. The local compose health - * check's 20 s start period sets the bound: 10 s leaves the server time to - * listen inside it. Without a deadline, a Sync backlog arriving during startup - * could keep the server from listening past the CLI's 120 s health wait. */ +/** Default for FileWatcherOptions.pollInterval. */ +const DEFAULT_POLL_INTERVAL_MS = 100 + +/** + * Default for FileWatcherOptions.heldChangesDeadline. The server starts + * listening only after watchVaultAndBuildIndex resolves, so the deadline caps + * how long a Sync backlog arriving during startup delays listening. + * - The local compose health check's 20 s start period sets the value: 10 s + * leaves the server time to listen inside it. + * - Without a deadline, the backlog could keep the server from listening past + * the CLI's 120 s health wait. + */ const DEFAULT_HELD_CHANGES_DEADLINE_MS = 10_000 type FileWatcherOptions = Readonly<{ @@ -52,8 +60,9 @@ type FileWatcherOptions = Readonly<{ * past the awaitWriteFinish window lets in-flight writes settle before the * rescan reads them. */ newDirectoryRescanDelay?: number - /** ms startup waits for the changes held during the rebuild before returning - * anyway (default 10000); the rest are applied in the background. */ + /** ms startup waits after the rebuild for the changes chokidar reported + * meanwhile to be indexed, before returning anyway (default 10000); the + * rest are applied in the background. */ heldChangesDeadline?: number }> @@ -76,8 +85,9 @@ type WatcherState = { mode: "holding"; heldPaths: Map } | { mode: "live" } | { mode: "closed" } /** A held path and the event token reserved for it when the watcher went live. - * The startup batch's removal or read of the path goes ahead only while the - * reservation is still the path's token. */ + * An event token is a symbol stored per path in currentEvents, which each + * newer event for the path replaces. The startup batch's removal or read of + * the path goes ahead only while the reservation is still the path's token. */ type ReservedPath = Readonly<{ filePath: string; reservation: symbol }> /** What indexing one changed file did. A note's embedding is still running when @@ -109,17 +119,17 @@ type HoldingWatcher = Readonly<{ * returns how many distinct paths that is. `applied` resolves once the * batch's index writes commit, and never rejects. */ applyHeldChanges: () => { heldPathCount: number; applied: Promise } - /** Stops watching, including the rescans and retries still scheduled. Used - * only when startup fails before the held changes apply. Never rejects, so - * the startup failure stays the error reported. */ + /** Stops watching; a rescan or retry still scheduled fires later and does + * nothing. Used only when startup fails before the held changes apply. + * Never rejects, so the startup failure stays the error reported. */ close: () => Promise }> /** - * Watches the vault and mirrors every file add/change/delete into the search - * index. Every change is held from the moment chokidar starts until - * applyHeldChanges goes live. After a successful startup the watcher runs for - * the process lifetime. + * Starts a vault watcher in holding mode: from the moment chokidar starts, it + * records each changed path instead of indexing it. applyHeldChanges switches + * it live, and from then on it mirrors every file add/change/delete into the + * search index for the process lifetime. */ const startHoldingWatcher = (params: WatchVaultParams, logger: Logger): HoldingWatcher => { const { vaultPath, search, watcherOptions } = params @@ -192,7 +202,8 @@ const startHoldingWatcher = (params: WatchVaultParams, logger: Logger): HoldingW } // Canvas files are always read — link extraction is unconditional. - // PDF and text files are only read when file content FTS is enabled. + // PDF and text files (INDEXABLE_TEXT_EXTENSIONS, which includes .pdf) are + // only read when file content FTS is enabled. const extension = extname(filePath) const isCanvas = extension === ".canvas" const isIndexableNonCanvas = @@ -204,6 +215,8 @@ const startHoldingWatcher = (params: WatchVaultParams, logger: Logger): HoldingW if (extension !== ".pdf") return await readFile(filePath, "utf8") const buffer = await readFile(filePath) + // pdfjs throws on a Node Buffer, so it gets a plain Uint8Array view + // over the same bytes. const pdfData = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength) const pdfResult = await extractPdfText(pdfData) return pdfResult.text @@ -222,8 +235,9 @@ const startHoldingWatcher = (params: WatchVaultParams, logger: Logger): HoldingW ) // Embed file content vectors — serialized per path via the same - // pendingEmbeds map (note paths end in .md, file paths don't). - // Reads the processed content from the file_content table. + // pendingEmbeds map. A note path ends in .md and a file path never + // does, so their keys never collide. Reads the processed content from + // the file_content table. const previousEmbed = pendingEmbeds.get(relativePath) ?? Promise.resolve() const currentEmbed = previousEmbed .catch((previousError) => { @@ -327,10 +341,9 @@ const startHoldingWatcher = (params: WatchVaultParams, logger: Logger): HoldingW }) pendingEmbeds.set(relativePath, currentEmbed) - // The caller reports the job's failure, so it receives the promise with - // the queue cleanup attached rather than currentEmbed: `.finally` passes - // a rejection through, and a cleanup promise nobody received would reject - // unhandled. The cleanup leaves a newer queued job registered. + // The caller gets the cleanup promise, not currentEmbed, because a + // promise nobody holds rejects unhandled: `.finally` passes currentEmbed's + // rejection through to it. The cleanup leaves a newer queued job registered. const embedding = currentEmbed.finally(() => { if (pendingEmbeds.get(relativePath) === currentEmbed) { pendingEmbeds.delete(relativePath) @@ -357,9 +370,9 @@ const startHoldingWatcher = (params: WatchVaultParams, logger: Logger): HoldingW return indexChangedNonMdFile(params) } - /** Releases a path's event token unless a newer event replaced it. A newer - * handler may already have finished and removed its entry; only the current - * event owns cleanup, and absence still invalidates older reads. */ + /** Releases a path's event token unless a newer event replaced it: deleting + * a newer event's token would make that event's read look superseded. An + * older read that finds no token still fails its check. */ const releaseEventToken = (relativePath: string, eventToken: symbol): void => { if (currentEvents.get(relativePath) === eventToken) currentEvents.delete(relativePath) } @@ -391,7 +404,9 @@ const startHoldingWatcher = (params: WatchVaultParams, logger: Logger): HoldingW } /** chokidar's add and change listener. While the watcher holds, it records - * the path for applyHeldChanges instead of indexing it. */ + * the path for applyHeldChanges instead of indexing it. chokidar ignores + * the returned promise, so each step logs its own failure and the listener + * never rejects. */ const handleChange = async (filePath: string): Promise => { if (watcherState.mode === "closed") return if (watcherState.mode === "holding") { @@ -519,7 +534,7 @@ const startHoldingWatcher = (params: WatchVaultParams, logger: Logger): HoldingW // indexing to avoid reading partial content awaitWriteFinish: { stabilityThreshold, - pollInterval: watcherOptions?.pollInterval ?? 100, + pollInterval: watcherOptions?.pollInterval ?? DEFAULT_POLL_INTERVAL_MS, }, }) @@ -750,6 +765,8 @@ const startHoldingWatcher = (params: WatchVaultParams, logger: Logger): HoldingW ): Promise => { const pathSegments = relative(vaultPath, filePath).split(sep) + // Walks down from the vault root: the vault root must list the first + // segment, that folder the second, and so on to the file's own name. // Sequential, so the first folder that lacks the next name ends the check. for (const [segmentIndex, segmentName] of pathSegments.entries()) { const folderPath = join(vaultPath, ...pathSegments.slice(0, segmentIndex)) @@ -757,11 +774,11 @@ const startHoldingWatcher = (params: WatchVaultParams, logger: Logger): HoldingW const missingFromListing = folderListing.status === "listed" && !folderListing.names.has(segmentName) + // A folder that could not be listed proves nothing either way, so the + // walk goes on and the stat below decides. if (folderListing.status === "missing" || missingFromListing) return "missing" } - // A folder that could not be listed proves nothing either way, so the stat - // below decides. try { const fileStat = await statOrNull(filePath) @@ -778,7 +795,9 @@ const startHoldingWatcher = (params: WatchVaultParams, logger: Logger): HoldingW } } - /** Removes a held path the startup batch found missing on disk. */ + /** Removes a held path the startup batch found missing on disk. A held + * deletion that applyHeldChanges already removed comes here again while its + * path is still gone; the second removal finds no rows and changes nothing. */ const removeHeldPath = (filePath: string): void => { logger.debug("removing a held path that is gone from disk", { path: relative(vaultPath, filePath), @@ -789,7 +808,7 @@ const startHoldingWatcher = (params: WatchVaultParams, logger: Logger): HoldingW /** * Reads one held path again for the startup batch, under its reservation, * without waiting for its embedding. - * - A live event for the path since the switch replaced the reservation, so + * - A live event for the path since going live replaced the reservation, so * the read drops what it read, as an older live read does. * - A path that vanished by its read is left to the `unlink` chokidar sends * for every file it tracks, as a live read leaves it. @@ -818,9 +837,13 @@ const startHoldingWatcher = (params: WatchVaultParams, logger: Logger): HoldingW } } - /** Reads held paths again at the rebuild's bounds: PDFs whose text is - * extracted PDF_EXTRACTION_CONCURRENCY at a time, after the others, which go - * INDEXING_FILE_READ_CONCURRENCY at a time. */ + /** + * Reads held paths again within the concurrency limits the rebuild also uses. + * - Other files go first, INDEXING_FILE_READ_CONCURRENCY at a time, so slow + * PDF extraction does not hold them back. + * - PDFs whose text is extracted go after, PDF_EXTRACTION_CONCURRENCY at a + * time. The groups run one after the other, so the two limits never add up. + */ const indexHeldPathsWithinBounds = async ( reservedPaths: ReadonlyArray, ): Promise => { @@ -876,10 +899,11 @@ const startHoldingWatcher = (params: WatchVaultParams, logger: Logger): HoldingW .map((check) => check.reservedPath) // Removals run before reads. upsertNote re-resolves only links whose target - // is not indexed, so when a renamed note's old path is removed first, the - // new path claims the links that pointed at the old one. + // is not indexed, so when both a renamed note's old and new paths were + // held, removing the old path first lets the new path's read claim the + // links that pointed at the old one. for (const { filePath, reservation } of missingPaths) { - // Skips a path a live event has taken since the switch, so a live `add` + // Skips a path a live event has taken since going live, so a live `add` // that indexed it is not removed. No await sits between this check and // the removal, so no live event can take the path in between. const reservationIsCurrent = currentEvents.get(relative(vaultPath, filePath)) === reservation @@ -897,9 +921,11 @@ const startHoldingWatcher = (params: WatchVaultParams, logger: Logger): HoldingW * can start: * * 1. The watcher goes live. - * 2. Each path whose latest held event is a deletion is removed, so a renamed - * note's new path, arriving live, claims the links that pointed at the old - * one. A file that came back is read again by the batch. + * 2. Each path whose latest held event is a deletion is removed. This covers + * a renamed note whose new path arrives as a live event: the old path is + * already gone, so the new path claims the links that pointed at the old + * one. reconcileHeldPaths covers a rename whose two paths were both held. + * A file that came back is read again by the batch. * 3. Each held path gets a reserved event token. It comes after step 2, * because removeFromIndex deletes the path's token. * From c6b80724df1c00bd42c3975a256ec640b6c94272 Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Mon, 19 Oct 2026 15:00:50 -0400 Subject: [PATCH 26/34] test(file-watcher): pin that a held change before a deletion does not stop the removal at going live MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The live-rename test now also holds a change to the old path before its deletion, so the latest held event, not the first, decides that going live removes the old path and incoming links follow the new one. Ship-Check: test-audit · claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- .../search/__tests__/file-watcher.test.ts | 52 +++++++++++-------- 1 file changed, 31 insertions(+), 21 deletions(-) diff --git a/src/vault-mcp/search/__tests__/file-watcher.test.ts b/src/vault-mcp/search/__tests__/file-watcher.test.ts index d17e4050d..b24980f06 100644 --- a/src/vault-mcp/search/__tests__/file-watcher.test.ts +++ b/src/vault-mcp/search/__tests__/file-watcher.test.ts @@ -1710,29 +1710,39 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { expect(linkFromSource()).toEqual([{ path: "folder/B.md", exists: true }]) }) - it("points incoming links at a renamed note whose new path arrives live during the batch", async () => { - const { testVault, search, fire, startup, finishRebuild } = await createControlledStartup({ - seedNotes: { "B.md": "old home\n", "source.md": "points at [[B]]\n" }, - }) - await writeFile(join(testVault, "source.md"), "points at [[B]]\n") - const listingGate = await gateRootListing(testVault) - const linkFromSource = () => { - const outgoingLinks = search.getOutgoingLinks({ path: "source.md" }, logger) - return outgoingLinks.map((link) => ({ path: link.path, exists: link.exists })) - } - expect(linkFromSource()).toEqual([{ path: "B.md", exists: true }]) + // A path's latest held event decides what going live does, so a change held + // before B.md's deletion must not keep B.md indexed. + it.each([ + { label: "its deletion", heldEvents: ["unlink"] }, + { label: "a change then its deletion", heldEvents: ["change", "unlink"] }, + ] as const)( + "points incoming links at a renamed note whose new path arrives live during the batch, after $label was held", + async ({ heldEvents }) => { + const { testVault, search, fire, startup, finishRebuild } = await createControlledStartup({ + seedNotes: { "B.md": "old home\n", "source.md": "points at [[B]]\n" }, + }) + await writeFile(join(testVault, "source.md"), "points at [[B]]\n") + const listingGate = await gateRootListing(testVault) + const linkFromSource = () => { + const outgoingLinks = search.getOutgoingLinks({ path: "source.md" }, logger) + return outgoingLinks.map((link) => ({ path: link.path, exists: link.exists })) + } + expect(linkFromSource()).toEqual([{ path: "B.md", exists: true }]) - await fire("unlink", "B.md") - finishRebuild() - await listingGate.reached - await mkdir(join(testVault, "folder")) - await writeFile(join(testVault, "folder", "B.md"), "new home\n") - await fire("add", "folder/B.md") - listingGate.release() - await startup + for (const heldEvent of heldEvents) { + await fire(heldEvent, "B.md") + } + finishRebuild() + await listingGate.reached + await mkdir(join(testVault, "folder")) + await writeFile(join(testVault, "folder", "B.md"), "new home\n") + await fire("add", "folder/B.md") + listingGate.release() + await startup - expect(linkFromSource()).toEqual([{ path: "folder/B.md", exists: true }]) - }) + expect(linkFromSource()).toEqual([{ path: "folder/B.md", exists: true }]) + }, + ) it("keeps a note that a live add indexed while the batch checked its held deletion", async () => { const { testVault, search, fire, startup, finishRebuild } = await createControlledStartup({ From 5c723f84acf1af6a1359bcde0ca62a7391780808 Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Mon, 19 Oct 2026 15:11:32 -0400 Subject: [PATCH 27/34] fix(file-watcher): describe the startup batch's superseded reads as read-then-drop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The batch reads each held path under its reservation and drops what it read when a live event replaced the reservation; only the removal is skipped. Four comments described that read as skipped or as starting after the replacement, and the listing-failure warning said the folder's files are read, when stat decides and a gone file is removed. Ship-Check: bug-check · claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- .../search/__tests__/file-watcher.test.ts | 2 +- src/vault-mcp/search/file-watcher.ts | 22 ++++++++++--------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/vault-mcp/search/__tests__/file-watcher.test.ts b/src/vault-mcp/search/__tests__/file-watcher.test.ts index b24980f06..c2c4f86af 100644 --- a/src/vault-mcp/search/__tests__/file-watcher.test.ts +++ b/src/vault-mcp/search/__tests__/file-watcher.test.ts @@ -2456,7 +2456,7 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { await startup expect(warnSpy).toHaveBeenCalledExactlyOnceWith( - "could not list a folder of held changes, so its files are read instead", + "could not list a folder of held changes, so stat checks its files", { path: "locked", error: "[Error]: controlled listing failure" }, ) expect(search.fullTextSearch({ query: "listedopal" }, logger).map((hit) => hit.path)).toEqual([ diff --git a/src/vault-mcp/search/file-watcher.ts b/src/vault-mcp/search/file-watcher.ts index 044539c3d..82bcb9e52 100644 --- a/src/vault-mcp/search/file-watcher.ts +++ b/src/vault-mcp/search/file-watcher.ts @@ -86,8 +86,8 @@ type WatcherState = /** A held path and the event token reserved for it when the watcher went live. * An event token is a symbol stored per path in currentEvents, which each - * newer event for the path replaces. The startup batch's removal or read of - * the path goes ahead only while the reservation is still the path's token. */ + * newer event for the path replaces. The startup batch removes the path, or + * indexes what it read, only while the reservation is still the path's token. */ type ReservedPath = Readonly<{ filePath: string; reservation: symbol }> /** What indexing one changed file did. A note's embedding is still running when @@ -97,8 +97,8 @@ type ChangeIndexResult = | { status: "file-indexed" } // The file was gone when read. | { status: "vanished" } - // A newer event or a removal for the same path arrived during the read, so - // this read's content was dropped. + // A newer event or a removal for the same path replaced this read's token + // before the read finished, so its content was dropped. | { status: "superseded" } // A note's read, stat or index write failed, or a non-md file's stat or // asset-table write did; already logged. A non-md file whose content alone @@ -147,8 +147,9 @@ const startHoldingWatcher = (params: WatchVaultParams, logger: Logger): HoldingW * - indexChangedFile stores a fresh token before it reads the file, and drops * what it read if a newer event or a removal replaced the token meanwhile. * - applyHeldChanges reserves one for each held path as the watcher goes - * live. The startup batch reads the path under it, so a later deletion or a - * newer event for the path cancels the batch's removal or read. + * live. The startup batch reads the path under it, so after a later + * deletion or a newer event for the path, the batch skips its removal or + * drops what it read. */ const currentEvents = new Map() @@ -741,7 +742,7 @@ const startHoldingWatcher = (params: WatchVaultParams, logger: Logger): HoldingW } catch (error) { if (isMissingPathError(error)) return { status: "missing" } - logger.warn("could not list a folder of held changes, so its files are read instead", { + logger.warn("could not list a folder of held changes, so stat checks its files", { path: relative(vaultPath, folderPath), error: describeError(error), }) @@ -808,7 +809,7 @@ const startHoldingWatcher = (params: WatchVaultParams, logger: Logger): HoldingW /** * Reads one held path again for the startup batch, under its reservation, * without waiting for its embedding. - * - A live event for the path since going live replaced the reservation, so + * - If a live event for the path replaced the reservation after going live, * the read drops what it read, as an older live read does. * - A path that vanished by its read is left to the `unlink` chokidar sends * for every file it tracks, as a live read leaves it. @@ -868,8 +869,9 @@ const startHoldingWatcher = (params: WatchVaultParams, logger: Logger): HoldingW } /** Checks each held path against the disk, removes the missing ones, then - * reads the rest again. The watcher is live meanwhile, so a path whose - * reservation a live event replaced is skipped: that event is newer. */ + * reads the rest again. The watcher is live meanwhile, so for a path whose + * reservation a live event replaced, the removal is skipped and the read's + * result is dropped: that event is newer. */ const reconcileHeldPaths = async (reservedPaths: ReadonlyArray): Promise => { const folderListings = new Map>() From 2a9fd8f9c81d2f2c515d47590e94cfe8d136d36c Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:09:15 -0400 Subject: [PATCH 28/34] test(file-watcher): pin removals before reads and the vanished read left to its unlink A rename whose old path's latest held event is a change is removed by the batch before its reads, so the new path still claims the incoming links. A batch read that finds the file gone leaves the rows for chokidar's unlink. The two rename comments now say which step covers which order. Co-Authored-By: Claude Opus 5.5 --- .../search/__tests__/file-watcher.test.ts | 88 ++++++++++++++----- src/vault-mcp/search/file-watcher.ts | 17 ++-- 2 files changed, 76 insertions(+), 29 deletions(-) diff --git a/src/vault-mcp/search/__tests__/file-watcher.test.ts b/src/vault-mcp/search/__tests__/file-watcher.test.ts index c2c4f86af..c7a08c8f5 100644 --- a/src/vault-mcp/search/__tests__/file-watcher.test.ts +++ b/src/vault-mcp/search/__tests__/file-watcher.test.ts @@ -1665,29 +1665,47 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { // chokidar sends an unlink for every file it tracks, so the batch leaves a // path that vanished by its read to that live event. - it.each([{ unlinkArrives: "before the read returns" }, { unlinkArrives: "after startup" }])( - "removes a held path deleted between its check and its read once its unlink arrives $unlinkArrives", - async ({ unlinkArrives }) => { - const { testVault, notePaths, fire, startup, finishRebuild } = await createControlledStartup({ - seedNotes: { "late.md": "oldtext\n" }, - }) - const notePath = join(testVault, "late.md") - await writeFile(notePath, "newer text\n") - const readGate = await gateFirstRead(notePath, "before-disk") - expect(notePaths()).toEqual([{ path: "late.md" }]) + it("removes a held path deleted between its check and its read when its unlink arrives during the read", async () => { + const { testVault, notePaths, fire, startup, finishRebuild } = await createControlledStartup({ + seedNotes: { "late.md": "oldtext\n" }, + }) + const notePath = join(testVault, "late.md") + await writeFile(notePath, "newer text\n") + const readGate = await gateFirstRead(notePath, "before-disk") + expect(notePaths()).toEqual([{ path: "late.md" }]) - await fire("change", "late.md") - finishRebuild() - await readGate.reached - await unlink(notePath) - if (unlinkArrives === "before the read returns") await fire("unlink", "late.md") - readGate.release() - await startup - if (unlinkArrives === "after startup") await fire("unlink", "late.md") + await fire("change", "late.md") + finishRebuild() + await readGate.reached + await unlink(notePath) + await fire("unlink", "late.md") + readGate.release() + await startup - expect(notePaths()).toEqual([]) - }, - ) + expect(notePaths()).toEqual([]) + }) + + it("leaves a held path deleted between its check and its read to its unlink after startup", async () => { + const { testVault, notePaths, fire, startup, finishRebuild } = await createControlledStartup({ + seedNotes: { "late.md": "oldtext\n" }, + }) + const notePath = join(testVault, "late.md") + await writeFile(notePath, "newer text\n") + const readGate = await gateFirstRead(notePath, "before-disk") + expect(notePaths()).toEqual([{ path: "late.md" }]) + + await fire("change", "late.md") + finishRebuild() + await readGate.reached + await unlink(notePath) + readGate.release() + await startup + // The batch's read found the file gone and changed nothing. + expect(notePaths()).toEqual([{ path: "late.md" }]) + await fire("unlink", "late.md") + + expect(notePaths()).toEqual([]) + }) it("removes a renamed note before reading its new path, so incoming links follow", async () => { const { testVault, search, fire, startup, finishRebuild } = await createControlledStartup({ @@ -1710,6 +1728,30 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { expect(linkFromSource()).toEqual([{ path: "folder/B.md", exists: true }]) }) + it("removes a renamed note's old path before reading its new one when the old path's latest held event is a change", async () => { + const { testVault, search, fire, startup, finishRebuild } = await createControlledStartup({ + seedNotes: { "B.md": "old home\n", "source.md": "points at [[B]]\n" }, + }) + await mkdir(join(testVault, "folder")) + await writeFile(join(testVault, "folder", "B.md"), "new home\n") + await writeFile(join(testVault, "source.md"), "points at [[B]]\n") + const linkFromSource = () => { + const outgoingLinks = search.getOutgoingLinks({ path: "source.md" }, logger) + return outgoingLinks.map((link) => ({ path: link.path, exists: link.exists })) + } + expect(linkFromSource()).toEqual([{ path: "B.md", exists: true }]) + + // The held change makes B.md's latest event a change, so going live does + // not remove it; the batch's removal step does, before the reads. + await fire("unlink", "B.md") + await fire("change", "B.md") + await fire("add", "folder/B.md") + finishRebuild() + await startup + + expect(linkFromSource()).toEqual([{ path: "folder/B.md", exists: true }]) + }) + // A path's latest held event decides what going live does, so a change held // before B.md's deletion must not keep B.md indexed. it.each([ @@ -2538,6 +2580,10 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { // The batch's read finds the file gone and leaves it to the unlink // chokidar sends for it. expect(checkedPaths).toEqual(new Set([contentPath])) + expect(fileRows()).toEqual({ + assets: [{ path: "content.txt" }], + contents: [{ path: "content.txt" }], + }) await fire("unlink", "content.txt") expect(fileRows()).toEqual({ assets: [], contents: [] }) diff --git a/src/vault-mcp/search/file-watcher.ts b/src/vault-mcp/search/file-watcher.ts index 82bcb9e52..6a68c4f26 100644 --- a/src/vault-mcp/search/file-watcher.ts +++ b/src/vault-mcp/search/file-watcher.ts @@ -901,9 +901,10 @@ const startHoldingWatcher = (params: WatchVaultParams, logger: Logger): HoldingW .map((check) => check.reservedPath) // Removals run before reads. upsertNote re-resolves only links whose target - // is not indexed, so when both a renamed note's old and new paths were - // held, removing the old path first lets the new path's read claim the - // links that pointed at the old one. + // is not indexed, so removing a renamed note's old path first lets the new + // path's read claim the links that pointed at it. Going live already + // removed an old path whose latest held event was its deletion; this covers + // one whose latest held event was a change. for (const { filePath, reservation } of missingPaths) { // Skips a path a live event has taken since going live, so a live `add` // that indexed it is not removed. No await sits between this check and @@ -923,11 +924,11 @@ const startHoldingWatcher = (params: WatchVaultParams, logger: Logger): HoldingW * can start: * * 1. The watcher goes live. - * 2. Each path whose latest held event is a deletion is removed. This covers - * a renamed note whose new path arrives as a live event: the old path is - * already gone, so the new path claims the links that pointed at the old - * one. reconcileHeldPaths covers a rename whose two paths were both held. - * A file that came back is read again by the batch. + * 2. Each path whose latest held event is a deletion is removed, so a renamed + * note's new path, held or arriving as a live event, claims the links that + * pointed at the old one. An old path whose latest held event is a change + * is removed by reconcileHeldPaths before its reads instead. A file that + * came back is read again by the batch. * 3. Each held path gets a reserved event token. It comes after step 2, * because removeFromIndex deletes the path's token. * From 48e5ee8174ca236cdde394508b12b2f627b42f45 Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:11:27 -0400 Subject: [PATCH 29/34] style(file-watcher): name the old path and the batch's reads in the rename-order comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ship-Check: code-quality · claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- src/vault-mcp/search/file-watcher.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/vault-mcp/search/file-watcher.ts b/src/vault-mcp/search/file-watcher.ts index 6a68c4f26..3a660f339 100644 --- a/src/vault-mcp/search/file-watcher.ts +++ b/src/vault-mcp/search/file-watcher.ts @@ -902,9 +902,9 @@ const startHoldingWatcher = (params: WatchVaultParams, logger: Logger): HoldingW // Removals run before reads. upsertNote re-resolves only links whose target // is not indexed, so removing a renamed note's old path first lets the new - // path's read claim the links that pointed at it. Going live already - // removed an old path whose latest held event was its deletion; this covers - // one whose latest held event was a change. + // path's read claim the links that pointed at the old path. Going live + // already removed an old path whose latest held event is a deletion; this + // loop removes one whose latest held event is a change. for (const { filePath, reservation } of missingPaths) { // Skips a path a live event has taken since going live, so a live `add` // that indexed it is not removed. No await sits between this check and @@ -926,9 +926,9 @@ const startHoldingWatcher = (params: WatchVaultParams, logger: Logger): HoldingW * 1. The watcher goes live. * 2. Each path whose latest held event is a deletion is removed, so a renamed * note's new path, held or arriving as a live event, claims the links that - * pointed at the old one. An old path whose latest held event is a change - * is removed by reconcileHeldPaths before its reads instead. A file that - * came back is read again by the batch. + * pointed at the old one. reconcileHeldPaths removes an old path whose + * latest held event is a change, before the batch reads any path. A file + * that came back is read again by the batch. * 3. Each held path gets a reserved event token. It comes after step 2, * because removeFromIndex deletes the path's token. * From 5c145387aa854eff62dd23ae2d8de3937e750c49 Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:14:53 -0400 Subject: [PATCH 30/34] test(file-watcher): pin that the startup batch completes when a held read finds its file gone MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A vanished read that threw instead of returning would fail the whole batch at error level while both "left to its unlink" tests still passed; each now asserts the batch's completion log. Ship-Check: test-audit · claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- src/vault-mcp/search/__tests__/file-watcher.test.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/vault-mcp/search/__tests__/file-watcher.test.ts b/src/vault-mcp/search/__tests__/file-watcher.test.ts index c7a08c8f5..e83f5d1b3 100644 --- a/src/vault-mcp/search/__tests__/file-watcher.test.ts +++ b/src/vault-mcp/search/__tests__/file-watcher.test.ts @@ -1692,6 +1692,7 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { const notePath = join(testVault, "late.md") await writeFile(notePath, "newer text\n") const readGate = await gateFirstRead(notePath, "before-disk") + const infoSpy = spyOnLogger("info") expect(notePaths()).toEqual([{ path: "late.md" }]) await fire("change", "late.md") @@ -1700,8 +1701,12 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { await unlink(notePath) readGate.release() await startup - // The batch's read found the file gone and changed nothing. + // The batch's read found the file gone and changed nothing, and the batch + // still completed. expect(notePaths()).toEqual([{ path: "late.md" }]) + expect(infoSpy).toHaveBeenCalledWith("applied the changes held during startup indexing", { + heldPaths: 1, + }) await fire("unlink", "late.md") expect(notePaths()).toEqual([]) @@ -2573,17 +2578,21 @@ describe("watchVaultAndBuildIndex — changes during startup", () => { return fileStat }) onTestFinished(() => vi.mocked(statOrNull).mockRestore()) + const infoSpy = spyOnLogger("info") await fire("change", "content.txt") finishRebuild() await startup // The batch's read finds the file gone and leaves it to the unlink - // chokidar sends for it. + // chokidar sends for it, and the batch still completes. expect(checkedPaths).toEqual(new Set([contentPath])) expect(fileRows()).toEqual({ assets: [{ path: "content.txt" }], contents: [{ path: "content.txt" }], }) + expect(infoSpy).toHaveBeenCalledWith("applied the changes held during startup indexing", { + heldPaths: 1, + }) await fire("unlink", "content.txt") expect(fileRows()).toEqual({ assets: [], contents: [] }) From 98dc559f5044caf9d2febf41f6d281ea0e5b9866 Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:10:56 -0400 Subject: [PATCH 31/34] docs(server): name the embedding promise startup leaves unawaited Co-Authored-By: Claude Opus 5.5 --- src/vault-mcp/server.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/vault-mcp/server.ts b/src/vault-mcp/server.ts index 49c7191a1..fa69c6e9d 100644 --- a/src/vault-mcp/server.ts +++ b/src/vault-mcp/server.ts @@ -151,7 +151,9 @@ const startServer = async (): Promise => { // Before the watcher starts, so its template writes are not held and re-read. await bootstrapMemoryIfEnabled(config, vaultPath) - // The returned embedding promise is not awaited: the rebuild logs its failure itself. + // watchVaultAndBuildIndex also returns the rebuild's embedding promise. Startup + // does not wait for it, so search answers by keyword until the vectors are + // ready, and rebuildFromVault logs an embedding failure itself. const { count } = await watchVaultAndBuildIndex( { vaultPath, search, watcherOptions: { usePolling: config.windowsBindMount } }, logger, From 3cb84987dfc61a51a4641caeb543ba6383b4eb46 Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:12:14 -0400 Subject: [PATCH 32/34] docs(server): say startup serves requests while the embedding pass runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Search does not answer by keyword alone until the pass finishes: a warm index keeps its earlier vectors, and hybrid search uses each note's vectors as soon as the pass stores them. Ship-Check: code-quality · claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- src/vault-mcp/server.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/vault-mcp/server.ts b/src/vault-mcp/server.ts index fa69c6e9d..9ff501b4f 100644 --- a/src/vault-mcp/server.ts +++ b/src/vault-mcp/server.ts @@ -152,8 +152,8 @@ const startServer = async (): Promise => { await bootstrapMemoryIfEnabled(config, vaultPath) // watchVaultAndBuildIndex also returns the rebuild's embedding promise. Startup - // does not wait for it, so search answers by keyword until the vectors are - // ready, and rebuildFromVault logs an embedding failure itself. + // does not wait for it, so the server takes requests while the embedding pass + // is still running, and rebuildFromVault logs an embedding failure itself. const { count } = await watchVaultAndBuildIndex( { vaultPath, search, watcherOptions: { usePolling: config.windowsBindMount } }, logger, From 568a32d99d302aebf2a2c5217713d9f706aae419 Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:13:45 -0400 Subject: [PATCH 33/34] docs: say search finds a note by keyword only until its vectors exist Three surfaces said search is keyword-only until the embedding pass finishes. Hybrid search falls back to keyword results only for a query with no vector match, and vectors kept from earlier runs are reused, so semantic matches fill in note by note. Co-Authored-By: Claude Opus 5.5 --- ARCHITECTURE.md | 2 +- deploy/local/README.md | 3 ++- src/vault-mcp/search/search-index.ts | 5 ++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index b97a3f876..69f526feb 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -496,7 +496,7 @@ When no embedder is configured (`EMBEDDING_ENABLED=false`), no vectors are index 1. **Pass 1** — index notes (FTS5 + metadata) 2. **Pass 2** — extract links (with the complete path list for resolution), then index file content (canvas, PDF, text → FTS5) -3. **Pass 3 (background)** — embed notes, then file content. Search works with FTS-only until vectors are ready +3. **Pass 3 (background)** — embed notes, then file content. Until a note's vectors exist, search finds it by keyword only **Startup order:** `watchVaultAndBuildIndex` starts the file watcher before the rebuild, so a change that lands while the rebuild reads the vault is neither missed nor replaced by the rebuild's older read: diff --git a/deploy/local/README.md b/deploy/local/README.md index e856e802e..e635347a5 100644 --- a/deploy/local/README.md +++ b/deploy/local/README.md @@ -27,7 +27,8 @@ What happens on first start: - The image is pulled (~150MB) - The keyword index builds in seconds — search works right away - The semantic (embedding) index builds in the background — expect a few - minutes on a large vault, with search served keyword-only until it finishes + minutes on a large vault. Keyword search works meanwhile, and semantic + matches appear as each note is embedded > **On Windows?** Set `WINDOWS_MODE=true` in your `.env` — then `VAULT_PATH` can point > at a normal Windows path like `C:\Users\you\MyVault`. With the CLI, edit the diff --git a/src/vault-mcp/search/search-index.ts b/src/vault-mcp/search/search-index.ts index 1f17af5d3..5bf55d728 100644 --- a/src/vault-mcp/search/search-index.ts +++ b/src/vault-mcp/search/search-index.ts @@ -2480,9 +2480,8 @@ export const createSearchIndex = ( // Every note read from disk, including notes that failed to index, unlike indexedNotes const readableNotePaths = new Set(noteContents.map((note) => note.relativePath)) - // Pass 3 runs in the background — the server can start accepting requests - // immediately after FTS indexing (Passes 1+2) finishes. Embedding is a - // progressive enhancement: search works with FTS-only until vectors are ready. + // Pass 3 runs in the background, so startup does not wait for it. Until a + // note's vectors exist, search finds that note by keyword only. const embeddingPromise = embedder && selectAllNoteChunkPathsStmt ? (async () => { From bc4aeeae3adc384ddda27637cafc74928f8f5865 Mon Sep 17 00:00:00 2001 From: Tanisha Aberdeen <32620895+aliasunder@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:15:00 -0400 Subject: [PATCH 34/34] docs(deploy): keep the first-start embedding bullet one sentence like its siblings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ship-Check: code-quality · claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 --- deploy/local/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/local/README.md b/deploy/local/README.md index e635347a5..b4fbe69af 100644 --- a/deploy/local/README.md +++ b/deploy/local/README.md @@ -27,7 +27,7 @@ What happens on first start: - The image is pulled (~150MB) - The keyword index builds in seconds — search works right away - The semantic (embedding) index builds in the background — expect a few - minutes on a large vault. Keyword search works meanwhile, and semantic + minutes on a large vault; keyword search works meanwhile, and semantic matches appear as each note is embedded > **On Windows?** Set `WINDOWS_MODE=true` in your `.env` — then `VAULT_PATH` can point