fix: atomic watcher updates and removals (1.17.2)

.claude-plugin/marketplace.json

@@ -13,7 +13,7 @@
       "name": "sensegrep",
       "source": "./plugin/sensegrep-plugin",
       "description": "Semantic code search for AI agents. Search code by meaning, not text patterns. Adds sensegrep MCP tools + smart usage instructions to Claude Code.",
-      "version": "1.17.1",
+      "version": "1.17.2",
       "author": {
         "name": "sensegrep"
       },

.cursor-plugin/marketplace.json

@@ -13,7 +13,7 @@
       "name": "sensegrep",
       "source": "plugin/sensegrep-cursor",
       "description": "Semantic code search for AI agents. Search code by meaning, not text patterns.",
-      "version": "1.17.1"
+      "version": "1.17.2"
     }
   ]
 }

packages/cli/CHANGELOG.md

@@ -1,5 +1,14 @@
 # @sensegrep/cli
 
+## 1.17.2
+
+### Patch Changes
+
+- [`62f4bb9`](https://github.com/Stahldavid/sensegrep/commit/62f4bb99102fcd332139a2331a36ff34e381acdd) Thanks [@Stahldavid](https://github.com/Stahldavid)! - Extend staged, atomic index activation to individual watched file updates and removals, including files that become ignored or empty. Validate the staged chunk count before committing metadata and leave the active generation untouched if persistence or metadata writing fails.
+
+- Updated dependencies [[`62f4bb9`](https://github.com/Stahldavid/sensegrep/commit/62f4bb99102fcd332139a2331a36ff34e381acdd)]:
+  - @sensegrep/core@1.17.2
+
 ## 1.17.1
 
 ### Patch Changes

packages/cli/README.md

@@ -58,4 +58,4 @@ Search defaults to one result per file; use `--max-per-file 2` for more snippets
 
 Duplicate JSON respects `--limit`. `summary.outputTruncated` means more groups were found than emitted; raise `--limit` to expose them. A continuation cursor advances the candidate scan, not output pagination. Candidate caps still require raising `--max-candidates` for full coverage.
 
-Incremental `index --no-watch` stages updates before atomically activating the new snapshot, preserving the old index if the process is interrupted. No-change runs avoid copying vectors. This protection does not yet extend to individual watcher updates.
+Incremental `index --no-watch` stages updates before atomically activating the new snapshot, preserving the old index if the process is interrupted. No-change runs avoid copying vectors. Since 1.17.2 this protection also covers individual watcher updates and removals, including files that become non-indexable.

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sensegrep/cli",
-  "version": "1.17.1",
+  "version": "1.17.2",
   "type": "module",
   "bin": {
     "sensegrep": "dist/main.js"
@@ -25,7 +25,7 @@
     "node": ">=20"
   },
   "dependencies": {
-    "@sensegrep/core": "^1.17.1"
+    "@sensegrep/core": "^1.17.2"
   },
   "publishConfig": {
     "access": "public"

packages/core/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @sensegrep/core
 
+## 1.17.2
+
+### Patch Changes
+
+- [`62f4bb9`](https://github.com/Stahldavid/sensegrep/commit/62f4bb99102fcd332139a2331a36ff34e381acdd) Thanks [@Stahldavid](https://github.com/Stahldavid)! - Extend staged, atomic index activation to individual watched file updates and removals, including files that become ignored or empty. Validate the staged chunk count before committing metadata and leave the active generation untouched if persistence or metadata writing fails.
+
 ## 1.17.1
 
 ### Patch Changes

packages/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sensegrep/core",
-  "version": "1.17.1",
+  "version": "1.17.2",
   "type": "module",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",

packages/core/src/semantic/indexer.test.ts

@@ -310,7 +310,7 @@ describe("Indexer incremental updates", () => {
 
     expect(embedDocumentsReusingFile).toHaveBeenCalledTimes(1)
     expect(embedDocuments).toHaveBeenCalledTimes(1)
-    expect(replaceFileDocuments).toHaveBeenCalledWith({}, "src/a.ts", expect.any(Array))
+    expect(replaceFileDocuments).toHaveBeenCalledWith({ staging: true }, "src/a.ts", expect.any(Array))
     expect(writeIndexMeta).toHaveBeenCalledTimes(1)
     expect(writeIndexMeta.mock.calls[0][1].files["src/a.ts"].collapsibleRegions).toEqual(regions)
     expect(writeIndexMeta.mock.calls[0][1].chunking).toEqual(testChunkingSignature)
@@ -326,19 +326,21 @@ describe("Indexer incremental updates", () => {
   })
 
   it("removes generated files from index metadata during file updates", async () => {
+    getStats.mockResolvedValue({ count: 0 })
     await writeFile(path.join(TEST_DIR, "src/a.ts"), "x".repeat(60_000))
     const { Indexer } = await import("./indexer.js")
 
     await Indexer.updateFile("src/a.ts")
 
-    expect(deleteByFile).toHaveBeenCalledWith({}, "src/a.ts")
+    expect(deleteByFile).toHaveBeenCalledWith({ staging: true }, "src/a.ts")
     expect(embedDocuments).not.toHaveBeenCalled()
     expect(embedDocumentsReusingFile).not.toHaveBeenCalled()
     expect(writeIndexMeta).toHaveBeenCalledTimes(1)
     expect(writeIndexMeta.mock.calls[0][1].files["src/a.ts"]).toBeUndefined()
   })
 
   it("removes stale metadata when a watched file is no longer indexable", async () => {
+    getStats.mockResolvedValue({ count: 0 })
     readIndexMeta.mockResolvedValueOnce({
       version: 1,
       root: TEST_DIR,
@@ -362,14 +364,45 @@ describe("Indexer incremental updates", () => {
 
     await Indexer.updateFile("src/a.ts.map")
 
-    expect(deleteByFile).toHaveBeenCalledWith({}, "src/a.ts.map")
+    expect(deleteByFile).toHaveBeenCalledWith({ staging: true }, "src/a.ts.map")
     expect(embedDocuments).not.toHaveBeenCalled()
     expect(embedDocumentsReusingFile).not.toHaveBeenCalled()
     expect(getConfig).not.toHaveBeenCalled()
     expect(writeIndexMeta).toHaveBeenCalledTimes(1)
     expect(writeIndexMeta.mock.calls[0][1].files["src/a.ts.map"]).toBeUndefined()
   })
 
+  it.each(["update", "remove"])("preserves active watcher metadata when %s persistence fails", async (operation) => {
+    const meta = await readIndexMeta()
+    const original = structuredClone(meta)
+    if (operation === "update") replaceFileDocuments.mockRejectedValueOnce(new Error("disk failure"))
+    else deleteByFile.mockRejectedValueOnce(new Error("disk failure"))
+    const { Indexer } = await import("./indexer.js")
+    await expect(operation === "update" ? Indexer.updateFile("src/a.ts") : Indexer.removeFile("src/a.ts")).rejects.toThrow("disk failure")
+    expect(meta).toEqual(original)
+    expect(writeIndexMeta).not.toHaveBeenCalled()
+    expect(dropCollectionTable).toHaveBeenCalledWith(TEST_DIR, "chunks_staging")
+  })
+
+  it("does not activate watcher changes with inconsistent staged counts", async () => {
+    getStats.mockResolvedValue({ count: 999 })
+    const { Indexer } = await import("./indexer.js")
+    await expect(Indexer.updateFile("src/a.ts")).rejects.toThrow("chunk mismatch")
+    expect(writeIndexMeta).not.toHaveBeenCalled()
+    expect(dropCollectionTable).toHaveBeenCalled()
+  })
+
+  it("discards watcher staging when the metadata commit fails", async () => {
+    writeIndexMeta.mockRejectedValueOnce(new Error("metadata failure"))
+    const meta = await readIndexMeta()
+    const original = structuredClone(meta)
+    const { Indexer } = await import("./indexer.js")
+    await expect(Indexer.updateFile("src/a.ts")).rejects.toThrow("metadata failure")
+    expect(meta).toEqual(original)
+    expect(clearProjectCache).not.toHaveBeenCalled()
+    expect(dropCollectionTable).toHaveBeenCalled()
+  })
+
   it("serializes concurrent indexing for the same project", async () => {
     let activeReads = 0
     let maxActiveReads = 0

packages/core/src/semantic/indexer.ts

@@ -566,31 +566,38 @@ export namespace Indexer {
     return { count: documents.length, chunkHashes: documents.map((d) => d.hash) }
   }
 
-  async function deleteFileFromIndexAndMeta(
+  async function commitWatchedMutation(
     collection: Awaited<ReturnType<typeof VectorStore.getCollection>>,
-    filePath: string,
+    meta: VectorStore.IndexMeta,
+    files: VectorStore.IndexMeta["files"],
+    mutate: (target: Awaited<ReturnType<typeof VectorStore.getCollection>>) => Promise<void>,
   ): Promise<void> {
-    await VectorStore.deleteByFile(collection, filePath)
-    const meta = await VectorStore.readIndexMeta(Instance.directory)
-    if (meta?.files) {
-      delete meta.files[filePath]
-      meta.updatedAt = Date.now()
-      await VectorStore.writeIndexMeta(Instance.directory, meta)
+    const staging = await VectorStore.createStagingCollection(Instance.directory, meta.embeddings.dimension)
+    let activated = false
+    try {
+      await VectorStore.copyCollection(collection, staging.collection)
+      await mutate(staging.collection)
+      const expected = Object.values(files).reduce((sum, file) => sum + (file.chunks?.length ?? 0), 0)
+      const actual = (await VectorStore.getStats(staging.collection)).count
+      if (actual !== expected) throw new Error(`Staged watcher index chunk mismatch: expected ${expected}, persisted ${actual}`)
+      await VectorStore.writeIndexMeta(Instance.directory, {
+        ...meta, files, tableName: staging.tableName, updatedAt: Date.now(),
+      })
+      activated = true
+      VectorStore.clearProjectCache(Instance.directory)
+      await VectorStore.cleanupInactiveTables(Instance.directory, staging.tableName)
+    } finally {
+      if (!activated) await VectorStore.dropCollectionTable(Instance.directory, staging.tableName).catch(() => {})
     }
   }
 
   async function deleteIndexedFileIfPresent(filePath: string): Promise<void> {
     const meta = await VectorStore.readIndexMeta(Instance.directory)
-    if (!meta?.files || !meta.files[filePath]) return
-
-    if (await VectorStore.hasCollection(Instance.directory)) {
-      const collection = await VectorStore.getCollectionUnsafe(Instance.directory, meta.embeddings?.dimension)
-      await VectorStore.deleteByFile(collection, filePath)
-    }
-
-    delete meta.files[filePath]
-    meta.updatedAt = Date.now()
-    await VectorStore.writeIndexMeta(Instance.directory, meta)
+    if (!meta?.files?.[filePath]) return
+    const collection = await VectorStore.getCollectionUnsafe(Instance.directory, meta.embeddings.dimension)
+    const files = { ...meta.files }
+    delete files[filePath]
+    await commitWatchedMutation(collection, meta, files, (target) => VectorStore.deleteByFile(target, filePath))
   }
 
   async function buildDocuments(input: {
@@ -1496,7 +1503,8 @@ export namespace Indexer {
     }
 
     const existingMeta = await VectorStore.readIndexMeta(Instance.directory)
-    if (existingMeta && !sameChunkingSignature(existingMeta.chunking, getChunkingSignature(Embeddings.getConfig()))) {
+    if (!existingMeta) throw new Error("No index metadata; run sensegrep index --no-watch before watched updates")
+    if (!sameChunkingSignature(existingMeta.chunking, getChunkingSignature(Embeddings.getConfig()))) {
       throw new Error("Chunking policy changed; run sensegrep index --no-watch to rebuild before applying watched updates")
     }
     const { collapsibleRegions, documents } = await analyzeFile(filePath, content)
@@ -1513,20 +1521,15 @@ export namespace Indexer {
       metadata,
     }))
     const prepared = await VectorStore.embedDocumentsReusingFile(collection, filePath, docsToAdd)
-    await VectorStore.replaceFileDocuments(collection, filePath, prepared.rows)
-    const meta = await VectorStore.readIndexMeta(Instance.directory)
-    if (meta?.files) {
-      meta.chunking = getChunkingSignature(Embeddings.getConfig())
-      meta.files[filePath] = {
-        size: stat.size,
-        mtimeMs: stat.mtimeMs,
-        hash: hashContent(content),
-        chunks: documents.map((d) => d.hash),
-        collapsibleRegions,
-      }
-      meta.updatedAt = Date.now()
-      await VectorStore.writeIndexMeta(Instance.directory, meta)
-    }
+    const nextFiles = { ...existingMeta.files, [filePath]: {
+      size: stat.size,
+      mtimeMs: stat.mtimeMs,
+      hash: hashContent(content),
+      chunks: documents.map((d) => d.hash),
+      collapsibleRegions,
+    } }
+    await commitWatchedMutation(collection, existingMeta, nextFiles,
+      (target) => VectorStore.replaceFileDocuments(target, filePath, prepared.rows))
   }
 
   export async function updateFile(filePath: string): Promise<void> {

packages/mcp/CHANGELOG.md

@@ -1,5 +1,14 @@
 # @sensegrep/mcp
 
+## 1.17.2
+
+### Patch Changes
+
+- [`62f4bb9`](https://github.com/Stahldavid/sensegrep/commit/62f4bb99102fcd332139a2331a36ff34e381acdd) Thanks [@Stahldavid](https://github.com/Stahldavid)! - Extend staged, atomic index activation to individual watched file updates and removals, including files that become ignored or empty. Validate the staged chunk count before committing metadata and leave the active generation untouched if persistence or metadata writing fails.
+
+- Updated dependencies [[`62f4bb9`](https://github.com/Stahldavid/sensegrep/commit/62f4bb99102fcd332139a2331a36ff34e381acdd)]:
+  - @sensegrep/core@1.17.2
+
 ## 1.17.1
 
 ### Patch Changes

packages/mcp/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@sensegrep/mcp",
   "mcpName": "io.github.Stahldavid/sensegrep",
-  "version": "1.17.1",
+  "version": "1.17.2",
   "type": "module",
   "main": "./dist/server.js",
   "bin": {
@@ -33,7 +33,7 @@
     "@modelcontextprotocol/node": "^2.0.0",
     "@modelcontextprotocol/sdk": "^1.29.0",
     "@modelcontextprotocol/server": "^2.0.0",
-    "@sensegrep/core": "^1.17.1",
+    "@sensegrep/core": "^1.17.2",
     "zod": "^4.4.3"
   },
   "publishConfig": {

packages/mcp/src/http-server.ts

@@ -39,7 +39,7 @@ export function createSensegrepHttpHandler(
   return createMcpHandler(async (context) => {
     options.onServerCreated?.(context);
     const server = new Server(
-      { name: "sensegrep", version: "1.17.1" },
+      { name: "sensegrep", version: "1.17.2" },
       { capabilities: { tools: {} } },
     );
 

packages/mcp/src/server.ts

@@ -939,7 +939,7 @@ export function createStdioMcpServer(): Server {
   const server = new Server(
     {
       name: "sensegrep",
-      version: "1.17.1",
+      version: "1.17.2",
     },
     {
       capabilities: {

packages/vscode/package.json

@@ -602,6 +602,6 @@
     "typescript": "^5.9.3"
   },
   "dependencies": {
-    "@sensegrep/core": "^1.17.1"
+    "@sensegrep/core": "^1.17.2"
   }
 }

plugin/sensegrep-cursor/.cursor-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "sensegrep",
-  "version": "1.17.1",
+  "version": "1.17.2",
   "description": "Semantic code search for AI agents. Search code by meaning, not text patterns. Replaces grep/ripgrep for 95% of code exploration tasks.",
   "author": {
     "name": "sensegrep"

plugin/sensegrep-cursor/.mcp.json

@@ -4,7 +4,7 @@
       "command": "npx",
       "args": [
         "-y",
-        "@sensegrep/mcp@1.17.1"
+        "@sensegrep/mcp@1.17.2"
       ]
     }
   }

plugin/sensegrep-plugin/.claude-plugin/marketplace.json

@@ -13,7 +13,7 @@
       "name": "sensegrep",
       "source": ".",
       "description": "Semantic code search for AI agents. Search code by meaning, not text patterns. Adds sensegrep MCP tools + smart usage instructions to Claude Code.",
-      "version": "1.17.1",
+      "version": "1.17.2",
       "author": {
         "name": "sensegrep"
       },

plugin/sensegrep-plugin/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "sensegrep",
-  "version": "1.17.1",
+  "version": "1.17.2",
   "description": "Semantic code search for AI agents. Search code by meaning, not text patterns. Replaces grep/ripgrep for 95% of code exploration tasks.",
   "author": {
     "name": "sensegrep",