diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cb32c5f2..3fa2473a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,11 @@ with the store-selected embedding model instead of the global default. This keeps chunk boundaries aligned with the model that creates and verifies the stored vectors without initializing an unrelated provider. +- Bench scoring no longer credits a result whose path merely ends with the + expected filename (`Vault Note Standards.md` for `Standards.md`), nor an + expected path longer than the result (`docs/api/README.md` for `README.md`). + `pathsMatch` now requires an exact normalized match or a suffix anchored at a + path separator (#943). ## [2.8.3] - 2026-08-16 diff --git a/src/bench/score.ts b/src/bench/score.ts index 86eccea5c..aed5a9afc 100644 --- a/src/bench/score.ts +++ b/src/bench/score.ts @@ -21,14 +21,13 @@ export function normalizePath(p: string): string { /** * Check if two paths refer to the same file. - * Handles different path formats by comparing normalized suffixes. + * Exact match after normalization, or the expected path as a whole + * path-segment suffix of the result (`dir/name`, never a mid-filename fragment). */ export function pathsMatch(result: string, expected: string): boolean { const nr = normalizePath(result); const ne = normalizePath(expected); - if (nr === ne) return true; - if (nr.endsWith(ne) || ne.endsWith(nr)) return true; - return false; + return nr === ne || nr.endsWith("/" + ne); } type ScoreMetrics = { diff --git a/test/bench-score.test.ts b/test/bench-score.test.ts index a0fe5e584..e6c79e23b 100644 --- a/test/bench-score.test.ts +++ b/test/bench-score.test.ts @@ -34,12 +34,20 @@ describe("pathsMatch", () => { expect(pathsMatch("Docs/README.md", "docs/readme.md")).toBe(true); }); - test("suffix match (result is longer)", () => { + test("suffix match anchored at a path separator (result is longer)", () => { expect(pathsMatch("/full/path/docs/readme.md", "docs/readme.md")).toBe(true); }); - test("suffix match (expected is longer)", () => { - expect(pathsMatch("readme.md", "docs/readme.md")).toBe(true); + test("expected longer than result does not match", () => { + expect(pathsMatch("readme.md", "docs/readme.md")).toBe(false); + }); + + test("basename does not match a longer basename", () => { + expect(pathsMatch("qmd://ai-live/Memory/Procedural/Vault Note Standards.md", "Standards.md")).toBe(false); + }); + + test("suffix must be anchored at a path separator", () => { + expect(pathsMatch("docs/api/readme.md", "i/readme.md")).toBe(false); }); test("qmd:// prefix handled", () => { @@ -101,7 +109,7 @@ describe("scoreResults", () => { test("reports recall@1/3/5 and matched documents", () => { const result = scoreResults( - ["x.md", "qmd://concepts/a.md", "docs/b.md", "docs/c.md", "docs/d.md"], + ["x.md", "qmd://col/concepts/a.md", "docs/b.md", "docs/c.md", "docs/d.md"], ["concepts/a.md", "b.md", "missing.md"], 3, );