Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 17 additions & 2 deletions src/search/algorithms/hybrid.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,29 @@ import { fuzzyAlgorithm } from "./fuzzy.js";
const W_BM25 = 0.7;
const W_FUZZY = 0.3;

/**
* Max of the scores with a floor, in loop form. `Math.max(...scores)` spreads
* the whole array into call arguments, which throws `RangeError: Maximum call
* stack size exceeded` once the doc count exceeds the V8 argument limit
* (~125K–130K on Node 22) — a 160K-doc corpus crashed the search instead of
* returning results (ranxianglei/acp-kernel#227).
*/
function maxScore(results: ScoredBlock[], floor: number): number {
let max = floor;
for (const r of results) {
if (r.score > max) max = r.score;
}
return max;
}

export const hybridAlgorithm: SearchAlgorithm = {
name: "hybrid",
description: "Weighted BM25(stem) + fuzzy n-gram. Default — best precision + recall.",
score(docs: SearchDoc[], query: string): ScoredBlock[] {
const bm = bm25Algorithm.score(docs, query);
const fz = fuzzyAlgorithm.score(docs, query);
const maxBm = Math.max(...bm.map((r) => r.score), 1e-9);
const maxFz = Math.max(...fz.map((r) => r.score), 1e-9);
const maxBm = maxScore(bm, 1e-9);
const maxFz = maxScore(fz, 1e-9);
const bmMap = new Map(bm.map((r) => [r.ref, r.score / maxBm]));
const fzMap = new Map(fz.map((r) => [r.ref, r.score / maxFz]));
return docs.map((d) => ({
Expand Down
39 changes: 39 additions & 0 deletions tests/hybrid-large-corpus.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
import test from "node:test";
import assert from "node:assert";
import { hybridAlgorithm } from "../src/search/algorithms/hybrid.js";
import type { SearchDoc } from "../src/search/types.js";

// 130K docs exceeds the V8 spread argument limit (~125K–130K on Node 22):
// the old `Math.max(...scores)` normalization threw `RangeError: Maximum call
// stack size exceeded` before returning any result (#227).
const N = 130_000;

function makeDocs(n: number): SearchDoc[] {
const docs: SearchDoc[] = new Array(n);
for (let i = 0; i < n; i += 1) {
docs[i] = {
kind: "message",
ref: `m${String(i).padStart(5, "0")}`,
text: `ledger entry ${i} alpha`,
title: `entry ${i}`,
role: "tool",
};
}
// one distinctive doc — scoring must still rank it first, not just not crash
docs[42] = {
kind: "message",
ref: "m00042",
text: "the zephyr marker document",
title: "entry 42",
role: "tool",
};
return docs;
}

test("hybridAlgorithm.score handles 130K docs without the spread RangeError (#227)", () => {
const docs = makeDocs(N);
const results = hybridAlgorithm.score(docs, "zephyr");
assert.equal(results.length, N);
const top = results.reduce((best, r) => (r.score > best.score ? r : best), results[0]!);
assert.equal(top.ref, "m00042");
});
Loading