From 4abf11a1e3e33826ab3031fa5bc014469fe26932 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Wed, 9 Sep 2026 00:53:52 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20protectedLatestTools=20=E2=80=94=20prot?= =?UTF-8?q?ect=20only=20the=20latest=20instance=20of=20a=20tool?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cumulative-snapshot tools (e.g. todo_list) keep their full state in the newest result, so every older result is strictly redundant. protectedTools over-protects them (all instances BLOCKED forever); preserveRecentMessages under-protects (N must span far enough back). New Config.protectedLatestTools (optional, glob patterns): for each pattern, only the LAST matching tool-call and its paired tool-result are protected from compression; older instances stay compressible. Superseded pairs keep their BLOCKED ref (never re-issued) but become foldable at apply time, since apply-time protection is re-checked against the current config. Wired into all protection sites: assignRefsNode (BLOCKED ref), filterProtectedToolMessages (range-apply exclusion), buildCompressibleRanges (recommendation), isAbsorbCandidate/appendAbsorbPrompts/applyAbsorb (absorb). Pre-flight: typecheck PASS; 637/637 tests (624 existing + 13 new in tests/protected-latest.test.ts); build PASS. --- src/absorb.ts | 26 ++- src/compress.ts | 24 ++- src/config.ts | 1 + src/index.ts | 8 +- src/protected.ts | 57 ++++++ src/recommend.ts | 12 +- src/types.ts | 6 + tests/protected-latest.test.ts | 357 +++++++++++++++++++++++++++++++++ 8 files changed, 481 insertions(+), 10 deletions(-) create mode 100644 tests/protected-latest.test.ts diff --git a/src/absorb.ts b/src/absorb.ts index d24a780..adc30dd 100644 --- a/src/absorb.ts +++ b/src/absorb.ts @@ -1,6 +1,12 @@ import { rawForRef, refForRaw, BLOCKED_REF } from "./refs.js"; import { ACP_TOOL_NAMES, ABSORB_TOOL_NAME } from "./compress-tools.js"; -import { isMessageProtected, matchToolPattern } from "./protected.js"; +import { + collectLatestProtected, + isMessageLatestProtected, + isMessageProtected, + matchToolPattern, + type LatestProtected, +} from "./protected.js"; import type { AbsorbConfig, AbsorbRecord, @@ -74,11 +80,16 @@ function isAcpOrConfiguredTool( /** True when a tool-result message is in scope for absorption prompting: * a tool-result of a non-ACP, non-excluded, non-protected tool. */ -export function isAbsorbCandidate(msg: CoreMessage, config: Config): boolean { +export function isAbsorbCandidate( + msg: CoreMessage, + config: Config, + latest?: LatestProtected, +): boolean { if (msg.contentType !== "tool-result" || !msg.toolCallId) return false; const cfg = resolveAbsorbConfig(config); if (isAcpOrConfiguredTool(msg.toolName, cfg)) return false; if (isMessageProtected(msg, config)) return false; + if (latest && isMessageLatestProtected(msg, latest)) return false; for (const pattern of cfg.excludeTools) { if (msg.toolName && matchToolPattern(msg.toolName, pattern)) return false; } @@ -136,8 +147,9 @@ export function appendAbsorbPrompts( } let promptedCount = 0; + const latest = collectLatestProtected(messages, config); const out = messages.map((msg) => { - if (!isAbsorbCandidate(msg, config)) return msg; + if (!isAbsorbCandidate(msg, config, latest)) return msg; if (absorbedIds.has(msg.id)) return msg; const text = msg.text ?? ""; if (text.includes(ABSORB_PROMPT_MARKER)) return msg; @@ -284,7 +296,13 @@ export function applyAbsorb(input: AbsorbInput): AbsorbOutcome { resultText: `absorb failed: ${target.toolName} is an ACP-managed tool result — it is not absorbable.`, }; } - if (isMessageProtected(target, input.config)) { + if ( + isMessageProtected(target, input.config) || + isMessageLatestProtected( + target, + collectLatestProtected(input.messages, input.config), + ) + ) { return { state: input.state, ok: false, diff --git a/src/compress.ts b/src/compress.ts index 51769ae..610df32 100644 --- a/src/compress.ts +++ b/src/compress.ts @@ -18,7 +18,11 @@ import { appendAbsorbPrompts, hideAbsorbedMessages } from "./absorb.js"; import { applyMessageFilters, listMessageFilters } from "./filter/index.js"; import { createRenderRefsNode } from "./render-refs.js"; import type { RenderStrategy } from "./render-refs.js"; -import { isMessageProtected } from "./protected.js"; +import { + collectLatestProtected, + isMessageLatestProtected, + isMessageProtected, +} from "./protected.js"; import { adjustBoundariesForToolPairs } from "./tool-pairs.js"; import { adjustBoundariesForReasoningPairs } from "./reasoning-pairs.js"; import { @@ -500,9 +504,16 @@ const assignRefsNode: PipelineNode = { name: "assign-refs", run(io, ctx) { const hasProtection = - ctx.config.protectedTools.length > 0 || !!ctx.config.isToolProtected; + ctx.config.protectedTools.length > 0 || + !!ctx.config.isToolProtected || + (ctx.config.protectedLatestTools?.length ?? 0) > 0; + const latest = hasProtection + ? collectLatestProtected(io.messages, ctx.config) + : undefined; const protectedFn = hasProtection - ? (m: CoreMessage) => isMessageProtected(m, ctx.config) + ? (m: CoreMessage) => + isMessageProtected(m, ctx.config) || + (latest ? isMessageLatestProtected(m, latest) : false) : undefined; const refResult = assignRefs(io.messages, { existing: io.state.messageRefs, @@ -992,6 +1003,8 @@ function filterProtectedToolMessages( // nothing auto-appended. const protectedCallIds = new Set(); const removedIds = new Set(); + const latest = collectLatestProtected(messages, config); + for (const id of latest.callIds) protectedCallIds.add(id); for (const msg of messages) { if (isMessageProtected(msg, config) && msg.toolCallId) { protectedCallIds.add(msg.toolCallId); @@ -1001,7 +1014,10 @@ function filterProtectedToolMessages( for (const id of directMessageIds) { const msg = messages.find((m) => m.id === id); if (!msg) continue; - if (isMessageProtected(msg, config)) { + if ( + isMessageProtected(msg, config) || + isMessageLatestProtected(msg, latest) + ) { removedIds.add(id); if (msg.toolCallId) protectedCallIds.add(msg.toolCallId); } diff --git a/src/config.ts b/src/config.ts index 0d5ef70..0856644 100644 --- a/src/config.ts +++ b/src/config.ts @@ -28,6 +28,7 @@ export function defaultConfig( minSummaryLength: 50, }, protectedTools: [], + protectedLatestTools: [], preserveRecentMessages: 5, preserveRecentTokens: 5000, modelContextLimit, diff --git a/src/index.ts b/src/index.ts index 5a92f65..85bffe0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -147,7 +147,13 @@ export { getSearchAlgorithm, listSearchAlgorithms, } from "./search.js"; -export { isMessageProtected, matchToolPattern } from "./protected.js"; +export { + collectLatestProtected, + isMessageLatestProtected, + isMessageProtected, + matchToolPattern, + type LatestProtected, +} from "./protected.js"; export { runPipeline, makeIO, diff --git a/src/protected.ts b/src/protected.ts index 9033273..aa830ba 100644 --- a/src/protected.ts +++ b/src/protected.ts @@ -112,3 +112,60 @@ export function isMessageProtectedWithPairing( } return false; } + +/** "Latest only" protection set: for each protectedLatestTools pattern, the + * LAST tool-call matching it (in message order) plus its paired result. Older + * instances of the same tool stay compressible. Use for cumulative-snapshot + * tools (e.g. todo_list) where only the newest result is the source of truth + * and every older result is strictly redundant. + * + * `callIds` holds the latest calls' toolCallIds (pairing covers the result + * half, including results projected without a toolName); `msgIds` holds + * latest calls that lack a toolCallId (pairing impossible — protect by id). */ +export interface LatestProtected { + callIds: Set; + msgIds: Set; +} + +export function collectLatestProtected( + messages: CoreMessage[], + config: Pick, +): LatestProtected { + const callIds = new Set(); + const msgIds = new Set(); + const patterns = config.protectedLatestTools ?? []; + if (patterns.length === 0) return { callIds, msgIds }; + for (const pattern of patterns) { + let last: CoreMessage | undefined; + for (const m of messages) { + if ( + m.contentType === "tool-call" && + m.toolName && + matchToolPattern(m.toolName, pattern) + ) { + last = m; + } + } + if (!last) continue; + if (last.toolCallId) callIds.add(last.toolCallId); + else msgIds.add(last.id); + } + return { callIds, msgIds }; +} + +/** True when msg is a latest-protected tool-call, or the tool-result paired to + * one (by toolCallId). */ +export function isMessageLatestProtected( + msg: CoreMessage, + latest: LatestProtected, +): boolean { + if (msg.contentType === "tool-call" && latest.msgIds.has(msg.id)) return true; + if ( + (msg.contentType === "tool-call" || msg.contentType === "tool-result") && + msg.toolCallId && + latest.callIds.has(msg.toolCallId) + ) { + return true; + } + return false; +} diff --git a/src/recommend.ts b/src/recommend.ts index b7b17f0..84082c7 100644 --- a/src/recommend.ts +++ b/src/recommend.ts @@ -21,7 +21,9 @@ import type { } from "./types.js"; import type { CompressionState } from "./types.js"; import { + collectLatestProtected, collectProtectedToolCallIds, + isMessageLatestProtected, isMessageProtectedWithPairing, isNeverPreserveRecent, } from "./protected.js"; @@ -164,6 +166,11 @@ export function buildCompressibleRanges( // Pairing: a tool-result may carry only toolCallId (no toolName). Collect the // callIds of protected tool-calls first, then protect matching results too. const protectedCallIds = collectProtectedToolCallIds(messages, config); + // Latest-only protected calls: their results are covered by the pairing + // union; the calls themselves need the explicit check below (pairing only + // matches tool-results). + const latest = collectLatestProtected(messages, config); + for (const id of latest.callIds) protectedCallIds.add(id); // Segmentation is array adjacency, never ref arithmetic: surface-replacing // hosts leave holes in the ref map (compressed messages leave the array, refs @@ -184,7 +191,10 @@ export function buildCompressibleRanges( continue; } - if (isMessageProtectedWithPairing(msg, config, protectedCallIds)) { + if ( + isMessageProtectedWithPairing(msg, config, protectedCallIds) || + isMessageLatestProtected(msg, latest) + ) { protectedMsgs.push({ ref, gapBefore: skipSinceProtected, diff --git a/src/types.ts b/src/types.ts index f7a4ea5..98fead7 100644 --- a/src/types.ts +++ b/src/types.ts @@ -178,6 +178,12 @@ export interface Config { compress: CompressValidationConfig; protectedTools: string[]; isToolProtected?: (toolName: string, toolInputText?: string) => boolean; + /** Tool-name patterns (glob suffix allowed) protected in their LATEST + * instance only: the newest matching tool-call and its paired tool-result + * are protected from compression; older instances remain compressible. For + * cumulative-snapshot tools (e.g. todo_list) where only the newest result + * is the source of truth. */ + protectedLatestTools?: string[]; preserveRecentMessages: number; preserveRecentTokens: number; modelContextLimit: number; diff --git a/tests/protected-latest.test.ts b/tests/protected-latest.test.ts new file mode 100644 index 0000000..48e0a60 --- /dev/null +++ b/tests/protected-latest.test.ts @@ -0,0 +1,357 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + createCore, + defaultConfig, + defaultCountTokens, + type CoreMessage, + type SessionState, +} from "../src/index.js"; +import { createInitialState } from "../src/state.js"; +import { assignRefs } from "../src/refs.js"; +import { + collectLatestProtected, + isMessageLatestProtected, +} from "../src/protected.js"; +import { buildCompressibleRanges } from "../src/recommend.js"; +import { applyAbsorb, isAbsorbCandidate } from "../src/absorb.js"; + +function msg(id: string, text: string): CoreMessage { + return { id, role: "user", contentType: "text", text }; +} + +function todoCall(id: string, callId: string, rev: number): CoreMessage { + return { + id, + role: "assistant", + contentType: "tool-call", + toolName: "todo_list", + toolCallId: callId, + text: JSON.stringify({ action: "update", revision: rev }), + }; +} + +function todoResult(id: string, callId: string, rev: number): CoreMessage { + return { + id, + role: "tool", + contentType: "tool-result", + toolCallId: callId, + text: JSON.stringify({ + revision: rev, + todos: [{ id: "t1", content: `task ${rev}`, status: "pending" }], + }), + }; +} + +function emptyState(): SessionState { + return { + blocks: [], + prune: { byMessageId: {}, activeBlockIds: [] }, + messageRefs: { byRaw: {}, byRef: {} }, + nudge: { + lastPerMessageNudgeTokens: 0, + lastNudgeShownTokens: 0, + pendingNudgeTurn: null, + baselineTokens: 0, + }, + stats: { totalTokensCompressed: 0 }, + compressionTiming: {}, + }; +} + +const validSummary = + "A meaningful summary that captures the key information of the compressed range including file paths and decisions."; + +function latestCfg(overrides: Record = {}) { + return defaultConfig(200000, { + compress: { minCompressRange: 0, maxSummaryLength: 0, minSummaryLength: 0 }, + preserveRecentMessages: 0, + preserveRecentTokens: 0, + protectedLatestTools: ["todo_list"], + ...overrides, + }); +} + +test("collectLatestProtected: no patterns → empty set", () => { + const messages = [todoCall("c1", "t1", 1), todoResult("r1", "t1", 1)]; + const latest = collectLatestProtected(messages, {}); + assert.equal(latest.callIds.size, 0); + assert.equal(latest.msgIds.size, 0); +}); + +test("collectLatestProtected: picks the LAST matching call per pattern", () => { + const messages = [ + todoCall("c1", "t1", 1), + todoResult("r1", "t1", 1), + msg("u", "continue"), + todoCall("c2", "t2", 2), + todoResult("r2", "t2", 2), + ]; + const latest = collectLatestProtected(messages, { + protectedLatestTools: ["todo_list"], + }); + assert.deepEqual([...latest.callIds], ["t2"]); + assert.equal(latest.msgIds.size, 0); +}); + +test("collectLatestProtected: glob suffix pattern matches", () => { + const messages = [todoCall("c1", "t1", 1)]; + const latest = collectLatestProtected(messages, { + protectedLatestTools: ["todo_*"], + }); + assert.deepEqual([...latest.callIds], ["t1"]); +}); + +test("collectLatestProtected: call without toolCallId is protected by message id", () => { + const call: CoreMessage = { + id: "c1", + role: "assistant", + contentType: "tool-call", + toolName: "todo_list", + text: "{}", + }; + const latest = collectLatestProtected([call], { + protectedLatestTools: ["todo_list"], + }); + assert.deepEqual([...latest.msgIds], ["c1"]); + assert.equal(latest.callIds.size, 0); +}); + +test("collectLatestProtected: no matching call → empty", () => { + const latest = collectLatestProtected([msg("u", "hi")], { + protectedLatestTools: ["todo_list"], + }); + assert.equal(latest.callIds.size, 0); + assert.equal(latest.msgIds.size, 0); +}); + +test("isMessageLatestProtected: matches latest call and its paired result only", () => { + const c1 = todoCall("c1", "t1", 1); + const r1 = todoResult("r1", "t1", 1); + const c2 = todoCall("c2", "t2", 2); + const r2 = todoResult("r2", "t2", 2); + const latest = collectLatestProtected([c1, r1, c2, r2], { + protectedLatestTools: ["todo_list"], + }); + assert.ok(isMessageLatestProtected(c2, latest), "latest call"); + assert.ok(isMessageLatestProtected(r2, latest), "latest result (paired)"); + assert.ok(!isMessageLatestProtected(c1, latest), "older call"); + assert.ok(!isMessageLatestProtected(r1, latest), "older result"); + assert.ok(!isMessageLatestProtected(msg("u", "hi"), latest), "text msg"); +}); + +test("processTurn: only the latest todo_list pair is BLOCKED; older pairs keep refs", () => { + const core = createCore(); + const messages: CoreMessage[] = [ + msg("a", "start"), + todoCall("c1", "t1", 1), + todoResult("r1", "t1", 1), + msg("b", "middle"), + todoCall("c2", "t2", 2), + todoResult("r2", "t2", 2), + ]; + const result = core.processTurn({ + messages, + state: emptyState(), + config: latestCfg(), + tokenCount: 0, + countTokens: defaultCountTokens, + }); + assert.equal(result.state.messageRefs.byRaw["a"], "m00001"); + assert.equal(result.state.messageRefs.byRaw["c1"], "m00002"); + assert.equal(result.state.messageRefs.byRaw["r1"], "m00003"); + assert.equal(result.state.messageRefs.byRaw["b"], "m00004"); + assert.equal(result.state.messageRefs.byRaw["c2"], "BLOCKED"); + assert.equal(result.state.messageRefs.byRaw["r2"], "BLOCKED"); +}); + +test("applyCompression: latest pair excluded from block, older pair folded", () => { + const core = createCore(); + const messages: CoreMessage[] = [ + msg("a", "x".repeat(6000)), + todoCall("c1", "t1", 1), + todoResult("r1", "t1", 1), + msg("b", "x".repeat(6000)), + todoCall("c2", "t2", 2), + todoResult("r2", "t2", 2), + ]; + const state = createInitialState(); + state.messageRefs = assignRefs(messages, { + existing: state.messageRefs, + nextIndex: 1, + }).map; + const result = core.applyCompression({ + ranges: [{ startRef: "m00001", endRef: "m00006", summary: validSummary }], + messages, + state, + config: latestCfg(), + }); + assert.equal(result.result.blocksCreated, 1); + assert.equal(result.result.errors.length, 0); + const block = result.state.blocks[0]!; + assert.ok(block.directMessageIds.includes("c1"), "older call folded"); + assert.ok(block.directMessageIds.includes("r1"), "older result folded"); + assert.ok(block.effectiveMessageIds.includes("c1")); + assert.ok(block.effectiveMessageIds.includes("r1")); + assert.ok(!block.directMessageIds.includes("c2"), "latest call excluded"); + assert.ok(!block.directMessageIds.includes("r2"), "latest result excluded"); + assert.ok(!block.effectiveMessageIds.includes("c2"), "latest call not covered"); + assert.ok(!block.effectiveMessageIds.includes("r2"), "latest result not covered"); +}); + +test("applyCompression: superseded pair is folded once a newer pair exists", () => { + const core = createCore(); + const messages: CoreMessage[] = [ + msg("a", "x".repeat(6000)), + todoCall("c1", "t1", 1), + todoResult("r1", "t1", 1), + msg("b", "x".repeat(6000)), + todoCall("c2", "t2", 2), + todoResult("r2", "t2", 2), + msg("d", "x".repeat(6000)), + todoCall("c3", "t3", 3), + todoResult("r3", "t3", 3), + ]; + const state = createInitialState(); + state.messageRefs = assignRefs(messages, { + existing: state.messageRefs, + nextIndex: 1, + }).map; + const result = core.applyCompression({ + ranges: [{ startRef: "m00001", endRef: "m00006", summary: validSummary }], + messages, + state, + config: latestCfg(), + }); + assert.equal(result.result.blocksCreated, 1); + const block = result.state.blocks[0]!; + assert.ok(block.directMessageIds.includes("c1"), "first pair folded"); + assert.ok(block.directMessageIds.includes("r1")); + assert.ok(block.directMessageIds.includes("c2"), "superseded pair folded"); + assert.ok(block.directMessageIds.includes("r2")); + assert.ok(!block.directMessageIds.includes("c3"), "latest pair untouched"); + assert.ok(!block.directMessageIds.includes("r3")); +}); + +test("applyCompression: protectedTools (hard) excludes ALL pairs, unlike protectedLatestTools", () => { + const core = createCore(); + const messages: CoreMessage[] = [ + msg("a", "x".repeat(6000)), + todoCall("c1", "t1", 1), + todoResult("r1", "t1", 1), + msg("b", "x".repeat(6000)), + todoCall("c2", "t2", 2), + todoResult("r2", "t2", 2), + ]; + const state = createInitialState(); + state.messageRefs = assignRefs(messages, { + existing: state.messageRefs, + nextIndex: 1, + }).map; + const result = core.applyCompression({ + ranges: [{ startRef: "m00001", endRef: "m00006", summary: validSummary }], + messages, + state, + config: latestCfg({ protectedLatestTools: [], protectedTools: ["todo_list"] }), + }); + assert.equal(result.result.blocksCreated, 1); + const block = result.state.blocks[0]!; + assert.ok(!block.directMessageIds.includes("c1"), "hard: first call excluded"); + assert.ok(!block.directMessageIds.includes("r1"), "hard: first result excluded"); + assert.ok(!block.directMessageIds.includes("c2"), "hard: latest call excluded"); + assert.ok(!block.directMessageIds.includes("r2"), "hard: latest result excluded"); +}); + +test("buildCompressibleRanges: latest pair protected, older pair compressible", () => { + const messages: CoreMessage[] = [ + msg("a", "x".repeat(2000)), + todoCall("c1", "t1", 1), + todoResult("r1", "t1", 1), + msg("b", "x".repeat(2000)), + todoCall("c2", "t2", 2), + todoResult("r2", "t2", 2), + ]; + const state = createInitialState(); + state.messageRefs = assignRefs(messages, { + existing: state.messageRefs, + nextIndex: 1, + }).map; + const ranges = buildCompressibleRanges(messages, state, latestCfg()); + // Older pair stays compressible. (Groups split at a user message once a + // group reaches 3 messages — pre-existing behavior — so a..r1 and b are + // two ranges.) + assert.equal(ranges.compressible.length, 2, "older pair stays compressible"); + assert.equal(ranges.compressible[0]!.startRef, "m00001"); + assert.equal(ranges.compressible[0]!.endRef, "m00003"); + assert.equal(ranges.compressible[0]!.count, 3); + assert.equal(ranges.compressible[1]!.startRef, "m00004"); + assert.equal(ranges.compressible[1]!.endRef, "m00004"); + assert.equal(ranges.compressible[1]!.count, 1); + assert.equal(ranges.protected.length, 1, "latest pair is one protected range"); + const pr = ranges.protected[0]!; + assert.equal(pr.startRef, "m00005"); + assert.equal(pr.endRef, "m00006"); + assert.deepEqual(pr.tools, ["todo_list"]); +}); + +test("isAbsorbCandidate: latest-protected result is not a candidate, older is", () => { + const messages: CoreMessage[] = [ + todoCall("c1", "t1", 1), + todoResult("r1", "t1", 1), + todoCall("c2", "t2", 2), + todoResult("r2", "t2", 2), + ]; + const config = latestCfg({ + absorb: { + enabled: true, + toolName: "absorb", + minToolTokens: 0, + contextThresholdPct: 0, + excludeTools: [], + }, + }); + const latest = collectLatestProtected(messages, config); + assert.ok(!isAbsorbCandidate(messages[3]!, config, latest), "latest result"); + assert.ok(isAbsorbCandidate(messages[1]!, config, latest), "older result"); +}); + +test("applyAbsorb: rejects the latest-protected result, accepts the older one", () => { + const messages: CoreMessage[] = [ + todoCall("c1", "t1", 1), + todoResult("r1", "t1", 1), + todoCall("c2", "t2", 2), + todoResult("r2", "t2", 2), + ]; + const state = createInitialState(); + state.messageRefs = assignRefs(messages, { + existing: state.messageRefs, + nextIndex: 1, + }).map; + const config = latestCfg({ + absorb: { + enabled: true, + toolName: "absorb", + minToolTokens: 0, + contextThresholdPct: 0, + excludeTools: [], + }, + }); + const rejected = applyAbsorb({ + ref: "m00004", + summary: "distilled essentials of the result", + messages, + state, + config, + }); + assert.equal(rejected.ok, false); + assert.match(rejected.resultText, /protected/); + const accepted = applyAbsorb({ + ref: "m00002", + summary: "distilled essentials of the result", + messages, + state, + config, + }); + assert.equal(accepted.ok, true); +});