fix(compress): normalize double-escaped summaries at ingest; flag unverifiable user quotes (closes #309)

package.json

@@ -61,7 +61,7 @@
 		"@earendil-works/pi-coding-agent": "0.83.0",
 		"@earendil-works/pi-tui": "0.83.0",
 		"@types/node": "^26.1.2",
-		"acp-kernel": "0.0.50",
+		"acp-kernel": "0.0.55",
 		"tsup": "^8.5.1",
 		"tsx": "^4.23.1",
 		"typescript": "^7.0.2"

scripts/e2e/scenarios/03-nudge-triggered.json

@@ -1,8 +1,11 @@
 {
     "name": "nudge-triggered",
-    "description": "Nudge-triggered compression: the model emits growth text across turns until billion-context-pi's context-usage nudge is injected, then the fake LLM detects the nudge and compresses. A low modelContextLimit makes the nudge fire quickly. Validates the nudge detection + baseline-recording path.",
+    "description": "Nudge-triggered compression: the model emits growth text across turns until billion-context-pi's context-usage nudge is injected, then the fake LLM detects the nudge and compresses. A low modelContextLimit makes the nudge fire quickly. Validates the nudge detection + baseline-recording path. minPressureBenefitTokens=0 (kernel #198 escape hatch): the 1500-token window is smaller than the default 5000-token benefit floor, which would suppress every pressure nudge; this scenario validates nudge detection + baseline recording, not the floor itself (unit-tested).",
     "acpConfig": {
-        "modelContextLimit": 1500
+        "modelContextLimit": 1500,
+        "compress": {
+            "minPressureBenefitTokens": 0
+        }
     },
     "turns": [
         {
@@ -40,4 +43,4 @@
         "minNudgeCount": 1,
         "nudgeBaselineSet": true
     }
-}
+}

src/compress-tool.ts

@@ -9,6 +9,7 @@ import { MAX_COMPRESS_ATTEMPTS } from "./runtime.js";
 import { debug, logError, logInfo, logThrow, logWarn } from "./log.js";
 import { estimateTokens, collectCoveredMessageIds, collectImageTokens, modelSupportsImages, lastUserMessageId, adjustedTokenCount } from "./tokens.js";
 import { defaultCountTokens, parseCompressArgs, viableRanges, formatRanges, type CompressionBlock, type CompressionState, type CompressParseDiagnostics, type NudgeDecision } from "acp-kernel";
+import { countUnicodeEscapes, findUnverifiableUserQuote, sanitizeSummary } from "./summary-sanitize.js";
 import { getSystemPromptText } from "./compat.js";
 import { OMP_UNSUPPORTED_MESSAGE } from "./omp.js";
 
@@ -303,6 +304,21 @@ async function handleCompress(args: CompressArgs, runtime: AcpRuntime, ctx: Exte
   const state = turn.state;
   const messages = turn.messages;
   const sid = ctx.sessionManager.getSessionId();
+  // Issue #309: normalize at ingest — the kernel stores/renders summaries
+  // verbatim, so double-escaped \uXXXX runs would persist into every future
+  // prompt. Unverifiable user-quote claims are logged as evidence only.
+  const sanitizedRanges = ranges.map((r) => {
+    const span = `${r.startId}..${r.endId}`;
+    const s = sanitizeSummary(r.summary);
+    if (s.unescaped) {
+      debug.event("compress", { sid, event: "summary-unescaped", span, escapes: countUnicodeEscapes(r.summary), beforeLen: r.summary.length, afterLen: s.text.length });
+    }
+    const unverifiedQuote = findUnverifiableUserQuote(s.text);
+    if (unverifiedQuote !== null) {
+      logWarn("compress", { sid, event: "summary-unverifiable-quote", span, claim: unverifiedQuote });
+    }
+    return s.text === r.summary ? r : { ...r, summary: s.text };
+  });
   const turnKey = lastUserMessageId(entries) ?? sid;
   const snapshot = compressibleSnapshotText(turn.nudge);
   if (runtime.compressRetryCappedFor(turnKey)) {
@@ -331,7 +347,7 @@ async function handleCompress(args: CompressArgs, runtime: AcpRuntime, ctx: Exte
     beforeTokens,
   });
   const applied = runtime.core.applyCompression({
-    ranges: ranges.map((r) => ({ startRef: r.startId, endRef: r.endId, summary: r.summary, topic: r.topic ?? topLevelTopic, summaryMaxChars, compressCallId: toolCallId })),
+    ranges: sanitizedRanges.map((r) => ({ startRef: r.startId, endRef: r.endId, summary: r.summary, topic: r.topic ?? topLevelTopic, summaryMaxChars, compressCallId: toolCallId })),
     messages,
     state,
     config,

src/config.ts

@@ -122,6 +122,13 @@ export interface CompressSettings {
   /** Token growth threshold for soft compression nudges. Default: 50000.
    *  Maps to kernel nudge.growthFloor + nudge.growthCap. */
   nudgeGrowthTokens?: number;
+  /** Minimum reclaimable tokens for a pressure-band nudge (kernel #198).
+   *  Default: max(5000, round(limit×0.01)). Explicit 0 restores the legacy
+   *  any-pending behavior — useful for tiny windows (e.g. e2e scenarios with
+   *  modelContextLimit 1500) where a fixed 5000-token floor exceeds the whole
+   *  window and would suppress every nudge. Maps to kernel
+   *  nudge.minPressureBenefitTokens. */
+  minPressureBenefitTokens?: number;
 }
 
 /** Per-provider compression overrides. Carries the same tuning fields as the
@@ -300,6 +307,7 @@ export function mergeCompress(
     maxContextLimit: model?.maxContextLimit ?? provider?.maxContextLimit ?? global?.maxContextLimit,
     emergencyThresholdPercent: model?.emergencyThresholdPercent ?? provider?.emergencyThresholdPercent ?? global?.emergencyThresholdPercent,
     nudgeGrowthTokens: model?.nudgeGrowthTokens ?? provider?.nudgeGrowthTokens ?? global?.nudgeGrowthTokens,
+    minPressureBenefitTokens: model?.minPressureBenefitTokens ?? provider?.minPressureBenefitTokens ?? global?.minPressureBenefitTokens,
   };
 }
 
@@ -347,6 +355,9 @@ export function resolveConfig(adapter: AdapterConfig, liveContextLimit: number,
     config.nudge.growthFloor = c.nudgeGrowthTokens;
     config.nudge.growthCap = c.nudgeGrowthTokens;
   }
+  if (c.minPressureBenefitTokens !== undefined) {
+    config.nudge.minPressureBenefitTokens = c.minPressureBenefitTokens;
+  }
   return config;
 }
 

src/summary-sanitize.ts

@@ -0,0 +1,79 @@
+// Issue #309: small models sometimes emit summaries whose CJK text arrived as
+// literal \uXXXX sequences (double-escaped on the wire, so one JSON.parse leaves
+// 6-char "\u5408" runs in the parsed string). The kernel stores and renders
+// summaries verbatim, so the corruption would persist into every future prompt
+// and re-contaminate tier-2/tier-3 distillations. Normalize at ingest instead.
+
+const UNICODE_ESCAPE_RE = /\\u[0-9a-fA-F]{4}/g;
+
+export function countUnicodeEscapes(s: string): number {
+  const m = s.match(UNICODE_ESCAPE_RE);
+  return m ? m.length : 0;
+}
+
+// Decodes ONLY \uXXXX runs (incl. surrogate pairs) — \n, \\ etc. stay literal,
+// so legitimate escape examples in prose are never mangled below the threshold.
+export function decodeUnicodeEscapes(s: string): string {
+  let out = "";
+  let i = 0;
+  while (i < s.length) {
+    if (s.charAt(i) === "\\" && s.charAt(i + 1) === "u") {
+      const hex = s.slice(i + 2, i + 6);
+      if (/^[0-9a-fA-F]{4}$/.test(hex)) {
+        const high = parseInt(hex, 16);
+        const next = s.slice(i + 6, i + 12);
+        if (high >= 0xd800 && high <= 0xdbff && /^\\u[0-9a-fA-F]{4}$/.test(next)) {
+          const low = parseInt(next.slice(2, 6), 16);
+          if (low >= 0xdc00 && low <= 0xdfff) {
+            out += String.fromCodePoint(((high - 0xd800) << 10) + (low - 0xdc00) + 0x10000);
+            i += 12;
+            continue;
+          }
+        }
+        out += String.fromCharCode(high);
+        i += 6;
+        continue;
+      }
+    }
+    out += s.charAt(i);
+    i++;
+  }
+  return out;
+}
+
+// Strictly-more-than semantics: a summary legitimately quoting a few \uXXXX
+// examples must survive; dense runs (>20) are the double-escape failure mode.
+export const UNESCAPE_THRESHOLD = 20;
+
+export function sanitizeSummary(s: string): { text: string; unescaped: boolean } {
+  if (countUnicodeEscapes(s) <= UNESCAPE_THRESHOLD) return { text: s, unescaped: false };
+  return { text: decodeUnicodeEscapes(s), unescaped: true };
+}
+
+// Issue #309 phenomenon B: a hallucinated user phrase was stored as
+// `user verbatim '...'` / `CURRENT TASK: user '...'`. Without an mNNNNN ref the
+// quote is unverifiable and later readers treat it as fact — the loop amplifier.
+// Detection only: callers log evidence ([warn] summary-unverifiable-quote),
+// they NEVER rewrite the model's text.
+const USER_QUOTE_CLAIMS: RegExp[] = [
+  /\buser\s+verbatim\b/i,
+  /\bverbatim\s+user\b/i,
+  /\buser\s*[:：]\s*["'"“”‘’]/i,
+  /\buser\s+["'"“”‘’]/i,
+  /\buser\s+said\s+["'"“”‘’]/i,
+  /["'"“”‘’][^"'\n]{1,200}["'"“”‘’]\s*\(\s*(?:from\s+)?user\s*\)/i,
+  /用户(?:的)?原话/,
+];
+
+const MESSAGE_REF_RE = /\bm\d{4,}\b/i;
+
+// null when no claim matches, or when the summary carries at least one mNNNNN
+// ref (quotes then point at verifiable messages).
+export function findUnverifiableUserQuote(summary: string): string | null {
+  if (MESSAGE_REF_RE.test(summary)) return null;
+  for (const re of USER_QUOTE_CLAIMS) {
+    const m = re.exec(summary);
+    if (m) return m[0].slice(0, 80);
+  }
+  return null;
+}

tests/compress-tool.test.ts

@@ -1,6 +1,6 @@
 import { test } from "node:test";
 import assert from "node:assert/strict";
-import { rm } from "node:fs/promises";
+import { readFile, rm } from "node:fs/promises";
 import { createAcpExtension } from "../src/index.js";
 
 // ─── helpers (mirror decompress-tool.test.ts) ──────────────────────────────
@@ -119,3 +119,40 @@ test("compress afterTokens is measured on the same sent-view scale as beforeToke
   assert.ok(reclaimed <= 180, `reclaimed ${reclaimed} over-claimed (raw afterTokens would be ~220): ${text}`);
   assert.equal(before - after, reclaimed, "reclaimed consistent with the arrow");
 });
+
+// issue #309: a model that emits double-escaped summaries (literal \uXXXX runs
+// in the parsed string) must not have that corruption stored — the kernel
+// renders summaries verbatim into every future prompt.
+test("compress normalizes double-escaped \\uXXXX summaries before storage", async () => {
+  const { api, handlers } = captureApi();
+  // minCompressRange gate needs ≥5000 chars in the range; the kernel's
+  // unconfigurable preserveRecentTokens (5000) protects any trailing window,
+  // so e2 carries ≥5000 tokens of its own and preserveRecentMessages:1 makes
+  // e1 (the compress target) fall outside every protected zone.
+  createAcpExtension({ modelContextLimit: 200_000, preserveRecentMessages: 1 })(api as any);
+  const stateFile = "/tmp/pai-acp-compress-unescape.session.json";
+  await rm(`${stateFile}.acp.json`, { force: true });
+  const entries = [userMsg("e1", "中".repeat(6000)), userMsg("e2", "中".repeat(6000))];
+  const ctx = fakeCtx(entries, stateFile);
+  ctx.__setUsage(100_000);
+  await runContextRound(handlers, ctx); // prime the context round
+
+  const compressTool = api.tools.find((t: any) => t.name === "compress")!;
+  // Padded so the DECODED form clears minSummaryLength (50): 4 + 25 + 25.
+  const escapedSummary = "摘要: " + "\\u5408".repeat(25) + " 结束。" + "尾".repeat(25);
+  assert.ok(escapedSummary.includes("\\u5408"), "precondition: literal escape runs in input");
+  const out = await compressTool.execute(
+    "tc1",
+    { content: [{ startId: "m00001", endId: "m00001", summary: escapedSummary }] },
+    undefined, undefined, ctx,
+  );
+  const text = typeof out === "string" ? out : out.content?.[0]?.text ?? String(out);
+  assert.ok(text.includes("▣ ACP"), `compress failed: ${text}`);
+  assert.ok(!text.includes("Errors:"), `compress was rejected: ${text}`);
+
+  const raw = JSON.parse(await readFile(`${stateFile}.acp.json`, "utf8"));
+  const block = (raw.blocks as any[]).find((b) => typeof b.summary === "string" && b.summary.length > 0);
+  assert.ok(block, "no block stored in acp state");
+  assert.ok(block.summary.includes("合".repeat(25)), "stored summary must contain decoded CJK");
+  assert.ok(!block.summary.includes("\\u5408"), "stored summary must not contain literal \\uXXXX runs");
+});

tests/config.test.ts

@@ -82,6 +82,18 @@ test("resolveConfig leaves growthFloor/growthCap at kernel defaults when compres
   assert.equal(cfg.nudge.growthCap, 50000);
 });
 
+test("resolveConfig maps compress.minPressureBenefitTokens to kernel nudge (0 = legacy any-pending)", () => {
+  const cfg = resolveConfig({ compress: { minPressureBenefitTokens: 0 } }, 1_000_000);
+  assert.equal(cfg.nudge.minPressureBenefitTokens, 0);
+  const cfg2 = resolveConfig({ compress: { minPressureBenefitTokens: 8000 } }, 1_000_000);
+  assert.equal(cfg2.nudge.minPressureBenefitTokens, 8000);
+});
+
+test("resolveConfig leaves minPressureBenefitTokens undefined (kernel default max(5000, limit×1%)) when omitted", () => {
+  const cfg = resolveConfig(EMPTY, 1_000_000);
+  assert.equal(cfg.nudge.minPressureBenefitTokens, undefined);
+});
+
 test("resolveConfig handles all three compress fields together", () => {
   const cfg = resolveConfig({ compress: { maxContextLimit: "70%", emergencyThresholdPercent: 0.9, nudgeGrowthTokens: 40000 } }, 1_000_000);
   assert.equal(cfg.nudge.maxContextLimitPct, 0.7);

tests/e2e-compress-config.test.ts

@@ -102,15 +102,18 @@ test("e2e compress config: without a config file the kernel defaults apply", asy
 
 // Behavioral: feed the real configFor() output into runtime.core.processTurn()
 // (src/index.ts:142) and assert shouldInject flips with the limit. The nudge
-// needs recommendedRanges > 0, not just a high usage ratio — hence the bulk text.
+// needs EFFECTIVE pending (merged ranges ≥ minCompressRange chars) over the
+// kernel's min-pressure-benefit floor (max(5000, limit×1%) — kernel #198),
+// not just a high usage ratio — hence the bulk text (8K chars/msg keeps every
+// range effective and the 2w-limit pending ≈24K tokens above the 5K floor).
 function compressibleMessages(): CoreMessage[] {
     const msgs: CoreMessage[] = [];
     for (let i = 0; i < 12; i++) {
         msgs.push({
             id: `h_${i}`,
             role: i % 2 === 0 ? "user" : "assistant",
             contentType: "text",
-            text: `historical detail ${i}. ${"x".repeat(3000)}`,
+            text: `historical detail ${i}. ${"x".repeat(8000)}`,
         });
     }
     return msgs;

tests/sent-view-arbitration.test.ts

@@ -31,6 +31,12 @@ function msg(id: string, role: string, text: string) {
 }
 
 const MID = "lorem ".repeat(3000);
+// Small bulk for idle-control tests: ~1.8K chars × 20 msgs ≈ 9K tokens ≈ 5%
+// usage at a 180K window — below kernel #194's first-sight bypass floor
+// (usage ≥ minContextLimitPct 45% with ready mass ≥ growth tokens), so the
+// control scenarios stay idle for the reason they were written to isolate
+// (the usage floor), not because a bypass is masking the difference.
+const SMALL = "lorem ".repeat(300);
 
 let branchEntries: any[] = [];
 
@@ -103,9 +109,9 @@ test("context transform stays idle when there is no provider usage to floor from
   createAcpExtension({ modelContextLimit: 180_000 })(api as any);
 
   const ctx = fakeCtx(500);
-  const entries = [msg("e0", "user", "start " + MID)];
-  for (let i = 1; i <= 18; i++) entries.push(msg(`e${i}`, i % 2 ? "assistant" : "user", `f${i} ` + MID));
-  entries.push(msg("e19", "assistant", "f19 " + MID));
+  const entries = [msg("e0", "user", "start " + SMALL)];
+  for (let i = 1; i <= 18; i++) entries.push(msg(`e${i}`, i % 2 ? "assistant" : "user", `f${i} ` + SMALL));
+  entries.push(msg("e19", "assistant", "f19 " + SMALL));
   branchEntries = entries;
   const r = await fire(handlers, entries, ctx);
   assert.equal(nudgeCount(r), 0, "no nudge: 42% estimate and no provider usage to floor from");
@@ -123,9 +129,9 @@ test("context transform skips the provider-usage floor while the anchor predates
   createAcpExtension({ modelContextLimit: 180_000 })(api as any);
 
   const ctx = fakeCtx(175_000);
-  const entries = [msg("e0", "user", "start " + MID)];
-  for (let i = 1; i <= 18; i++) entries.push(msg(`e${i}`, i % 2 ? "assistant" : "user", `f${i} ` + MID));
-  entries.push({ type: "message", id: "e19", parentId: null, timestamp: "", message: { role: "assistant", content: "f19 " + MID, timestamp: Date.now(), usage: { input: 175_000, cacheRead: 0, cacheWrite: 0 } } });
+  const entries = [msg("e0", "user", "start " + SMALL)];
+  for (let i = 1; i <= 18; i++) entries.push(msg(`e${i}`, i % 2 ? "assistant" : "user", `f${i} ` + SMALL));
+  entries.push({ type: "message", id: "e19", parentId: null, timestamp: "", message: { role: "assistant", content: "f19 " + SMALL, timestamp: Date.now(), usage: { input: 175_000, cacheRead: 0, cacheWrite: 0 } } });
   entries.push({ type: "message", id: "e20", parentId: null, timestamp: "", message: { role: "toolResult", toolName: "compress", toolCallId: "c1", content: [{ type: "text", text: "▣ ACP | 42.3K → 18.9K tokens (~23.4K reclaimed, 3 blocks)" }], timestamp: Date.now() } });
   branchEntries = entries;
   const r = await fire(handlers, entries, ctx);