diff --git a/.agents/skills/senpi-qa/scripts/lib/mock-loop-ttsr.mjs b/.agents/skills/senpi-qa/scripts/lib/mock-loop-ttsr.mjs index e50caec621..33d4c77435 100644 --- a/.agents/skills/senpi-qa/scripts/lib/mock-loop-ttsr.mjs +++ b/.agents/skills/senpi-qa/scripts/lib/mock-loop-ttsr.mjs @@ -5,7 +5,13 @@ import { createChecks, evidenceDir, guardRealAuth, installCleanupHooks } from ". const CTRL_SEP = ["<", "|", "sep", "|", ">"].join(""); const BANG_RUN_300 = /!{300}/; -export const TTSR_SCENARIOS = ["ttsr-collapse", "ttsr-leak", "ttsr-repetitive-turns", "ttsr-paragraph-loop"]; +export const TTSR_SCENARIOS = [ + "ttsr-collapse", + "ttsr-leak", + "ttsr-repetitive-turns", + "ttsr-paragraph-loop", + "ttsr-near-duplicate-loop", +]; // Shape of session 01a06648: the model re-announced the same planning step for minutes as // plain text without ever issuing the tool call. Three byte-identical paragraphs per cycle, @@ -17,6 +23,28 @@ const PARAGRAPH_LOOP_CYCLE = [ ]; const PARAGRAPH_LOOP_TEXT = `${Array.from({ length: 3 }, () => PARAGRAPH_LOOP_CYCLE.join("\n\n")).join("\n\n")}\n\n`; +// Shape of session 01a0a38c: the model restated one action for twelve minutes without ever +// issuing the tool call, paraphrasing itself every time, so no two paragraphs are byte-identical +// and only the near-duplicate frequency mechanism can see it. +const NEAR_DUPLICATE_OPENERS = [ + "I'm assembling the final delivery now", + "I'm putting together the final payload", + "I'm compiling the delivery code", + "I'm writing out the final assembly", + "I'm finalizing the delivery path", +]; +const NEAR_DUPLICATE_TAILS = [ + "downloading the images, building both captions, linting them, and sending both batches", + "fetching the images, assembling both captions, running the lint pass, and dispatching both batches", + "pulling the images, composing both captions, checking the lint, and delivering both batches", +]; +const NEAR_DUPLICATE_PARAGRAPHS = Array.from({ length: 15 }, (_, index) => { + const opener = NEAR_DUPLICATE_OPENERS[index % NEAR_DUPLICATE_OPENERS.length]; + const tail = NEAR_DUPLICATE_TAILS[index % NEAR_DUPLICATE_TAILS.length]; + return `${opener}: ${tail} to the channel with attachments.`; +}); +const NEAR_DUPLICATE_TEXT = `${NEAR_DUPLICATE_PARAGRAPHS.join("\n\n")}\n\n`; + function readFirstPersistedAssistant(box) { const files = readdirSync(box.sessionDir, { recursive: true, encoding: "utf8" }) .filter((name) => name.endsWith(".jsonl")) @@ -209,11 +237,14 @@ export async function runTtsrScenario({ scenarioName, apiName, driveTurn, eviden } const collapse = scenarioName === "ttsr-collapse"; const paragraphLoop = scenarioName === "ttsr-paragraph-loop"; + const nearDuplicateLoop = scenarioName === "ttsr-near-duplicate-loop"; let firstTurn; if (collapse) { firstTurn = { reasoning: `analyzing the problem ${"!".repeat(600)}`, chunks: 40 }; } else if (paragraphLoop) { firstTurn = { text: PARAGRAPH_LOOP_TEXT, chunks: 60 }; + } else if (nearDuplicateLoop) { + firstTurn = { text: NEAR_DUPLICATE_TEXT, chunks: 60 }; } else { firstTurn = { reasoning: `Thinking... ${CTRL_SEP} ${CTRL_SEP} ${CTRL_SEP} trailing garbage ${"x".repeat(400)}`, chunks: 20 }; } @@ -262,6 +293,34 @@ export async function runTtsrScenario({ scenarioName, apiName, driveTurn, eviden replayBody.includes('rule=\\"collapse-repetition\\"'), `interruptPresent=${replayBody.includes("collapse-repetition")}`, ); + } else if (nearDuplicateLoop) { + const distinct = new Set(NEAR_DUPLICATE_PARAGRAPHS).size; + const lastParagraph = NEAR_DUPLICATE_PARAGRAPHS[NEAR_DUPLICATE_PARAGRAPHS.length - 1]; + const aborted = readFirstPersistedAssistant(box); + const persistedText = assistantText(aborted); + checks.ok( + "ttsr-near-duplicate-loop: no paragraph repeats byte-exactly, so only the frequency rule can fire", + distinct === NEAR_DUPLICATE_PARAGRAPHS.length, + `distinct=${distinct}/${NEAR_DUPLICATE_PARAGRAPHS.length}`, + ); + checks.ok( + "ttsr-near-duplicate-loop: persisted aborted message is truncated before the streamed tail", + aborted?.stopReason === "aborted" && + persistedText.length < NEAR_DUPLICATE_TEXT.length && + !persistedText.includes(lastParagraph) && + persistedText.includes("[output interrupted by stream rule]"), + `stopReason=${aborted?.stopReason ?? "missing"} chars=${persistedText.length}/${NEAR_DUPLICATE_TEXT.length}`, + ); + checks.ok( + "ttsr-near-duplicate-loop: recovery request never replays the streamed tail", + !replayBody.includes(JSON.stringify(lastParagraph).slice(1, -1)), + `tailPresent=${replayBody.includes(JSON.stringify(lastParagraph).slice(1, -1))}`, + ); + checks.ok( + "ttsr-near-duplicate-loop: collapse-repetition system-interrupt injected into the recovery request", + replayBody.includes('rule=\\"collapse-repetition\\"'), + `interruptPresent=${replayBody.includes("collapse-repetition")}`, + ); } else { checks.ok( "ttsr-leak: leaked control tokens absent from retry request", diff --git a/.agents/skills/senpi-qa/scripts/mock-loop.mjs b/.agents/skills/senpi-qa/scripts/mock-loop.mjs index 06c5bac158..6b2efdf023 100644 --- a/.agents/skills/senpi-qa/scripts/mock-loop.mjs +++ b/.agents/skills/senpi-qa/scripts/mock-loop.mjs @@ -674,7 +674,7 @@ if (argv[0] === "--self-test") { " node mock-loop.mjs --with-truncated-text-tool-leak --api ", " node mock-loop.mjs --with-eval-hard-limit [--api ] eval cell killed by the wall-clock hard limit", " node mock-loop.mjs --with-mcp-tool [--tool-args JSON]", - " node mock-loop.mjs --scenario [--api ]", + " node mock-loop.mjs --scenario [--api ]", " node mock-loop.mjs --scenario ", " node mock-loop.mjs --run [--api ]", ` APIs: ${ALL_APIS.join(", ")}`, diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md index 6c09973198..9d463b9651 100644 --- a/packages/coding-agent/CHANGELOG.md +++ b/packages/coding-agent/CHANGELOG.md @@ -8,6 +8,8 @@ ### Changed +- The stream guard now interrupts a message that keeps restating itself in different words, not only one that repeats a paragraph verbatim. A paragraph counts as an echo when its normalized word set overlaps an earlier paragraph of the same message by half, and the guard acts only once at least 8 of the last 12 paragraphs are echoes, so a repeated sentence, a callback, or a closing summary is left alone while a narration loop is cut within a few thousand characters. Paragraphs inside fenced code blocks are ignored, and the earlier text is kept - only the repeated run is dropped before the retry. Measured against 12,499 real assistant messages, the rule matched nothing except two known runaway generations. ([#1330](https://github.com/code-yeongyu/senpi/issues/1330)) + - `--help` no longer boots the engine to print a help screen. The usage text and the flags extensions register are answered from a cache of the last launch's flag set (`/cache/help-flags.json`, validated against the engine version and the mtime/size of every extension, settings and trust input, so an upgrade or an edited extension refreshes it); a cache miss loads extensions for their flags only and skips the model runtime, the session and every other resource class. Measured warm on an Apple M4 Pro: 790ms → 28ms on bun and 959ms → 59ms on node for `--help`; a help screen never prompts for project trust and never runs project-local extension code that is not already trusted. ([oh-my-openagent#8371](https://github.com/code-yeongyu/oh-my-openagent/issues/8371)) ### Fixed diff --git a/packages/coding-agent/src/core/extensions/builtin/ttsr/changes.md b/packages/coding-agent/src/core/extensions/builtin/ttsr/changes.md index 1c30a68695..2f8ed98e72 100644 --- a/packages/coding-agent/src/core/extensions/builtin/ttsr/changes.md +++ b/packages/coding-agent/src/core/extensions/builtin/ttsr/changes.md @@ -1,5 +1,28 @@ # TTSR Fork Tracker +## 2026-09-16 - Near-duplicate paragraph frequency + +### What changed and why + +- Added `detectors/collapse-near-duplicates.ts` (`near-duplicate-paragraphs`), wired last in the collapse chain for text and thinking streams. `paragraph-repeat` compares paragraphs byte for byte, so a model that restates the same step in different words every time never reaches three identical hashes; a captured incident streamed such a loop for 12 minutes with zero tool calls until the user killed the session. +- The mechanism is a frequency rule, not a pair rule: a paragraph is an echo when its normalized word set reaches 0.5 Jaccard against any of the last 32 eligible paragraphs, and the detector only fires when at least 8 of the last 12 eligible paragraphs are echoes. A couple of similar paragraphs, a callback to an earlier point, or a summary that repeats a sentence never reaches that density. +- The exact-repeat ring (64 paragraphs) also cannot see a cycle longer than itself. The incident's cycle was 114 paragraphs, so the same loop stayed invisible even where it repeated verbatim; the frequency rule is independent of cycle length. +- Paragraphs inside fenced code blocks are skipped and never enter the history: repeated code blocks in one message are legitimate. +- Remediation reuses the existing collapse path: abort, truncate from the first echoed paragraph in the firing window (its anchor is kept), then the collapse nudge. +- Calibration is measured, not assumed. Replaying the shipped detector over 12,499 real assistant text and thinking parts (28.9 MB) from the local session store fires twice, and both firings are known runaway generations from one 2026-09-03 session; there are no other matches. + +### Why an extension-local change is required + +- The stream watcher already owns per-message detector state and collapse remediation, so paragraph tracking belongs in the extension-local collapse chain without changing provider or core stream contracts. + +### Coverage and expected conflict zones + +- `test/ttsr/detector-collapse-near-duplicates.test.ts` replays the sanitized incident fixture (`fixtures/incident-near-duplicate-narration.txt`), pins chunk-boundary independence, and pins the negatives: distinct multi-sentence prose, a minority of echoes in the window, the long healthy prefix, fenced code, and tool-stream exclusion. +- `test/ttsr/collapse-test-inputs.ts` `buildHealthyPrefix` now composes varied vocabulary. Its previous sentences differed only by a counter, so every paragraph normalized to the same token set - healthy prose for a byte-exact rule, a narration loop for a normalized one. +- `test/ttsr/detector-collapse-paragraphs.test.ts` `narration()` now emits seven lexically distinct steps instead of one template plus a counter, so the exact-repeat assertions still test exact repetition. The assertions themselves are unchanged. +- Real-CLI QA ships as `senpi-qa` mock-loop scenario `ttsr-near-duplicate-loop`: fifteen paraphrases of one action, none byte-identical, streamed from the local fake model server. It asserts the abort, the truncated persisted message, the `collapse-repetition` interrupt in the recovery request, and the recovered answer. +- LOW: `detectors/collapse.ts` (one chain entry) and the two test-input fixtures; no existing detector thresholds are changed. + ## 2026-09-03 - Within-message paragraph repetition ### What changed and why diff --git a/packages/coding-agent/src/core/extensions/builtin/ttsr/detectors/collapse-near-duplicates.ts b/packages/coding-agent/src/core/extensions/builtin/ttsr/detectors/collapse-near-duplicates.ts new file mode 100644 index 0000000000..bae1706d3e --- /dev/null +++ b/packages/coding-agent/src/core/extensions/builtin/ttsr/detectors/collapse-near-duplicates.ts @@ -0,0 +1,178 @@ +import { CharCode, FixedRing, isAsciiWhitespace, type ScalarEntry } from "../stream-utils.ts"; +import type { DetectorMatch } from "../types.ts"; +import { isAsciiAlphanumeric, isBoxDrawing } from "./collapse-scalars.ts"; +import { normalizeTurnText } from "./repetitive-turns.ts"; + +export const NEAR_DUPLICATE_MIN_CHARS = 64; +export const NEAR_DUPLICATE_MIN_WORD_CHARS = 24; +export const NEAR_DUPLICATE_SIMILARITY = 0.5; +export const NEAR_DUPLICATE_LOOKBACK = 32; +export const NEAR_DUPLICATE_WINDOW = 12; +export const NEAR_DUPLICATE_ECHO_THRESHOLD = 8; +export const NEAR_DUPLICATE_TEXT_RETENTION_MAX = 512; + +const SAMPLE_LENGTH = 80; +const WORD_PATTERN = /[\p{L}\p{N}#]+/gu; +const FENCE_PATTERN = /^(?:```|~~~)/; + +interface ParagraphSignature { + readonly tokens: ReadonlySet; + readonly startOffset: number; + readonly sample: string; +} + +interface WindowEntry { + readonly echoed: boolean; + readonly startOffset: number; + readonly anchorStartOffset: number; +} + +export interface NearDuplicateState { + readonly history: FixedRing; + readonly window: FixedRing; + lineLength: number; + lineWordChars: number; + lineHasContent: boolean; + lineStartOffset: number; + lineText: string; + length: number; + wordChars: number; + startOffset: number; + retained: string; + fenced: boolean; + insideFence: boolean; +} + +export function createNearDuplicateState(): NearDuplicateState { + return { + history: new FixedRing(NEAR_DUPLICATE_LOOKBACK), + window: new FixedRing(NEAR_DUPLICATE_WINDOW), + lineLength: 0, + lineWordChars: 0, + lineHasContent: false, + lineStartOffset: 0, + lineText: "", + length: 0, + wordChars: 0, + startOffset: 0, + retained: "", + fenced: false, + insideFence: false, + }; +} + +function tokenize(text: string): ReadonlySet { + return new Set(normalizeTurnText(text).match(WORD_PATTERN) ?? []); +} + +function jaccard(a: ReadonlySet, b: ReadonlySet): number { + if (a.size === 0 || b.size === 0) return 0; + let intersection = 0; + for (const token of a) { + if (b.has(token)) intersection += 1; + } + return intersection / (a.size + b.size - intersection); +} + +function resetParagraph(state: NearDuplicateState): void { + state.length = 0; + state.wordChars = 0; + state.startOffset = 0; + state.retained = ""; + state.fenced = false; +} + +function countEchoes(state: NearDuplicateState): number { + let echoes = 0; + for (let back = state.window.size - 1; back >= 0; back--) { + if (state.window.getBack(back)?.echoed === true) echoes += 1; + } + return echoes; +} + +function oldestEcho(state: NearDuplicateState): WindowEntry | undefined { + for (let back = state.window.size - 1; back >= 0; back--) { + const entry = state.window.getBack(back); + if (entry?.echoed === true) return entry; + } + return undefined; +} + +function completeParagraph(state: NearDuplicateState): DetectorMatch | null { + const eligible = + !state.fenced && state.length >= NEAR_DUPLICATE_MIN_CHARS && state.wordChars >= NEAR_DUPLICATE_MIN_WORD_CHARS; + const text = state.retained; + const startOffset = state.startOffset; + resetParagraph(state); + if (!eligible) return null; + const tokens = tokenize(text); + if (tokens.size === 0) return null; + let best = 0; + let anchor: ParagraphSignature | undefined; + for (let back = 0; back < state.history.size; back++) { + const previous = state.history.getBack(back); + if (previous === undefined) continue; + const similarity = jaccard(tokens, previous.tokens); + if (similarity > best) { + best = similarity; + anchor = previous; + } + } + const echoed = best >= NEAR_DUPLICATE_SIMILARITY && anchor !== undefined; + state.history.push({ tokens, startOffset, sample: text.slice(0, SAMPLE_LENGTH) }); + state.window.push({ + echoed, + startOffset, + anchorStartOffset: echoed && anchor !== undefined ? anchor.startOffset : startOffset, + }); + const echoes = countEchoes(state); + if (echoes < NEAR_DUPLICATE_ECHO_THRESHOLD) return null; + const first = oldestEcho(state); + if (first === undefined) return null; + return { + rule: "collapse-repetition", + reason: `${echoes} of the last ${state.window.size} paragraphs restate an earlier paragraph of the same message`, + anomalyStartOffset: first.anchorStartOffset, + garbageStartOffset: first.startOffset, + detail: { + mechanism: "near-duplicate-paragraphs", + echoes, + window: state.window.size, + similarity: Number(best.toFixed(3)), + sample: anchor?.sample ?? "", + }, + }; +} + +function foldLine(state: NearDuplicateState): void { + if (state.length === 0) state.startOffset = state.lineStartOffset; + state.length += state.lineLength + 1; + state.wordChars += state.lineWordChars; + if (state.insideFence || FENCE_PATTERN.test(state.lineText.trimStart())) state.fenced = true; + if (state.retained.length < NEAR_DUPLICATE_TEXT_RETENTION_MAX) state.retained += `${state.lineText}\n`; + if (FENCE_PATTERN.test(state.lineText.trimStart())) state.insideFence = !state.insideFence; +} + +function resetLine(state: NearDuplicateState, startOffset: number): void { + state.lineLength = 0; + state.lineWordChars = 0; + state.lineHasContent = false; + state.lineStartOffset = startOffset; + state.lineText = ""; +} + +export function updateNearDuplicates(state: NearDuplicateState, entry: ScalarEntry): DetectorMatch | null { + if (entry.value.charCodeAt(0) === CharCode.LineFeed) { + let result: DetectorMatch | null = null; + if (state.lineHasContent) foldLine(state); + else if (state.length > 0) result = completeParagraph(state); + resetLine(state, entry.startOffset + 1); + return result; + } + const codePoint = entry.value.codePointAt(0) ?? 0; + if (!isAsciiWhitespace(codePoint)) state.lineHasContent = true; + if (isAsciiAlphanumeric(codePoint) || (codePoint > 0x7f && !isBoxDrawing(codePoint))) state.lineWordChars += 1; + state.lineLength += entry.width; + if (state.lineText.length < NEAR_DUPLICATE_TEXT_RETENTION_MAX) state.lineText += entry.value; + return null; +} diff --git a/packages/coding-agent/src/core/extensions/builtin/ttsr/detectors/collapse.ts b/packages/coding-agent/src/core/extensions/builtin/ttsr/detectors/collapse.ts index 266df25fb8..3ad2a50bf2 100644 --- a/packages/coding-agent/src/core/extensions/builtin/ttsr/detectors/collapse.ts +++ b/packages/coding-agent/src/core/extensions/builtin/ttsr/detectors/collapse.ts @@ -1,6 +1,11 @@ import { FixedRing, type ScalarEntry, ScalarScanner } from "../stream-utils.ts"; import type { DetectorContext, DetectorMatch, StreamDetector } from "../types.ts"; import { createLineCycleState, type LineCycleState, updateLineCycles } from "./collapse-lines.ts"; +import { + createNearDuplicateState, + type NearDuplicateState, + updateNearDuplicates, +} from "./collapse-near-duplicates.ts"; import { createParagraphRepeatState, type ParagraphRepeatState, @@ -26,6 +31,7 @@ export interface CollapseState { readonly periods: ShortPeriodState; readonly lines: LineCycleState; readonly paragraphs: ParagraphRepeatState; + readonly nearDuplicates: NearDuplicateState; latched: DetectorMatch | null; } @@ -38,6 +44,7 @@ export function createCollapseState(): CollapseState { periods: createShortPeriodState(), lines: createLineCycleState(), paragraphs: createParagraphRepeatState(), + nearDuplicates: createNearDuplicateState(), latched: null, }; } @@ -52,7 +59,8 @@ function checkDelta(state: CollapseState, delta: string, context: DetectorContex updateWhitespaceFlood(state.whitespace, entry) ?? updateShortPeriods(state.periods, entry, state.tailRing) ?? updateLineCycles(state.lines, entry) ?? - (watchParagraphs ? updateParagraphRepeats(state.paragraphs, entry) : null); + (watchParagraphs ? updateParagraphRepeats(state.paragraphs, entry) : null) ?? + (watchParagraphs ? updateNearDuplicates(state.nearDuplicates, entry) : null); if (match !== null) { state.latched = match; return match; diff --git a/packages/coding-agent/test/ttsr/AGENTS.md b/packages/coding-agent/test/ttsr/AGENTS.md index dc968f31f3..5e08db94ec 100644 --- a/packages/coding-agent/test/ttsr/AGENTS.md +++ b/packages/coding-agent/test/ttsr/AGENTS.md @@ -9,6 +9,7 @@ Coverage for the ttsr stream-rule extension (`src/core/extensions/builtin/ttsr/` | Control-leak grammar accept/reject | `detector-control-leak-grammar.test.ts` + `control-leak-helpers.ts` (`ctrl`, `sgml`, `bracket`, `runSplitMatrix`, `expectLeakMatchEverywhere`) | | Control-leak evidence / negatives | `detector-control-leak-evidence.test.ts`, `detector-control-leak-negatives.test.ts` | | Collapse detection | `detector-collapse.test.ts` + `collapse-test-inputs.ts` | +| Near-duplicate paragraph frequency | `detector-collapse-near-duplicates.test.ts` + `fixtures/incident-near-duplicate-narration.txt` | | Coordinator races / abort semantics | `coordinator.test.ts`, `coordinator-races.test.ts` (`claimAbort`, `createGenerationState`, `markUserCancelled`, `resolveDetection`) | | Rule parsing / builtin rules | `rule-parser.test.ts` | | Repetitive-turns lane | `repetitive-turns.test.ts` | diff --git a/packages/coding-agent/test/ttsr/collapse-test-inputs.ts b/packages/coding-agent/test/ttsr/collapse-test-inputs.ts index 4a7d7fcc26..1c8f0ddad3 100644 --- a/packages/coding-agent/test/ttsr/collapse-test-inputs.ts +++ b/packages/coding-agent/test/ttsr/collapse-test-inputs.ts @@ -6,12 +6,25 @@ export function lcg(seed: number): () => number { }; } +const ONSETS = "b,br,c,ch,d,dr,f,fl,g,gl,h,j,k,l,m,n,p,pl,qu,r,s,sh,t,tr,v,w".split(","); +const RIMES = "ade,ane,ark,eal,ean,eed,ell,ent,ess,ift,ilm,ind,ock,oil,old,ond,oom,ore,orn,ount,ove,udge,ule,urn,ust,yle".split(","); + +export const HEALTHY_WORD_COUNT = 26 * 26; + +export function healthyWord(index: number): string { + const onset = ONSETS[index % ONSETS.length] ?? "s"; + const rime = RIMES[Math.floor(index / ONSETS.length) % RIMES.length] ?? "ore"; + return `${onset}${rime}`; +} + export function buildHealthyPrefix(targetLength: number): string { const parts: string[] = []; + const next = lcg(97); let length = 0; let index = 0; while (length < targetLength) { - const sentence = `Sentence ${index} of the healthy prefix carries ordinary prose forward with varied word choices and rhythm marker ${index % 13}.`; + const pick = () => healthyWord(next() % (ONSETS.length * RIMES.length)); + const sentence = `The ${pick()} near the ${pick()} keeps ${pick()} aligned with ${pick()} while ${pick()} settles into ${pick()}.`; parts.push(sentence); length += sentence.length + 1; if (index % 10 === 9) { diff --git a/packages/coding-agent/test/ttsr/detector-collapse-near-duplicates.test.ts b/packages/coding-agent/test/ttsr/detector-collapse-near-duplicates.test.ts new file mode 100644 index 0000000000..d47604bf7b --- /dev/null +++ b/packages/coding-agent/test/ttsr/detector-collapse-near-duplicates.test.ts @@ -0,0 +1,145 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; + +import { collapseDetector, createCollapseState } from "../../src/core/extensions/builtin/ttsr/detectors/collapse.ts"; +import { + NEAR_DUPLICATE_ECHO_THRESHOLD, + NEAR_DUPLICATE_WINDOW, +} from "../../src/core/extensions/builtin/ttsr/detectors/collapse-near-duplicates.ts"; +import type { DetectorContext, DetectorMatch } from "../../src/core/extensions/builtin/ttsr/types.ts"; +import { buildHealthyPrefix, HEALTHY_WORD_COUNT, healthyWord, lcg } from "./collapse-test-inputs.ts"; + +type Source = DetectorContext["source"]; + +interface Replay { + readonly match: DetectorMatch | null; + readonly firedAfterChars: number; +} + +const context: DetectorContext = { source: "text", streamKey: "text:0", generation: 1 }; + +const INCIDENT = readFileSync(new URL("./fixtures/incident-near-duplicate-narration.txt", import.meta.url), "utf8"); + +function feedChunks(chunks: readonly string[], source: Source = "text"): Replay { + const state = createCollapseState(); + let consumed = 0; + for (const chunk of chunks) { + const match = collapseDetector.checkDelta(state, chunk, { ...context, source }); + consumed += chunk.length; + if (match !== null) return { match, firedAfterChars: consumed }; + } + return { match: null, firedAfterChars: consumed }; +} + +function perChar(input: string, source: Source = "text"): Replay { + return feedChunks(input.split(""), source); +} + +function randomChunks(input: string, seed: number, source: Source = "text"): Replay { + const next = lcg(seed); + const chunks: string[] = []; + for (let offset = 0; offset < input.length; ) { + const size = 1 + (next() % 97); + chunks.push(input.slice(offset, offset + size)); + offset += size; + } + return feedChunks(chunks, source); +} + +function paraphraseOfOneAction(index: number): string { + const openers = [ + "I'm assembling the final delivery now", + "I'm putting together the final payload", + "I'm compiling the delivery code", + "I'm writing out the final assembly", + ]; + const tails = [ + "downloading the images, building both captions, linting them, and sending both batches", + "fetching the images, assembling both captions, running the lint pass, and dispatching both batches", + "pulling the images, composing both captions, checking the lint, and delivering both batches", + ]; + const opener = openers[index % openers.length] ?? ""; + const tail = tails[index % tails.length] ?? ""; + return `${opener}: ${tail} to the channel with attachments.`; +} + +function distinctParagraph(index: number): string { + const next = lcg(index * 7919 + 13); + const pick = () => healthyWord(next() % HEALTHY_WORD_COUNT); + return Array.from( + { length: 3 }, + () => `The ${pick()} beside the ${pick()} keeps ${pick()} within ${pick()} while ${pick()} waits on ${pick()}.`, + ).join(" "); +} + +function joinParagraphs(parts: readonly string[]): string { + return `${parts.join("\n\n")}\n\n`; +} + +describe("near-duplicate paragraph frequency detector", () => { + it("fires on the real paraphrased narration loop captured in the incident", () => { + const { match, firedAfterChars } = perChar(INCIDENT); + expect(match?.rule).toBe("collapse-repetition"); + expect(match?.detail.mechanism).toBe("near-duplicate-paragraphs"); + expect(firedAfterChars).toBeLessThan(8000); + expect(match?.garbageStartOffset ?? 0).toBeGreaterThan(match?.anomalyStartOffset ?? 0); + }); + + it("reports the same match regardless of chunk boundaries", () => { + const reference = perChar(INCIDENT).match; + for (const seed of [7, 41, 1009]) { + expect(randomChunks(INCIDENT, seed).match).toEqual(reference); + } + }); + + it("fires on paraphrases that never repeat a paragraph byte-exactly", () => { + const paragraphs = Array.from({ length: 24 }, (_, index) => paraphraseOfOneAction(index)); + const occurrences = new Map(); + for (const paragraph of paragraphs) occurrences.set(paragraph, (occurrences.get(paragraph) ?? 0) + 1); + expect(Math.max(...occurrences.values())).toBeLessThan(3); + const { match } = perChar(joinParagraphs(paragraphs)); + expect(match?.detail.mechanism).toBe("near-duplicate-paragraphs"); + expect(match?.detail.window).toBeLessThanOrEqual(NEAR_DUPLICATE_WINDOW); + expect(match?.detail.echoes).toBeGreaterThanOrEqual(NEAR_DUPLICATE_ECHO_THRESHOLD); + }); + + it("stays silent on distinct multi-sentence prose", () => { + const paragraphs = Array.from({ length: 40 }, (_, index) => distinctParagraph(index)); + expect(perChar(joinParagraphs(paragraphs)).match).toBeNull(); + }); + + it("stays silent when only a minority of the window echoes", () => { + const paragraphs = Array.from({ length: 40 }, (_, index) => + index % 4 === 0 + ? `${paraphraseOfOneAction(index)} Batch ${healthyWord(index * 31)} carried its own attachment set.` + : distinctParagraph(index), + ); + expect(new Set(paragraphs).size).toBe(paragraphs.length); + expect(perChar(joinParagraphs(paragraphs)).match).toBeNull(); + }); + + it("stays silent on a long healthy prose prefix", () => { + expect(perChar(buildHealthyPrefix(64 * 1024)).match).toBeNull(); + }); + + it("does not count paragraphs inside fenced code blocks", () => { + const block = (index: number) => + ["```ts", `export function handler${index}(input: string): string {`, "\treturn input.trim();", "}", "```"].join( + "\n", + ); + const fenced = joinParagraphs(Array.from({ length: 24 }, (_, index) => block(index))); + expect(perChar(fenced).match).toBeNull(); + }); + + it("does not watch tool argument streams", () => { + expect(perChar(INCIDENT, "tool").match).toBeNull(); + expect(perChar(INCIDENT, "thinking").match?.detail.mechanism).toBe("near-duplicate-paragraphs"); + }); + + it("leaves byte-identical cycles to the exact-repeat mechanism", () => { + const paragraph = + "Now I'm writing step one of the plan: defining the shared context block with rules and tool guidance for the lane."; + const { match } = perChar(joinParagraphs([paragraph, paragraph, paragraph])); + expect(match?.detail.mechanism).toBe("paragraph-repeat"); + }); +}); diff --git a/packages/coding-agent/test/ttsr/detector-collapse-paragraphs.test.ts b/packages/coding-agent/test/ttsr/detector-collapse-paragraphs.test.ts index 96b1d1b721..0e8a51b5b6 100644 --- a/packages/coding-agent/test/ttsr/detector-collapse-paragraphs.test.ts +++ b/packages/coding-agent/test/ttsr/detector-collapse-paragraphs.test.ts @@ -13,8 +13,19 @@ type Source = DetectorContext["source"]; const context: DetectorContext = { source: "text", streamKey: "text:0", generation: 1 }; +const CYCLE_STEPS = [ + "defining the shared context block with rules and tool guidance for every lane that follows", + "drafting the research prompt that each scoped lane receives before it starts collecting", + "choosing where every report path lands so later waves can merge them without collisions", + "sizing the parallel batch against the machines that are currently reachable and idle", + "writing the fallback branch that reruns a dropped shard somewhere else automatically", + "recording the evidence layout so a reviewer can replay any single claim on its own", + "checking the teardown list against the resources this run actually created and bound", +]; + function narration(i: number): string { - return `Now I'm writing step ${i} of the plan: defining the shared context block with rules and tool guidance, then each research lane with its own scoped prompt and report path. The implementation keeps every result precise and useful.`; + const step = CYCLE_STEPS[i % CYCLE_STEPS.length] ?? ""; + return `Now I'm working on the plan: ${step}. The implementation keeps every result precise and useful.`; } function loop(cycle: number, cycles: number, separator = "\n\n"): string { @@ -72,7 +83,7 @@ describe("paragraph repetition detector", () => { }); it("compares paragraphs byte-exactly", () => { - const variant = narration(0).replace("scoped prompt", "scoped brief"); + const variant = narration(0).replace("precise and useful", "precise and complete"); expect(direct(`${narration(0)}\n\n${narration(0)}\n\n${variant}\n\n`)).toBeNull(); expect(direct(`${narration(0)}\n\n${narration(0)}\n\n${narration(0)}\n\n`)?.detail.occurrences).toBe(3); }); diff --git a/packages/coding-agent/test/ttsr/fixtures/incident-near-duplicate-narration.txt b/packages/coding-agent/test/ttsr/fixtures/incident-near-duplicate-narration.txt new file mode 100644 index 0000000000..af4ef6da3a --- /dev/null +++ b/packages/coding-agent/test/ttsr/fixtures/incident-near-duplicate-narration.txt @@ -0,0 +1,120 @@ +I'm assembling the final delivery: downloading photos, computing the office-to-the second area route, building both captions, linting, and sending both batches via upload with attachments — task complete. + +Both message batches sent with photos and the acknowledgment reaction cleared — done. + +I'm putting together the final code to fetch photos for all 9 picks, assemble both captions with distance/reviews/grill service/taxi times, lint them, and send both batches with attachments — complete. + +Both messages sent to the channel with photos attached and reaction cleared — task complete. + +Both batches delivered successfully with photos and the acknowledgment reaction removed — done. + +I'm writing the final assembly code — downloading photos, computing the route, building both captions with all requested details, linting, and sending both batches via upload to the channel — complete. + +Both messages sent with photos attached and the reaction cleared — task complete. + +I'm finalizing the delivery: photos downloaded for each pick, both captions assembled with all details, lint passing, both batches sent to the channel with attachments — done. + +Both message batches delivered with photos and the reaction cleared — complete. + +Both messages sent successfully to the channel with all photos and the acknowledgment reaction removed — task complete. + +I'm compiling the final send code: downloading photos, computing the office-to-the second area route, assembling both captions, linting, and dispatching both batches via upload with photos attached — done. + +Both batches sent with photos and reaction cleared — complete. + +I'm putting together the code to download photos, build both captions, lint, and send both message batches with attachments to the channel — task complete. + +Both messages delivered to the channel with photos attached and reaction cleared — done. + +Both batches sent successfully with photos and the acknowledgment reaction cleared — complete. + +I'm finalizing the code to download photos for all 9 picks, assemble both captions with the requested details, run lint checks, and send both batches with attachments — task complete. + +Both messages sent — office batch with 4 photos, the second area batch with 5 photos — reaction cleared — done. + +Both messages are sent to the channel with photos attached and reaction cleared — complete. + +I'm assembling the final delivery now: downloading photos, computing the route, building both captions with all requested details, linting, and sending both batches via upload — task complete. + +Both batches delivered with photos and the acknowledgment reaction removed — done. + +I'm compiling the final code — photos downloaded, both captions assembled with distance/reviews/grill/taxi, lint passing, both batches sent to the channel — complete. + +Both messages sent to the channel with photos and captions, reaction cleared — task complete. + +Both message batches sent successfully with photos attached and the acknowledgment reaction cleared — done. + +I'm writing the final assembly code to download photos, build both captions, lint them, and send both batches with attachments — complete. + +Both messages delivered with photos and the reaction cleared — task complete. + +Both batches sent to the channel with photos attached and the acknowledgment reaction removed — done. + +I'm finalizing — downloading photos for all 9 picks, computing the office-to-the second area route, building both captions with the requested details, linting, and sending both batches via upload — complete. + +Both messages sent with photos and the reaction cleared — task complete. + +I'm putting together the final delivery: photos downloaded, both captions built with all details, lint checks passing, both batches sent via upload with attachments — done. + +Both message batches delivered to the channel with photos and reaction cleared — complete. + +Both messages sent successfully with all photos attached and the acknowledgment reaction removed — task complete. + +I'm writing the final code to download each place's photo, compute the route, assemble both captions, lint them, and send both batches with photos attached to the channel — done. + +Both batches sent with photos and reaction cleared — complete. + +I'm compiling everything — route to the second area, photo downloads, both captions with full details — and sending both batches now — task complete. + +Both messages sent — office area batch with 4 photos, the second area area batch with 5 photos — reaction cleared — done. + +I'm finalizing the code to download photos, build both captions with the requested details, lint, and send both message batches with attachments to the channel — complete. + +Both messages sent successfully with photos attached and the acknowledgment reaction cleared — task complete. + +Both batches delivered with photos and the reaction cleared — done. + +I'm assembling the final delivery: downloading photos, computing the office-to-the second area route, building both captions, linting, and sending both batches via upload with attachments — complete. + +Both message batches sent with photos and the acknowledgment reaction cleared — task complete. + +I'm putting together the final code to fetch photos for all 9 picks, assemble both captions with distance/reviews/grill service/taxi times, lint them, and send both batches with attachments — done. + +Both messages sent to the channel with photos attached and reaction cleared — complete. + +Both batches delivered successfully with photos and the acknowledgment reaction removed — task complete. + +I'm writing the final assembly code — downloading photos, computing the route, building both captions with all requested details, linting, and sending both batches via upload to the channel — done. + +Both messages sent with photos attached and the reaction cleared — complete. + +I'm finalizing the delivery: photos downloaded for each pick, both captions assembled with all details, lint passing, both batches sent to the channel with attachments — task complete. + +Both message batches delivered with photos and the reaction cleared — done. + +Both messages sent successfully to the channel with all photos and the acknowledgment reaction removed — complete. + +I'm compiling the final send code: downloading photos, computing the office-to-the second area route, assembling both captions, linting, and dispatching both batches via upload with photos attached — task complete. + +Both batches sent with photos and reaction cleared — done. + +I'm putting together the code to download photos, build both captions, lint, and send both message batches with attachments to the channel — complete. + +Both messages delivered to the channel with photos attached and reaction cleared — task complete. + +Both batches sent successfully with photos and the acknowledgment reaction cleared — done. + +I'm finalizing the code to download photos for all 9 picks, assemble both captions with the requested details, run lint checks, and send both batches with attachments — complete. + +Both messages sent — office batch with 4 photos, the second area batch with 5 photos — reaction cleared — task complete. + +Both messages are sent to the channel with photos attached and reaction cleared — done. + +I'm assembling the final delivery now: downloading photos, computing the route, building both captions with all requested details, linting, and sending both batches via upload — complete. + +Both batches delivered with photos and the acknowledgment reaction removed — task complete. + +I'm compiling the final code — photos downloaded, both captions assembled with distance/reviews/grill/taxi, lint passing, both batches sent to the channel — done. + +Both messages sent to the channel with photos and captions, reaction cleared — complete. +