fix(ttsr): interrupt paragraph loops by near-duplicate frequency

.agents/skills/senpi-qa/scripts/lib/mock-loop-ttsr.mjs

@@ -5,7 +5,13 @@ import { createChecks, evidenceDir, guardRealAuth, installCleanupHooks } from ".
 const CTRL_SEP = ["<", "|", "sep", "|", ">"].join("");
 const BANG_RUN_300 = /!{300}/;
 
-export const TTSR_SCENARIOS = ["ttsr-collapse", "ttsr-leak", "ttsr-repetitive-turns", "ttsr-paragraph-loop"];
+export const TTSR_SCENARIOS = [
+	"ttsr-collapse",
+	"ttsr-leak",
+	"ttsr-repetitive-turns",
+	"ttsr-paragraph-loop",
+	"ttsr-near-duplicate-loop",
+];
 
 // Shape of session 01a06648: the model re-announced the same planning step for minutes as
 // plain text without ever issuing the tool call. Three byte-identical paragraphs per cycle,
@@ -17,6 +23,28 @@ const PARAGRAPH_LOOP_CYCLE = [
 ];
 const PARAGRAPH_LOOP_TEXT = `${Array.from({ length: 3 }, () => PARAGRAPH_LOOP_CYCLE.join("\n\n")).join("\n\n")}\n\n`;
 
+// Shape of session 01a0a38c: the model restated one action for twelve minutes without ever
+// issuing the tool call, paraphrasing itself every time, so no two paragraphs are byte-identical
+// and only the near-duplicate frequency mechanism can see it.
+const NEAR_DUPLICATE_OPENERS = [
+	"I'm assembling the final delivery now",
+	"I'm putting together the final payload",
+	"I'm compiling the delivery code",
+	"I'm writing out the final assembly",
+	"I'm finalizing the delivery path",
+];
+const NEAR_DUPLICATE_TAILS = [
+	"downloading the images, building both captions, linting them, and sending both batches",
+	"fetching the images, assembling both captions, running the lint pass, and dispatching both batches",
+	"pulling the images, composing both captions, checking the lint, and delivering both batches",
+];
+const NEAR_DUPLICATE_PARAGRAPHS = Array.from({ length: 15 }, (_, index) => {
+	const opener = NEAR_DUPLICATE_OPENERS[index % NEAR_DUPLICATE_OPENERS.length];
+	const tail = NEAR_DUPLICATE_TAILS[index % NEAR_DUPLICATE_TAILS.length];
+	return `${opener}: ${tail} to the channel with attachments.`;
+});
+const NEAR_DUPLICATE_TEXT = `${NEAR_DUPLICATE_PARAGRAPHS.join("\n\n")}\n\n`;
+
 function readFirstPersistedAssistant(box) {
 	const files = readdirSync(box.sessionDir, { recursive: true, encoding: "utf8" })
 		.filter((name) => name.endsWith(".jsonl"))
@@ -209,11 +237,14 @@ export async function runTtsrScenario({ scenarioName, apiName, driveTurn, eviden
 	}
 	const collapse = scenarioName === "ttsr-collapse";
 	const paragraphLoop = scenarioName === "ttsr-paragraph-loop";
+	const nearDuplicateLoop = scenarioName === "ttsr-near-duplicate-loop";
 	let firstTurn;
 	if (collapse) {
 		firstTurn = { reasoning: `analyzing the problem ${"!".repeat(600)}`, chunks: 40 };
 	} else if (paragraphLoop) {
 		firstTurn = { text: PARAGRAPH_LOOP_TEXT, chunks: 60 };
+	} else if (nearDuplicateLoop) {
+		firstTurn = { text: NEAR_DUPLICATE_TEXT, chunks: 60 };
 	} else {
 		firstTurn = { reasoning: `Thinking... ${CTRL_SEP} ${CTRL_SEP} ${CTRL_SEP} trailing garbage ${"x".repeat(400)}`, chunks: 20 };
 	}
@@ -262,6 +293,34 @@ export async function runTtsrScenario({ scenarioName, apiName, driveTurn, eviden
 				replayBody.includes('rule=\\"collapse-repetition\\"'),
 				`interruptPresent=${replayBody.includes("collapse-repetition")}`,
 			);
+		} else if (nearDuplicateLoop) {
+			const distinct = new Set(NEAR_DUPLICATE_PARAGRAPHS).size;
+			const lastParagraph = NEAR_DUPLICATE_PARAGRAPHS[NEAR_DUPLICATE_PARAGRAPHS.length - 1];
+			const aborted = readFirstPersistedAssistant(box);
+			const persistedText = assistantText(aborted);
+			checks.ok(
+				"ttsr-near-duplicate-loop: no paragraph repeats byte-exactly, so only the frequency rule can fire",
+				distinct === NEAR_DUPLICATE_PARAGRAPHS.length,
+				`distinct=${distinct}/${NEAR_DUPLICATE_PARAGRAPHS.length}`,
+			);
+			checks.ok(
+				"ttsr-near-duplicate-loop: persisted aborted message is truncated before the streamed tail",
+				aborted?.stopReason === "aborted" &&
+					persistedText.length < NEAR_DUPLICATE_TEXT.length &&
+					!persistedText.includes(lastParagraph) &&
+					persistedText.includes("[output interrupted by stream rule]"),
+				`stopReason=${aborted?.stopReason ?? "missing"} chars=${persistedText.length}/${NEAR_DUPLICATE_TEXT.length}`,
+			);
+			checks.ok(
+				"ttsr-near-duplicate-loop: recovery request never replays the streamed tail",
+				!replayBody.includes(JSON.stringify(lastParagraph).slice(1, -1)),
+				`tailPresent=${replayBody.includes(JSON.stringify(lastParagraph).slice(1, -1))}`,
+			);
+			checks.ok(
+				"ttsr-near-duplicate-loop: collapse-repetition system-interrupt injected into the recovery request",
+				replayBody.includes('rule=\\"collapse-repetition\\"'),
+				`interruptPresent=${replayBody.includes("collapse-repetition")}`,
+			);
 		} else {
 			checks.ok(
 				"ttsr-leak: leaked control tokens absent from retry request",

.agents/skills/senpi-qa/scripts/mock-loop.mjs

@@ -674,7 +674,7 @@ if (argv[0] === "--self-test") {
 			"  node mock-loop.mjs --with-truncated-text-tool-leak --api <anthropic-messages|openai-completions>",
 			"  node mock-loop.mjs --with-eval-hard-limit [--api <name>]  eval cell killed by the wall-clock hard limit",
 			"  node mock-loop.mjs --with-mcp-tool <tool> [--tool-args JSON]",
-			"  node mock-loop.mjs --scenario <transient-recover|budget-exhaust|server-error-fallback|long-retry-after|billing-swap|anthropic-policy-refusal-fallback|kimi-xtml-thinking-recover|model-request-rejected-recover|ttsr-collapse|ttsr-leak|ttsr-repetitive-turns|ttsr-paragraph-loop> [--api <name>]",
+			"  node mock-loop.mjs --scenario <transient-recover|budget-exhaust|server-error-fallback|long-retry-after|billing-swap|anthropic-policy-refusal-fallback|kimi-xtml-thinking-recover|model-request-rejected-recover|ttsr-collapse|ttsr-leak|ttsr-repetitive-turns|ttsr-paragraph-loop|ttsr-near-duplicate-loop> [--api <name>]",
 			"  node mock-loop.mjs --scenario <hinted-429-in-turn|no-hint-429-fast-fallback|hinted-429-probe-back|no-hint-429-no-chain>",
 			"  node mock-loop.mjs --run <prompt> [--api <name>]",
 			`  APIs: ${ALL_APIS.join(", ")}`,

packages/coding-agent/CHANGELOG.md

@@ -8,6 +8,8 @@
 
 ### Changed
 
+- The stream guard now interrupts a message that keeps restating itself in different words, not only one that repeats a paragraph verbatim. A paragraph counts as an echo when its normalized word set overlaps an earlier paragraph of the same message by half, and the guard acts only once at least 8 of the last 12 paragraphs are echoes, so a repeated sentence, a callback, or a closing summary is left alone while a narration loop is cut within a few thousand characters. Paragraphs inside fenced code blocks are ignored, and the earlier text is kept - only the repeated run is dropped before the retry. Measured against 12,499 real assistant messages, the rule matched nothing except two known runaway generations. ([#1330](https://github.com/code-yeongyu/senpi/issues/1330))
+
 - `--help` no longer boots the engine to print a help screen. The usage text and the flags extensions register are answered from a cache of the last launch's flag set (`<agentDir>/cache/help-flags.json`, validated against the engine version and the mtime/size of every extension, settings and trust input, so an upgrade or an edited extension refreshes it); a cache miss loads extensions for their flags only and skips the model runtime, the session and every other resource class. Measured warm on an Apple M4 Pro: 790ms → 28ms on bun and 959ms → 59ms on node for `--help`; a help screen never prompts for project trust and never runs project-local extension code that is not already trusted. ([oh-my-openagent#8371](https://github.com/code-yeongyu/oh-my-openagent/issues/8371))
 
 ### Fixed

packages/coding-agent/src/core/extensions/builtin/ttsr/changes.md

@@ -1,5 +1,28 @@
 # TTSR Fork Tracker
 
+## 2026-09-16 - Near-duplicate paragraph frequency
+
+### What changed and why
+
+- Added `detectors/collapse-near-duplicates.ts` (`near-duplicate-paragraphs`), wired last in the collapse chain for text and thinking streams. `paragraph-repeat` compares paragraphs byte for byte, so a model that restates the same step in different words every time never reaches three identical hashes; a captured incident streamed such a loop for 12 minutes with zero tool calls until the user killed the session.
+- The mechanism is a frequency rule, not a pair rule: a paragraph is an echo when its normalized word set reaches 0.5 Jaccard against any of the last 32 eligible paragraphs, and the detector only fires when at least 8 of the last 12 eligible paragraphs are echoes. A couple of similar paragraphs, a callback to an earlier point, or a summary that repeats a sentence never reaches that density.
+- The exact-repeat ring (64 paragraphs) also cannot see a cycle longer than itself. The incident's cycle was 114 paragraphs, so the same loop stayed invisible even where it repeated verbatim; the frequency rule is independent of cycle length.
+- Paragraphs inside fenced code blocks are skipped and never enter the history: repeated code blocks in one message are legitimate.
+- Remediation reuses the existing collapse path: abort, truncate from the first echoed paragraph in the firing window (its anchor is kept), then the collapse nudge.
+- Calibration is measured, not assumed. Replaying the shipped detector over 12,499 real assistant text and thinking parts (28.9 MB) from the local session store fires twice, and both firings are known runaway generations from one 2026-09-03 session; there are no other matches.
+
+### Why an extension-local change is required
+
+- The stream watcher already owns per-message detector state and collapse remediation, so paragraph tracking belongs in the extension-local collapse chain without changing provider or core stream contracts.
+
+### Coverage and expected conflict zones
+
+- `test/ttsr/detector-collapse-near-duplicates.test.ts` replays the sanitized incident fixture (`fixtures/incident-near-duplicate-narration.txt`), pins chunk-boundary independence, and pins the negatives: distinct multi-sentence prose, a minority of echoes in the window, the long healthy prefix, fenced code, and tool-stream exclusion.
+- `test/ttsr/collapse-test-inputs.ts` `buildHealthyPrefix` now composes varied vocabulary. Its previous sentences differed only by a counter, so every paragraph normalized to the same token set - healthy prose for a byte-exact rule, a narration loop for a normalized one.
+- `test/ttsr/detector-collapse-paragraphs.test.ts` `narration()` now emits seven lexically distinct steps instead of one template plus a counter, so the exact-repeat assertions still test exact repetition. The assertions themselves are unchanged.
+- Real-CLI QA ships as `senpi-qa` mock-loop scenario `ttsr-near-duplicate-loop`: fifteen paraphrases of one action, none byte-identical, streamed from the local fake model server. It asserts the abort, the truncated persisted message, the `collapse-repetition` interrupt in the recovery request, and the recovered answer.
+- LOW: `detectors/collapse.ts` (one chain entry) and the two test-input fixtures; no existing detector thresholds are changed.
+
 ## 2026-09-03 - Within-message paragraph repetition
 
 ### What changed and why

packages/coding-agent/src/core/extensions/builtin/ttsr/detectors/collapse-near-duplicates.ts

@@ -0,0 +1,178 @@
+import { CharCode, FixedRing, isAsciiWhitespace, type ScalarEntry } from "../stream-utils.ts";
+import type { DetectorMatch } from "../types.ts";
+import { isAsciiAlphanumeric, isBoxDrawing } from "./collapse-scalars.ts";
+import { normalizeTurnText } from "./repetitive-turns.ts";
+
+export const NEAR_DUPLICATE_MIN_CHARS = 64;
+export const NEAR_DUPLICATE_MIN_WORD_CHARS = 24;
+export const NEAR_DUPLICATE_SIMILARITY = 0.5;
+export const NEAR_DUPLICATE_LOOKBACK = 32;
+export const NEAR_DUPLICATE_WINDOW = 12;
+export const NEAR_DUPLICATE_ECHO_THRESHOLD = 8;
+export const NEAR_DUPLICATE_TEXT_RETENTION_MAX = 512;
+
+const SAMPLE_LENGTH = 80;
+const WORD_PATTERN = /[\p{L}\p{N}#]+/gu;
+const FENCE_PATTERN = /^(?:```|~~~)/;
+
+interface ParagraphSignature {
+	readonly tokens: ReadonlySet<string>;
+	readonly startOffset: number;
+	readonly sample: string;
+}
+
+interface WindowEntry {
+	readonly echoed: boolean;
+	readonly startOffset: number;
+	readonly anchorStartOffset: number;
+}
+
+export interface NearDuplicateState {
+	readonly history: FixedRing<ParagraphSignature>;
+	readonly window: FixedRing<WindowEntry>;
+	lineLength: number;
+	lineWordChars: number;
+	lineHasContent: boolean;
+	lineStartOffset: number;
+	lineText: string;
+	length: number;
+	wordChars: number;
+	startOffset: number;
+	retained: string;
+	fenced: boolean;
+	insideFence: boolean;
+}
+
+export function createNearDuplicateState(): NearDuplicateState {
+	return {
+		history: new FixedRing<ParagraphSignature>(NEAR_DUPLICATE_LOOKBACK),
+		window: new FixedRing<WindowEntry>(NEAR_DUPLICATE_WINDOW),
+		lineLength: 0,
+		lineWordChars: 0,
+		lineHasContent: false,
+		lineStartOffset: 0,
+		lineText: "",
+		length: 0,
+		wordChars: 0,
+		startOffset: 0,
+		retained: "",
+		fenced: false,
+		insideFence: false,
+	};
+}
+
+function tokenize(text: string): ReadonlySet<string> {
+	return new Set(normalizeTurnText(text).match(WORD_PATTERN) ?? []);
+}
+
+function jaccard(a: ReadonlySet<string>, b: ReadonlySet<string>): number {
+	if (a.size === 0 || b.size === 0) return 0;
+	let intersection = 0;
+	for (const token of a) {
+		if (b.has(token)) intersection += 1;
+	}
+	return intersection / (a.size + b.size - intersection);
+}
+
+function resetParagraph(state: NearDuplicateState): void {
+	state.length = 0;
+	state.wordChars = 0;
+	state.startOffset = 0;
+	state.retained = "";
+	state.fenced = false;
+}
+
+function countEchoes(state: NearDuplicateState): number {
+	let echoes = 0;
+	for (let back = state.window.size - 1; back >= 0; back--) {
+		if (state.window.getBack(back)?.echoed === true) echoes += 1;
+	}
+	return echoes;
+}
+
+function oldestEcho(state: NearDuplicateState): WindowEntry | undefined {
+	for (let back = state.window.size - 1; back >= 0; back--) {
+		const entry = state.window.getBack(back);
+		if (entry?.echoed === true) return entry;
+	}
+	return undefined;
+}
+
+function completeParagraph(state: NearDuplicateState): DetectorMatch | null {
+	const eligible =
+		!state.fenced && state.length >= NEAR_DUPLICATE_MIN_CHARS && state.wordChars >= NEAR_DUPLICATE_MIN_WORD_CHARS;
+	const text = state.retained;
+	const startOffset = state.startOffset;
+	resetParagraph(state);
+	if (!eligible) return null;
+	const tokens = tokenize(text);
+	if (tokens.size === 0) return null;
+	let best = 0;
+	let anchor: ParagraphSignature | undefined;
+	for (let back = 0; back < state.history.size; back++) {
+		const previous = state.history.getBack(back);
+		if (previous === undefined) continue;
+		const similarity = jaccard(tokens, previous.tokens);
+		if (similarity > best) {
+			best = similarity;
+			anchor = previous;
+		}
+	}
+	const echoed = best >= NEAR_DUPLICATE_SIMILARITY && anchor !== undefined;
+	state.history.push({ tokens, startOffset, sample: text.slice(0, SAMPLE_LENGTH) });
+	state.window.push({
+		echoed,
+		startOffset,
+		anchorStartOffset: echoed && anchor !== undefined ? anchor.startOffset : startOffset,
+	});
+	const echoes = countEchoes(state);
+	if (echoes < NEAR_DUPLICATE_ECHO_THRESHOLD) return null;
+	const first = oldestEcho(state);
+	if (first === undefined) return null;
+	return {
+		rule: "collapse-repetition",
+		reason: `${echoes} of the last ${state.window.size} paragraphs restate an earlier paragraph of the same message`,
+		anomalyStartOffset: first.anchorStartOffset,
+		garbageStartOffset: first.startOffset,
+		detail: {
+			mechanism: "near-duplicate-paragraphs",
+			echoes,
+			window: state.window.size,
+			similarity: Number(best.toFixed(3)),
+			sample: anchor?.sample ?? "",
+		},
+	};
+}
+
+function foldLine(state: NearDuplicateState): void {
+	if (state.length === 0) state.startOffset = state.lineStartOffset;
+	state.length += state.lineLength + 1;
+	state.wordChars += state.lineWordChars;
+	if (state.insideFence || FENCE_PATTERN.test(state.lineText.trimStart())) state.fenced = true;
+	if (state.retained.length < NEAR_DUPLICATE_TEXT_RETENTION_MAX) state.retained += `${state.lineText}\n`;
+	if (FENCE_PATTERN.test(state.lineText.trimStart())) state.insideFence = !state.insideFence;
+}
+
+function resetLine(state: NearDuplicateState, startOffset: number): void {
+	state.lineLength = 0;
+	state.lineWordChars = 0;
+	state.lineHasContent = false;
+	state.lineStartOffset = startOffset;
+	state.lineText = "";
+}
+
+export function updateNearDuplicates(state: NearDuplicateState, entry: ScalarEntry): DetectorMatch | null {
+	if (entry.value.charCodeAt(0) === CharCode.LineFeed) {
+		let result: DetectorMatch | null = null;
+		if (state.lineHasContent) foldLine(state);
+		else if (state.length > 0) result = completeParagraph(state);
+		resetLine(state, entry.startOffset + 1);
+		return result;
+	}
+	const codePoint = entry.value.codePointAt(0) ?? 0;
+	if (!isAsciiWhitespace(codePoint)) state.lineHasContent = true;
+	if (isAsciiAlphanumeric(codePoint) || (codePoint > 0x7f && !isBoxDrawing(codePoint))) state.lineWordChars += 1;
+	state.lineLength += entry.width;
+	if (state.lineText.length < NEAR_DUPLICATE_TEXT_RETENTION_MAX) state.lineText += entry.value;
+	return null;
+}

packages/coding-agent/src/core/extensions/builtin/ttsr/detectors/collapse.ts

@@ -1,6 +1,11 @@
 import { FixedRing, type ScalarEntry, ScalarScanner } from "../stream-utils.ts";
 import type { DetectorContext, DetectorMatch, StreamDetector } from "../types.ts";
 import { createLineCycleState, type LineCycleState, updateLineCycles } from "./collapse-lines.ts";
+import {
+	createNearDuplicateState,
+	type NearDuplicateState,
+	updateNearDuplicates,
+} from "./collapse-near-duplicates.ts";
 import {
 	createParagraphRepeatState,
 	type ParagraphRepeatState,
@@ -26,6 +31,7 @@ export interface CollapseState {
 	readonly periods: ShortPeriodState;
 	readonly lines: LineCycleState;
 	readonly paragraphs: ParagraphRepeatState;
+	readonly nearDuplicates: NearDuplicateState;
 	latched: DetectorMatch | null;
 }
 
@@ -38,6 +44,7 @@ export function createCollapseState(): CollapseState {
 		periods: createShortPeriodState(),
 		lines: createLineCycleState(),
 		paragraphs: createParagraphRepeatState(),
+		nearDuplicates: createNearDuplicateState(),
 		latched: null,
 	};
 }
@@ -52,7 +59,8 @@ function checkDelta(state: CollapseState, delta: string, context: DetectorContex
 			updateWhitespaceFlood(state.whitespace, entry) ??
 			updateShortPeriods(state.periods, entry, state.tailRing) ??
 			updateLineCycles(state.lines, entry) ??
-			(watchParagraphs ? updateParagraphRepeats(state.paragraphs, entry) : null);
+			(watchParagraphs ? updateParagraphRepeats(state.paragraphs, entry) : null) ??
+			(watchParagraphs ? updateNearDuplicates(state.nearDuplicates, entry) : null);
 		if (match !== null) {
 			state.latched = match;
 			return match;

packages/coding-agent/test/ttsr/AGENTS.md

@@ -9,6 +9,7 @@ Coverage for the ttsr stream-rule extension (`src/core/extensions/builtin/ttsr/`
 | Control-leak grammar accept/reject | `detector-control-leak-grammar.test.ts` + `control-leak-helpers.ts` (`ctrl`, `sgml`, `bracket`, `runSplitMatrix`, `expectLeakMatchEverywhere`) |
 | Control-leak evidence / negatives | `detector-control-leak-evidence.test.ts`, `detector-control-leak-negatives.test.ts` |
 | Collapse detection | `detector-collapse.test.ts` + `collapse-test-inputs.ts` |
+| Near-duplicate paragraph frequency | `detector-collapse-near-duplicates.test.ts` + `fixtures/incident-near-duplicate-narration.txt` |
 | Coordinator races / abort semantics | `coordinator.test.ts`, `coordinator-races.test.ts` (`claimAbort`, `createGenerationState`, `markUserCancelled`, `resolveDetection`) |
 | Rule parsing / builtin rules | `rule-parser.test.ts` |
 | Repetitive-turns lane | `repetitive-turns.test.ts` |