diff --git a/packages/core/package.json b/packages/core/package.json index 8ef4b2638c..3228aa46c5 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -119,6 +119,7 @@ "./usage-record-schema": "./dist/usage-record-schema.js", "./session-send-projection": "./dist/session-send-projection.js", "./session-name": "./dist/session-name.js", + "./text-sanitize": "./dist/text-sanitize.js", "./thread-search": "./dist/thread-search.js", "./agent-graph-timeline": "./dist/agent-graph-timeline.js", "./agent-swarm": "./dist/agent-swarm.js", diff --git a/packages/core/src/__tests__/text-sanitize.test.ts b/packages/core/src/__tests__/text-sanitize.test.ts new file mode 100644 index 0000000000..c770b7d4c0 --- /dev/null +++ b/packages/core/src/__tests__/text-sanitize.test.ts @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { strict as assert } from 'node:assert'; +import { describe, it } from 'node:test'; +import { truncateUtf16Safe } from '../text-sanitize.js'; + +describe('truncateUtf16Safe', () => { + it('returns text under the budget unchanged', () => { + assert.equal(truncateUtf16Safe('abc', 3), 'abc'); + assert.equal(truncateUtf16Safe('', 5), ''); + }); + + it('cuts at the budget when the boundary is clean', () => { + assert.equal(truncateUtf16Safe('abcdef', 4), 'abcd'); + // A whole pair that fits exactly is kept. + assert.equal(truncateUtf16Safe('ab\u{1f98a}cd', 4), 'ab\u{1f98a}'); + }); + + it('drops a dangling high surrogate when the cut splits a pair', () => { + // 🦊 is U+1F98A: two code units, so a budget of 3 lands mid-pair. + assert.equal(truncateUtf16Safe('ab\u{1f98a}cd', 3), 'ab'); + // Astral-only input: every odd budget lands mid-pair. + const foxes = '\u{1f98a}'.repeat(4); + assert.equal(truncateUtf16Safe(foxes, 5), '\u{1f98a}\u{1f98a}'); + assert.doesNotMatch( + truncateUtf16Safe(foxes, 5), + /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(? { + assert.equal(truncateUtf16Safe('abc', 0), ''); + assert.equal(truncateUtf16Safe('abc', -1), ''); + }); + + it('leaves a lone surrogate already inside the budget alone', () => { + // Pre-existing malformed input is the caller's concern; only the cut + // boundary is guaranteed. + assert.equal(truncateUtf16Safe('ab\uD83Dcd', 10), 'ab\uD83Dcd'); + }); +}); diff --git a/packages/core/src/local-memory.ts b/packages/core/src/local-memory.ts index ffefc91f8b..943ec2ff08 100644 --- a/packages/core/src/local-memory.ts +++ b/packages/core/src/local-memory.ts @@ -26,6 +26,7 @@ import type { Sha256Digest } from './oauth-subscription.js'; import { redactSecrets } from './redaction.js'; +import { truncateUtf16Safe } from './text-sanitize.js'; export type { Sha256Digest }; @@ -307,8 +308,7 @@ export function buildLocalMemoryPromptBody( const body = blocks.join('\n\n').trim(); if (body.length === 0) return undefined; if (body.length <= LOCAL_MEMORY_PROMPT_MAX_CHARS) return body; - const truncated = body.slice(0, LOCAL_MEMORY_PROMPT_MAX_CHARS); - const boundarySafe = /[\uD800-\uDBFF]$/.test(truncated) ? truncated.slice(0, -1) : truncated; + const boundarySafe = truncateUtf16Safe(body, LOCAL_MEMORY_PROMPT_MAX_CHARS); return `${boundarySafe.trimEnd()}\n\n${LOCAL_MEMORY_PROMPT_TRUNCATION_MARKER}`; } diff --git a/packages/core/src/text-sanitize.ts b/packages/core/src/text-sanitize.ts index 3b93cb0460..92a41690f7 100644 --- a/packages/core/src/text-sanitize.ts +++ b/packages/core/src/text-sanitize.ts @@ -111,3 +111,21 @@ export function sanitizeUnicodeText(text: string, opts: SanitizeUnicodeOptions): if (points.length <= opts.maxCodePoints) return cleaned; return points.slice(0, opts.maxCodePoints).join('') + suffix; } + +/** + * Truncate to at most `maxUnits` UTF-16 code units without ending on an + * unpaired high surrogate: when the cut lands inside a surrogate pair, the + * dangling high half is dropped with the rest of the clipped tail, so the + * result survives a UTF-8 round trip (durable storage, model requests) + * instead of decoding as U+FFFD. + * + * Budgets here are code *units*, not code points — callers clip against + * provider/storage limits measured in UTF-16 lengths. Marker/suffix policy + * stays with the caller, per the boundary note above. + */ +export function truncateUtf16Safe(text: string, maxUnits: number): string { + if (maxUnits <= 0) return ''; + if (text.length <= maxUnits) return text; + const kept = text.slice(0, maxUnits); + return /[\uD800-\uDBFF]$/.test(kept) ? kept.slice(0, -1) : kept; +} diff --git a/packages/mcp/src/sep-2243.ts b/packages/mcp/src/sep-2243.ts index 260bc2d0ed..e6cc04c20e 100644 --- a/packages/mcp/src/sep-2243.ts +++ b/packages/mcp/src/sep-2243.ts @@ -17,6 +17,7 @@ * under the License. */ +import { truncateUtf16Safe } from '@maka/core/text-sanitize'; import { formatMcpDiagnosticText } from './diagnostic-text.js'; const X_MCP_HEADER = 'x-mcp-header'; @@ -281,9 +282,8 @@ function valueAtPath(root: unknown, path: readonly string[]): unknown { function truncateRenderedWarning(value: string): string { if (value.length <= MCP_HEADER_WARNING_LENGTH_LIMIT) return value; - const end = MCP_HEADER_WARNING_LENGTH_LIMIT - 1; - const safeEnd = end > 0 && /[\uD800-\uDBFF]/u.test(value[end - 1] ?? '') ? end - 1 : end; - return `${value.slice(0, safeEnd)}\u2026`; + // The ellipsis is one BMP code unit, so it fits in the unit it reclaims. + return `${truncateUtf16Safe(value, MCP_HEADER_WARNING_LENGTH_LIMIT - 1)}\u2026`; } function isRecord(value: unknown): value is Record { diff --git a/packages/runtime/src/__tests__/mcp-tools.test.ts b/packages/runtime/src/__tests__/mcp-tools.test.ts index b3ffdeb2b3..560dcf7058 100644 --- a/packages/runtime/src/__tests__/mcp-tools.test.ts +++ b/packages/runtime/src/__tests__/mcp-tools.test.ts @@ -188,6 +188,42 @@ test('MCP annotations cannot lower permissions and model output has aggregate bo assert.doesNotMatch(text, /secretBlob/u); }); +test('MCP text clipping never leaves an unpaired surrogate at the boundary', async () => { + const provider = fakeProvider( + [boundTool(descriptor('untrusted', 'claims-read-only', true), binding('surrogate-binding'))], + async () => ({ + // Every code point is astral, so a clip boundary inside any pair would + // surface as an unpaired surrogate ahead of the truncation marker. + content: [{ type: 'text', text: '🦊'.repeat(120_000) }], + }), + ); + const [tool] = buildMcpTools(provider); + const output = await tool?.impl( + {}, + { + sessionId: 's', + turnId: 't', + cwd: '/tmp', + toolCallId: 'call', + abortSignal: new AbortController().signal, + emitOutput() {}, + }, + ); + const model = await tool?.toModelOutput?.({ toolCallId: 'call', input: {}, output }); + assert.equal(model?.type, 'content'); + if (model?.type !== 'content') throw new Error('expected content tool output'); + const text = model.value + .filter((item) => item.type === 'text') + .map((item) => (item.type === 'text' ? item.text : '')) + .join(''); + assert.ok(text.length <= 200_000); + assert.match(text, /…\[truncated by Maka\]/u); + assert.doesNotMatch( + text, + /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(? { const tools = buildMcpTools( fakeProvider( diff --git a/packages/runtime/src/bots/__tests__/telegram-utf16.test.ts b/packages/runtime/src/bots/__tests__/telegram-utf16.test.ts index 4d8b23240c..7dc237364f 100644 --- a/packages/runtime/src/bots/__tests__/telegram-utf16.test.ts +++ b/packages/runtime/src/bots/__tests__/telegram-utf16.test.ts @@ -22,19 +22,9 @@ import { describe, it } from 'node:test'; import { __TEST__ } from '../telegram-bridge.js'; -const { utf16Len, prefixWithinUtf16, splitForTelegram } = __TEST__; +const { utf16Len, splitForTelegram } = __TEST__; describe('Telegram UTF-16 limits', () => { - it('truncates prefixes without splitting surrogate pairs', () => { - for (const [text, limit, expected] of [ - ['hello', 100, 'hello'], - ['abcdef', 3, 'abc'], - ['a😀', 2, 'a'], - ] as const) { - assert.equal(prefixWithinUtf16(text, limit), expected, text); - } - }); - it('splits oversized text within limits, on code points, and preferably at newlines', () => { assert.deepEqual(splitForTelegram('hello world'), ['hello world']); diff --git a/packages/runtime/src/bots/telegram-bridge.ts b/packages/runtime/src/bots/telegram-bridge.ts index b4d7cc4557..8a19e218df 100644 --- a/packages/runtime/src/bots/telegram-bridge.ts +++ b/packages/runtime/src/bots/telegram-bridge.ts @@ -28,6 +28,7 @@ import type { BotAttachmentKind } from '@maka/core/bot-events'; import type { BotChannelSettings } from '@maka/core/bot-chat-settings'; import { generalizedErrorMessage } from '@maka/core/redaction'; +import { truncateUtf16Safe } from '@maka/core/text-sanitize'; import { BaseBotAdapter, botReadinessFromSettings } from './base-adapter.js'; import type { BotPlatform, @@ -63,28 +64,6 @@ function utf16Len(s: string): number { return s.length === 0 ? 0 : Buffer.byteLength(s, 'utf16le') / 2; } -/** - * Return the longest prefix of `s` whose UTF-16 length is ≤ `cap`, - * respecting surrogate-pair boundaries (we never slice a - * multi-code-unit character in half). We iterate codepoint-by- - * codepoint instead of binary-searching slices: the cost of - * mistakenly splitting an emoji is far worse than the O(n) cost. - */ -function prefixWithinUtf16(s: string, cap: number): string { - if (utf16Len(s) <= cap) return s; - let used = 0; - let end = 0; - for (let i = 0; i < s.length; ) { - const code = s.codePointAt(i)!; - const units = code > 0xffff ? 2 : 1; - if (used + units > cap) break; - used += units; - i += units; - end = i; - } - return s.slice(0, end); -} - /** * Split `text` into UTF-16-bounded chunks for Telegram delivery. * Prefers breaking on a newline within the last ~10% of the chunk; @@ -100,7 +79,7 @@ function splitForTelegram(text: string): string[] { const pieces: string[] = []; let remaining = text; while (utf16Len(remaining) > cap) { - let chunk = prefixWithinUtf16(remaining, cap); + let chunk = truncateUtf16Safe(remaining, cap); const minBoundary = Math.floor(chunk.length * 0.9); const nl = chunk.lastIndexOf('\n'); if (nl >= minBoundary) chunk = chunk.slice(0, nl); @@ -264,7 +243,6 @@ function classifyTelegramSendResponse(response: any): TelegramSendClassification export const __TEST__ = { utf16Len, - prefixWithinUtf16, splitForTelegram, buildTelegramSendBody, normalizeTelegramReplyToMessageId, @@ -374,7 +352,7 @@ export class TelegramBotBridge extends BaseBotAdapter implements SendCapable { return createTelegramReplyStream({ chatId, streamId: options.streamId, - prepareDraftText: (text) => prefixWithinUtf16(text, TELEGRAM_MAX_UTF16_PER_MESSAGE), + prepareDraftText: (text) => truncateUtf16Safe(text, TELEGRAM_MAX_UTF16_PER_MESSAGE), sendDraft: async (draftId, text) => { const response = await telegramApi(token, 'sendMessageDraft', { chat_id: privateChatId, diff --git a/packages/runtime/src/mcp-tools.ts b/packages/runtime/src/mcp-tools.ts index a7c48941db..5ce298b858 100644 --- a/packages/runtime/src/mcp-tools.ts +++ b/packages/runtime/src/mcp-tools.ts @@ -27,6 +27,7 @@ import type { McpToolSnapshot, } from '@maka/core/mcp'; import type { PermissionMode, ToolCategory } from '@maka/core/permission'; +import { truncateUtf16Safe } from '@maka/core/text-sanitize'; import type { ExecutionBoundary } from '@maka/core/sandbox-boundary'; import type { ToolRecoveryMode } from '@maka/core/runtime-event'; import type { ToolResultContentPart, ToolResultOutput } from './model-protocol.js'; @@ -284,8 +285,9 @@ function summarizeNonVisualBlock(block: McpCallResult['content'][number]): unkno function clipModelText(value: string, limit: number): string { if (value.length <= limit) return value; + // The marker is all-BMP, so slicing it can't split a pair. if (limit <= TRUNCATION_MARKER.length) return TRUNCATION_MARKER.slice(0, limit); - return `${value.slice(0, limit - TRUNCATION_MARKER.length)}${TRUNCATION_MARKER}`; + return `${truncateUtf16Safe(value, limit - TRUNCATION_MARKER.length)}${TRUNCATION_MARKER}`; } function safeJsonStringify(value: unknown): string {