diff --git a/src/normalize.ts b/src/normalize.ts index 0641df8..12e2214 100644 --- a/src/normalize.ts +++ b/src/normalize.ts @@ -2,6 +2,7 @@ import anyAscii from 'any-ascii'; import { MAX_INPUT_BYTES } from './defaults.js'; +import { PLACEHOLDER_SENTINEL_LEFT, PLACEHOLDER_SENTINEL_RIGHT } from './placeholder-escape.js'; /** Adversarial-resistant input normalisation with offset map. * Steps in order: whitespace-PII collapse (gated by digit/@ post-check), @@ -22,6 +23,12 @@ const ZERO_WIDTH_CHARS = new Set([ '', // ZERO WIDTH NO-BREAK SPACE / BOM '⁠', // WORD JOINER '­', // SOFT HYPHEN + // Placeholder-escape PUA sentinels — strip before detection so the + // GLiNER tokenizer never sees them. Leaving them in causes spurious + // private_person spans on the sentinel bytes themselves, which then + // corrupt user-authored `{{...}}` templates after vault substitution. + PLACEHOLDER_SENTINEL_LEFT, + PLACEHOLDER_SENTINEL_RIGHT, ]); /** Same characters as the Python `_SPACED_PII_RE`. Word-anchored on diff --git a/src/placeholder-escape.ts b/src/placeholder-escape.ts index cd43ac5..9bd384d 100644 --- a/src/placeholder-escape.ts +++ b/src/placeholder-escape.ts @@ -7,7 +7,11 @@ */ export const PLACEHOLDER_OPEN = '{{'; -export const PLACEHOLDER_OPEN_ESCAPED = ''; +/** First char of the 2-char PUA sentinel pair (U+E000). */ +export const PLACEHOLDER_SENTINEL_LEFT = '\uE000'; +/** Second char of the 2-char PUA sentinel pair (U+E001). */ +export const PLACEHOLDER_SENTINEL_RIGHT = '\uE001'; +export const PLACEHOLDER_OPEN_ESCAPED = PLACEHOLDER_SENTINEL_LEFT + PLACEHOLDER_SENTINEL_RIGHT; export function escapePlaceholders(text: string): string { return text.split(PLACEHOLDER_OPEN).join(PLACEHOLDER_OPEN_ESCAPED); diff --git a/test/normalize.test.ts b/test/normalize.test.ts index 9e946c9..a2b9bc5 100644 --- a/test/normalize.test.ts +++ b/test/normalize.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from 'vitest'; import { normalizeForDetection, remapSpan } from '../src/normalize.js'; +import { PLACEHOLDER_OPEN_ESCAPED, escapePlaceholders } from '../src/placeholder-escape.js'; describe('normalizeForDetection', () => { it('passes through ASCII unchanged', () => { @@ -13,6 +14,30 @@ describe('normalizeForDetection', () => { } }); + it('strips placeholder-escape PUA sentinels (U+E000 / U+E001)', () => { + // The escape mechanism rewrites user-authored `{{` to a 2-char PUA + // pair before detection. Normalise must strip those bytes so the + // GLiNER tokenizer never tags them as PII — leaving them in produces + // spurious `private_person` spans on the sentinel itself, which + // corrupts the user's `{{...}}` template after vault substitution. + const escaped = escapePlaceholders('{{name}}'); + const r = normalizeForDetection(escaped); + expect(r.normalized).toBe('name}}'); + // A span over just `name` in normalised coords [0, 4) remaps to the + // word's location in escaped coords [2, 6) — sentinel positions are + // skipped, so vault substitution never touches them. + const [origStart, origEnd] = remapSpan(0, 4, r.normToOrig); + expect(origStart).toBe(2); + expect(origEnd).toBe(6); + }); + + it('does not pass-through inputs that contain the PUA sentinel', () => { + // Bare sentinel triggers the non-ASCII path; ensure it is stripped. + const text = `prefix${PLACEHOLDER_OPEN_ESCAPED}suffix`; + const r = normalizeForDetection(text); + expect(r.normalized).toBe('prefixsuffix'); + }); + it('strips zero-width characters', () => { // ZWSP between `john` and `@`. Original length 18, normalised 17. const text = 'john​@example.com'; diff --git a/test/nullpii.test.ts b/test/nullpii.test.ts index 15c75d2..e91252d 100644 --- a/test/nullpii.test.ts +++ b/test/nullpii.test.ts @@ -179,6 +179,25 @@ describe('NullPii e2e pipeline (mocked ONNX)', () => { await b.dispose(); }); + it('preserves user-authored {{...}} templates around a real PII hit', async () => { + // Regression: gateway report showed `{{short-kebab-case-slug}}` in a + // system prompt was corrupted to `{{PII_PRIVATE_PERSON_2_…}}short-kebab-case-slug}}` + // because the escape PUA sentinel was tagged as `private_person`. + // With sentinels stripped before detection, the template survives the + // round-trip and the real PII (email) is the only thing replaced. + const n = new NullPii({ modelDir: '/fake', backend: 'cpu' }); + const text = 'name: {{short-kebab-case-slug}} — contact alice@acme.io'; + const out = await n.sanitize(text); + // Exactly one span — the email — fires from the recognizer pack. + // The template syntax is untouched. + expect(out.spans.find((s) => s.label === 'private_email')?.text).toBe('alice@acme.io'); + expect(out.sanitized).toContain('{{short-kebab-case-slug}}'); + expect(out.sanitized).not.toContain('{{PII_PRIVATE_PERSON'); + const restored = n.restore(out.sanitized, out.sessionId); + expect(restored.restored).toBe(text); + await n.dispose(); + }); + it('init runs once across many sanitize calls', async () => { const n = new NullPii({ modelDir: '/fake', backend: 'cpu' }); const r1 = await n.sanitize('First email: a@acme.io');