From b528ec2ccfc2af9ec4bcb23870575bbc3901ff10 Mon Sep 17 00:00:00 2001 From: lBroth Date: Thu, 21 May 2026 08:31:27 +0200 Subject: [PATCH 1/2] fix(normalize): strip placeholder-escape PUA sentinels before detection GLiNER tags the bare PUA pair (U+E000 U+E001) used to escape user-authored `{{` as `private_person`. Vault substitution then replaces the leading sentinel with `{{PII_PRIVATE_PERSON_*}}`, corrupting templates like `{{short-kebab-case-slug}}` into `{{PII_..._}}short-kebab-case-slug}}`. Root cause: any-ascii returns "" for PUA chars; transcodeChar falls through to preserve them verbatim in the normalized text, so the tokenizer sees them as UNK tokens that the model labels as person. Fix: add the sentinels to ZERO_WIDTH_CHARS so normalizeForDetection strips them before detection. Model never sees the sentinels; offset remap covers the original positions, so vault substitution skips them. Co-Authored-By: Claude Opus 4.7 --- src/normalize.ts | 7 +++++++ src/placeholder-escape.ts | 6 +++++- test/normalize.test.ts | 25 +++++++++++++++++++++++++ test/nullpii.test.ts | 19 +++++++++++++++++++ 4 files changed, 56 insertions(+), 1 deletion(-) diff --git a/src/normalize.ts b/src/normalize.ts index 0641df8..12e2214 100644 --- a/src/normalize.ts +++ b/src/normalize.ts @@ -2,6 +2,7 @@ import anyAscii from 'any-ascii'; import { MAX_INPUT_BYTES } from './defaults.js'; +import { PLACEHOLDER_SENTINEL_LEFT, PLACEHOLDER_SENTINEL_RIGHT } from './placeholder-escape.js'; /** Adversarial-resistant input normalisation with offset map. * Steps in order: whitespace-PII collapse (gated by digit/@ post-check), @@ -22,6 +23,12 @@ const ZERO_WIDTH_CHARS = new Set([ '', // ZERO WIDTH NO-BREAK SPACE / BOM '⁠', // WORD JOINER '­', // SOFT HYPHEN + // Placeholder-escape PUA sentinels — strip before detection so the + // GLiNER tokenizer never sees them. Leaving them in causes spurious + // private_person spans on the sentinel bytes themselves, which then + // corrupt user-authored `{{...}}` templates after vault substitution. + PLACEHOLDER_SENTINEL_LEFT, + PLACEHOLDER_SENTINEL_RIGHT, ]); /** Same characters as the Python `_SPACED_PII_RE`. Word-anchored on diff --git a/src/placeholder-escape.ts b/src/placeholder-escape.ts index cd43ac5..bb34f64 100644 --- a/src/placeholder-escape.ts +++ b/src/placeholder-escape.ts @@ -7,7 +7,11 @@ */ export const PLACEHOLDER_OPEN = '{{'; -export const PLACEHOLDER_OPEN_ESCAPED = ''; +/** First char of the 2-char PUA sentinel pair (U+E000). */ +export const PLACEHOLDER_SENTINEL_LEFT = ''; +/** Second char of the 2-char PUA sentinel pair (U+E001). */ +export const PLACEHOLDER_SENTINEL_RIGHT = ''; +export const PLACEHOLDER_OPEN_ESCAPED = PLACEHOLDER_SENTINEL_LEFT + PLACEHOLDER_SENTINEL_RIGHT; export function escapePlaceholders(text: string): string { return text.split(PLACEHOLDER_OPEN).join(PLACEHOLDER_OPEN_ESCAPED); diff --git a/test/normalize.test.ts b/test/normalize.test.ts index 9e946c9..a2b9bc5 100644 --- a/test/normalize.test.ts +++ b/test/normalize.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from 'vitest'; import { normalizeForDetection, remapSpan } from '../src/normalize.js'; +import { PLACEHOLDER_OPEN_ESCAPED, escapePlaceholders } from '../src/placeholder-escape.js'; describe('normalizeForDetection', () => { it('passes through ASCII unchanged', () => { @@ -13,6 +14,30 @@ describe('normalizeForDetection', () => { } }); + it('strips placeholder-escape PUA sentinels (U+E000 / U+E001)', () => { + // The escape mechanism rewrites user-authored `{{` to a 2-char PUA + // pair before detection. Normalise must strip those bytes so the + // GLiNER tokenizer never tags them as PII — leaving them in produces + // spurious `private_person` spans on the sentinel itself, which + // corrupts the user's `{{...}}` template after vault substitution. + const escaped = escapePlaceholders('{{name}}'); + const r = normalizeForDetection(escaped); + expect(r.normalized).toBe('name}}'); + // A span over just `name` in normalised coords [0, 4) remaps to the + // word's location in escaped coords [2, 6) — sentinel positions are + // skipped, so vault substitution never touches them. + const [origStart, origEnd] = remapSpan(0, 4, r.normToOrig); + expect(origStart).toBe(2); + expect(origEnd).toBe(6); + }); + + it('does not pass-through inputs that contain the PUA sentinel', () => { + // Bare sentinel triggers the non-ASCII path; ensure it is stripped. + const text = `prefix${PLACEHOLDER_OPEN_ESCAPED}suffix`; + const r = normalizeForDetection(text); + expect(r.normalized).toBe('prefixsuffix'); + }); + it('strips zero-width characters', () => { // ZWSP between `john` and `@`. Original length 18, normalised 17. const text = 'john​@example.com'; diff --git a/test/nullpii.test.ts b/test/nullpii.test.ts index 15c75d2..e91252d 100644 --- a/test/nullpii.test.ts +++ b/test/nullpii.test.ts @@ -179,6 +179,25 @@ describe('NullPii e2e pipeline (mocked ONNX)', () => { await b.dispose(); }); + it('preserves user-authored {{...}} templates around a real PII hit', async () => { + // Regression: gateway report showed `{{short-kebab-case-slug}}` in a + // system prompt was corrupted to `{{PII_PRIVATE_PERSON_2_…}}short-kebab-case-slug}}` + // because the escape PUA sentinel was tagged as `private_person`. + // With sentinels stripped before detection, the template survives the + // round-trip and the real PII (email) is the only thing replaced. + const n = new NullPii({ modelDir: '/fake', backend: 'cpu' }); + const text = 'name: {{short-kebab-case-slug}} — contact alice@acme.io'; + const out = await n.sanitize(text); + // Exactly one span — the email — fires from the recognizer pack. + // The template syntax is untouched. + expect(out.spans.find((s) => s.label === 'private_email')?.text).toBe('alice@acme.io'); + expect(out.sanitized).toContain('{{short-kebab-case-slug}}'); + expect(out.sanitized).not.toContain('{{PII_PRIVATE_PERSON'); + const restored = n.restore(out.sanitized, out.sessionId); + expect(restored.restored).toBe(text); + await n.dispose(); + }); + it('init runs once across many sanitize calls', async () => { const n = new NullPii({ modelDir: '/fake', backend: 'cpu' }); const r1 = await n.sanitize('First email: a@acme.io'); From cacd6b8662d17c00a31f5bfcb1d93ea922a7af81 Mon Sep 17 00:00:00 2001 From: lBroth Date: Thu, 21 May 2026 08:43:06 +0200 Subject: [PATCH 2/2] =?UTF-8?q?style(escape):=20use=20=EE=80=80=20/=20?= =?UTF-8?q?=EE=80=81=20source=20escapes=20instead=20of=20raw=20PUA=20chars?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same runtime bytes — but raw PUA in source rendered as random emojis on GitHub mobile (iOS substitutes PUA with Apple's private emoji font) and is invisible / unsearchable in most editors. Escape syntax is grep-friendly and self-documents the codepoint. Co-Authored-By: Claude Opus 4.7 --- src/placeholder-escape.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/placeholder-escape.ts b/src/placeholder-escape.ts index bb34f64..9bd384d 100644 --- a/src/placeholder-escape.ts +++ b/src/placeholder-escape.ts @@ -8,9 +8,9 @@ export const PLACEHOLDER_OPEN = '{{'; /** First char of the 2-char PUA sentinel pair (U+E000). */ -export const PLACEHOLDER_SENTINEL_LEFT = ''; +export const PLACEHOLDER_SENTINEL_LEFT = '\uE000'; /** Second char of the 2-char PUA sentinel pair (U+E001). */ -export const PLACEHOLDER_SENTINEL_RIGHT = ''; +export const PLACEHOLDER_SENTINEL_RIGHT = '\uE001'; export const PLACEHOLDER_OPEN_ESCAPED = PLACEHOLDER_SENTINEL_LEFT + PLACEHOLDER_SENTINEL_RIGHT; export function escapePlaceholders(text: string): string {