diff --git a/src/nullpii.ts b/src/nullpii.ts index 2ea9a1f..2fcd7b5 100644 --- a/src/nullpii.ts +++ b/src/nullpii.ts @@ -28,6 +28,7 @@ import { ModelManager } from './model-manager.js'; import { normalizeForDetection, remapSpan } from './normalize.js'; import { escapePlaceholders, unescapePlaceholders } from './placeholder-escape.js'; import { runRecognizers } from './recognizers.js'; +import { dropSpansInsideTemplates, findTemplateRanges } from './template-mask.js'; import { GLINER_MODEL_CATEGORIES, GLINER_ZERO_SHOT_EXTRA, @@ -230,8 +231,17 @@ export class NullPii { this.config.threshold ?? DEFAULT_POST_FILTER_THRESHOLD, this.config.categoryThresholds ?? {}, ); + // Drop spans that fall inside user-authored template syntax + // (`{{...}}`, `${...}`, `<%...%>`, `{%...%}`). The model routinely + // tags template variable names as `private_person`, which after + // vault substitution would produce nested-brace garbage like + // `{{{{PII_..._}}}}`. Template ranges are computed on the original + // input — escape is length-preserving so offsets transfer to the + // escaped-text coordinate space the spans live in. + const templateRanges = findTemplateRanges(text); + const masked = dropSpansInsideTemplates(merged, templateRanges); const refineOn = this.config.boundaryRefine ?? DEFAULT_BOUNDARY_REFINE; - const spans = refineOn ? refineSpanBoundaries(escaped, merged) : merged; + const spans = refineOn ? refineSpanBoundaries(escaped, masked) : masked; const session = sessionId ?? this.vault.createSession(); logf(LOG_SCOPE, 'sanitize', { diff --git a/src/template-mask.ts b/src/template-mask.ts new file mode 100644 index 0000000..05c4aea --- /dev/null +++ b/src/template-mask.ts @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: Apache-2.0 + +import type { PiiSpan } from './types/index.js'; + +/** + * Scan `text` for common templating syntax ranges. Spans falling inside + * one of these ranges are dropped before vault substitution so the model + * never gets to flag a template variable name as PII. + * + * Supported syntaxes (deliberately conservative): + * - `{{ ... }}` — Mustache / Handlebars / Vue / Jinja2 expression + * - `${ ... }` — JS / TS template literal + * - `<% ... %>` — ERB / EJS + * - `{% ... %}` — Jinja2 / Twig statement + * + * Each match covers the FULL syntactic span including the delimiters, + * so a recognizer span partially overlapping the opener / closer is + * also dropped (the alternative leaves bracket count off after vault + * substitution). + * + * Non-greedy match — nested `{{...}}` resolves on the inner pair, which + * is the desired outcome (the outer braces become bare braces, which + * GLiNER no longer mistakes for sentinels). + */ +const TEMPLATE_PATTERNS: readonly RegExp[] = [ + /\{\{[\s\S]*?\}\}/g, + /\$\{[^{}]*\}/g, + /<%[\s\S]*?%>/g, + /\{%[\s\S]*?%\}/g, +]; + +export interface TemplateRange { + readonly start: number; + readonly end: number; +} + +export function findTemplateRanges(text: string): TemplateRange[] { + const ranges: TemplateRange[] = []; + for (const re of TEMPLATE_PATTERNS) { + re.lastIndex = 0; + let m: RegExpExecArray | null = re.exec(text); + while (m !== null) { + ranges.push({ start: m.index, end: m.index + m[0].length }); + m = re.exec(text); + } + } + return mergeRanges(ranges); +} + +/** Drop spans that overlap ANY template range. Partial overlap is enough — + * a span that crosses a `{{` / `}}` boundary corrupts bracket count after + * vault substitution, so it must go too. */ +export function dropSpansInsideTemplates( + spans: readonly PiiSpan[], + ranges: readonly TemplateRange[], +): PiiSpan[] { + if (ranges.length === 0) return [...spans]; + return spans.filter((s) => !ranges.some((r) => overlaps(s.start, s.end, r.start, r.end))); +} + +function overlaps(aStart: number, aEnd: number, bStart: number, bEnd: number): boolean { + return aStart < bEnd && bStart < aEnd; +} + +function mergeRanges(ranges: readonly TemplateRange[]): TemplateRange[] { + if (ranges.length <= 1) return [...ranges]; + const sorted = [...ranges].sort((a, b) => a.start - b.start || a.end - b.end); + const out: TemplateRange[] = []; + for (const r of sorted) { + const last = out[out.length - 1]; + if (last !== undefined && r.start < last.end) { + out[out.length - 1] = { start: last.start, end: Math.max(last.end, r.end) }; + } else { + out.push(r); + } + } + return out; +} diff --git a/test/nullpii.test.ts b/test/nullpii.test.ts index e91252d..dcebe51 100644 --- a/test/nullpii.test.ts +++ b/test/nullpii.test.ts @@ -198,6 +198,46 @@ describe('NullPii e2e pipeline (mocked ONNX)', () => { await n.dispose(); }); + it('drops spans inside user-authored template syntax', async () => { + // Regression: even with PUA sentinels stripped (PR #43), the model + // still tagged template variable names (`short-kebab-case-slug`) as + // `private_person`, producing nested-brace garbage like + // `{{{{PII_..._}}}}` after vault substitution. The template-mask + // post-filter drops spans inside `{{...}}` / `${...}` / `<%...%>`. + // The recognizer pack does NOT match `short-kebab-case-slug` so this + // test relies on the post-filter, not the ML mock (which returns no + // spans). To simulate the ML false-positive we register a recognizer + // that fires inside the template — same code path. + const n = new NullPii({ + modelDir: '/fake', + backend: 'cpu', + recognizers: [ + { + id: 'test:fake-person', + pattern: /short-kebab-case-slug/g, + label: 'private_person', + confidence: 0.99, + }, + ], + }); + const text = 'name: {{short-kebab-case-slug}} email: alice@acme.io'; + // Re-register the default email recognizer so the email still fires. + n.addRecognizer({ + id: 'core:email', + pattern: /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/g, + label: 'private_email', + confidence: 0.95, + }); + const out = await n.sanitize(text); + // Only the email survives; the template content is masked out. + expect(out.spans).toHaveLength(1); + expect(out.spans[0]?.label).toBe('private_email'); + expect(out.sanitized).toContain('{{short-kebab-case-slug}}'); + // No nested-brace corruption. + expect(out.sanitized).not.toMatch(/\{\{\{\{/); + await n.dispose(); + }); + it('init runs once across many sanitize calls', async () => { const n = new NullPii({ modelDir: '/fake', backend: 'cpu' }); const r1 = await n.sanitize('First email: a@acme.io'); diff --git a/test/template-mask.test.ts b/test/template-mask.test.ts new file mode 100644 index 0000000..2753ac6 --- /dev/null +++ b/test/template-mask.test.ts @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from 'vitest'; +import { dropSpansInsideTemplates, findTemplateRanges } from '../src/template-mask.js'; +import type { PiiSpan } from '../src/types/index.js'; + +function mkSpan(start: number, end: number, text: string): PiiSpan { + return { label: 'private_person', start, end, text, score: 0.9 }; +} + +describe('findTemplateRanges', () => { + it('detects Mustache / Handlebars `{{...}}`', () => { + const text = 'hello {{name}} world'; + expect(findTemplateRanges(text)).toEqual([{ start: 6, end: 14 }]); + }); + + it('detects JS template literal `${...}`', () => { + const text = 'hi ${user.name}!'; + expect(findTemplateRanges(text)).toEqual([{ start: 3, end: 15 }]); + }); + + it('detects ERB / EJS `<%...%>`', () => { + const text = '<%= user.email %>'; + expect(findTemplateRanges(text)).toEqual([{ start: 0, end: 17 }]); + }); + + it('detects Jinja2 / Twig `{%...%}`', () => { + const text = '{% if x %}body{% endif %}'; + expect(findTemplateRanges(text)).toEqual([ + { start: 0, end: 10 }, + { start: 14, end: 25 }, + ]); + }); + + it('handles mixed templates and merges overlaps', () => { + const text = '{% if x %}{{name}}{% endif %}'; + const ranges = findTemplateRanges(text); + expect(ranges).toEqual([ + { start: 0, end: 10 }, + { start: 10, end: 18 }, + { start: 18, end: 29 }, + ]); + }); + + it('non-greedy match — separate `{{a}}` and `{{b}}` produce two ranges', () => { + const text = '{{a}} between {{b}}'; + expect(findTemplateRanges(text)).toEqual([ + { start: 0, end: 5 }, + { start: 14, end: 19 }, + ]); + }); + + it('returns empty for text without template syntax', () => { + expect(findTemplateRanges('plain text only')).toEqual([]); + }); +}); + +describe('dropSpansInsideTemplates', () => { + // Reference text: 'hello {{user_name}} send to alice@acme.io' + // Template range = [6, 19); email = [28, 41). + const templateSpan = mkSpan(8, 17, 'user_name'); + const emailSpan: PiiSpan = { + label: 'private_email', + start: 28, + end: 41, + text: 'alice@acme.io', + score: 0.95, + }; + + it('drops spans fully inside a template range', () => { + const out = dropSpansInsideTemplates([templateSpan, emailSpan], [{ start: 6, end: 19 }]); + expect(out).toEqual([emailSpan]); + }); + + it('drops spans partially overlapping a template boundary', () => { + // Span that crosses `}}` corrupts brackets after vault substitution. + const crossing = mkSpan(15, 22, 'name}} s'); + const out = dropSpansInsideTemplates([crossing, emailSpan], [{ start: 6, end: 19 }]); + expect(out).toEqual([emailSpan]); + }); + + it('keeps spans entirely outside template ranges', () => { + const out = dropSpansInsideTemplates([emailSpan], [{ start: 6, end: 19 }]); + expect(out).toEqual([emailSpan]); + }); + + it('no ranges → passthrough', () => { + const out = dropSpansInsideTemplates([emailSpan, templateSpan], []); + expect(out).toEqual([emailSpan, templateSpan]); + }); +});