From 51291d34a27bf08c4b3b2ecfda0cc7d7123dcf72 Mon Sep 17 00:00:00 2001 From: lBroth Date: Thu, 21 May 2026 20:19:27 +0200 Subject: [PATCH] feat(url): allowlist public-reference hosts from private_url output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit URLs pointing at public reference / documentation surfaces (github.com, anthropic.com, docs.python.org, …) are never PII on their own. Redacting them in a system prompt strips signal the upstream model needs (which repo, which API doc) without protecting anyone — and it eats the token budget for the placeholder + the restore-stream housekeeping on every roundtrip. This adds `PUBLIC_URL_HOSTS` (Set of canonical reference domains, subdomain-matching) plus a `dropPublicUrlSpans` post-filter wired default-on. Override via `NullPiiConfig.urlAllowlist: 'none'` to keep the strict per-URL redaction behaviour. Co-Authored-By: Claude Opus 4.7 --- src/defaults.ts | 5 +- src/nullpii.ts | 8 ++- src/types/config.ts | 6 +++ src/url-filter.ts | 112 ++++++++++++++++++++++++++++++++++++++++ test/nullpii.test.ts | 28 ++++++++++ test/url-filter.test.ts | 59 +++++++++++++++++++++ 6 files changed, 215 insertions(+), 3 deletions(-) create mode 100644 src/url-filter.ts create mode 100644 test/url-filter.test.ts diff --git a/src/defaults.ts b/src/defaults.ts index 8aac542..cedade2 100644 --- a/src/defaults.ts +++ b/src/defaults.ts @@ -62,8 +62,9 @@ export const DEFAULT_MODEL_REVISION = 'main'; export const DEFAULT_RECOGNIZERS: readonly Recognizer[] = [ // ─── URL / Email ────────────────────────────────────────────── // URL: only http(s) + www. — bare-domain.tld dropped (FP-prone). - // The optional URL whitelist filter (PUBLIC_URL_HOSTS) lives in - // `src/url-filter.ts` and is opt-in. + // The public-host allowlist (`PUBLIC_URL_HOSTS` in `src/url-filter.ts`) + // runs as a post-filter and is on by default; opt out via + // `NullPiiConfig.urlAllowlist: 'none'`. { id: 'core:url', pattern: /\b(?:https?:\/\/|www\.)[^\s<>"]+/g, diff --git a/src/nullpii.ts b/src/nullpii.ts index 2ea9a1f..8948f08 100644 --- a/src/nullpii.ts +++ b/src/nullpii.ts @@ -40,6 +40,7 @@ import { type SanitizeOptions, type SanitizeResult, } from './types/index.js'; +import { dropPublicUrlSpans } from './url-filter.js'; import { PiiVault } from './vault.js'; const LOG_SCOPE = 'nullpii'; @@ -230,8 +231,13 @@ export class NullPii { this.config.threshold ?? DEFAULT_POST_FILTER_THRESHOLD, this.config.categoryThresholds ?? {}, ); + // Drop URLs pointing at well-known public reference / documentation + // hosts (github.com, anthropic.com, docs.*, …). These leak no PII + // on their own and otherwise erode the user's token budget when a + // system prompt cites a project repo or API doc page. + const filteredUrls = this.config.urlAllowlist === 'none' ? merged : dropPublicUrlSpans(merged); const refineOn = this.config.boundaryRefine ?? DEFAULT_BOUNDARY_REFINE; - const spans = refineOn ? refineSpanBoundaries(escaped, merged) : merged; + const spans = refineOn ? refineSpanBoundaries(escaped, filteredUrls) : filteredUrls; const session = sessionId ?? this.vault.createSession(); logf(LOG_SCOPE, 'sanitize', { diff --git a/src/types/config.ts b/src/types/config.ts index 15c69f8..13be84d 100644 --- a/src/types/config.ts +++ b/src/types/config.ts @@ -64,4 +64,10 @@ export interface NullPiiConfig { /** Trim whitespace + common punctuation from span edges as a final * post-pass. Default: `true`. */ readonly boundaryRefine?: boolean; + /** Public-URL allowlist mode. Default: built-in {@link + * src/url-filter.PUBLIC_URL_HOSTS} drops URLs to well-known reference + * domains (github.com, anthropic.com, docs.python.org, …) from the + * `private_url` output. Pass `'none'` to disable and have every URL + * the recognizer pack and model emit treated as PII. */ + readonly urlAllowlist?: 'none'; } diff --git a/src/url-filter.ts b/src/url-filter.ts new file mode 100644 index 0000000..7d1a0c4 --- /dev/null +++ b/src/url-filter.ts @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: Apache-2.0 + +import type { PiiSpan } from './types/index.js'; + +/** + * Hosts whose URLs are public reference / documentation surfaces and + * are never PII on their own. Matching `private_url` spans are dropped + * from the final span set so that, e.g., a link to + * `https://github.com/anthropics/claude-code/issues` in a system prompt + * stays verbatim instead of becoming `{{PII_PRIVATE_URL_*}}`. + * + * Match is on the URL's host (case-insensitive). A leading `www.` is + * stripped before lookup. Subdomains of an allowlisted host also match + * (`docs.python.org` is covered by `python.org`). + */ +export const PUBLIC_URL_HOSTS: ReadonlySet = new Set([ + // Source hosting + 'github.com', + 'gitlab.com', + 'bitbucket.org', + 'codeberg.org', + 'sourceforge.net', + // Package registries + 'npmjs.com', + 'pypi.org', + 'crates.io', + 'rubygems.org', + 'packagist.org', + 'nuget.org', + 'maven.org', + // Docs / encyclopedias + 'wikipedia.org', + 'wikimedia.org', + 'mozilla.org', + 'developer.mozilla.org', + 'w3.org', + 'whatwg.org', + 'rfc-editor.org', + 'ietf.org', + // AI / ML vendors (public marketing / docs surfaces) + 'anthropic.com', + 'openai.com', + 'huggingface.co', + 'deepmind.com', + 'mistral.ai', + // Language / runtime official sites + 'python.org', + 'nodejs.org', + 'rust-lang.org', + 'golang.org', + 'go.dev', + 'oracle.com', + 'kotlinlang.org', + 'scala-lang.org', + // Cloud vendor docs (the marketing top-level — not customer subdomains) + 'aws.amazon.com', + 'cloud.google.com', + 'azure.microsoft.com', + 'docs.microsoft.com', + 'learn.microsoft.com', + // Q&A + 'stackoverflow.com', + 'stackexchange.com', + 'serverfault.com', + 'superuser.com', + // Standards bodies + 'iso.org', + 'unicode.org', +]); + +/** + * Returns `true` when `urlText` (a substring matched by the URL + * recognizer) targets one of the {@link PUBLIC_URL_HOSTS}, OR a + * subdomain of one. Returns `false` on malformed input — over-redact + * rather than under-redact. + */ +export function isPublicUrl(urlText: string): boolean { + const host = extractHost(urlText); + if (host === null) return false; + const lower = host.toLowerCase(); + const stripped = lower.startsWith('www.') ? lower.slice(4) : lower; + if (PUBLIC_URL_HOSTS.has(stripped)) return true; + // Subdomain match: walk parents (`docs.python.org` → `python.org`). + let cursor = stripped; + while (cursor.includes('.')) { + const dot = cursor.indexOf('.'); + cursor = cursor.slice(dot + 1); + if (PUBLIC_URL_HOSTS.has(cursor)) return true; + } + return false; +} + +/** Drop `private_url` spans whose host is in the public-URL allowlist. */ +export function dropPublicUrlSpans(spans: readonly PiiSpan[]): PiiSpan[] { + return spans.filter((s) => s.label !== 'private_url' || !isPublicUrl(s.text)); +} + +function extractHost(urlText: string): string | null { + // The recognizer pattern allows `https?://` and `www.` prefixes. Try + // both shapes — URL constructor needs a scheme. + try { + if (/^https?:\/\//i.test(urlText)) { + return new URL(urlText).host; + } + if (/^www\./i.test(urlText)) { + return new URL(`http://${urlText}`).host; + } + } catch { + return null; + } + return null; +} diff --git a/test/nullpii.test.ts b/test/nullpii.test.ts index e91252d..30961ad 100644 --- a/test/nullpii.test.ts +++ b/test/nullpii.test.ts @@ -198,6 +198,34 @@ describe('NullPii e2e pipeline (mocked ONNX)', () => { await n.dispose(); }); + it('keeps public-host URLs verbatim, redacts private hosts', async () => { + // github.com / docs.python.org are in PUBLIC_URL_HOSTS — they should + // survive the round-trip untouched. A URL on an unknown host stays + // redacted as `private_url`. + const n = new NullPii({ modelDir: '/fake', backend: 'cpu' }); + const text = + 'See https://github.com/anthropics/claude-code/issues and https://acme.io/internal'; + const out = await n.sanitize(text); + // Only the acme.io URL is redacted. + const urlSpans = out.spans.filter((s) => s.label === 'private_url'); + expect(urlSpans).toHaveLength(1); + expect(urlSpans[0]?.text).toBe('https://acme.io/internal'); + expect(out.sanitized).toContain('https://github.com/anthropics/claude-code/issues'); + expect(out.sanitized).not.toContain('https://acme.io/internal'); + const restored = n.restore(out.sanitized, out.sessionId); + expect(restored.restored).toBe(text); + await n.dispose(); + }); + + it('urlAllowlist: "none" redacts every URL including public hosts', async () => { + const n = new NullPii({ modelDir: '/fake', backend: 'cpu', urlAllowlist: 'none' }); + const text = 'See https://github.com/foo and https://acme.io/internal'; + const out = await n.sanitize(text); + const urlSpans = out.spans.filter((s) => s.label === 'private_url'); + expect(urlSpans).toHaveLength(2); + await n.dispose(); + }); + it('init runs once across many sanitize calls', async () => { const n = new NullPii({ modelDir: '/fake', backend: 'cpu' }); const r1 = await n.sanitize('First email: a@acme.io'); diff --git a/test/url-filter.test.ts b/test/url-filter.test.ts new file mode 100644 index 0000000..5fe4c45 --- /dev/null +++ b/test/url-filter.test.ts @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from 'vitest'; +import type { PiiSpan } from '../src/types/index.js'; +import { dropPublicUrlSpans, isPublicUrl } from '../src/url-filter.js'; + +function urlSpan(text: string): PiiSpan { + return { label: 'private_url', start: 0, end: text.length, text, score: 0.95 }; +} + +describe('isPublicUrl', () => { + it('matches direct hosts (github.com, anthropic.com)', () => { + expect(isPublicUrl('https://github.com/foo/bar')).toBe(true); + expect(isPublicUrl('https://anthropic.com/news')).toBe(true); + }); + + it('matches subdomains of allowlisted hosts', () => { + expect(isPublicUrl('https://docs.python.org/3/library/')).toBe(true); + expect(isPublicUrl('https://developer.mozilla.org/en-US/')).toBe(true); + }); + + it('strips `www.` prefix before lookup', () => { + expect(isPublicUrl('https://www.wikipedia.org/wiki/Foo')).toBe(true); + expect(isPublicUrl('www.github.com/issues')).toBe(true); + }); + + it('rejects unrelated hosts', () => { + expect(isPublicUrl('https://acme.io/internal/dashboard')).toBe(false); + expect(isPublicUrl('https://example.com/foo')).toBe(false); + }); + + it('rejects malformed input (over-redact stance)', () => { + expect(isPublicUrl('not a url')).toBe(false); + expect(isPublicUrl('javascript:alert(1)')).toBe(false); + expect(isPublicUrl('')).toBe(false); + }); +}); + +describe('dropPublicUrlSpans', () => { + it('drops private_url spans matching the allowlist, keeps others', () => { + const spans: PiiSpan[] = [ + urlSpan('https://github.com/foo'), + urlSpan('https://acme.io/internal'), + { label: 'private_email', start: 0, end: 13, text: 'a@b.com', score: 0.95 }, + ]; + const out = dropPublicUrlSpans(spans); + expect(out).toHaveLength(2); + expect(out.find((s) => s.text === 'https://github.com/foo')).toBeUndefined(); + expect(out.find((s) => s.text === 'https://acme.io/internal')).toBeDefined(); + expect(out.find((s) => s.label === 'private_email')).toBeDefined(); + }); + + it('passes through when no private_url spans are present', () => { + const spans: PiiSpan[] = [ + { label: 'private_email', start: 0, end: 7, text: 'a@b.com', score: 0.95 }, + ]; + expect(dropPublicUrlSpans(spans)).toEqual(spans); + }); +});