feat(url): allowlist public-reference hosts from private_url output

src/defaults.ts

@@ -62,8 +62,9 @@ export const DEFAULT_MODEL_REVISION = 'main';
 export const DEFAULT_RECOGNIZERS: readonly Recognizer[] = [
   // ─── URL / Email ──────────────────────────────────────────────
   // URL: only http(s) + www. — bare-domain.tld dropped (FP-prone).
-  // The optional URL whitelist filter (PUBLIC_URL_HOSTS) lives in
-  // `src/url-filter.ts` and is opt-in.
+  // The public-host allowlist (`PUBLIC_URL_HOSTS` in `src/url-filter.ts`)
+  // runs as a post-filter and is on by default; opt out via
+  // `NullPiiConfig.urlAllowlist: 'none'`.
   {
     id: 'core:url',
     pattern: /\b(?:https?:\/\/|www\.)[^\s<>"]+/g,

src/nullpii.ts

@@ -40,6 +40,7 @@ import {
   type SanitizeOptions,
   type SanitizeResult,
 } from './types/index.js';
+import { dropPublicUrlSpans } from './url-filter.js';
 import { PiiVault } from './vault.js';
 
 const LOG_SCOPE = 'nullpii';
@@ -230,8 +231,13 @@ export class NullPii {
       this.config.threshold ?? DEFAULT_POST_FILTER_THRESHOLD,
       this.config.categoryThresholds ?? {},
     );
+    // Drop URLs pointing at well-known public reference / documentation
+    // hosts (github.com, anthropic.com, docs.*, …). These leak no PII
+    // on their own and otherwise erode the user's token budget when a
+    // system prompt cites a project repo or API doc page.
+    const filteredUrls = this.config.urlAllowlist === 'none' ? merged : dropPublicUrlSpans(merged);
     const refineOn = this.config.boundaryRefine ?? DEFAULT_BOUNDARY_REFINE;
-    const spans = refineOn ? refineSpanBoundaries(escaped, merged) : merged;
+    const spans = refineOn ? refineSpanBoundaries(escaped, filteredUrls) : filteredUrls;
 
     const session = sessionId ?? this.vault.createSession();
     logf(LOG_SCOPE, 'sanitize', {

src/types/config.ts

@@ -64,4 +64,10 @@ export interface NullPiiConfig {
   /** Trim whitespace + common punctuation from span edges as a final
    * post-pass. Default: `true`. */
   readonly boundaryRefine?: boolean;
+  /** Public-URL allowlist mode. Default: built-in {@link
+   * src/url-filter.PUBLIC_URL_HOSTS} drops URLs to well-known reference
+   * domains (github.com, anthropic.com, docs.python.org, …) from the
+   * `private_url` output. Pass `'none'` to disable and have every URL
+   * the recognizer pack and model emit treated as PII. */
+  readonly urlAllowlist?: 'none';
 }

src/url-filter.ts

@@ -0,0 +1,112 @@
+// SPDX-License-Identifier: Apache-2.0
+
+import type { PiiSpan } from './types/index.js';
+
+/**
+ * Hosts whose URLs are public reference / documentation surfaces and
+ * are never PII on their own. Matching `private_url` spans are dropped
+ * from the final span set so that, e.g., a link to
+ * `https://github.com/anthropics/claude-code/issues` in a system prompt
+ * stays verbatim instead of becoming `{{PII_PRIVATE_URL_*}}`.
+ *
+ * Match is on the URL's host (case-insensitive). A leading `www.` is
+ * stripped before lookup. Subdomains of an allowlisted host also match
+ * (`docs.python.org` is covered by `python.org`).
+ */
+export const PUBLIC_URL_HOSTS: ReadonlySet<string> = new Set([
+  // Source hosting
+  'github.com',
+  'gitlab.com',
+  'bitbucket.org',
+  'codeberg.org',
+  'sourceforge.net',
+  // Package registries
+  'npmjs.com',
+  'pypi.org',
+  'crates.io',
+  'rubygems.org',
+  'packagist.org',
+  'nuget.org',
+  'maven.org',
+  // Docs / encyclopedias
+  'wikipedia.org',
+  'wikimedia.org',
+  'mozilla.org',
+  'developer.mozilla.org',
+  'w3.org',
+  'whatwg.org',
+  'rfc-editor.org',
+  'ietf.org',
+  // AI / ML vendors (public marketing / docs surfaces)
+  'anthropic.com',
+  'openai.com',
+  'huggingface.co',
+  'deepmind.com',
+  'mistral.ai',
+  // Language / runtime official sites
+  'python.org',
+  'nodejs.org',
+  'rust-lang.org',
+  'golang.org',
+  'go.dev',
+  'oracle.com',
+  'kotlinlang.org',
+  'scala-lang.org',
+  // Cloud vendor docs (the marketing top-level — not customer subdomains)
+  'aws.amazon.com',
+  'cloud.google.com',
+  'azure.microsoft.com',
+  'docs.microsoft.com',
+  'learn.microsoft.com',
+  // Q&A
+  'stackoverflow.com',
+  'stackexchange.com',
+  'serverfault.com',
+  'superuser.com',
+  // Standards bodies
+  'iso.org',
+  'unicode.org',
+]);
+
+/**
+ * Returns `true` when `urlText` (a substring matched by the URL
+ * recognizer) targets one of the {@link PUBLIC_URL_HOSTS}, OR a
+ * subdomain of one. Returns `false` on malformed input — over-redact
+ * rather than under-redact.
+ */
+export function isPublicUrl(urlText: string): boolean {
+  const host = extractHost(urlText);
+  if (host === null) return false;
+  const lower = host.toLowerCase();
+  const stripped = lower.startsWith('www.') ? lower.slice(4) : lower;
+  if (PUBLIC_URL_HOSTS.has(stripped)) return true;
+  // Subdomain match: walk parents (`docs.python.org` → `python.org`).
+  let cursor = stripped;
+  while (cursor.includes('.')) {
+    const dot = cursor.indexOf('.');
+    cursor = cursor.slice(dot + 1);
+    if (PUBLIC_URL_HOSTS.has(cursor)) return true;
+  }
+  return false;
+}
+
+/** Drop `private_url` spans whose host is in the public-URL allowlist. */
+export function dropPublicUrlSpans(spans: readonly PiiSpan[]): PiiSpan[] {
+  return spans.filter((s) => s.label !== 'private_url' || !isPublicUrl(s.text));
+}
+
+function extractHost(urlText: string): string | null {
+  // The recognizer pattern allows `https?://` and `www.` prefixes. Try
+  // both shapes — URL constructor needs a scheme.
+  try {
+    if (/^https?:\/\//i.test(urlText)) {
+      return new URL(urlText).host;
+    }
+    if (/^www\./i.test(urlText)) {
+      return new URL(`http://${urlText}`).host;
+    }
+  } catch {
+    return null;
+  }
+  return null;
+}

test/nullpii.test.ts

@@ -198,6 +198,34 @@ describe('NullPii e2e pipeline (mocked ONNX)', () => {
     await n.dispose();
   });
 
+  it('keeps public-host URLs verbatim, redacts private hosts', async () => {
+    // github.com / docs.python.org are in PUBLIC_URL_HOSTS — they should
+    // survive the round-trip untouched. A URL on an unknown host stays
+    // redacted as `private_url`.
+    const n = new NullPii({ modelDir: '/fake', backend: 'cpu' });
+    const text =
+      'See https://github.com/anthropics/claude-code/issues and https://acme.io/internal';
+    const out = await n.sanitize(text);
+    // Only the acme.io URL is redacted.
+    const urlSpans = out.spans.filter((s) => s.label === 'private_url');
+    expect(urlSpans).toHaveLength(1);
+    expect(urlSpans[0]?.text).toBe('https://acme.io/internal');
+    expect(out.sanitized).toContain('https://github.com/anthropics/claude-code/issues');
+    expect(out.sanitized).not.toContain('https://acme.io/internal');
+    const restored = n.restore(out.sanitized, out.sessionId);
+    expect(restored.restored).toBe(text);
+    await n.dispose();
+  });
+
+  it('urlAllowlist: "none" redacts every URL including public hosts', async () => {
+    const n = new NullPii({ modelDir: '/fake', backend: 'cpu', urlAllowlist: 'none' });
+    const text = 'See https://github.com/foo and https://acme.io/internal';
+    const out = await n.sanitize(text);
+    const urlSpans = out.spans.filter((s) => s.label === 'private_url');
+    expect(urlSpans).toHaveLength(2);
+    await n.dispose();
+  });
+
   it('init runs once across many sanitize calls', async () => {
     const n = new NullPii({ modelDir: '/fake', backend: 'cpu' });
     const r1 = await n.sanitize('First email: a@acme.io');

test/url-filter.test.ts

@@ -0,0 +1,59 @@
+// SPDX-License-Identifier: Apache-2.0
+
+import { describe, expect, it } from 'vitest';
+import type { PiiSpan } from '../src/types/index.js';
+import { dropPublicUrlSpans, isPublicUrl } from '../src/url-filter.js';
+
+function urlSpan(text: string): PiiSpan {
+  return { label: 'private_url', start: 0, end: text.length, text, score: 0.95 };
+}
+
+describe('isPublicUrl', () => {
+  it('matches direct hosts (github.com, anthropic.com)', () => {
+    expect(isPublicUrl('https://github.com/foo/bar')).toBe(true);
+    expect(isPublicUrl('https://anthropic.com/news')).toBe(true);
+  });
+
+  it('matches subdomains of allowlisted hosts', () => {
+    expect(isPublicUrl('https://docs.python.org/3/library/')).toBe(true);
+    expect(isPublicUrl('https://developer.mozilla.org/en-US/')).toBe(true);
+  });
+
+  it('strips `www.` prefix before lookup', () => {
+    expect(isPublicUrl('https://www.wikipedia.org/wiki/Foo')).toBe(true);
+    expect(isPublicUrl('www.github.com/issues')).toBe(true);
+  });
+
+  it('rejects unrelated hosts', () => {
+    expect(isPublicUrl('https://acme.io/internal/dashboard')).toBe(false);
+    expect(isPublicUrl('https://example.com/foo')).toBe(false);
+  });
+
+  it('rejects malformed input (over-redact stance)', () => {
+    expect(isPublicUrl('not a url')).toBe(false);
+    expect(isPublicUrl('javascript:alert(1)')).toBe(false);
+    expect(isPublicUrl('')).toBe(false);
+  });
+});
+
+describe('dropPublicUrlSpans', () => {
+  it('drops private_url spans matching the allowlist, keeps others', () => {
+    const spans: PiiSpan[] = [
+      urlSpan('https://github.com/foo'),
+      urlSpan('https://acme.io/internal'),
+      { label: 'private_email', start: 0, end: 13, text: 'a@b.com', score: 0.95 },
+    ];
+    const out = dropPublicUrlSpans(spans);
+    expect(out).toHaveLength(2);
+    expect(out.find((s) => s.text === 'https://github.com/foo')).toBeUndefined();
+    expect(out.find((s) => s.text === 'https://acme.io/internal')).toBeDefined();
+    expect(out.find((s) => s.label === 'private_email')).toBeDefined();
+  });
+
+  it('passes through when no private_url spans are present', () => {
+    const spans: PiiSpan[] = [
+      { label: 'private_email', start: 0, end: 7, text: 'a@b.com', score: 0.95 },
+    ];
+    expect(dropPublicUrlSpans(spans)).toEqual(spans);
+  });
+});