diff --git a/bun.lock b/bun.lock index 2b6652c..1fb4188 100644 --- a/bun.lock +++ b/bun.lock @@ -18,7 +18,7 @@ }, "packages/cli": { "name": "@sowdb/cli", - "version": "0.1.9", + "version": "0.1.14", "bin": { "sow": "dist/cli.js", }, @@ -35,7 +35,7 @@ }, "packages/core": { "name": "@sowdb/core", - "version": "0.1.9", + "version": "0.1.14", "dependencies": { "@faker-js/faker": "^9.6.0", "better-sqlite3": "^11.9.0", @@ -53,7 +53,7 @@ }, "packages/mcp": { "name": "@sowdb/mcp", - "version": "0.1.9", + "version": "0.1.14", "bin": { "sow-mcp": "./dist/index.js", }, diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index e71e74a..2666a38 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -22,6 +22,8 @@ Options: --exclude Comma-separated tables to exclude --full Copy all rows (no sampling, slower but complete) --no-sanitize Skip PII sanitization + --allow-unsafe NULL out columns with unknown Postgres types + instead of aborting (default: abort) -c, --config Path to .sow.yml config file --json Output as JSON events (for agents) -q, --quiet Minimal output, no spinners @@ -97,6 +99,7 @@ Examples: exclude: { type: "string" }, full: { type: "boolean", default: false }, noSanitize: { type: "boolean", default: false }, + allowUnsafe: { type: "boolean", default: false }, config: { type: "string", shortFlag: "c" }, json: { type: "boolean", default: false }, quiet: { type: "boolean", shortFlag: "q", default: false }, diff --git a/packages/cli/src/commands/connect.ts b/packages/cli/src/commands/connect.ts index 9c3faf7..551dbcf 100644 --- a/packages/cli/src/commands/connect.ts +++ b/packages/cli/src/commands/connect.ts @@ -244,6 +244,7 @@ export async function runConnect( ? merged.samplingConfig.excludeTables : undefined, noSanitize: !merged.sanitizationConfig.enabled, + allowUnsafe: !!flags.allowUnsafe, seed: merged.samplingConfig.seed, }, log); diff --git a/packages/core/src/branching/connector.ts b/packages/core/src/branching/connector.ts index b9f66e1..cfec3d1 100644 --- a/packages/core/src/branching/connector.ts +++ b/packages/core/src/branching/connector.ts @@ -67,6 +67,7 @@ export async function createConnector( enabled: !opts.noSanitize, rules: [] as { table: string; column: string; type: any }[], skipColumns: [] as string[], + allowUnsafe: !!opts.allowUnsafe, }; if (!opts.noSanitize) { @@ -75,6 +76,7 @@ export async function createConnector( const sanitizer = createSanitizer({ config: sanitizationConfig, tables: analysis.schema.tables, + enumTypes: analysis.schema.enums, onProgress, }); const sanitized = sanitizer.sanitize(sampled.tables); diff --git a/packages/core/src/branching/types.ts b/packages/core/src/branching/types.ts index 83a256a..be6ef05 100644 --- a/packages/core/src/branching/types.ts +++ b/packages/core/src/branching/types.ts @@ -68,6 +68,11 @@ export interface ConnectorCreateOptions { seed?: number; /** Copy all rows instead of sampling. Overrides maxRowsPerTable. */ full?: boolean; + /** + * When true, columns whose Postgres type the sanitizer cannot verify are + * NULLed out instead of aborting the connect. Maps to CLI `--allow-unsafe`. + */ + allowUnsafe?: boolean; } export interface ConnectorCreateResult { diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 45f7cb1..7edf4c2 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -6,7 +6,7 @@ export { analyze } from "./analyzer/index.js"; export type { AnalyzeOptions } from "./analyzer/index.js"; export { createSampler } from "./sampler/index.js"; export type { SamplerOptions } from "./sampler/index.js"; -export { createSanitizer } from "./sanitizer/index.js"; +export { createSanitizer, SanitizationAbort } from "./sanitizer/index.js"; export type { SanitizerOptions } from "./sanitizer/index.js"; export { createExporter } from "./exporter/index.js"; export type { ExporterOptions } from "./exporter/index.js"; diff --git a/packages/core/src/sanitizer/detector.test.ts b/packages/core/src/sanitizer/detector.test.ts index 9faa9cf..ef1c1a5 100644 --- a/packages/core/src/sanitizer/detector.test.ts +++ b/packages/core/src/sanitizer/detector.test.ts @@ -1,4 +1,10 @@ -import { detectColumnPII, detectTablePII } from "./detector.js"; +import { + detectColumnPII, + detectTablePII, + classifyPgType, + stripArraySuffix, + pgTypeToPIIType, +} from "./detector.js"; describe("detectColumnPII", () => { describe("email detection", () => { @@ -121,3 +127,74 @@ describe("detectTablePII", () => { expect(results.every((r) => r.table === "users")).toBe(true); }); }); + +describe("Postgres type classification", () => { + it("classifies known numeric/date/bool as safe", () => { + expect(classifyPgType("int4")).toBe("safe"); + expect(classifyPgType("bigint")).toBe("safe"); + expect(classifyPgType("timestamp")).toBe("safe"); + expect(classifyPgType("boolean")).toBe("safe"); + expect(classifyPgType("uuid")).toBe("safe"); + expect(classifyPgType("money")).toBe("safe"); + expect(classifyPgType("interval")).toBe("safe"); + expect(classifyPgType("int4range")).toBe("safe"); + expect(classifyPgType("bytea")).toBe("safe"); + }); + + it("classifies text-like types as handled", () => { + expect(classifyPgType("text")).toBe("handled"); + expect(classifyPgType("varchar")).toBe("handled"); + expect(classifyPgType("citext")).toBe("handled"); + }); + + it("classifies jsonb/inet/macaddr/xml as handled", () => { + expect(classifyPgType("jsonb")).toBe("handled"); + expect(classifyPgType("json")).toBe("handled"); + expect(classifyPgType("inet")).toBe("handled"); + expect(classifyPgType("cidr")).toBe("handled"); + expect(classifyPgType("macaddr")).toBe("handled"); + expect(classifyPgType("xml")).toBe("handled"); + }); + + it("classifies pg_lsn / hstore / tsvector as unknown", () => { + expect(classifyPgType("pg_lsn")).toBe("unknown"); + expect(classifyPgType("hstore")).toBe("unknown"); + expect(classifyPgType("tsvector")).toBe("unknown"); + }); + + it("treats custom enum types as safe when provided in the enum set", () => { + const enums = new Set(["user_role"]); + expect(classifyPgType("user_role", enums)).toBe("safe"); + expect(classifyPgType("user_role")).toBe("unknown"); + }); + + it("strips array suffix and classifies via base type", () => { + expect(stripArraySuffix("text[]")).toEqual({ baseType: "text", isArray: true }); + expect(stripArraySuffix("_int4")).toEqual({ baseType: "int4", isArray: true }); + expect(stripArraySuffix("text")).toEqual({ baseType: "text", isArray: false }); + expect(classifyPgType("text[]")).toBe("handled"); + expect(classifyPgType("int4[]")).toBe("safe"); + }); + + it("maps jsonb/inet/xml/bytea/macaddr to an intrinsic PIIType", () => { + expect(pgTypeToPIIType("jsonb")).toBe("jsonb"); + expect(pgTypeToPIIType("inet")).toBe("ip_address"); + expect(pgTypeToPIIType("cidr")).toBe("ip_address"); + expect(pgTypeToPIIType("macaddr")).toBe("mac_address"); + expect(pgTypeToPIIType("xml")).toBe("xml_text"); + expect(pgTypeToPIIType("bytea")).toBe("binary_blob"); + expect(pgTypeToPIIType("text")).toBeNull(); + }); + + it("detects a jsonb column as PII by type alone", () => { + const result = detectColumnPII("metadata", "jsonb", []); + expect(result.isPII).toBe(true); + expect(result.type).toBe("jsonb"); + }); + + it("detects an inet column as ip_address by type alone", () => { + const result = detectColumnPII("last_seen_from", "inet", []); + expect(result.isPII).toBe(true); + expect(result.type).toBe("ip_address"); + }); +}); diff --git a/packages/core/src/sanitizer/detector.ts b/packages/core/src/sanitizer/detector.ts index 858aa50..f9d1523 100644 --- a/packages/core/src/sanitizer/detector.ts +++ b/packages/core/src/sanitizer/detector.ts @@ -36,6 +36,106 @@ function matchValues( */ const NUMERIC_TYPES = /^(int|float|double|decimal|numeric|real|serial|bigint|smallint|money|int2|int4|int8|float4|float8)/i; +/** + * Postgres types that are inherently safe: no PII, no transformation needed, + * can be copied to the sandbox verbatim. Used by the fail-closed gate. + */ +const SAFE_PASSTHROUGH_TYPES = new Set([ + // numeric + "int", "int2", "int4", "int8", "smallint", "integer", "bigint", + "serial", "bigserial", "smallserial", + "real", "float4", "float8", "double precision", "numeric", "decimal", + "money", + // boolean + "bool", "boolean", + // date/time + "date", "time", "timetz", "timestamp", "timestamptz", + "interval", + // uuid (sanitized separately if name matches) + "uuid", + // binary / large + "bytea", + // ranges + "int4range", "int8range", "numrange", "tsrange", "tstzrange", "daterange", + // geometric / network / other + "point", "line", "lseg", "box", "path", "polygon", "circle", + // bit + "bit", "varbit", + // oid family + "oid", "regproc", "regclass", "regtype", +]); + +/** + * Postgres types the sanitizer actively handles via the detector + * (either through value-pattern matching on text, or a dedicated transformer). + */ +const HANDLED_TEXT_TYPES = new Set([ + "text", "varchar", "char", "character", "character varying", "citext", "name", +]); + +const HANDLED_SPECIAL_TYPES = new Set([ + "jsonb", "json", + "inet", "cidr", + "macaddr", "macaddr8", + "xml", +]); + +/** Strip the `[]` suffix (or leading `_`) from a Postgres array type. */ +export function stripArraySuffix(pgType: string): { baseType: string; isArray: boolean } { + const t = pgType.trim(); + if (t.endsWith("[]")) return { baseType: t.slice(0, -2), isArray: true }; + if (t.startsWith("_")) return { baseType: t.slice(1), isArray: true }; + return { baseType: t, isArray: false }; +} + +/** + * Classify a Postgres type for the fail-closed sanitization gate. + * + * - "safe": known non-PII, passthrough OK + * - "handled": the sanitizer has a detector or transformer for this type + * - "unknown": not in any known set — must be reported to the gate + */ +export function classifyPgType( + pgType: string, + knownEnumTypes: Set = new Set(), +): "safe" | "handled" | "unknown" { + const normalized = pgType.trim().toLowerCase(); + const { baseType } = stripArraySuffix(normalized); + + if (SAFE_PASSTHROUGH_TYPES.has(baseType)) return "safe"; + if (HANDLED_TEXT_TYPES.has(baseType)) return "handled"; + if (HANDLED_SPECIAL_TYPES.has(baseType)) return "handled"; + if (NUMERIC_TYPES.test(baseType)) return "safe"; + if (knownEnumTypes.has(baseType)) return "safe"; + // Common qualified enum names like "public.user_role" + if (knownEnumTypes.has(baseType.replace(/^public\./, ""))) return "safe"; + return "unknown"; +} + +/** Map a Postgres data_type string to a PIIType when that mapping is intrinsic + * to the type (ignoring column name). Used both by the gate and the detector. + */ +export function pgTypeToPIIType(pgType: string): import("../types.js").PIIType | null { + const { baseType } = stripArraySuffix(pgType.trim().toLowerCase()); + switch (baseType) { + case "jsonb": + case "json": + return "jsonb"; + case "inet": + case "cidr": + return "ip_address"; + case "macaddr": + case "macaddr8": + return "mac_address"; + case "xml": + return "xml_text"; + case "bytea": + return "binary_blob"; + default: + return null; + } +} + export function detectColumnPII( columnName: string, columnType: string, @@ -54,6 +154,19 @@ export function detectColumnPII( } } + // Type-intrinsic PII (jsonb, inet, macaddr, xml, etc.) — no column name needed. + const intrinsic = pgTypeToPIIType(columnType); + if (intrinsic) { + return { + isPII: true, + type: intrinsic, + confidence: "high", + matchedBy: "column_name", + sampleMatches: 0, + totalSampled: sampleValues.length, + }; + } + if (NUMERIC_TYPES.test(columnType)) { return { isPII: false, diff --git a/packages/core/src/sanitizer/index.ts b/packages/core/src/sanitizer/index.ts index 5f15e5c..9ab295a 100644 --- a/packages/core/src/sanitizer/index.ts +++ b/packages/core/src/sanitizer/index.ts @@ -7,18 +7,61 @@ import type { SampledTable, TableInfo, ProgressCallback, + UnhandledColumn, + EnumType, } from "../types.js"; -import { detectTablePII } from "./detector.js"; +import { detectTablePII, classifyPgType } from "./detector.js"; import { transformRows } from "./transformer.js"; export interface SanitizerOptions { config: SanitizationConfig; tables: TableInfo[]; + /** Custom enum types captured by the analyzer — treated as safe passthrough. */ + enumTypes?: EnumType[]; onProgress?: ProgressCallback; } +/** + * Thrown when fail-closed sanitization detects columns the sanitizer + * cannot verify. Includes the list of offending columns so callers can + * render a clear error or re-run with --allow-unsafe. + */ +export class SanitizationAbort extends Error { + readonly unhandledColumns: UnhandledColumn[]; + constructor(unhandledColumns: UnhandledColumn[]) { + super(formatAbortMessage(unhandledColumns)); + this.name = "SanitizationAbort"; + this.unhandledColumns = unhandledColumns; + } +} + +function formatAbortMessage(cols: UnhandledColumn[]): string { + const lines: string[] = []; + lines.push( + `Sanitization aborted — ${cols.length} column${cols.length === 1 ? "" : "s"} ha${cols.length === 1 ? "s" : "ve"} types sow cannot verify:`, + ); + for (const c of cols) { + lines.push(` - ${c.table}.${c.column} (${c.pgType}) — ${c.reason}`); + } + lines.push(""); + lines.push("These columns would be copied to the sandbox AS-IS, potentially leaking"); + lines.push("PII that exists in them. Pass --allow-unsafe to sow connect to skip"); + lines.push("sanitization of these columns (they will be NULLed out in the branch)."); + lines.push(""); + lines.push("To add explicit handling, edit .sow.yml:"); + lines.push(" sanitize:"); + lines.push(" rules:"); + if (cols[0]) { + lines.push(` - table: ${cols[0].table}`); + lines.push(` column: ${cols[0].column}`); + lines.push(` type: ${cols[0].pgType}`); + } + return lines.join("\n"); +} + export function createSanitizer(options: SanitizerOptions) { - const { config, tables, onProgress } = options; + const { config, tables, enumTypes = [], onProgress } = options; + const knownEnums = new Set(enumTypes.map((e) => e.name)); function sanitize( sampledTables: SampledTable[], @@ -32,11 +75,46 @@ export function createSanitizer(options: SanitizerOptions) { })), rulesApplied: [], columnsSkipped: config.skipColumns, + unhandledColumns: [], + warnings: [], }; } const skipSet = new Set(config.skipColumns); + const ruleKeys = new Set( + config.rules.map((r) => `${r.table}.${r.column}`), + ); + + // ------------------------------------------------------------- + // Fail-closed gate — walk every column of every sampled table. + // ------------------------------------------------------------- + const unhandledColumns: UnhandledColumn[] = []; + for (const st of sampledTables) { + const tableInfo = tables.find((t) => t.name === st.table); + if (!tableInfo) continue; + for (const col of tableInfo.columns) { + const key = `${st.table}.${col.name}`; + if (skipSet.has(key)) continue; + if (ruleKeys.has(key)) continue; // user explicitly handled + const classification = classifyPgType(col.type, knownEnums); + if (classification === "unknown") { + unhandledColumns.push({ + table: st.table, + column: col.name, + pgType: col.type, + reason: `no handler configured for ${col.type}`, + }); + } + } + } + + if (unhandledColumns.length > 0 && !config.allowUnsafe) { + throw new SanitizationAbort(unhandledColumns); + } + // ------------------------------------------------------------- + // Normal PII detection + transformation. + // ------------------------------------------------------------- const allPII: PIIColumnInfo[] = []; for (const st of sampledTables) { const tableInfo = tables.find((t) => t.name === st.table); @@ -66,6 +144,17 @@ export function createSanitizer(options: SanitizerOptions) { detail: { columnCount: columnTypeMap.size }, }); + // Build per-table set of columns to NULL out (unhandled + allowUnsafe). + const nullOutByTable = new Map>(); + for (const u of unhandledColumns) { + let s = nullOutByTable.get(u.table); + if (!s) { + s = new Set(); + nullOutByTable.set(u.table, s); + } + s.add(u.column); + } + const sanitizedTables: SanitizedTable[] = []; for (const st of sampledTables) { @@ -77,20 +166,42 @@ export function createSanitizer(options: SanitizerOptions) { } } - if (tableColumns.size === 0) { - sanitizedTables.push({ - table: st.table, - rows: st.rows, - sanitizedColumns: [], - }); - } else { - const rows = transformRows(st.rows, tableColumns); - sanitizedTables.push({ - table: st.table, - rows, - sanitizedColumns: Array.from(tableColumns.keys()), + const nullCols = nullOutByTable.get(st.table); + let rows = st.rows; + + if (tableColumns.size > 0) { + rows = transformRows(rows, tableColumns); + } + + if (nullCols && nullCols.size > 0) { + rows = rows.map((row) => { + const newRow = { ...row }; + for (const c of nullCols) { + if (c in newRow) newRow[c] = null; + } + return newRow; }); } + + const sanitizedColumns = [ + ...Array.from(tableColumns.keys()), + ...(nullCols ? Array.from(nullCols) : []), + ]; + + sanitizedTables.push({ + table: st.table, + rows, + sanitizedColumns, + }); + } + + const warnings: string[] = []; + if (unhandledColumns.length > 0 && config.allowUnsafe) { + warnings.push( + `${unhandledColumns.length} column(s) NULLed out due to unknown Postgres types (allowUnsafe=true): ${unhandledColumns + .map((c) => `${c.table}.${c.column} (${c.pgType})`) + .join(", ")}`, + ); } return { @@ -102,6 +213,8 @@ export function createSanitizer(options: SanitizerOptions) { }, ), columnsSkipped: config.skipColumns, + unhandledColumns, + warnings, }; } diff --git a/packages/core/src/sanitizer/sanitizer-gate.test.ts b/packages/core/src/sanitizer/sanitizer-gate.test.ts new file mode 100644 index 0000000..a7b982b --- /dev/null +++ b/packages/core/src/sanitizer/sanitizer-gate.test.ts @@ -0,0 +1,160 @@ +import { createSanitizer, SanitizationAbort } from "./index.js"; +import type { SampledTable, TableInfo, SanitizationConfig } from "../types.js"; + +function col(name: string, type: string) { + return { name, type, nullable: true, defaultValue: null, maxLength: null, isGenerated: false }; +} + +function makeTable(name: string, columns: ReturnType[]): TableInfo { + return { name, schema: "public", columns, primaryKey: [], constraints: [] }; +} + +function baseConfig(overrides: Partial = {}): SanitizationConfig { + return { enabled: true, rules: [], skipColumns: [], ...overrides }; +} + +describe("sanitizer fail-closed gate", () => { + it("throws SanitizationAbort when an unknown type is encountered by default", () => { + const tables = [makeTable("events", [col("id", "int4"), col("payload", "pg_lsn")])]; + const sampled: SampledTable[] = [ + { table: "events", rows: [{ id: 1, payload: "0/3000000" }], totalRowsInSource: 1, edgeCasesIncluded: [] }, + ]; + const sanitizer = createSanitizer({ config: baseConfig(), tables }); + expect(() => sanitizer.sanitize(sampled)).toThrow(SanitizationAbort); + }); + + it("error message lists each unhandled column with its pg type", () => { + const tables = [ + makeTable("events", [col("props", "hstore")]), + makeTable("users", [col("role", "user_role_unknown")]), + ]; + const sampled: SampledTable[] = [ + { table: "events", rows: [{ props: "a=>1" }], totalRowsInSource: 1, edgeCasesIncluded: [] }, + { table: "users", rows: [{ role: "admin" }], totalRowsInSource: 1, edgeCasesIncluded: [] }, + ]; + const sanitizer = createSanitizer({ config: baseConfig(), tables }); + let err: SanitizationAbort | null = null; + try { + sanitizer.sanitize(sampled); + } catch (e) { + err = e as SanitizationAbort; + } + expect(err).toBeInstanceOf(SanitizationAbort); + expect(err!.message).toContain("events.props"); + expect(err!.message).toContain("hstore"); + expect(err!.message).toContain("users.role"); + expect(err!.message).toContain("user_role_unknown"); + expect(err!.unhandledColumns).toHaveLength(2); + expect(err!.message).toContain("--allow-unsafe"); + }); + + it("does not throw when allowUnsafe=true and NULLs the offending columns", () => { + const tables = [makeTable("events", [col("id", "int4"), col("payload", "pg_lsn")])]; + const sampled: SampledTable[] = [ + { + table: "events", + rows: [ + { id: 1, payload: "0/3000000" }, + { id: 2, payload: "0/4000000" }, + ], + totalRowsInSource: 2, + edgeCasesIncluded: [], + }, + ]; + const sanitizer = createSanitizer({ + config: baseConfig({ allowUnsafe: true }), + tables, + }); + const result = sanitizer.sanitize(sampled); + expect(result.tables[0].rows[0].payload).toBeNull(); + expect(result.tables[0].rows[1].payload).toBeNull(); + expect(result.tables[0].rows[0].id).toBe(1); + expect(result.unhandledColumns).toHaveLength(1); + expect(result.warnings && result.warnings.length).toBeGreaterThan(0); + }); + + it("explicit config rule takes precedence over type detection (gate skips it)", () => { + const tables = [makeTable("events", [col("weird", "pg_lsn")])]; + const sampled: SampledTable[] = [ + { table: "events", rows: [{ weird: "0/3000000" }], totalRowsInSource: 1, edgeCasesIncluded: [] }, + ]; + const sanitizer = createSanitizer({ + config: baseConfig({ + rules: [{ table: "events", column: "weird", type: "free_text" }], + }), + tables, + }); + const result = sanitizer.sanitize(sampled); + // No throw, no unhandled, the rule was applied. + expect(result.unhandledColumns).toEqual([]); + expect(result.tables[0].rows[0].weird).not.toBe("0/3000000"); + }); + + it("known Postgres types (int4, timestamp, bool, text, uuid) do not trip the gate", () => { + const tables = [ + makeTable("users", [ + col("id", "int4"), + col("created_at", "timestamp"), + col("active", "bool"), + col("bio", "text"), + col("uid", "uuid"), + ]), + ]; + const sampled: SampledTable[] = [ + { + table: "users", + rows: [{ id: 1, created_at: new Date(), active: true, bio: "hello", uid: "11111111-1111-1111-1111-111111111111" }], + totalRowsInSource: 1, + edgeCasesIncluded: [], + }, + ]; + const sanitizer = createSanitizer({ config: baseConfig(), tables }); + expect(() => sanitizer.sanitize(sampled)).not.toThrow(); + }); + + it("custom enum types from analyzer are treated as safe", () => { + const tables = [makeTable("users", [col("role", "user_role")])]; + const sampled: SampledTable[] = [ + { table: "users", rows: [{ role: "admin" }], totalRowsInSource: 1, edgeCasesIncluded: [] }, + ]; + const sanitizer = createSanitizer({ + config: baseConfig(), + tables, + enumTypes: [{ name: "user_role", schema: "public", values: ["admin", "user"] }], + }); + const result = sanitizer.sanitize(sampled); + expect(result.tables[0].rows[0].role).toBe("admin"); + }); + + it("sanitizes a jsonb column end-to-end through createSanitizer", () => { + const tables = [makeTable("audit", [col("id", "int4"), col("metadata", "jsonb")])]; + const sampled: SampledTable[] = [ + { + table: "audit", + rows: [{ id: 1, metadata: JSON.stringify({ email: "pii@leak.com", action: "login" }) }], + totalRowsInSource: 1, + edgeCasesIncluded: [], + }, + ]; + const sanitizer = createSanitizer({ config: baseConfig(), tables }); + const result = sanitizer.sanitize(sampled); + const out = JSON.parse(result.tables[0].rows[0].metadata as string); + expect(out.action).toBe("login"); + expect(out.email).not.toBe("pii@leak.com"); + expect(out.email).toContain("@"); + }); + + it("passes allowUnsafe config without errors when no unknown types exist", () => { + const tables = [makeTable("users", [col("id", "int4")])]; + const sampled: SampledTable[] = [ + { table: "users", rows: [{ id: 1 }], totalRowsInSource: 1, edgeCasesIncluded: [] }, + ]; + const sanitizer = createSanitizer({ + config: baseConfig({ allowUnsafe: true }), + tables, + }); + const result = sanitizer.sanitize(sampled); + expect(result.unhandledColumns).toEqual([]); + expect(result.warnings).toEqual([]); + }); +}); diff --git a/packages/core/src/sanitizer/transformer.test.ts b/packages/core/src/sanitizer/transformer.test.ts index 16c2be6..6b15e21 100644 --- a/packages/core/src/sanitizer/transformer.test.ts +++ b/packages/core/src/sanitizer/transformer.test.ts @@ -109,3 +109,118 @@ describe("transformRows", () => { expect(rows[0].email).toBe("test@test.com"); }); }); + +describe("jsonb transformation", () => { + it("replaces an email field inside a simple object", () => { + const input = JSON.stringify({ email: "alice@example.com", role: "admin" }); + const out = transformValue(input, "jsonb") as string; + const parsed = JSON.parse(out); + expect(parsed.role).toBe("admin"); + expect(parsed.email).not.toBe("alice@example.com"); + expect(parsed.email).toContain("@"); + }); + + it("recurses into nested objects", () => { + const input = JSON.stringify({ + user: { contact: { email: "deep@example.com" }, id: 42 }, + }); + const out = transformValue(input, "jsonb") as string; + const parsed = JSON.parse(out); + expect(parsed.user.id).toBe(42); + expect(parsed.user.contact.email).not.toBe("deep@example.com"); + expect(parsed.user.contact.email).toContain("@"); + }); + + it("walks arrays of objects", () => { + const input = JSON.stringify([ + { email: "a@a.com", age: 1 }, + { email: "b@b.com", age: 2 }, + ]); + const out = transformValue(input, "jsonb") as string; + const parsed = JSON.parse(out); + expect(parsed).toHaveLength(2); + expect(parsed[0].age).toBe(1); + expect(parsed[1].age).toBe(2); + expect(parsed[0].email).not.toBe("a@a.com"); + expect(parsed[1].email).not.toBe("b@b.com"); + expect(parsed[0].email).toContain("@"); + }); + + it("passes scalar JSONB values through (bare string, number, null)", () => { + expect(transformValue(JSON.stringify("hello"), "jsonb")).toBe('"hello"'); + expect(transformValue(JSON.stringify(42), "jsonb")).toBe("42"); + // null at the top level bypasses the transformer entirely + expect(transformValue(null, "jsonb")).toBeNull(); + }); + + it("returns invalid JSON unchanged without throwing", () => { + const bogus = "{ not valid json"; + expect(() => transformValue(bogus, "jsonb")).not.toThrow(); + expect(transformValue(bogus, "jsonb")).toBe(bogus); + }); + + it("preserves non-PII fields and only replaces PII-keyed values", () => { + const input = JSON.stringify({ + role: "admin", + created_at: "2024-01-01", + email: "x@y.com", + }); + const out = transformValue(input, "jsonb") as string; + const parsed = JSON.parse(out); + expect(parsed.role).toBe("admin"); + expect(parsed.created_at).toBe("2024-01-01"); + expect(parsed.email).not.toBe("x@y.com"); + }); + + it("accepts already-parsed object (not a string)", () => { + const out = transformValue( + { phone: "+1-555-0000", id: 99 }, + "jsonb", + ) as string; + const parsed = JSON.parse(out); + expect(parsed.id).toBe(99); + expect(parsed.phone).not.toBe("+1-555-0000"); + }); +}); + +describe("new Postgres type transformers", () => { + it("mac_address returns a mac-like string", () => { + const out = transformValue("aa:bb:cc:dd:ee:ff", "mac_address") as string; + expect(typeof out).toBe("string"); + expect(out).toMatch(/^[0-9a-f]{2}(:[0-9a-f]{2}){5}$/i); + expect(out).not.toBe("aa:bb:cc:dd:ee:ff"); + }); + + it("ip_address preserves CIDR suffix when present", () => { + const out = transformValue("10.0.0.1/24", "ip_address") as string; + expect(out).toMatch(/\/24$/); + }); + + it("ip_address produces an IPv4 for a v4 input", () => { + const out = transformValue("10.0.0.1", "ip_address") as string; + expect(out).toMatch(/^\d+\.\d+\.\d+\.\d+$/); + }); + + it("xml_text returns a root-wrapped lorem paragraph", () => { + const out = transformValue("hi", "xml_text") as string; + expect(out.startsWith("")).toBe(true); + expect(out.endsWith("")).toBe(true); + expect(out).not.toContain("secret"); + }); + + it("binary_blob passes through unchanged", () => { + const buf = Buffer.from([1, 2, 3, 4]); + expect(transformValue(buf, "binary_blob")).toBe(buf); + }); + + it("passthrough type returns value unchanged", () => { + expect(transformValue("USD 100.00", "passthrough")).toBe("USD 100.00"); + expect(transformValue("[1,5)", "passthrough")).toBe("[1,5)"); + }); + + it("mac_address is deterministic for the same input", () => { + const a = transformValue("aa:bb:cc:dd:ee:ff", "mac_address"); + const b = transformValue("aa:bb:cc:dd:ee:ff", "mac_address"); + expect(a).toBe(b); + }); +}); diff --git a/packages/core/src/sanitizer/transformer.ts b/packages/core/src/sanitizer/transformer.ts index 2f66814..a850f93 100644 --- a/packages/core/src/sanitizer/transformer.ts +++ b/packages/core/src/sanitizer/transformer.ts @@ -1,9 +1,9 @@ import { faker } from "@faker-js/faker"; import type { PIIType } from "../types.js"; import { deterministicSeed } from "./consistency.js"; +import { BUILTIN_PII_RULES } from "./rules.js"; type TransformFn = (value: unknown) => unknown; -type AsyncTransformFn = (value: unknown) => Promise; function withDeterministicSeed(value: unknown, fn: () => unknown): unknown { const str = String(value ?? ""); @@ -12,6 +12,42 @@ function withDeterministicSeed(value: unknown, fn: () => unknown): unknown { return fn(); } +/** + * Given an object key, return the PIIType (if any) that its name matches + * against the built-in column-name patterns. Reuses the exact patterns from + * rules.ts so JSONB detection is consistent with column detection. + */ +function keyToPIIType(key: string): PIIType | null { + for (const rule of BUILTIN_PII_RULES) { + for (const p of rule.columnNamePatterns) { + if (p.test(key)) return rule.type; + } + } + return null; +} + +function sanitizeJsonValue(value: unknown): unknown { + if (value === null || value === undefined) return value; + if (Array.isArray(value)) { + return value.map((v) => sanitizeJsonValue(v)); + } + if (typeof value === "object") { + const out: Record = {}; + for (const [k, v] of Object.entries(value as Record)) { + const piiType = keyToPIIType(k); + if (piiType && v !== null && v !== undefined && typeof v !== "object") { + out[k] = transformValue(v, piiType); + } else if (v !== null && typeof v === "object") { + out[k] = sanitizeJsonValue(v); + } else { + out[k] = v; + } + } + return out; + } + return value; +} + const transformers: Record = { email: (value) => withDeterministicSeed(value, () => faker.internet.email().toLowerCase()), @@ -40,9 +76,21 @@ const transformers: Record = { withDeterministicSeed(value, () => { const str = String(value ?? ""); if (str.includes(":")) return faker.internet.ipv6(); - return faker.internet.ip(); + return faker.internet.ipv4(); + }), + + ip_address: (value) => + withDeterministicSeed(value, () => { + const str = String(value ?? ""); + if (str.includes(":")) return faker.internet.ipv6(); + const cidrMatch = str.match(/\/(\d+)$/); + const ip = faker.internet.ipv4(); + return cidrMatch ? `${ip}${cidrMatch[0]}` : ip; }), + mac_address: (value) => + withDeterministicSeed(value, () => faker.internet.mac()), + url: (value) => withDeterministicSeed(value, () => faker.internet.url()), @@ -71,6 +119,31 @@ const transformers: Record = { free_text: (value) => withDeterministicSeed(value, () => faker.lorem.paragraph()), + jsonb: (value) => { + // Accept already-parsed objects (many pg drivers hand jsonb back parsed) + // OR a JSON string. If string is invalid, pass through unchanged. + let parsed: unknown; + if (typeof value === "string") { + try { + parsed = JSON.parse(value); + } catch { + // Invalid JSON — return as-is, caller may warn. + return value; + } + } else { + parsed = value; + } + const sanitized = sanitizeJsonValue(parsed); + return JSON.stringify(sanitized); + }, + + xml_text: (value) => + withDeterministicSeed(value, () => `${faker.lorem.paragraph()}`), + + binary_blob: (value) => value, + + passthrough: (value) => value, + custom: (value) => value, }; diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index 99362b7..4616c77 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -140,11 +140,17 @@ export type PIIType = | "ssn" | "credit_card" | "ip" + | "ip_address" + | "mac_address" | "url" | "uuid" | "date_of_birth" | "password" | "free_text" + | "jsonb" + | "xml_text" + | "binary_blob" + | "passthrough" | "custom"; export type PIIConfidence = "high" | "medium" | "low" | "uncertain"; @@ -271,6 +277,18 @@ export interface SanitizationConfig { enabled: boolean; rules: SanitizationRule[]; skipColumns: string[]; + /** + * When true, columns with unknown Postgres types are NULLed out rather + * than aborting. When false (default), sanitize() throws SanitizationAbort. + */ + allowUnsafe?: boolean; +} + +export interface UnhandledColumn { + table: string; + column: string; + pgType: string; + reason: string; } export interface SanitizationRule { @@ -290,6 +308,14 @@ export interface SanitizationResult { tables: SanitizedTable[]; rulesApplied: SanitizationRule[]; columnsSkipped: string[]; + /** + * Columns whose Postgres type the sanitizer could not verify. + * Populated whenever allowUnsafe is true (these columns are NULLed). + * Empty when allowUnsafe is false (sanitize() will throw instead). + */ + unhandledColumns?: UnhandledColumn[]; + /** Human-readable warnings surfaced by the sanitizer (for `sow doctor`). */ + warnings?: string[]; } // ---------------------------------------------------------------------------