From 45e3b176ba15e3f61573da1dd0e4df92952a01e3 Mon Sep 17 00:00:00 2001 From: unsiqasik Date: Sat, 30 May 2026 01:25:08 +0000 Subject: [PATCH] feat: add log scrubbing utility and expand logger redaction paths Closes #1014 Changes: - Expand pino redaction paths in logger.ts to cover API keys, private keys, master keys, service keys, connection strings, webhook secrets, and more - Add src/utils/scrub.ts utility for scrubbing secrets from console output (env-based + pattern-based scrubbing for Bearer tokens, connection strings, query-string credentials, key=value pairs) - Apply scrub() to database.ts error logging to prevent connection string leaks - Add comprehensive test suite for scrub utility --- src/config/database.ts | 5 +- src/utils/__tests__/scrub.test.ts | 126 ++++++++++++++++++++++++++++ src/utils/logger.ts | 49 ++++++++++- src/utils/scrub.ts | 135 ++++++++++++++++++++++++++++++ 4 files changed, 311 insertions(+), 4 deletions(-) create mode 100644 src/utils/__tests__/scrub.test.ts create mode 100644 src/utils/scrub.ts diff --git a/src/config/database.ts b/src/config/database.ts index 42a34ecc..9edd424f 100644 --- a/src/config/database.ts +++ b/src/config/database.ts @@ -1,6 +1,7 @@ import { Pool, QueryConfig, QueryResult, QueryResultRow, PoolClient } from "pg"; import { isReadOnlyQuery } from "../utils/readOnlyDetector"; import { IS_SANDBOX, SANDBOX_DATABASE_URL, DATABASE_URL } from "./env"; +import { scrub } from "../utils/scrub"; // Configuration for slow query logging @@ -230,7 +231,7 @@ export async function queryRead( return result; } catch (err) { // Log replica failure and fall back to primary - console.warn("Read replica query failed, falling back to primary:", err); + console.warn("Read replica query failed, falling back to primary:", scrub(String(err))); } finally { client?.release(); } @@ -327,7 +328,7 @@ export async function getPgBouncerStats(): Promise<{ clientConnections: (parseInt(row.cl_active || 0) + parseInt(row.cl_idle || 0)), }; } catch (err) { - console.warn("Failed to get PgBouncer stats:", err); + console.warn("Failed to get PgBouncer stats:", scrub(String(err))); return { activeConnections: 0, idleConnections: 0, diff --git a/src/utils/__tests__/scrub.test.ts b/src/utils/__tests__/scrub.test.ts new file mode 100644 index 00000000..2eb21662 --- /dev/null +++ b/src/utils/__tests__/scrub.test.ts @@ -0,0 +1,126 @@ +import { scrub, scrubObject, refreshScrubMap } from '../utils/scrub'; + +describe('Log Scrubbing Utility', () => { + const ORIGINAL_ENV = process.env; + + beforeEach(() => { + process.env = { ...ORIGINAL_ENV }; + // Set some test secrets + process.env.SUPABASE_SERVICE_KEY = 'sb-test-secret-key-1234567890abcdef'; + process.env.DATABASE_URL = 'postgres://user:supersecretpassword@db.example.com:5432/mydb'; + process.env.API_KEY = 'ak_live_1234567890abcdef1234'; + process.env.JWT_SECRET = 'jwt-super-secret-value-123456'; + refreshScrubMap(); + }); + + afterEach(() => { + process.env = ORIGINAL_ENV; + refreshScrubMap(); + }); + + describe('env-based scrubbing', () => { + it('should redact SUPABASE_SERVICE_KEY value', () => { + const input = `Connecting with key ${process.env.SUPABASE_SERVICE_KEY}`; + const result = scrub(input); + expect(result).not.toContain('sb-test-secret-key-1234567890abcdef'); + expect(result).toContain('[REDACTED:SUPABASE_SERVICE_KEY]'); + }); + + it('should redact DATABASE_URL value', () => { + const input = `Failed to connect to ${process.env.DATABASE_URL}`; + const result = scrub(input); + expect(result).not.toContain('supersecretpassword'); + expect(result).toContain('[REDACTED:DATABASE_URL]'); + }); + + it('should redact API_KEY value', () => { + const input = `Using key ${process.env.API_KEY} for request`; + const result = scrub(input); + expect(result).not.toContain('ak_live_1234567890abcdef1234'); + expect(result).toContain('[REDACTED:API_KEY]'); + }); + + it('should not redact short env values (< 8 chars)', () => { + process.env.SHORT_SECRET = 'abc'; + refreshScrubMap(); + const result = scrub('Value is abc'); + expect(result).toBe('Value is abc'); + }); + }); + + describe('pattern-based scrubbing', () => { + it('should redact Bearer tokens', () => { + const input = 'Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U'; + const result = scrub(input); + expect(result).not.toContain('eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9'); + expect(result).toContain('Bearer [REDACTED]'); + }); + + it('should redact Basic auth headers', () => { + const input = 'Authorization: Basic dXNlcjpwYXNzd29yZA=='; + const result = scrub(input); + expect(result).not.toContain('dXNlcjpwYXNzd29yZA=='); + expect(result).toContain('Basic [REDACTED]'); + }); + + it('should redact passwords in connection strings', () => { + const input = 'postgres://admin:mysecretpwd@localhost:5432/db'; + const result = scrub(input); + expect(result).not.toContain('mysecretpwd'); + expect(result).toContain('[REDACTED]@'); + }); + + it('should redact redis connection string passwords', () => { + const input = 'redis://:r3d1s_s3cr3t@redis.internal:6379/0'; + const result = scrub(input); + expect(result).not.toContain('r3d1s_s3cr3t'); + expect(result).toContain('[REDACTED]@'); + }); + + it('should redact api_key in query strings', () => { + const input = 'https://api.example.com/data?api_key=sk_live_abcdef123456&format=json'; + const result = scrub(input); + expect(result).not.toContain('sk_live_abcdef123456'); + expect(result).toContain('[REDACTED]'); + }); + + it('should redact key=value pairs with secret-looking keys', () => { + const input = 'Config: api_key=supersecret123, other=visible'; + const result = scrub(input); + expect(result).not.toContain('supersecret123'); + expect(result).toContain('other=visible'); + }); + }); + + describe('scrubObject', () => { + it('should scrub string values in objects', () => { + const input = { + name: 'test', + apiKey: process.env.API_KEY!, + count: 42, + }; + const result = scrubObject(input); + expect(result.name).toBe('test'); + expect(result.apiKey).not.toContain('ak_live_1234567890abcdef1234'); + expect(result.count).toBe(42); + }); + + it('should not mutate the original object', () => { + const input = { secret: process.env.API_KEY! }; + const original = input.secret; + scrubObject(input); + expect(input.secret).toBe(original); + }); + }); + + describe('safe strings', () => { + it('should not modify strings without secrets', () => { + const input = 'User logged in successfully from 192.168.1.1'; + expect(scrub(input)).toBe(input); + }); + + it('should handle empty strings', () => { + expect(scrub('')).toBe(''); + }); + }); +}); diff --git a/src/utils/logger.ts b/src/utils/logger.ts index 64f0d0fc..316eb6ee 100644 --- a/src/utils/logger.ts +++ b/src/utils/logger.ts @@ -116,16 +116,61 @@ const logger: Logger = pino( // Redact sensitive fields before any transport sees them redact: { paths: [ + // Authentication & authorization 'password', 'token', - 'accountNumber', 'secret', 'authorization', 'req.headers.authorization', '*.password', '*.token', - '*.accountNumber', '*.secret', + + // PII — financial identifiers + 'accountNumber', + '*.accountNumber', + + // API keys & credentials + 'apiKey', + 'api_key', + 'apiSecret', + 'api_secret', + 'privateKey', + 'private_key', + 'masterKey', + 'master_key', + 'serviceKey', + 'service_key', + 'accessKey', + 'access_key', + 'secretKey', + 'secret_key', + '*.apiKey', + '*.api_key', + '*.apiSecret', + '*.api_secret', + '*.privateKey', + '*.private_key', + '*.masterKey', + '*.master_key', + + // Connection strings & DSNs (may embed credentials) + 'databaseUrl', + 'database_url', + 'dbUrl', + 'db_url', + 'connectionString', + 'connection_string', + 'redisUrl', + 'redis_url', + '*.databaseUrl', + '*.database_url', + + // Webhook / callback secrets + 'webhookSecret', + 'webhook_secret', + '*.webhookSecret', + '*.webhook_secret', ], placeholder: '[REDACTED]', censor: '[REDACTED]', diff --git a/src/utils/scrub.ts b/src/utils/scrub.ts new file mode 100644 index 00000000..ba9d84d4 --- /dev/null +++ b/src/utils/scrub.ts @@ -0,0 +1,135 @@ +/** + * Log scrubbing utility — sanitises strings before they reach console output. + * + * Replaces values of known secret-bearing environment variables and common + * credential patterns with `[REDACTED]` so that `console.log()` calls in + * legacy code paths cannot leak secrets to stdout / log aggregators. + * + * Usage: + * import { scrub } from '../utils/scrub'; + * console.log(scrub(`Connecting to ${dbUrl}`)); + * + * This module is intentionally dependency-free and synchronous so it can be + * imported anywhere without circular-dependency concerns. + */ + +// --------------------------------------------------------------------------- +// 1. Env-var based scrubbing — values from the process environment +// --------------------------------------------------------------------------- + +/** Environment variable names whose values should never appear in logs. */ +const SENSITIVE_ENV_KEYS = [ + 'SUPABASE_SERVICE_KEY', + 'SUPABASE_URL', // URL may contain embedded key in query params + 'DATABASE_URL', + 'DB_URL', + 'REDIS_URL', + 'STELLAR_SECRET_KEY', + 'STELLAR_SECRET', + 'JWT_SECRET', + 'JWT_REFRESH_SECRET', + 'API_KEY', + 'API_SECRET', + 'WEBHOOK_SECRET', + 'PRIVATE_KEY', + 'MASTER_KEY', + 'SERVICE_KEY', + 'ACCESS_KEY', + 'SECRET_KEY', + 'ENCRYPTION_KEY', + 'SIGNING_KEY', + 'LOKI_HOST', // may contain credentials in URL +] as const; + +/** + * Build a scrub map from the current environment. + * Called once at module load; re-exported for test overrides. + */ +function buildEnvScrubMap(): Map { + const map = new Map(); + for (const key of SENSITIVE_ENV_KEYS) { + const val = process.env[key]; + if (val && val.length >= 8) { + // Only scrub values that are long enough to be real secrets + // (avoids redacting common short words like "test"). + map.set(val, `[REDACTED:${key}]`); + } + } + return map; +} + +let envScrubMap = buildEnvScrubMap(); + +/** Rebuild the env scrub map (call after mutating process.env in tests). */ +export function refreshScrubMap(): void { + envScrubMap = buildEnvScrubMap(); +} + +// --------------------------------------------------------------------------- +// 2. Pattern-based scrubbing — regex for common credential formats +// --------------------------------------------------------------------------- + +const CREDENTIAL_PATTERNS: Array<{ pattern: RegExp; replacement: string }> = [ + // Bearer tokens (eyJ... JWTs, generic) + { pattern: /Bearer\s+[A-Za-z0-9\-._~+/]+=*/gi, replacement: 'Bearer [REDACTED]' }, + + // Basic auth header (base64) + { pattern: /Basic\s+[A-Za-z0-9+/]+=*/gi, replacement: 'Basic [REDACTED]' }, + + // Connection strings with embedded password: + // postgres://user:password@host / redis://:secret@host + { pattern: /((?:postgres|mysql|redis|mongodb|amqp)(?:ql|s)?:\/\/[^:\s]+):([^\s@]+)@/gi, + replacement: '$1:[REDACTED]@' }, + + // Generic key=value pairs where key looks secret + { pattern: /((?:api[_-]?key|api[_-]?secret|secret[_-]?key|private[_-]?key|access[_-]?token|auth[_-]?token|webhook[_-]?secret|master[_-]?key|service[_-]?key|encryption[_-]?key|signing[_-]?key)\s*[=:]\s*)([^\s,;}{)\]]+)/gi, + replacement: '$1[REDACTED]' }, + + // Query-string secrets: ?api_key=xxx &token=yyy + { pattern: /([?&](?:api[_-]?key|token|secret|key|password|access_token)=)([^&\s]+)/gi, + replacement: '$1[REDACTED]' }, +]; + +// --------------------------------------------------------------------------- +// 3. Public API +// --------------------------------------------------------------------------- + +/** + * Scrub a single string, removing any known secrets or credential patterns. + * + * @param input - The raw string to sanitise + * @returns The sanitised string with secrets replaced by `[REDACTED]` + */ +export function scrub(input: string): string { + let result = input; + + // Replace known env-var values + for (const [secret, replacement] of envScrubMap) { + if (result.includes(secret)) { + result = result.replaceAll(secret, replacement); + } + } + + // Apply regex patterns + for (const { pattern, replacement } of CREDENTIAL_PATTERNS) { + // Reset lastIndex for global regexes + pattern.lastIndex = 0; + result = result.replace(pattern, replacement); + } + + return result; +} + +/** + * Scrub all string values in a flat object (shallow). + * Non-string values are returned unchanged. + */ +export function scrubObject>(obj: T): T { + const out = { ...obj }; + for (const key of Object.keys(out)) { + if (typeof out[key] === 'string') { + (out as Record)[key] = scrub(out[key] as string); + } + } + return out; +}