Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ services:
DATABASE_URL: postgresql://${POSTGRES_USER:-fides}:${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}@postgres:5432/${POSTGRES_DB:-fides}
DISCOVERY_PORT: "3100"
DISCOVERY_DB_AUTO_MIGRATE: ${DISCOVERY_DB_AUTO_MIGRATE:-true}
DISCOVERY_DB_SCHEMA: ${DISCOVERY_DB_SCHEMA:-discovery}

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Keep Compose upgrades on the existing discovery schema

For existing Docker Compose deployments with a persisted Postgres volume and no DISCOVERY_DB_SCHEMA in .env, this new default silently moves discovery from the previously used public schema to a fresh discovery schema. On upgrade, runDiscoveryMigrations will create empty tables there, so registered identities/agents appear to be lost until operators discover and override the variable or manually migrate data. Make the schema opt-in for Compose upgrades or provide an explicit migration/compatibility path.

Useful? React with 👍 / 👎.

NODE_ENV: production
CORS_ORIGIN: ${CORS_ORIGIN:-}
SERVICE_API_KEY: ${SERVICE_API_KEY:-}
Expand Down
2 changes: 2 additions & 0 deletions docs/deployment.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,12 +44,14 @@ cp .env.example .env
| Variable | Default | Required | Description |
| --------------------------- | ------- | -------- | ----------- |
| `DISCOVERY_DB_AUTO_MIGRATE` | `true` | no | Runs idempotent discovery migrations on startup and records applied ids plus checksums in `discovery_schema_migrations`. Set `false` when migrations are managed externally. |
| `DISCOVERY_DB_SCHEMA` | _(empty)_ | no | Optional Postgres schema name for discovery tables when multiple services share one database. Must be a simple identifier. |

### Trust Graph Store

| Variable | Default | Required | Description |
| ----------------------------- | ------- | -------- | ----------- |
| `TRUST_GRAPH_DB_AUTO_MIGRATE` | `true` | no | Runs idempotent trust graph migrations on startup and records applied ids plus checksums in `trust_graph_schema_migrations`. Set `false` when migrations are managed externally. |
| `TRUST_GRAPH_DB_SCHEMA` | `trust_graph` in Docker Compose, otherwise _(empty)_ | no | Optional Postgres schema name for trust graph tables when multiple services share one database. Must be a simple identifier. |

### Agentd Authority Store

Expand Down
34 changes: 32 additions & 2 deletions services/discovery/src/db/client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,31 @@ const DEV_FALLBACK = 'postgresql://fides:fides@localhost:5432/fides'

function getConnectionString(): string {
const url = process.env.DATABASE_URL
if (url) return url
const schemaName = process.env.DISCOVERY_DB_SCHEMA
if (url) return withSearchPath(url, schemaName)

if (process.env.NODE_ENV === 'production') {
throw new Error('DATABASE_URL must be set in production')
}

console.warn('DATABASE_URL not set — using development fallback')
return DEV_FALLBACK
return withSearchPath(DEV_FALLBACK, schemaName)
}

function withSearchPath(connectionString: string, schemaName?: string): string {
if (!schemaName) return connectionString

assertValidSchemaName(schemaName)

const url = new URL(connectionString)
url.searchParams.set('options', `-c search_path=${schemaName},public`)

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Quote schema names in the search_path

When DISCOVERY_DB_SCHEMA contains uppercase letters, which assertValidSchemaName currently allows, this sends an unquoted search_path=Foo,public; PostgreSQL folds that to lowercase, while ensureConfiguredSchema creates the quoted schema "Foo". In that configuration the new schema is created but ignored, and migrations/queries fall back to public or a lowercase schema. Restrict the value to lowercase identifiers or quote/escape it consistently in the search path.

Useful? React with 👍 / 👎.

return url.toString()
}

function assertValidSchemaName(schemaName: string): void {
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(schemaName)) {
throw new Error('DISCOVERY_DB_SCHEMA must be a simple Postgres identifier')
}
}

const poolConfig = {
Expand All @@ -28,6 +45,10 @@ const connectionString = getConnectionString()
export const sql = postgres(connectionString, poolConfig)
export const db = drizzle(sql, { schema })

export function createRawClient(): postgres.Sql {
return postgres(getConnectionString(), poolConfig)
}

export const DISCOVERY_MIGRATIONS = [
{
id: '001_initial',
Expand Down Expand Up @@ -108,6 +129,7 @@ export async function runDiscoveryMigrations(client: postgres.Sql): Promise<void
await client`SELECT pg_advisory_lock(hashtext('discovery_migrations'))`

try {
await ensureConfiguredSchema(client)
await ensureDiscoveryMigrationLedger(client)
for (const migration of DISCOVERY_MIGRATIONS) {
const checksum = discoveryMigrationChecksum(migration)
Expand Down Expand Up @@ -143,6 +165,14 @@ export async function runDiscoveryMigrations(client: postgres.Sql): Promise<void
}
}

async function ensureConfiguredSchema(client: postgres.Sql): Promise<void> {
const schemaName = process.env.DISCOVERY_DB_SCHEMA
if (!schemaName) return

assertValidSchemaName(schemaName)
await client.unsafe(`CREATE SCHEMA IF NOT EXISTS "${schemaName}"`)
}

async function ensureDiscoveryMigrationLedger(client: postgres.Sql): Promise<void> {
await client`
CREATE TABLE IF NOT EXISTS discovery_schema_migrations (
Expand Down
3 changes: 2 additions & 1 deletion services/discovery/src/migrate.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { runDiscoveryMigrations, sql } from './db/client.js'
import { createRawClient, runDiscoveryMigrations } from './db/client.js'

async function main() {
const sql = createRawClient()
try {
await runDiscoveryMigrations(sql)
console.log('discovery migrations applied')
Expand Down
40 changes: 40 additions & 0 deletions services/discovery/test/storage.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,46 @@ describe.skipIf(!postgresUrl)('discovery migrations', () => {
}
}, 30_000)

it('creates and uses the configured discovery schema', async () => {
const schema = `discovery_configured_${crypto.randomUUID().replaceAll('-', '')}`
const schemaIdentifier = quoteIdentifier(schema)
const adminSql = postgres(postgresUrl, { max: 1 })
const scopedSql = postgres(postgresUrlWithSearchPath(postgresUrl, schema), { max: 1 })
const previousSchema = process.env.DISCOVERY_DB_SCHEMA

try {
process.env.DISCOVERY_DB_SCHEMA = schema
await runDiscoveryMigrations(scopedSql)

const schemaRows = await adminSql`
SELECT schema_name
FROM information_schema.schemata
WHERE schema_name = ${schema}
`
expect(schemaRows).toHaveLength(1)

const tableRows = await adminSql`
SELECT table_schema, table_name
FROM information_schema.tables
WHERE table_schema = ${schema}
AND table_name IN ('identities', 'agents', 'discovery_schema_migrations')
`
expect(tableRows).toHaveLength(3)

const currentSchema = await scopedSql`SELECT current_schema() AS schema`
expect(currentSchema[0]?.schema).toBe(schema)
} finally {
if (previousSchema === undefined) {
delete process.env.DISCOVERY_DB_SCHEMA
} else {
process.env.DISCOVERY_DB_SCHEMA = previousSchema
}
await scopedSql.end()
await adminSql.unsafe(`DROP SCHEMA IF EXISTS ${schemaIdentifier} CASCADE`)
await adminSql.end()
}
}, 30_000)

it('fails closed when an applied discovery migration checksum drifts', async () => {
const schema = `discovery_checksum_${crypto.randomUUID().replaceAll('-', '')}`
const schemaIdentifier = quoteIdentifier(schema)
Expand Down