diff --git a/architecture/persistence/postgres-implementation.md b/architecture/persistence/postgres-implementation.md new file mode 100644 index 0000000..ccb4871 --- /dev/null +++ b/architecture/persistence/postgres-implementation.md @@ -0,0 +1,521 @@ +--- +Document Status: ✅ Stable — DCM implementation +Document Type: Architecture Reference — PostgreSQL Implementation +Established: 2026-05-26 +Maps to: udlm/design-principles/infrastructure-optimization.md +--- + +# PostgreSQL Implementation + +> **Implements contracts defined in UDLM**: +> [udlm/design-principles/infrastructure-optimization.md](https://github.com/croadfeldt/udlm/blob/main/design-principles/infrastructure-optimization.md). +> UDLM requires that the four data domains be persistently queryable with +> declared immutability invariants. This document specifies DCM's +> PostgreSQL realization: schema, enforcement mechanisms, query optimization, +> and retention policies. + +> See [`postgres-mandate.md`](postgres-mandate.md) for the architectural +> decision and rationale. + +--- + +## 1. Schema design + +DCM's database schema enforces the four-domain contracts through PostgreSQL +native features. + +### 1.1 Intent domain + +```sql +-- Append-only. Raw consumer declarations. Never modified after write. + +CREATE TABLE intent_records ( + intent_uuid UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + entity_uuid UUID NOT NULL, + tenant_uuid UUID NOT NULL REFERENCES tenants(tenant_uuid), + catalog_item_uuid UUID NOT NULL, + submitted_by UUID NOT NULL, + submitted_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + submitted_via VARCHAR(32) NOT NULL + CHECK (submitted_via IN ('api', 'gitops', 'cli', 'message_bus')), + intent_version INTEGER NOT NULL DEFAULT 1, + fields JSONB NOT NULL DEFAULT '{}', + provenance JSONB NOT NULL DEFAULT '{}' +); + +CREATE INDEX idx_intent_entity ON intent_records(entity_uuid, intent_version); +CREATE INDEX idx_intent_tenant ON intent_records(tenant_uuid, submitted_at); + +REVOKE UPDATE, DELETE ON intent_records FROM dcm_app; +``` + +### 1.2 Requested domain + +```sql +-- Append-only. Assembled, policy-evaluated, placed payloads. + +CREATE TABLE requested_records ( + requested_uuid UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + entity_uuid UUID NOT NULL, + tenant_uuid UUID NOT NULL REFERENCES tenants(tenant_uuid), + operation_uuid UUID NOT NULL REFERENCES operations(operation_uuid), + intent_uuid UUID NOT NULL REFERENCES intent_records(intent_uuid), + resource_type VARCHAR(256) NOT NULL, + provider_uuid UUID NOT NULL, + assembled_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + assembled_payload JSONB NOT NULL DEFAULT '{}', + layer_sources JSONB NOT NULL DEFAULT '[]', + policy_results JSONB NOT NULL DEFAULT '{}', + placement_result JSONB NOT NULL DEFAULT '{}', + provenance JSONB NOT NULL DEFAULT '{}' +); + +CREATE INDEX idx_requested_entity ON requested_records(entity_uuid); +CREATE INDEX idx_requested_tenant ON requested_records(tenant_uuid); +CREATE INDEX idx_requested_operation ON requested_records(operation_uuid); + +REVOKE UPDATE, DELETE ON requested_records FROM dcm_app; +``` + +### 1.3 Realized domain + +```sql +-- Versioned rows. is_current flag. Append-on-change semantics. + +CREATE TABLE realized_entities ( + realized_uuid UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + entity_uuid UUID NOT NULL, + tenant_uuid UUID NOT NULL REFERENCES tenants(tenant_uuid), + resource_type VARCHAR(256) NOT NULL, + provider_uuid UUID NOT NULL, + requested_uuid UUID NOT NULL REFERENCES requested_records(requested_uuid), + realized_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + version_major INTEGER NOT NULL, + version_minor INTEGER NOT NULL, + version_revision INTEGER NOT NULL, + is_current BOOLEAN NOT NULL, + lifecycle_state VARCHAR(32) NOT NULL, + realized_payload JSONB NOT NULL DEFAULT '{}', + provider_metadata JSONB NOT NULL DEFAULT '{}', + provenance JSONB NOT NULL DEFAULT '{}' +); + +CREATE INDEX idx_realized_entity_current ON realized_entities(entity_uuid) + WHERE is_current = true; +CREATE INDEX idx_realized_entity_history ON realized_entities(entity_uuid, realized_at); +CREATE INDEX idx_realized_tenant ON realized_entities(tenant_uuid, realized_at); +CREATE INDEX idx_realized_provider ON realized_entities(provider_uuid, realized_at); + +-- Append-on-change enforced via trigger that updates is_current on previous version +-- when a new version is inserted; the previous row's is_current flips to false. +``` + +### 1.4 Discovered domain + +```sql +-- Ephemeral snapshots from provider discovery runs. + +CREATE TABLE discovered_records ( + discovery_uuid UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + entity_uuid UUID, -- null for orphans + tenant_uuid UUID REFERENCES tenants(tenant_uuid), + provider_uuid UUID NOT NULL, + resource_type VARCHAR(256) NOT NULL, + discovered_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + discovery_run_uuid UUID NOT NULL, + discovered_fields JSONB NOT NULL DEFAULT '{}', + provider_native_id VARCHAR(512), + match_confidence VARCHAR(16) DEFAULT 'exact' + CHECK (match_confidence IN ('exact', 'high', 'low', 'unmatched')) +); + +CREATE INDEX idx_discovered_entity ON discovered_records(entity_uuid, discovered_at); +CREATE INDEX idx_discovered_run ON discovered_records(discovery_run_uuid); +CREATE INDEX idx_discovered_orphans ON discovered_records(entity_uuid) + WHERE entity_uuid IS NULL; +``` + +### 1.5 Pipeline events + +```sql +-- Append-only event log. Replaces Kafka for pipeline routing in standard deployments. +-- LISTEN/NOTIFY provides real-time notification to pipeline consumers. + +CREATE TABLE pipeline_events ( + event_uuid UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + event_type VARCHAR(128) NOT NULL, + entity_uuid UUID, + request_uuid UUID, + tenant_uuid UUID, + actor_uuid UUID, + payload JSONB NOT NULL DEFAULT '{}', + published_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + consumed_by JSONB NOT NULL DEFAULT '[]', + consumed_at TIMESTAMPTZ +); + +CREATE INDEX idx_events_type ON pipeline_events(event_type, published_at); +CREATE INDEX idx_events_entity ON pipeline_events(entity_uuid, published_at); +CREATE INDEX idx_events_unconsumed ON pipeline_events(event_type, published_at) + WHERE consumed_at IS NULL; + +REVOKE UPDATE, DELETE ON pipeline_events FROM dcm_app; + +-- Notify function for real-time pipeline routing +CREATE OR REPLACE FUNCTION notify_pipeline_event() RETURNS TRIGGER AS $$ +BEGIN + PERFORM pg_notify('dcm_pipeline', json_build_object( + 'event_uuid', NEW.event_uuid, + 'event_type', NEW.event_type, + 'entity_uuid', NEW.entity_uuid + )::text); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER pipeline_event_notify + AFTER INSERT ON pipeline_events + FOR EACH ROW EXECUTE FUNCTION notify_pipeline_event(); +``` + +--- + +## 2. Enforcement mechanisms + +### 2.1 Append-only via REVOKE + +The application role (`dcm_app`) has only INSERT and SELECT permissions on +append-only tables. UPDATE and DELETE are revoked: + +```sql +REVOKE UPDATE, DELETE ON intent_records FROM dcm_app; +REVOKE UPDATE, DELETE ON requested_records FROM dcm_app; +REVOKE UPDATE, DELETE ON pipeline_events FROM dcm_app; +REVOKE UPDATE, DELETE ON audit_records FROM dcm_app; +``` + +Database administration roles retain full access for operational concerns +(point-in-time recovery, planned schema migrations), but the application +cannot mutate append-only data. + +### 2.2 Row-Level Security (tenant isolation) + +Every table with tenant data has RLS enforced: + +```sql +ALTER TABLE intent_records ENABLE ROW LEVEL SECURITY; +ALTER TABLE requested_records ENABLE ROW LEVEL SECURITY; +ALTER TABLE realized_entities ENABLE ROW LEVEL SECURITY; +ALTER TABLE discovered_records ENABLE ROW LEVEL SECURITY; +ALTER TABLE pipeline_events ENABLE ROW LEVEL SECURITY; + +CREATE POLICY tenant_isolation_intent ON intent_records + FOR ALL TO dcm_app + USING (tenant_uuid = current_setting('dcm.current_tenant_uuid')::uuid); + +CREATE POLICY tenant_isolation_requested ON requested_records + FOR ALL TO dcm_app + USING (tenant_uuid = current_setting('dcm.current_tenant_uuid')::uuid); + +CREATE POLICY tenant_isolation_realized ON realized_entities + FOR ALL TO dcm_app + USING (tenant_uuid = current_setting('dcm.current_tenant_uuid')::uuid); + +CREATE POLICY tenant_isolation_discovered ON discovered_records + FOR ALL TO dcm_app + USING (tenant_uuid = current_setting('dcm.current_tenant_uuid')::uuid); + +CREATE POLICY tenant_isolation_events ON pipeline_events + FOR SELECT TO dcm_app + USING (tenant_uuid = current_setting('dcm.current_tenant_uuid')::uuid); +``` + +The API Gateway sets `dcm.current_tenant_uuid` at session startup per the +authenticated request's tenant scope. RLS enforces that no query can return +rows from other tenants — even with a buggy WHERE clause (STI-001, STI-002). + +Platform admin queries set `dcm.current_tenant_uuid = '*'` (resolved via a +separate RLS policy that grants cross-tenant access only to platform_admin +role). + +### 2.3 Hash chain (audit integrity) + +The `audit_records` table has a SHA-256 hash chain. Each record's +`record_hash` includes the previous record's hash: + +```sql +CREATE TABLE audit_records ( + audit_uuid UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + entity_uuid UUID, + tenant_uuid UUID, + action VARCHAR(64) NOT NULL, + actor_uuid UUID, + actor_type VARCHAR(32) NOT NULL, + timestamp TIMESTAMPTZ NOT NULL DEFAULT NOW(), + payload JSONB NOT NULL DEFAULT '{}', + previous_record_hash CHAR(64), + record_hash CHAR(64) NOT NULL +); + +CREATE INDEX idx_audit_entity ON audit_records(entity_uuid, timestamp); +CREATE INDEX idx_audit_actor ON audit_records(actor_uuid, timestamp); +CREATE INDEX idx_audit_action ON audit_records(action, timestamp); + +REVOKE UPDATE, DELETE ON audit_records FROM dcm_app; +``` + +The per-entity hash chain enables targeted integrity verification without +requiring a full-database recompute. Each entity's chain is independently +verifiable. + +### 2.4 Append-on-change via trigger (Realized) + +```sql +CREATE OR REPLACE FUNCTION update_realized_current() RETURNS TRIGGER AS $$ +BEGIN + -- Flip previous version's is_current to false + UPDATE realized_entities + SET is_current = false + WHERE entity_uuid = NEW.entity_uuid + AND realized_uuid != NEW.realized_uuid + AND is_current = true; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER realized_version_update + AFTER INSERT ON realized_entities + FOR EACH ROW WHEN (NEW.is_current = true) + EXECUTE FUNCTION update_realized_current(); +``` + +DCM never UPDATEs realized rows in-place — it always INSERTs a new version +with `is_current = true`. The trigger atomically flips the previous version's +flag. + +--- + +## 3. Query optimization and indexing + +### 3.1 Hot paths + +| Query | Index strategy | +|---|---| +| Current state of entity | `idx_realized_entity_current` (partial index WHERE is_current = true) | +| Entity history | `idx_realized_entity_history` (entity_uuid, realized_at) | +| Tenant catalog browse | `idx_realized_tenant` | +| Provider drift comparison | `idx_realized_provider` joined with `idx_discovered_entity` | +| Pipeline event delivery | `idx_events_unconsumed` (partial index WHERE consumed_at IS NULL) | +| Audit chain verification (per entity) | `idx_audit_entity` | +| Policy evaluation context lookup | `idx_policies_active` (partial WHERE status = 'active') | + +### 3.2 JSONB GIN indexes + +For policy evaluation and complex queries on assembled_payload: + +```sql +CREATE INDEX idx_requested_payload_gin ON requested_records USING GIN (assembled_payload); +CREATE INDEX idx_realized_payload_gin ON realized_entities USING GIN (realized_payload); +``` + +These enable fast `assembled_payload @> '{...}'` queries used by policy +evaluation and audit search. + +### 3.3 Materialized views (catalog browse) + +```sql +CREATE MATERIALIZED VIEW catalog_browse_view AS + SELECT + ci.catalog_item_uuid, + ci.handle, + ci.display_name, + ci.resource_type, + ci.allowed_locations, -- pre-joined from layer_reference + ci.allowed_versions, + ci.tenant_visibility, + rts.schema_summary, + pco.cost_estimate + FROM catalog_items ci + LEFT JOIN resource_type_specs rts USING (resource_type) + LEFT JOIN provider_cost_overview pco USING (catalog_item_uuid) + WHERE ci.status = 'active'; + +CREATE UNIQUE INDEX idx_catalog_browse ON catalog_browse_view(catalog_item_uuid); + +-- Refresh on catalog change events via pipeline_events trigger +``` + +Materialized views handle catalog browse and other read-heavy lookups. They +refresh on `catalog.changed` events; staleness window is bounded by the +event delivery latency (PT5S in standard deployments). + +For workloads requiring sub-second refresh, Redis cache is an optional +deployment enhancement (see [`postgres-mandate.md` §3](postgres-mandate.md)). + +--- + +## 4. Connection pooling + +DCM uses PgBouncer in transaction-pooling mode for connection management: + +```ini +[databases] +dcm = host=postgres.dcm.svc port=5432 dbname=dcm_prod + +[pgbouncer] +pool_mode = transaction +default_pool_size = 50 +min_pool_size = 10 +max_client_conn = 10000 +server_idle_timeout = 600 +``` + +Each DCM control plane service instance establishes a connection through +PgBouncer; PgBouncer multiplexes onto a smaller backend pool. This supports +thousands of concurrent API requests without exhausting PostgreSQL backend +connections. + +--- + +## 5. Data retention and archival policies + +DCM applies per-domain retention policies: + +| Domain | Default retention | Notes | +|---|---|---| +| Intent | Indefinite | Audit and reproducibility; small per-record size | +| Requested | Indefinite | Provenance chain for active entities; archived per profile after entity decommission | +| Realized (historical versions) | Per profile: P365D (minimal) → P10Y (fsi/sovereign) | `is_current = false` rows | +| Realized (current) | While entity active; permanent post-decommission for audit | `is_current = true` rows; immutable after decommission | +| Discovered | P30D rolling | Discovery snapshots; not retained as authoritative state | +| Pipeline events | P7D rolling for delivered events; indefinite for replay-eligible | Consumed events purged; never-consumed events retained for replay | +| Audit | Indefinite | Tamper-evident chain; never deleted; archived to cold storage per profile | + +### 5.1 Archival mechanism + +For long-retention data (audit, decommissioned entity records), DCM +supports archival to cold storage: + +```yaml +archival_policy: + archive_to: s3 | gcs | azure_blob | filesystem + archive_after: P1Y # archive after 1 year (configurable) + archive_format: jsonl + sha256 manifest + archive_encryption: AES-256-GCM with archived-data-encryption-key (HSM) + verification_schedule: P30D # periodic random-sample verification + retention_in_archive: P10Y (default; fsi/sovereign extends to P30Y) +``` + +Archived data remains queryable through DCM's audit-archive API, with +multi-second latency vs sub-second for hot data. + +### 5.2 Profile-governed retention defaults + +| Profile | Historical Realized retention | Audit retention | Archival enabled | +|---|---|---|---| +| minimal | P90D | P1Y | No | +| dev | P180D | P1Y | No | +| standard | P365D | P3Y | Optional | +| prod | P3Y | P7Y | Yes | +| fsi | P7Y | P10Y | Yes; PCI DSS, SOX requirements | +| sovereign | P10Y | P30Y | Yes; sovereign data residency in archive | + +--- + +## 6. High availability and disaster recovery + +### 6.1 Standard HA + +DCM standard deployments use PostgreSQL streaming replication or +Patroni-managed clusters: + +- 1 primary + 2 streaming replicas +- Synchronous replication to at least one replica (`synchronous_commit = remote_apply` + for fsi/sovereign) +- Automatic failover via Patroni leader election +- PgBouncer reads from primary; failover transparent to DCM services + +### 6.2 Backup strategy + +- Continuous archiving via WAL streaming to object storage (S3 / GCS / etc.) +- Daily base backups (pg_basebackup or Velero with Crunchy Operator) +- Point-in-time recovery to any second within the WAL retention window +- Encrypted backups (AES-256-GCM with HSM-managed KEK for fsi/sovereign) + +### 6.3 Cross-zone DR + +For sovereign deployments, each sovereignty zone has independent PostgreSQL +HA. Cross-zone DR uses the DCM federation mechanism (signed export bundles) +rather than database-level replication — preserves the sovereignty boundary. + +--- + +## 7. Sovereignty partitioning + +``` +Sovereign deployment: + zone-1 (EU): PostgreSQL HA cluster, DCM control plane, local providers + zone-2 (US): PostgreSQL HA cluster, DCM control plane, local providers + zone-3 (APAC): PostgreSQL HA cluster, DCM control plane, local providers + +Federation between zones: DCM-to-DCM mTLS tunnels per +runtime-features/federation-runtime.md +No cross-zone database replication. +``` + +Each zone's PostgreSQL is independent. The database boundary IS the +sovereignty boundary. RLS still applies within each zone for tenant isolation. + +--- + +## 8. Schema migration + +DCM uses a forward-only migration tool (`golang-migrate` or `dbmate`): + +``` +schemas/sql/ + 001-initial.sql # base schema + 002-add-conformance.sql + 003-add-archival.sql + ... +``` + +Migrations run as part of DCM control plane bootstrap. The application role +does not have schema-change permissions; only the migration tool's +dedicated role (with DDL grants) runs migrations. + +Major schema changes require coordinated DCM version bumps and are +documented in `../reference/implementation-specifications.md`. + +--- + +## 9. Operational queries + +Standard operator queries that should be fast (< 100ms p99): + +| Query | Expected response | +|---|---| +| `GET /api/v1/resources/{entity_uuid}` | Current Realized State for entity | +| `GET /api/v1/resources/{entity_uuid}/audit` | Per-entity audit chain | +| `GET /api/v1/catalog` | Browse-able catalog items for current tenant | +| `GET /api/v1/admin/policies` | Active policies list (platform admin) | +| `GET /api/v1/admin/drift?since=PT1H` | Recent drift events | +| `GET /api/v1/admin/orphans` | Orphan candidate review queue | + +Queries that may take longer (< 5s acceptable): + +| Query | Notes | +|---|---| +| `GET /api/v1/admin/audit/search?q=...` | Full-text audit search; JSONB GIN | +| `GET /api/v1/admin/cost/attribution?range=...` | Aggregated cost analysis | +| `POST /api/v1/admin/audit/verify` | Hash chain verification for a tenant | + +--- + +## 10. Realization note + +The schemas, indexes, and operational patterns above are **DCM's specific +realization choices**. A peer DCM realization using a different storage +technology would have its own equivalent enforcement mechanisms — different +SQL, different indexes, potentially different concurrency models — while +satisfying the same UDLM persistence contract. diff --git a/architecture/persistence/postgres-mandate.md b/architecture/persistence/postgres-mandate.md new file mode 100644 index 0000000..cea0040 --- /dev/null +++ b/architecture/persistence/postgres-mandate.md @@ -0,0 +1,186 @@ +--- +Document Status: ✅ Stable — DCM architectural decision +Document Type: Architecture Reference — Persistence Decision +Established: 2026-05-26 +Maps to: udlm/design-principles/infrastructure-optimization.md +--- + +# PostgreSQL Mandate + +> **Implements contracts defined in UDLM**: +> [udlm/design-principles/infrastructure-optimization.md](https://github.com/croadfeldt/udlm/blob/main/design-principles/infrastructure-optimization.md). +> UDLM requires that all four data domains (Intent, Requested, Realized, +> Discovered) be persistently queryable. The technology choice is not +> specified by UDLM — it is a realization-layer decision. DCM mandates +> **PostgreSQL** (or any PostgreSQL-compatible database: CockroachDB, Aurora +> PostgreSQL, Crunchy Postgres) as its required persistence infrastructure. + +This is a **DCM-level architectural decision**, not a UDLM contract. A peer +DCM realization could pick a different SQL category (or a non-SQL category +entirely) and still satisfy the UDLM persistence contract, provided it +honors immutability, queryability, and the wire-level data formats. + +--- + +## 1. The decision + +**DCM mandates PostgreSQL as its sole required external infrastructure.** + +All other dependencies — identity, secrets, event streaming, caching, Git +ingress, service mesh — can either be handled internally by DCM or +optionally delegated to external systems. PostgreSQL is the floor. + +Specifically: any PostgreSQL-compatible database satisfies the mandate. + +| Acceptable | Reason | +|---|---| +| PostgreSQL (vanilla) | Reference implementation | +| CockroachDB | Wire-compatible; native HA; sovereignty-friendly partitioning | +| Aurora PostgreSQL | AWS-managed; PostgreSQL wire-compatible | +| Crunchy Postgres | K8s-native operator; PostgreSQL upstream | +| YugabyteDB (PG mode) | Distributed; PostgreSQL wire-compatible | + +What disqualifies a database: missing JSONB, missing `LISTEN/NOTIFY`, missing +RLS, missing append-only tables (REVOKE UPDATE/DELETE), missing `pgcrypto`, +or any subset of these. DCM's contract enforcement assumes all of them. + +--- + +## 2. Why PostgreSQL (the rationale) + +DCM's design principle is: prescribe **data contracts** (schemas, +immutability rules, versioning, hash chains) — not infrastructure products. +Where a contract maps directly to a single well-understood infrastructure +category, DCM prescribes the category and the contract, not an abstraction +layer over it. + +Abstraction layers earn their place when the underlying implementations have +genuinely different interaction contracts — different APIs, different +lifecycle semantics, different operational models. When the implementations +share a standard protocol (SQL, OIDC, AMQP), the protocol is the +abstraction. Adding a DCM-specific abstraction on top of a standard +protocol is unnecessary indirection. + +PostgreSQL satisfies every UDLM persistence contract obligation through +native features: + +| UDLM contract obligation | PostgreSQL native feature | +|---|---| +| Append-only on Intent / Requested / Audit | `REVOKE UPDATE, DELETE` + audit trigger | +| Versioning on Realized | Row versioning with semantic version columns + `is_current` flag | +| Tamper-evident audit | SHA-256 hash chain in `audit_records` table | +| Tenant isolation | Row-Level Security (RLS) policies | +| Event-driven pipeline routing | `LISTEN/NOTIFY` | +| JSONB document storage | Native JSONB with GIN indexes | +| Strong transactional consistency | ACID transactions across all DCM tables | +| Sovereignty partitioning | One PostgreSQL instance per sovereignty zone | +| Air-gapped deployment | Single dependency to operate offline | + +### 2.1 Why not separate stores per domain + +| Concern | Four-store answer (Git + Kafka + Redis + PostgreSQL) | Single-store answer (PostgreSQL) | +|---|---|---| +| Immutability | Git commits are immutable | Append-only tables + REVOKE UPDATE, DELETE + audit trigger | +| Version history | Git log | Row versioning with semantic version fields | +| Audit trail | Git commit metadata | SHA-256 hash chain (stronger — explicit cryptographic chain vs Git's graph integrity) | +| PR-based review | Native Git workflow | DCM's Policy Engine + Scoring Model + Authority Tier routing (more sophisticated) | +| Tamper evidence | Git SHA integrity | Per-record hash chain (per-record, not per-repo) | +| Transactional consistency | Cross-store sync required | Native — intent + audit + operation in same transaction | +| Sovereignty partitioning | Separate Git/Kafka/Redis per zone | Separate PostgreSQL instance per zone (one thing to deploy, not four) | +| Air-gapped deployment | Git + Kafka + Redis + PostgreSQL (4 infra dependencies) | PostgreSQL only (1 dependency) | +| Operations skill set | Git admin + Kafka admin + Redis admin + DBA | DBA only | + +The four-store approach was historically considered but rejected on +operational and integrity grounds. A single well-understood database +provides stronger guarantees with one-fourth the operational surface. + +### 2.2 The "no abstraction layer over standard protocols" principle + +DCM does NOT abstract over SQL. The Catalog Manager, Policy Manager, +Request Orchestrator, and all other services use PostgreSQL directly. The +schema is defined by DCM; the queries are DCM-native. + +This is the same pattern DCM applies elsewhere: + +- OIDC is the abstraction over identity providers; DCM does not add a + DCM-specific identity abstraction +- AMQP/Kafka is the abstraction over message buses; DCM does not add a + DCM-specific bus abstraction +- gRPC/HTTP+JSON is the abstraction over RPC; DCM does not add a + DCM-specific RPC layer + +For persistence, SQL is the abstraction. PostgreSQL is the implementation. +DCM mandates PostgreSQL because writing PostgreSQL-flavored SQL is more +direct, more debuggable, and more operationally stable than writing +SQL-via-an-abstraction. + +--- + +## 3. Optional infrastructure (deployment enhancements) + +PostgreSQL is required. Everything else is optional: + +| Infrastructure | When to add | What it provides | +|---|---|---| +| OIDC IdP | Enterprise auth; multi-tenant federation | External authentication via registered auth_provider | +| Vault | Existing Vault infrastructure; dynamic secrets; full HSM seal | External secrets backend | +| Kafka | >1000 events/sec; multiple consumer groups with independent replay | Replaces pipeline_events + LISTEN/NOTIFY | +| Redis | Read-heavy catalog/placement workloads; geo-distributed reads | Replaces materialized views | +| Git repository | CI/CD integration; PR-based ingress | Adds Git as ingress path alongside API/CLI | +| Service mesh | Production mTLS between services | Replaces application-level TLS config | + +Each can be added per-deployment without changing the architectural mandate. +None are required. + +--- + +## 4. Deployment profiles + +| Profile | Required | Optional | +|---|---|---| +| **Minimal** (homelab/dev) | PostgreSQL (single instance) | — | +| **Standard** (production) | PostgreSQL (HA) | Keycloak, Vault, Service mesh, Redis | +| **Enterprise** (large scale) | PostgreSQL (HA + read replicas) | Keycloak (HA), Vault (HA + HSM), Service mesh, Kafka, Redis, Git | +| **Sovereign** (air-gapped) | PostgreSQL (per-zone) | Keycloak (per-zone), Vault (per-zone + HSM seal), Service mesh | + +--- + +## 5. Sovereignty partitioning + +DCM supports sovereignty zones by deploying separate PostgreSQL instances +per zone (one DCM control plane plus database per sovereignty zone). The +zones do not share a database; federation between zones uses the +DCM-to-DCM federation mechanism (see +[`../runtime-features/federation-runtime.md`](../runtime-features/federation-runtime.md)), +not database replication. + +This means a `sovereign` deployment has N PostgreSQL instances for N +sovereignty zones — never one database serving multiple zones. The single +database per zone simplifies sovereignty enforcement (the database +boundary IS the sovereignty boundary). + +--- + +## 6. Realization note + +PostgreSQL is **DCM's choice**, not a UDLM requirement. UDLM requires: + +- All four data domains are persistently queryable +- Wire-level data formats are honored +- Immutability invariants are maintained for Intent / Requested / Audit +- Versioning is supported for Realized +- Schema-sharing protocol permits federation peers to exchange schemas + +A peer DCM realization could pick: + +- A purpose-built database (e.g., a wide-column store + audit chain) +- A multi-engine architecture (e.g., separate ledger + queryable cache) +- A different SQL category (MySQL, MariaDB, SQL Server) + +...and remain UDLM-conformant, provided the four-domain queryability and +immutability invariants are honored. This DCM realization deliberately +picks a single well-understood SQL category and avoids abstraction-over-SQL. + +For implementation details — table structures, schema, query optimization, +indexing, data retention — see +[`postgres-implementation.md`](postgres-implementation.md). diff --git a/schemas/sql/001-initial.sql b/schemas/sql/001-initial.sql new file mode 100644 index 0000000..43a20f6 --- /dev/null +++ b/schemas/sql/001-initial.sql @@ -0,0 +1,591 @@ +-- DCM PostgreSQL Schema — Initial +-- Spec ref: DCM data model doc 49, doc 51 (Infrastructure Optimization) +-- Implements: STI-001 (mandatory tenant_uuid predicate), STI-002 (RLS) +-- +-- This schema implements ALL FOUR DCM data domains in a single database: +-- Intent Domain — append-only consumer declarations +-- Requested Domain — append-only assembled/validated payloads +-- Realized Domain — versioned provider-confirmed state +-- Discovered Domain — ephemeral discovery snapshots +-- Plus: audit records (hash chain), operations (LRO), pipeline events, subscriptions + +\connect dcm + +-- ─── Extensions ────────────────────────────────────────────────────────────── + +CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; +CREATE EXTENSION IF NOT EXISTS "pgcrypto"; + +-- ─── Tenants ────────────────────────────────────────────────────────────────── + +CREATE TABLE tenants ( + tenant_uuid UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + handle VARCHAR(64) UNIQUE NOT NULL, + display_name VARCHAR(256) NOT NULL, + status VARCHAR(32) NOT NULL DEFAULT 'ACTIVE' + CHECK (status IN ('ACTIVE', 'SUSPENDED', 'DECOMMISSIONED')), + profile VARCHAR(32) NOT NULL DEFAULT 'dev' + CHECK (profile IN ('minimal', 'dev', 'standard', 'prod', 'fsi', 'sovereign')), + data_classifications_permitted JSONB NOT NULL DEFAULT '["internal"]', + sovereignty_zones JSONB NOT NULL DEFAULT '[]', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +-- ─── Actors (users and service accounts) ────────────────────────────────────── + +CREATE TABLE actors ( + actor_uuid UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + tenant_uuid UUID NOT NULL REFERENCES tenants(tenant_uuid), + actor_type VARCHAR(32) NOT NULL + CHECK (actor_type IN ('human', 'service_account', 'system_component', 'provider')), + handle VARCHAR(256) NOT NULL, + display_name VARCHAR(256) NOT NULL, + auth_method VARCHAR(32) NOT NULL DEFAULT 'internal' + CHECK (auth_method IN ('internal', 'external')), + password_hash VARCHAR(256), -- argon2id hash (internal auth only) + totp_secret_ref VARCHAR(256), -- reference to TOTP secret in secrets table (optional MFA) + external_id VARCHAR(512), -- IdP subject claim (external auth only) + auth_provider_uuid UUID, -- which auth_provider (external auth only) + roles JSONB NOT NULL DEFAULT '[]', -- direct role assignments (internal); merged with IdP claims (external) + status VARCHAR(32) NOT NULL DEFAULT 'ACTIVE', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + last_login_at TIMESTAMPTZ, + UNIQUE(tenant_uuid, handle) +); + +-- ─── Entities (Realized State) ──────────────────────────────────────────────── +-- Spec ref: DCM data model doc 01 (Entity Types), doc 02 (Four States) +-- Realized domain — one row per realized entity per version. + +CREATE TABLE realized_entities ( + realized_uuid UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + entity_uuid UUID NOT NULL, -- Stable identifier across versions + tenant_uuid UUID NOT NULL REFERENCES tenants(tenant_uuid), + resource_type VARCHAR(256) NOT NULL, -- FQN e.g. Compute.VirtualMachine + resource_type_uuid UUID, + entity_type VARCHAR(64) NOT NULL + CHECK (entity_type IN ('infrastructure_resource', 'composite_resource', + 'process_resource', 'shared_resource', 'allocatable_pool')), + lifecycle_state VARCHAR(32) NOT NULL + CHECK (lifecycle_state IN ('PROVISIONING', 'OPERATIONAL', 'DEGRADED', + 'SUSPENDED', 'FAILED', 'DECOMMISSIONED', + 'INGESTED', 'INGESTION_PENDING')), + request_uuid UUID, -- The request that created/updated this + provider_uuid UUID, -- Which provider owns this entity + realized_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + realized_by UUID REFERENCES actors(actor_uuid), + fields JSONB NOT NULL DEFAULT '{}', -- Full realized field set + provenance JSONB NOT NULL DEFAULT '{}', -- Field-level provenance map + provider_metadata JSONB NOT NULL DEFAULT '{}', -- Provider-supplied metadata + sovereignty_zones JSONB NOT NULL DEFAULT '[]', + tags JSONB NOT NULL DEFAULT '{}', + version_major INTEGER NOT NULL DEFAULT 1, + version_minor INTEGER NOT NULL DEFAULT 0, + version_revision INTEGER NOT NULL DEFAULT 0, + is_current BOOLEAN NOT NULL DEFAULT TRUE -- Only one current per entity_uuid +); + +CREATE INDEX idx_realized_tenant ON realized_entities(tenant_uuid); +CREATE INDEX idx_realized_entity ON realized_entities(entity_uuid); +CREATE INDEX idx_realized_lifecycle ON realized_entities(tenant_uuid, lifecycle_state); +CREATE INDEX idx_realized_resource_type ON realized_entities(tenant_uuid, resource_type); +CREATE INDEX idx_realized_current ON realized_entities(entity_uuid, is_current) WHERE is_current = TRUE; + +-- ─── Operations (LRO tracking) ──────────────────────────────────────────────── +-- Spec ref: doc 25 §2 (Request Orchestrator), doc 49 §7.1 (operation_uuid issuer) +-- operation_uuid == request_uuid, issued by API Gateway at ingress. + +CREATE TABLE operations ( + operation_uuid UUID PRIMARY KEY, -- == request_uuid, issued by API Gateway + tenant_uuid UUID NOT NULL REFERENCES tenants(tenant_uuid), + resource_uuid UUID, -- AEP convention: the resource this operation acts on + operation_type VARCHAR(64) NOT NULL -- Lifecycle operation vocabulary (doc B §2.2) + CHECK (operation_type IN ('initial_provisioning', 'update', 'scale', + 'rehydration', 'decommission', 'ownership_transfer', + 'subscription_renewal', 'drift_remediation', + 'provider_migration', 'compliance_rescan')), + changed_fields JSONB, -- Array of field paths that changed (update/scale ops) + status VARCHAR(32) NOT NULL DEFAULT 'INITIATED' + CHECK (status IN ('INITIATED', 'ASSEMBLING', 'POLICY_EVALUATION', + 'POLICY_BLOCKED', 'PENDING_OVERRIDE', + 'PLACEMENT', 'DISPATCHED', 'PROVISIONING', + 'OPERATIONAL', 'FAILED', 'CANCELLED', + 'OVERRIDE_DENIED', 'OVERRIDE_TIMEOUT')), + actor_uuid UUID REFERENCES actors(actor_uuid), + catalog_item_uuid UUID, + resource_type VARCHAR(256), + metadata JSONB NOT NULL DEFAULT '{}', + score NUMERIC(5,2), + selected_provider_uuid UUID, + error_code VARCHAR(128), + error_message TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + completed_at TIMESTAMPTZ +); + +CREATE INDEX idx_operations_tenant ON operations(tenant_uuid, status); + +-- ─── Override Requests ─────────────────────────────────────────────────────── +-- Spec ref: doc B §18.8 (Override Approval Flow) + +CREATE TABLE override_requests ( + override_request_uuid UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + request_uuid UUID NOT NULL REFERENCES operations(operation_uuid), + tenant_uuid UUID NOT NULL REFERENCES tenants(tenant_uuid), + blocking_policy_handle VARCHAR(256) NOT NULL, + blocking_policy_enforcement VARCHAR(16) NOT NULL CHECK (blocking_policy_enforcement IN ('hard', 'soft')), + blocking_reason TEXT NOT NULL, + resolution_guidance JSONB NOT NULL DEFAULT '{}', -- compliant_values, suggestions per field + required_approval_type VARCHAR(16) NOT NULL CHECK (required_approval_type IN ('single', 'dual')), + eligible_approver_roles JSONB NOT NULL DEFAULT '[]', + -- Consumer-initiated override request + consumer_justification TEXT, + consumer_compensating_controls JSONB, + resolution_action VARCHAR(32) CHECK (resolution_action IN ('modify', 'request_override', 'cancel', 'escalate')), + status VARCHAR(16) NOT NULL DEFAULT 'blocked' + CHECK (status IN ('blocked', 'pending', 'approved', 'rejected', 'expired', 'resolved', 'cancelled')), + timeout_at TIMESTAMPTZ NOT NULL, + -- First approval + first_approver_uuid UUID REFERENCES actors(actor_uuid), + first_approver_role VARCHAR(64), + first_justification TEXT, + first_compensating_controls JSONB, + first_approved_at TIMESTAMPTZ, + -- Second approval (dual-approval only) + second_approver_uuid UUID REFERENCES actors(actor_uuid), + second_approver_role VARCHAR(64), + second_approved_at TIMESTAMPTZ, + -- Metadata + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + resolved_at TIMESTAMPTZ +); + +CREATE INDEX idx_override_status ON override_requests(status, timeout_at); +CREATE INDEX idx_override_request ON override_requests(request_uuid); + +CREATE INDEX idx_operations_tenant ON operations(tenant_uuid); +CREATE INDEX idx_operations_status ON operations(tenant_uuid, status); +CREATE INDEX idx_operations_resource ON operations(resource_uuid); + +-- ─── Audit Records ──────────────────────────────────────────────────────────── +-- Spec ref: DCM data model doc 16 (Universal Audit), doc 49 §3 (Hash Chain) +-- Implements: Tamper-evident hash chain with SHA-256 + +CREATE TABLE audit_records ( + record_uuid UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + record_timestamp TIMESTAMPTZ NOT NULL DEFAULT NOW(), + entity_uuid UUID, -- Subject entity (if applicable) + entity_type VARCHAR(64), + tenant_uuid UUID REFERENCES tenants(tenant_uuid), + action VARCHAR(128) NOT NULL, -- Closed vocabulary + -- WHO + immediate_actor_uuid UUID, + immediate_actor_type VARCHAR(32), + authorized_by_uuid UUID, + session_uuid UUID, + signer_uuid UUID NOT NULL, -- Service or actor that signed this leaf + signer_type VARCHAR(16) NOT NULL CHECK (signer_type IN ('service', 'actor', 'provider')), + -- WHAT + subject_handle VARCHAR(512), + stage VARCHAR(64), -- Pipeline stage (intent_submitted, layer_applied, etc.) + source VARCHAR(256), -- Which layer/policy/service + source_type VARCHAR(64), -- layer_merge, policy_gatekeeper, policy_transformation, etc. + decision VARCHAR(32), -- allow, deny, applied, resolved, etc. + -- CHAIN OF CUSTODY — payload integrity + input_payload_hash VARCHAR(64), -- SHA-256 of payload BEFORE this mutation + output_payload_hash VARCHAR(64), -- SHA-256 of payload AFTER this mutation + context_hash VARCHAR(64), -- Evaluation context hash (policy stages only) + -- FIELD-LEVEL DETAIL (mutation/field granularity only) + fields_changed JSONB, -- Array of field paths changed (mutation+) + field_mutations JSONB, -- Per-field old/new hashes (field granularity only) + -- MERKLE TREE (RFC 9162 pattern) + leaf_index BIGINT NOT NULL, -- Position in global Merkle tree + record_hash VARCHAR(64) NOT NULL, -- SHA-256 of record content + previous_leaf_hash VARCHAR(64), -- Hash of previous leaf for this request + signature TEXT NOT NULL, -- Ed25519/ECDSA-P256 over all fields + -- METADATA + dcm_version VARCHAR(32), + request_uuid UUID, + policy_uuid UUID, + provider_uuid UUID +); + +CREATE INDEX idx_audit_entity ON audit_records(entity_uuid, leaf_index); +CREATE INDEX idx_audit_tenant ON audit_records(tenant_uuid, record_timestamp); +CREATE INDEX idx_audit_action ON audit_records(action, record_timestamp); +CREATE INDEX idx_audit_request ON audit_records(request_uuid, leaf_index); +CREATE INDEX idx_audit_leaf ON audit_records(leaf_index); + +-- Signed Tree Heads (RFC 9162 pattern) +CREATE TABLE signed_tree_heads ( + sth_uuid UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + tree_size BIGINT NOT NULL, -- Number of leaves when STH was computed + timestamp TIMESTAMPTZ NOT NULL DEFAULT NOW(), + sha256_root_hash VARCHAR(64) NOT NULL, -- Merkle root hash + signature TEXT NOT NULL, -- Signed by DCM audit signing key + signing_key_id VARCHAR(128) NOT NULL -- Identifies which key signed this STH +); + +CREATE INDEX idx_sth_tree_size ON signed_tree_heads(tree_size); + +-- Merkle tree intermediate nodes (materialized mode — optional) +CREATE TABLE merkle_tree_nodes ( + level INT NOT NULL, -- Tree level (0 = leaves) + position BIGINT NOT NULL, -- Position at this level + hash VARCHAR(64) NOT NULL, -- SHA-256 hash of this node + PRIMARY KEY (level, position) +); + +-- Audit table is append-only — enforce via policy +REVOKE UPDATE, DELETE ON audit_records FROM dcm_app; +GRANT INSERT, SELECT ON audit_records TO dcm_app; +GRANT INSERT, SELECT ON audit_records TO dcm_audit; + +-- ─── Providers ──────────────────────────────────────────────────────────────── + +CREATE TABLE providers ( + provider_uuid UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + handle VARCHAR(128) UNIQUE NOT NULL, + display_name VARCHAR(256) NOT NULL, + provider_type VARCHAR(64) NOT NULL + CHECK (provider_type IN ('service_provider', 'information_provider', + 'auth_provider', + 'peer_dcm', 'process_provider')), + status VARCHAR(32) NOT NULL DEFAULT 'PENDING' + CHECK (status IN ('PENDING', 'ACTIVE', 'SUSPENDED', 'DEREGISTERED', 'SANDBOX')), + endpoint VARCHAR(512) NOT NULL, -- mTLS endpoint URL + public_key_pem TEXT, + capabilities JSONB NOT NULL DEFAULT '{}', + supported_resource_types JSONB NOT NULL DEFAULT '[]', + sovereignty_declarations JSONB NOT NULL DEFAULT '[]', + registered_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + last_health_check TIMESTAMPTZ, + health_status VARCHAR(32) DEFAULT 'UNKNOWN' +); + +CREATE INDEX idx_providers_status ON providers(status); +CREATE INDEX idx_providers_type ON providers(provider_type, status); + +-- ─── Service Catalog ────────────────────────────────────────────────────────── + +CREATE TABLE catalog_items ( + catalog_item_uuid UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + handle VARCHAR(128) UNIQUE NOT NULL, + display_name VARCHAR(256) NOT NULL, + description TEXT, + resource_type VARCHAR(256) NOT NULL, + provider_uuid UUID REFERENCES providers(provider_uuid), + field_schema JSONB NOT NULL DEFAULT '{}', -- JSON Schema for request fields + cost_estimate JSONB, + visibility_policy JSONB NOT NULL DEFAULT '{}', -- RBAC / group visibility rules + status VARCHAR(32) NOT NULL DEFAULT 'ACTIVE', + version_major INTEGER NOT NULL DEFAULT 1, + version_minor INTEGER NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX idx_catalog_status ON catalog_items(status); +CREATE INDEX idx_catalog_resource_type ON catalog_items(resource_type); + +-- ─── Row-Level Security (STI-001, STI-002) ──────────────────────────────────── +-- Enforce tenant isolation at storage layer. +-- dcm_app cannot query across tenant boundaries. +-- dcm_admin bypasses RLS for platform admin operations (separately audited). + +ALTER TABLE realized_entities ENABLE ROW LEVEL SECURITY; +ALTER TABLE operations ENABLE ROW LEVEL SECURITY; +ALTER TABLE audit_records ENABLE ROW LEVEL SECURITY; + +CREATE POLICY tenant_isolation_realized + ON realized_entities + FOR ALL + TO dcm_app + USING (tenant_uuid = current_setting('dcm.current_tenant_uuid')::uuid); + +CREATE POLICY tenant_isolation_operations + ON operations + FOR ALL + TO dcm_app + USING (tenant_uuid = current_setting('dcm.current_tenant_uuid')::uuid); + +CREATE POLICY tenant_isolation_audit + ON audit_records + FOR SELECT + TO dcm_app + USING (tenant_uuid = current_setting('dcm.current_tenant_uuid')::uuid); + +-- dcm_admin bypasses RLS (explicit grant) +ALTER TABLE realized_entities FORCE ROW LEVEL SECURITY; +ALTER TABLE operations FORCE ROW LEVEL SECURITY; +GRANT ALL ON ALL TABLES IN SCHEMA public TO dcm_admin; +ALTER ROLE dcm_admin BYPASSRLS; + +-- ─── Triggers ──────────────────────────────────────────────────────────────── + +-- Auto-set updated_at +CREATE OR REPLACE FUNCTION update_updated_at() +RETURNS TRIGGER AS $$ +BEGIN + NEW.updated_at = NOW(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER operations_updated_at + BEFORE UPDATE ON operations + FOR EACH ROW EXECUTE FUNCTION update_updated_at(); + +-- Enforce append-only on audit (belt-and-suspenders beyond REVOKE) +CREATE OR REPLACE FUNCTION prevent_audit_modification() +RETURNS TRIGGER AS $$ +BEGIN + RAISE EXCEPTION 'Audit records are immutable. Record UUID: %', OLD.record_uuid; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER audit_immutable + BEFORE UPDATE OR DELETE ON audit_records + FOR EACH ROW EXECUTE FUNCTION prevent_audit_modification(); + +-- ─── Intent Domain (doc 51 §2.3) ──────────────────────────────────────────── +-- Append-only. Raw consumer declarations. Never modified after write. + +CREATE TABLE intent_records ( + intent_uuid UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + entity_uuid UUID NOT NULL, + tenant_uuid UUID NOT NULL REFERENCES tenants(tenant_uuid), + catalog_item_uuid UUID NOT NULL, + submitted_by UUID NOT NULL, + submitted_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + submitted_via VARCHAR(32) NOT NULL + CHECK (submitted_via IN ('api', 'gitops', 'cli', 'message_bus')), + intent_version INTEGER NOT NULL DEFAULT 1, + fields JSONB NOT NULL DEFAULT '{}', + provenance JSONB NOT NULL DEFAULT '{}' +); + +CREATE INDEX idx_intent_entity ON intent_records(entity_uuid, intent_version); +CREATE INDEX idx_intent_tenant ON intent_records(tenant_uuid, submitted_at); +REVOKE UPDATE, DELETE ON intent_records FROM dcm_app; + +-- ─── Requested Domain (doc 51 §2.3) ───────────────────────────────────────── +-- Append-only. Assembled, policy-evaluated, placed payloads. + +CREATE TABLE requested_records ( + requested_uuid UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + entity_uuid UUID NOT NULL, + tenant_uuid UUID NOT NULL REFERENCES tenants(tenant_uuid), + operation_uuid UUID NOT NULL REFERENCES operations(operation_uuid), + intent_uuid UUID NOT NULL REFERENCES intent_records(intent_uuid), + resource_type VARCHAR(256) NOT NULL, + provider_uuid UUID NOT NULL, + assembled_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + assembled_payload JSONB NOT NULL DEFAULT '{}', + layer_sources JSONB NOT NULL DEFAULT '[]', + policy_results JSONB NOT NULL DEFAULT '{}', + placement_result JSONB NOT NULL DEFAULT '{}', + provenance JSONB NOT NULL DEFAULT '{}' +); + +CREATE INDEX idx_requested_entity ON requested_records(entity_uuid); +CREATE INDEX idx_requested_tenant ON requested_records(tenant_uuid); +CREATE INDEX idx_requested_operation ON requested_records(operation_uuid); +REVOKE UPDATE, DELETE ON requested_records FROM dcm_app; + +-- ─── Discovered Domain (doc 51 §2.3) ──────────────────────────────────────── +-- Ephemeral snapshots from provider discovery runs. + +CREATE TABLE discovered_records ( + discovery_uuid UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + entity_uuid UUID, + tenant_uuid UUID REFERENCES tenants(tenant_uuid), + provider_uuid UUID NOT NULL, + resource_type VARCHAR(256) NOT NULL, + discovered_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + discovery_run_uuid UUID NOT NULL, + discovered_fields JSONB NOT NULL DEFAULT '{}', + provider_native_id VARCHAR(512), + match_confidence VARCHAR(16) DEFAULT 'exact' + CHECK (match_confidence IN ('exact', 'high', 'low', 'unmatched')) +); + +CREATE INDEX idx_discovered_entity ON discovered_records(entity_uuid, discovered_at); +CREATE INDEX idx_discovered_run ON discovered_records(discovery_run_uuid); +CREATE INDEX idx_discovered_orphans ON discovered_records(entity_uuid) WHERE entity_uuid IS NULL; + +-- ─── Pipeline Events (doc 51 §2.3) ────────────────────────────────────────── +-- Append-only event log. LISTEN/NOTIFY for real-time pipeline routing. + +CREATE TABLE pipeline_events ( + event_uuid UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + event_type VARCHAR(128) NOT NULL, + entity_uuid UUID, + request_uuid UUID, + tenant_uuid UUID, + actor_uuid UUID, + payload JSONB NOT NULL DEFAULT '{}', + published_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + consumed_by JSONB NOT NULL DEFAULT '[]', + consumed_at TIMESTAMPTZ +); + +CREATE INDEX idx_events_type ON pipeline_events(event_type, published_at); +CREATE INDEX idx_events_entity ON pipeline_events(entity_uuid, published_at); +CREATE INDEX idx_events_unconsumed ON pipeline_events(event_type, published_at) + WHERE consumed_at IS NULL; +REVOKE UPDATE, DELETE ON pipeline_events FROM dcm_app; + +-- Notify trigger for real-time pipeline routing +CREATE OR REPLACE FUNCTION notify_pipeline_event() RETURNS TRIGGER AS $$ +BEGIN + PERFORM pg_notify('dcm_pipeline', json_build_object( + 'event_uuid', NEW.event_uuid, + 'event_type', NEW.event_type, + 'entity_uuid', NEW.entity_uuid + )::text); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER pipeline_event_notify + AFTER INSERT ON pipeline_events + FOR EACH ROW EXECUTE FUNCTION notify_pipeline_event(); + +-- ─── RLS on new tables ────────────────────────────────────────────────────── + +ALTER TABLE intent_records ENABLE ROW LEVEL SECURITY; +ALTER TABLE requested_records ENABLE ROW LEVEL SECURITY; +ALTER TABLE discovered_records ENABLE ROW LEVEL SECURITY; +ALTER TABLE pipeline_events ENABLE ROW LEVEL SECURITY; + +CREATE POLICY tenant_isolation_intent ON intent_records + FOR ALL TO dcm_app + USING (tenant_uuid = current_setting('dcm.current_tenant_uuid')::uuid); +CREATE POLICY tenant_isolation_requested ON requested_records + FOR ALL TO dcm_app + USING (tenant_uuid = current_setting('dcm.current_tenant_uuid')::uuid); +CREATE POLICY tenant_isolation_discovered ON discovered_records + FOR ALL TO dcm_app + USING (tenant_uuid = current_setting('dcm.current_tenant_uuid')::uuid); +CREATE POLICY tenant_isolation_events ON pipeline_events + FOR SELECT TO dcm_app + USING (tenant_uuid = current_setting('dcm.current_tenant_uuid')::uuid); + +-- ─── Secrets (doc 31, doc 51 §4.2) ─────────────────────────────────────────── +-- Internal secrets management using envelope encryption. +-- Each secret value is AES-256-GCM encrypted with a per-secret DEK. +-- DEKs are encrypted with the master KEK from the deployment environment. +-- External mode (Vault) bypasses this table entirely. + +CREATE TABLE secrets ( + secret_uuid UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + tenant_uuid UUID REFERENCES tenants(tenant_uuid), -- null for system secrets + secret_path VARCHAR(512) NOT NULL, -- hierarchical path (e.g., providers/{uuid}/credentials) + secret_type VARCHAR(64) NOT NULL + CHECK (secret_type IN ('credential', 'encryption_key', + 'signing_key', 'certificate', 'generic')), + encrypted_value BYTEA NOT NULL, -- AES-256-GCM encrypted + encrypted_dek BYTEA NOT NULL, -- DEK encrypted with KEK + encryption_algorithm VARCHAR(32) NOT NULL DEFAULT 'AES-256-GCM', + kek_id VARCHAR(128) NOT NULL, -- identifies which KEK encrypted the DEK + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + rotated_at TIMESTAMPTZ, + expires_at TIMESTAMPTZ, + lifecycle_state VARCHAR(16) NOT NULL DEFAULT 'ACTIVE' + CHECK (lifecycle_state IN ('ACTIVE', 'ROTATING', 'REVOKED', 'EXPIRED')), + UNIQUE(secret_path) +); + +CREATE INDEX idx_secrets_tenant ON secrets(tenant_uuid); +CREATE INDEX idx_secrets_path ON secrets(secret_path); +CREATE INDEX idx_secrets_expiry ON secrets(expires_at) WHERE lifecycle_state = 'ACTIVE'; + +-- Secrets table: no UPDATE on encrypted_value (rotation creates new row, revokes old) +-- SELECT restricted to dcm_app via RLS +ALTER TABLE secrets ENABLE ROW LEVEL SECURITY; +CREATE POLICY tenant_isolation_secrets ON secrets + FOR ALL TO dcm_app + USING (tenant_uuid IS NULL OR tenant_uuid = current_setting('dcm.current_tenant_uuid')::uuid); + +-- ─── Subscriptions (doc 50) ────────────────────────────────────────────────── + +CREATE TABLE subscriptions ( + subscription_uuid UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + tenant_uuid UUID NOT NULL REFERENCES tenants(tenant_uuid), + handle VARCHAR(256) NOT NULL, + display_name VARCHAR(256) NOT NULL, + catalog_item_uuid UUID NOT NULL, + resource_type VARCHAR(256) NOT NULL, + provider_uuid UUID NOT NULL, + lifecycle_state VARCHAR(32) NOT NULL DEFAULT 'PENDING' + CHECK (lifecycle_state IN ( + 'PENDING', 'PROVISIONING', 'ACTIVE', + 'SUSPENDED', 'RENEWAL_PENDING', 'TIER_CHANGE_PENDING', + 'EXPIRED', 'CANCELLED', 'DECOMMISSIONING', 'DECOMMISSIONED' + )), + terms JSONB NOT NULL DEFAULT '{}', + entitlements JSONB NOT NULL DEFAULT '{}', + update_channels JSONB NOT NULL DEFAULT '[]', + terms_version VARCHAR(32) NOT NULL DEFAULT '1.0.0', + started_at TIMESTAMPTZ, + expires_at TIMESTAMPTZ, + grace_period INTERVAL NOT NULL DEFAULT '30 days', + auto_renew BOOLEAN NOT NULL DEFAULT true, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE(tenant_uuid, handle) +); + +CREATE INDEX idx_subscriptions_tenant ON subscriptions(tenant_uuid, lifecycle_state); +CREATE INDEX idx_subscriptions_provider ON subscriptions(provider_uuid); +CREATE INDEX idx_subscriptions_expiry ON subscriptions(expires_at) WHERE lifecycle_state = 'ACTIVE'; + +CREATE TABLE subscription_entities ( + subscription_uuid UUID NOT NULL REFERENCES subscriptions(subscription_uuid), + entity_uuid UUID NOT NULL, + role VARCHAR(64) NOT NULL DEFAULT 'managed', + bound_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (subscription_uuid, entity_uuid) +); + +CREATE TABLE subscription_updates ( + update_uuid UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + subscription_uuid UUID NOT NULL REFERENCES subscriptions(subscription_uuid), + entity_uuid UUID NOT NULL, + provider_uuid UUID NOT NULL, + channel VARCHAR(64) NOT NULL, + status VARCHAR(32) NOT NULL DEFAULT 'PENDING' + CHECK (status IN ('PENDING', 'APPROVED', 'REJECTED', + 'APPLIED', 'FAILED', 'EXPIRED')), + update_payload JSONB NOT NULL DEFAULT '{}', + submitted_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + decided_at TIMESTAMPTZ, + decided_by UUID, + applied_at TIMESTAMPTZ, + auto_applied BOOLEAN NOT NULL DEFAULT false +); + +CREATE INDEX idx_sub_updates_subscription ON subscription_updates(subscription_uuid, status); +CREATE INDEX idx_sub_updates_pending ON subscription_updates(status, submitted_at) WHERE status = 'PENDING'; + +ALTER TABLE subscriptions ENABLE ROW LEVEL SECURITY; +ALTER TABLE subscription_entities ENABLE ROW LEVEL SECURITY; +ALTER TABLE subscription_updates ENABLE ROW LEVEL SECURITY; + +CREATE POLICY tenant_isolation_subscriptions ON subscriptions + FOR ALL TO dcm_app + USING (tenant_uuid = current_setting('dcm.current_tenant_uuid')::uuid); +CREATE POLICY tenant_isolation_sub_entities ON subscription_entities + FOR ALL TO dcm_app + USING (subscription_uuid IN ( + SELECT subscription_uuid FROM subscriptions + WHERE tenant_uuid = current_setting('dcm.current_tenant_uuid')::uuid + )); +CREATE POLICY tenant_isolation_sub_updates ON subscription_updates + FOR ALL TO dcm_app + USING (subscription_uuid IN ( + SELECT subscription_uuid FROM subscriptions + WHERE tenant_uuid = current_setting('dcm.current_tenant_uuid')::uuid + ));