diff --git a/CHANGELOG.md b/CHANGELOG.md index 0153c8e..2a215a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ All notable changes to pgclone are documented in this file. +## [4.1.0] + +### Added +- **Schema diff (`pgclone.diff(source_conninfo, schema_name)`)** — read-only DDL drift detection between source and the local target. Returns a JSON document with summary counts plus per-category arrays of `only_in_source` / `only_in_target` / `modified` for tables (with per-column `type` / `not_null` / `default` drift), indexes (excluding those backing constraints), constraints, user-defined triggers, views and materialized views, and sequences. Both source and local connections run inside `BEGIN ... READ ONLY` transactions; the function never executes DDL or DML on either side. + +### Internal +- New isolated translation unit `src/pgclone_diff.c`. The diff feature does not share helpers with `src/pgclone.c`, keeping the surface area additive and trivially auditable. +- Catalog queries explicitly use `ORDER BY ... COLLATE "C"` to guarantee identical sort order on both sides regardless of the local lc_collate setting, so the merge-walk comparison is deterministic. + ## [4.0.1] ### Fixed diff --git a/META.json b/META.json index 89e67e3..a370d5d 100644 --- a/META.json +++ b/META.json @@ -2,14 +2,14 @@ "name": "pgclone", "abstract": "Clone PostgreSQL databases, schemas, tables between staging, test, dev and prod environments", "description": "PostgreSQL extension for easily cloning your DB, Schemas, Tables and more between environments", - "version": "4.0.1", + "version": "4.1.0", "maintainer": "Valeh Agayev ", "license": "postgresql", "provides": { "pgclone": { "abstract": "Clone PostgreSQL databases, schemas, and tables across environments", - "file": "sql/pgclone--4.0.1.sql", - "version": "4.0.1" + "file": "sql/pgclone--4.1.0.sql", + "version": "4.1.0" } }, "prereqs": { @@ -42,6 +42,8 @@ "database", "table", "masking", - "anonymization" + "anonymization", + "diff", + "drift" ] } \ No newline at end of file diff --git a/Makefile b/Makefile index 5278e1c..3082928 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ EXTENSION = pgclone MODULE_big = pgclone -OBJS = src/pgclone.o src/pgclone_bgw.o +OBJS = src/pgclone.o src/pgclone_bgw.o src/pgclone_diff.o DATA = $(wildcard sql/pgclone--*.sql) PG_CPPFLAGS = -I$(shell $(PG_CONFIG) --includedir) -Isrc diff --git a/README.md b/README.md index 5339e62..fb6a221 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![CI](https://github.com/valehdba/pgclone/actions/workflows/ci.yml/badge.svg)](https://github.com/valehdba/pgclone/actions/workflows/ci.yml) [![Postgres 14–18](https://img.shields.io/badge/Postgres-14%E2%80%9318-336791?logo=postgresql&logoColor=white)](https://github.com/valehdba/pgclone) [![License](https://img.shields.io/badge/License-PostgreSQL-blue.svg)](https://github.com/valehdba/pgclone/blob/main/LICENSE) -[![Version](https://img.shields.io/badge/version-4.0.1-orange)](https://github.com/valehdba/pgclone/releases/tag/v4.0.1) +[![Version](https://img.shields.io/badge/version-4.1.0-orange)](https://github.com/valehdba/pgclone/releases/tag/v4.1.0) A PostgreSQL extension that clones databases, schemas, tables, and functions between PostgreSQL instances — directly from SQL. No `pg_dump`, no `pg_restore`, no shell scripts. @@ -141,10 +141,10 @@ pgclone uses Unix domain sockets for local loopback connections, so the default - [x] v3.4.0: Clone roles with permissions and passwords - [x] v3.5.0: Clone verification - compare row counts across source and target - [x] v3.6.0: GDPR/Compliance masking report -- [x] v4.0.0: Schema namespace - all functions under `pgclone` schema (`pgclone.table()`, `pgclone.schema()`, etc.) -- [ ] v4.1.0: Schema diff - DDL drift detection between source and target (`pgclone.diff`) -- [ ] v4.2.0: Pre-flight validator - connection, space, permissions, version, name-conflict checks before clone (`pgclone.preflight`) -- [ ] v4.3.0: FK-aware referential sampling - sample N rows and follow foreign keys to keep child rows consistent (`pgclone.table_sample`) +- [x] v4.0.0: Schema namespace — all functions under `pgclone` schema (`pgclone.table()`, `pgclone.schema()`, etc.) +- [x] v4.1.0: Schema diff — DDL drift detection between source and target (`pgclone.diff`) +- [ ] v4.2.0: Pre-flight validator — connection, space, permissions, version, name-conflict checks before clone (`pgclone.preflight`) +- [ ] v4.3.0: FK-aware referential sampling — sample N rows and follow foreign keys to keep child rows consistent (`pgclone.table_sample`) ## License diff --git a/docs/USAGE.md b/docs/USAGE.md index 68a08a8..b88da39 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -657,6 +657,112 @@ SELECT * FROM pgclone.masking_report('public'); --- +## Schema Diff (v4.1.0) + +Detect DDL drift between a source database and the local target without +modifying either side. Useful for answering "is dev still in sync with prod?" +before a release, after a hotfix, or as a scheduled health check. + +### Usage + +```sql +SELECT pgclone.diff( + 'host=source-server dbname=prod user=postgres password=secret', + 'app_schema' +)::jsonb; +``` + +The function compares the named schema on the source against the same-named +schema on the database the call is issued from. It returns a single JSON +document — pipe through `::jsonb` (as above) or `jsonb_pretty()` for +readability. + +### What is compared + +| Category | Compared by | Drift detected on | +|-------------|--------------------------------------------------|------------------------------| +| Tables | `relname` | (presence) | +| Columns | `(table, column)` | type, `NOT NULL`, default | +| Indexes | `index name` | `pg_get_indexdef` | +| Constraints | `constraint name` | `pg_get_constraintdef` | +| Triggers | `trigger name` (user-defined only) | `pg_get_triggerdef` | +| Views | `relname` (regular + materialized) | `pg_get_viewdef` | +| Sequences | `relname` | (presence) | + +Indexes that back a primary key or unique constraint are reported under the +**constraints** category, not the indexes category, to avoid double-counting. + +### Output shape + +```json +{ + "schema": "app_schema", + "in_sync": false, + "diff_count": 4, + "summary": { + "tables_only_in_source": 1, "tables_only_in_target": 0, "tables_modified": 1, + "indexes_only_in_source": 1, "indexes_only_in_target": 0, "indexes_modified": 0, + "constraints_only_in_source": 0, "constraints_only_in_target": 0, "constraints_modified": 0, + "triggers_only_in_source": 0, "triggers_only_in_target": 0, "triggers_modified": 0, + "views_only_in_source": 0, "views_only_in_target": 0, "views_modified": 0, + "sequences_only_in_source": 1, "sequences_only_in_target": 0 + }, + "tables": { + "only_in_source": ["audit_log"], + "only_in_target": [], + "modified": [ + { + "name": "customers", + "columns_only_in_source": [ + {"name": "loyalty_tier", "type": "text", "not_null": false, "default": null} + ], + "columns_only_in_target": [], + "columns_drift": [ + {"name": "id", + "source_type": "bigint", "target_type": "integer", + "source_not_null": true, "target_not_null": true, + "source_default": "nextval('customers_id_seq'::regclass)", + "target_default": "nextval('customers_id_seq'::regclass)"} + ] + } + ] + }, + "indexes": { "only_in_source": [...], "only_in_target": [], "modified": [] }, + "constraints": { "only_in_source": [], "only_in_target": [], "modified": [] }, + "triggers": { "only_in_source": [], "only_in_target": [], "modified": [] }, + "views": { "only_in_source": [], "only_in_target": [], "modified": [] }, + "sequences": { "only_in_source": ["audit_log_seq"], "only_in_target": [] } +} +``` + +`in_sync` is `true` only when `diff_count` is `0`. + +### Quick boolean check + +```sql +SELECT (pgclone.diff(:src, 'app_schema')::jsonb ->> 'in_sync')::boolean AS in_sync; +``` + +### Notes + +- **Read-only on both sides.** Both source and local connections run inside a + `BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY` transaction; the function + never executes DDL or DML. +- **Permissions.** The calling role only needs read access to `pg_catalog` + on both sides — the same access required by `\d` in psql. +- **Sort stability.** Catalog rows are ordered with `COLLATE "C"` so the + comparison is deterministic regardless of the cluster's `lc_collate`. +- **Schema must exist on the source.** If it does not exist on the target, + every source object is reported under `only_in_source` (and vice versa). +- **Currently scoped to a single schema per call.** Loop in SQL to compare + multiple schemas: + ```sql + SELECT n, pgclone.diff(:src, n)::jsonb + FROM unnest(ARRAY['public','app','reporting']) AS n; + ``` + +--- + ## JSON Options Reference | Option | Type | Default | Description | @@ -698,6 +804,7 @@ SELECT * FROM pgclone.masking_report('public'); | `pgclone.verify(conninfo)` | table | Compare row counts for all tables across source and target | | `pgclone.verify(conninfo, schema)` | table | Compare row counts for tables in a specific schema | | `pgclone.masking_report(schema)` | table | GDPR/compliance audit: sensitive columns, mask status, recommendations | +| `pgclone.diff(conninfo, schema)` | text (JSON) | DDL drift report: tables, columns, indexes, constraints, triggers, views, sequences | | `pgclone.table_async(...)` | int | Async table clone (returns job_id) | | `pgclone.schema_async(...)` | int | Async schema clone (returns job_id) | | `pgclone.progress(job_id)` | json | Job progress as JSON | diff --git a/pgclone.control b/pgclone.control index 01cb011..832314c 100644 --- a/pgclone.control +++ b/pgclone.control @@ -1,6 +1,6 @@ # pgclone extension comment = 'Clone PostgreSQL databases, schemas, tables, roles and permissions with selective columns, data filtering, data masking/anonymization, async, parallel cloning, materialized views, resume, and conflict resolution' -default_version = '4.0.1' +default_version = '4.1.0' module_pathname = '$libdir/pgclone' relocatable = false superuser = true diff --git a/sql/pgclone--4.0.1--4.1.0.sql b/sql/pgclone--4.0.1--4.1.0.sql new file mode 100644 index 0000000..7d31b39 --- /dev/null +++ b/sql/pgclone--4.0.1--4.1.0.sql @@ -0,0 +1,19 @@ +/* pgclone--4.0.1--4.1.0.sql */ +\echo Use "ALTER EXTENSION pgclone UPDATE" to load this file. \quit + +-- v4.1.0: Schema diff — DDL drift detection between source and target. +-- +-- Adds pgclone.diff(source_conninfo, schema_name): a read-only +-- comparison of catalog metadata across source and local target. +-- Returns a JSON document describing tables/indexes/constraints/ +-- triggers/views/sequences that exist on only one side or differ. + +CREATE FUNCTION pgclone.diff(source_conninfo TEXT, schema_name TEXT) +RETURNS TEXT AS 'MODULE_PATHNAME', 'pgclone_diff' +LANGUAGE C VOLATILE STRICT; + +COMMENT ON FUNCTION pgclone.diff(TEXT, TEXT) IS + 'Compare DDL of a schema between source and the local target. ' + 'Returns JSON drift report listing objects only_in_source / only_in_target / modified ' + 'across tables (with per-column type/nullability/default drift), indexes, ' + 'constraints, triggers, views, and sequences. Read-only on both sides.'; diff --git a/sql/pgclone--4.1.0.sql b/sql/pgclone--4.1.0.sql new file mode 100644 index 0000000..cf69d6d --- /dev/null +++ b/sql/pgclone--4.1.0.sql @@ -0,0 +1,151 @@ +/* pgclone--4.1.0.sql */ +\echo Use "CREATE EXTENSION pgclone" to load this file. \quit + +-- v4.0.0: All functions now live under the 'pgclone' schema. +-- Usage: SELECT pgclone.table(...), pgclone.schema(...), etc. +CREATE SCHEMA IF NOT EXISTS pgclone; + +-- SYNCHRONOUS +CREATE FUNCTION pgclone.table(source_conninfo TEXT, schema_name TEXT, table_name TEXT, include_data BOOLEAN DEFAULT true) RETURNS TEXT AS 'MODULE_PATHNAME', 'pgclone_table' LANGUAGE C VOLATILE STRICT; +CREATE FUNCTION pgclone.table(source_conninfo TEXT, schema_name TEXT, table_name TEXT, include_data BOOLEAN, target_table_name TEXT) RETURNS TEXT AS 'MODULE_PATHNAME', 'pgclone_table' LANGUAGE C VOLATILE; +CREATE FUNCTION pgclone.table(source_conninfo TEXT, schema_name TEXT, table_name TEXT, include_data BOOLEAN, target_table_name TEXT, options TEXT) RETURNS TEXT AS 'MODULE_PATHNAME', 'pgclone_table' LANGUAGE C VOLATILE; +COMMENT ON FUNCTION pgclone.table(TEXT, TEXT, TEXT, BOOLEAN, TEXT, TEXT) IS 'Clone table with JSON options: {"columns":["col1","col2"], "where":"status=''active''", "indexes":false, "constraints":false, "triggers":false, "mask":{"email":"email","name":"name","phone":"phone","col":{"type":"partial","prefix":2,"suffix":3},"col2":"hash","col3":"null","col4":{"type":"random_int","min":0,"max":100},"col5":{"type":"constant","value":"REDACTED"}}}'; +CREATE FUNCTION pgclone.table_ex(source_conninfo TEXT, schema_name TEXT, table_name TEXT, include_data BOOLEAN, target_table_name TEXT, include_indexes BOOLEAN DEFAULT true, include_constraints BOOLEAN DEFAULT true, include_triggers BOOLEAN DEFAULT true) RETURNS TEXT AS 'MODULE_PATHNAME', 'pgclone_table_ex' LANGUAGE C VOLATILE; +CREATE FUNCTION pgclone.schema(source_conninfo TEXT, schema_name TEXT, include_data BOOLEAN DEFAULT true) RETURNS TEXT AS 'MODULE_PATHNAME', 'pgclone_schema' LANGUAGE C VOLATILE STRICT; +CREATE FUNCTION pgclone.schema(source_conninfo TEXT, schema_name TEXT, include_data BOOLEAN, options TEXT) RETURNS TEXT AS 'MODULE_PATHNAME', 'pgclone_schema' LANGUAGE C VOLATILE; +CREATE FUNCTION pgclone.schema_ex(source_conninfo TEXT, schema_name TEXT, include_data BOOLEAN, include_indexes BOOLEAN DEFAULT true, include_constraints BOOLEAN DEFAULT true, include_triggers BOOLEAN DEFAULT true) RETURNS TEXT AS 'MODULE_PATHNAME', 'pgclone_schema_ex' LANGUAGE C VOLATILE; +CREATE FUNCTION pgclone.functions(source_conninfo TEXT, schema_name TEXT) RETURNS TEXT AS 'MODULE_PATHNAME', 'pgclone_functions' LANGUAGE C VOLATILE STRICT; +CREATE FUNCTION pgclone.database(source_conninfo TEXT, include_data BOOLEAN DEFAULT true) RETURNS TEXT AS 'MODULE_PATHNAME', 'pgclone_database' LANGUAGE C VOLATILE STRICT; +CREATE FUNCTION pgclone.database(source_conninfo TEXT, include_data BOOLEAN, options TEXT) RETURNS TEXT AS 'MODULE_PATHNAME', 'pgclone_database' LANGUAGE C VOLATILE; + +-- v2.0.1: Create target database and clone into it +CREATE FUNCTION pgclone.database_create(source_conninfo TEXT, target_dbname TEXT, include_data BOOLEAN DEFAULT true) RETURNS TEXT AS 'MODULE_PATHNAME', 'pgclone_database_create' LANGUAGE C VOLATILE; +CREATE FUNCTION pgclone.database_create(source_conninfo TEXT, target_dbname TEXT, include_data BOOLEAN, options TEXT) RETURNS TEXT AS 'MODULE_PATHNAME', 'pgclone_database_create' LANGUAGE C VOLATILE; +COMMENT ON FUNCTION pgclone.database_create(TEXT, TEXT, BOOLEAN) IS 'Create target database if not exists, then clone all schemas/tables/functions from source. Run from postgres DB.'; + +-- ASYNC (require shared_preload_libraries = 'pgclone') +CREATE FUNCTION pgclone.table_async(source_conninfo TEXT, schema_name TEXT, table_name TEXT, include_data BOOLEAN DEFAULT true, target_table_name TEXT DEFAULT NULL, options TEXT DEFAULT NULL) RETURNS INTEGER AS 'MODULE_PATHNAME', 'pgclone_table_async' LANGUAGE C VOLATILE; +CREATE FUNCTION pgclone.schema_async(source_conninfo TEXT, schema_name TEXT, include_data BOOLEAN DEFAULT true, options TEXT DEFAULT NULL) RETURNS INTEGER AS 'MODULE_PATHNAME', 'pgclone_schema_async' LANGUAGE C VOLATILE; + +-- PROGRESS & JOB MANAGEMENT +CREATE FUNCTION pgclone.progress(job_id INTEGER) RETURNS TEXT AS 'MODULE_PATHNAME', 'pgclone_progress' LANGUAGE C VOLATILE STRICT; +CREATE FUNCTION pgclone.cancel(job_id INTEGER) RETURNS TEXT AS 'MODULE_PATHNAME', 'pgclone_cancel' LANGUAGE C VOLATILE STRICT; +CREATE FUNCTION pgclone.resume(job_id INTEGER) RETURNS INTEGER AS 'MODULE_PATHNAME', 'pgclone_resume' LANGUAGE C VOLATILE STRICT; +CREATE FUNCTION pgclone.jobs() RETURNS TEXT AS 'MODULE_PATHNAME', 'pgclone_jobs' LANGUAGE C VOLATILE STRICT; +CREATE FUNCTION pgclone.clear_jobs() RETURNS INTEGER AS 'MODULE_PATHNAME', 'pgclone_clear_jobs' LANGUAGE C VOLATILE STRICT; +COMMENT ON FUNCTION pgclone.clear_jobs() IS 'Clear completed/failed/cancelled job slots from shared memory'; + +-- v2.1.0+v2.1.1+v2.1.2: Progress Tracking View with progress bar, elapsed time, ETA +CREATE FUNCTION pgclone.progress_detail() +RETURNS TABLE ( + job_id INTEGER, + status TEXT, + op_type TEXT, + schema_name TEXT, + table_name TEXT, + current_phase TEXT, + current_table TEXT, + tables_total BIGINT, + tables_completed BIGINT, + rows_copied BIGINT, + bytes_copied BIGINT, + elapsed_ms BIGINT, + start_time TIMESTAMPTZ, + end_time TIMESTAMPTZ, + error_message TEXT, + pct_complete DOUBLE PRECISION, + progress_bar TEXT, + elapsed_time TEXT +) AS 'MODULE_PATHNAME', 'pgclone_progress_view' +LANGUAGE C VOLATILE STRICT; + +COMMENT ON FUNCTION pgclone.progress_detail() IS 'Returns tabular progress with visual progress bar and elapsed time for all clone jobs'; + +-- VIEW: convenient wrapper +CREATE VIEW pgclone.jobs_view AS + SELECT * FROM pgclone.progress_detail(); + +COMMENT ON VIEW pgclone.jobs_view IS 'Live progress tracking view with progress bar and elapsed time for all pgclone async clone jobs'; + +-- v3.1.0: Auto-discovery of sensitive data +CREATE FUNCTION pgclone.discover_sensitive(source_conninfo TEXT, schema_name TEXT) +RETURNS TEXT AS 'MODULE_PATHNAME', 'pgclone_discover_sensitive' +LANGUAGE C VOLATILE STRICT; +COMMENT ON FUNCTION pgclone.discover_sensitive(TEXT, TEXT) IS 'Scan source schema for columns matching sensitive data patterns (email, name, phone, ssn, salary, etc.) and return suggested mask rules as JSON'; + +-- v3.2.0: Static data masking on local tables +CREATE FUNCTION pgclone.mask_in_place(schema_name TEXT, table_name TEXT, mask_json TEXT) +RETURNS TEXT AS 'MODULE_PATHNAME', 'pgclone_mask_in_place' +LANGUAGE C VOLATILE STRICT; +COMMENT ON FUNCTION pgclone.mask_in_place(TEXT, TEXT, TEXT) IS 'Apply data masking to an existing local table via UPDATE. mask_json uses same format as clone mask option: {"email": "email", "name": "name", "ssn": "null"}'; + +-- v3.3.0: Dynamic data masking via views and role-based access +CREATE FUNCTION pgclone.create_masking_policy(schema_name TEXT, table_name TEXT, mask_json TEXT, privileged_role TEXT) +RETURNS TEXT AS 'MODULE_PATHNAME', 'pgclone_create_masking_policy' +LANGUAGE C VOLATILE STRICT; +COMMENT ON FUNCTION pgclone.create_masking_policy(TEXT, TEXT, TEXT, TEXT) IS 'Create a dynamic masking policy: creates a masked view, revokes base table access from PUBLIC, grants view to PUBLIC, grants base table to privileged role'; + +CREATE FUNCTION pgclone.drop_masking_policy(schema_name TEXT, table_name TEXT) +RETURNS TEXT AS 'MODULE_PATHNAME', 'pgclone_drop_masking_policy' +LANGUAGE C VOLATILE STRICT; +COMMENT ON FUNCTION pgclone.drop_masking_policy(TEXT, TEXT) IS 'Remove a dynamic masking policy: drops the masked view and restores base table access to PUBLIC'; + +-- v3.4.0: Clone roles with permissions and passwords +CREATE FUNCTION pgclone.clone_roles(source_conninfo TEXT) +RETURNS TEXT AS 'MODULE_PATHNAME', 'pgclone_clone_roles' +LANGUAGE C VOLATILE STRICT; +COMMENT ON FUNCTION pgclone.clone_roles(TEXT) IS 'Clone all non-system roles from source with encrypted passwords, attributes, memberships, and all permissions. Requires superuser on both source and target.'; + +CREATE FUNCTION pgclone.clone_roles(source_conninfo TEXT, role_names TEXT) +RETURNS TEXT AS 'MODULE_PATHNAME', 'pgclone_clone_roles' +LANGUAGE C VOLATILE; +COMMENT ON FUNCTION pgclone.clone_roles(TEXT, TEXT) IS 'Clone specific roles (comma-separated) from source with encrypted passwords, attributes, memberships, and permissions. If role exists on target, syncs password and attributes without dropping.'; + +-- v3.5.0: Clone verification — compare row counts +CREATE FUNCTION pgclone.verify(source_conninfo TEXT, schema_name TEXT) +RETURNS TABLE ( + schema_name TEXT, + table_name TEXT, + source_rows BIGINT, + target_rows BIGINT, + match TEXT +) AS 'MODULE_PATHNAME', 'pgclone_verify' +LANGUAGE C VOLATILE STRICT; +COMMENT ON FUNCTION pgclone.verify(TEXT, TEXT) IS 'Compare row counts between source and local target for all tables in a schema. Returns side-by-side comparison with match status.'; + +CREATE FUNCTION pgclone.verify(source_conninfo TEXT) +RETURNS TABLE ( + schema_name TEXT, + table_name TEXT, + source_rows BIGINT, + target_rows BIGINT, + match TEXT +) AS 'MODULE_PATHNAME', 'pgclone_verify' +LANGUAGE C VOLATILE STRICT; +COMMENT ON FUNCTION pgclone.verify(TEXT) IS 'Compare row counts between source and local target for all user tables across all schemas. Returns side-by-side comparison with match status.'; + +-- v3.6.0: GDPR/Compliance masking report +CREATE FUNCTION pgclone.masking_report(schema_name TEXT) +RETURNS TABLE ( + schema_name TEXT, + table_name TEXT, + column_name TEXT, + sensitivity TEXT, + mask_status TEXT, + recommendation TEXT +) AS 'MODULE_PATHNAME', 'pgclone_masking_report' +LANGUAGE C VOLATILE STRICT; +COMMENT ON FUNCTION pgclone.masking_report(TEXT) IS 'Generate GDPR/compliance audit report: lists sensitive columns, their masking status, and recommendations. Checks for masked views.'; + +-- v4.1.0: Schema diff — DDL drift detection between source and local target +CREATE FUNCTION pgclone.diff(source_conninfo TEXT, schema_name TEXT) +RETURNS TEXT AS 'MODULE_PATHNAME', 'pgclone_diff' +LANGUAGE C VOLATILE STRICT; +COMMENT ON FUNCTION pgclone.diff(TEXT, TEXT) IS + 'Compare DDL of a schema between source and the local target. ' + 'Returns JSON drift report listing objects only_in_source / only_in_target / modified ' + 'across tables (with per-column type/nullability/default drift), indexes, ' + 'constraints, triggers, views, and sequences. Read-only on both sides.'; + +-- VERSION +CREATE FUNCTION pgclone.version() RETURNS TEXT AS 'MODULE_PATHNAME', 'pgclone_version' LANGUAGE C IMMUTABLE STRICT; diff --git a/src/pgclone.c b/src/pgclone.c index fe55fba..4e7a0f3 100644 --- a/src/pgclone.c +++ b/src/pgclone.c @@ -4082,7 +4082,7 @@ PG_FUNCTION_INFO_V1(pgclone_version); Datum pgclone_version(PG_FUNCTION_ARGS) { - PG_RETURN_TEXT_P(cstring_to_text("pgclone 4.0.1")); + PG_RETURN_TEXT_P(cstring_to_text("pgclone 4.1.0")); } /* =============================================================== diff --git a/src/pgclone_diff.c b/src/pgclone_diff.c new file mode 100644 index 0000000..578b927 --- /dev/null +++ b/src/pgclone_diff.c @@ -0,0 +1,925 @@ +/* + * pgclone_diff.c — Schema drift / DDL diff between source and local target. + * + * Implements pgclone.diff(source_conninfo, schema_name): a read-only + * comparison of catalog metadata. Returns a JSON document describing + * objects present in only one side and modified objects on both sides. + * + * Categories compared (per schema): + * - tables (and per-table column drift) + * - indexes (excluding those backing constraints) + * - constraints + * - triggers (user-defined only; AI/RI internal triggers excluded) + * - views and materialized views + * - sequences + * + * Both connections run in READ ONLY transactions; this function + * never executes DDL or DML on either side. + * + * Copyright (c) 2026, Valeh Agayev pgclone contributors + * Licensed under PostgreSQL License + */ + +#include "postgres.h" +#include "fmgr.h" +#include "funcapi.h" +#include "utils/builtins.h" +#include "utils/elog.h" +#include "libpq-fe.h" +#include "miscadmin.h" +#include "utils/guc.h" +#include "commands/dbcommands.h" + +/* --------------------------------------------------------------- + * Forward decls of local helpers borrowed in spirit from pgclone.c. + * We do NOT reuse symbols from pgclone.c — keeping this translation + * unit fully self-contained makes the feature trivially auditable + * and reversible. + * --------------------------------------------------------------- */ +static PGconn *pgclone_diff_connect_source(const char *conninfo); +static PGconn *pgclone_diff_connect_local(void); +static void pgclone_diff_normalize_session(PGconn *conn); +static void pgclone_diff_begin_readonly(PGconn *conn); +static void pgclone_diff_rollback(PGconn *conn); +static PGresult *pgclone_diff_select(PGconn *conn, const char *query); +static void pgclone_diff_validate_identifier(const char *ident, const char *what); +static void pgclone_diff_append_local_host(StringInfo conninfo); + +/* Per-category JSON emitters */ +static void diff_tables_and_columns(PGconn *src, PGconn *tgt, const char *schema, + StringInfo out, int *out_diff_count, + int *only_src, int *only_tgt, int *modified); +static void diff_named_objects(PGconn *src, PGconn *tgt, const char *category_label, + const char *query, StringInfo out, int *out_diff_count, + int *only_src, int *only_tgt, int *modified); +static void diff_sequences(PGconn *src, PGconn *tgt, const char *schema, + StringInfo out, int *out_diff_count, + int *only_src, int *only_tgt); + +/* =============================================================== + * Connection helpers (self-contained copy of the same logic in + * pgclone.c so this file can be compiled and reasoned about + * independently). All errors raise via ereport(ERROR, ...). + * =============================================================== */ + +static void +pgclone_diff_normalize_session(PGconn *conn) +{ + PGresult *res = PQexec(conn, "SET search_path = pg_catalog"); + if (PQresultStatus(res) != PGRES_COMMAND_OK) + ereport(WARNING, + (errmsg("pgclone.diff: could not set search_path: %s", + PQerrorMessage(conn)))); + PQclear(res); +} + +static void +pgclone_diff_begin_readonly(PGconn *conn) +{ + PGresult *res = PQexec(conn, + "BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY"); + if (PQresultStatus(res) != PGRES_COMMAND_OK) + { + char *msg = pstrdup(PQerrorMessage(conn)); + PQclear(res); + ereport(ERROR, + (errcode(ERRCODE_EXTERNAL_ROUTINE_EXCEPTION), + errmsg("pgclone.diff: could not begin read-only transaction: %s", msg))); + } + PQclear(res); +} + +static void +pgclone_diff_rollback(PGconn *conn) +{ + PGresult *res = PQexec(conn, "ROLLBACK"); + /* ROLLBACK shouldn't fail on a read-only tx; ignore status, just free. */ + if (res) + PQclear(res); +} + +static PGconn * +pgclone_diff_connect_source(const char *conninfo) +{ + PGconn *conn = PQconnectdb(conninfo); + if (PQstatus(conn) != CONNECTION_OK) + { + char *msg = pstrdup(PQerrorMessage(conn)); + PQfinish(conn); + ereport(ERROR, + (errcode(ERRCODE_CONNECTION_FAILURE), + errmsg("pgclone.diff: could not connect to source: %s", msg))); + } + pgclone_diff_normalize_session(conn); + pgclone_diff_begin_readonly(conn); + return conn; +} + +static void +pgclone_diff_append_local_host(StringInfo conninfo) +{ + const char *socket_dir = GetConfigOption("unix_socket_directories", false, false); + + if (socket_dir && socket_dir[0]) + { + char *first_dir = pstrdup(socket_dir); + char *comma = strchr(first_dir, ','); + int len; + + if (comma) + *comma = '\0'; + len = (int) strlen(first_dir); + while (len > 0 && first_dir[len - 1] == ' ') + first_dir[--len] = '\0'; + appendStringInfo(conninfo, "host=%s", first_dir); + pfree(first_dir); + } + else + { + appendStringInfoString(conninfo, "host=127.0.0.1"); + } +} + +static PGconn * +pgclone_diff_connect_local(void) +{ + PGconn *conn; + StringInfoData conninfo; + const char *dbname; + const char *port; + const char *username; + + dbname = get_database_name(MyDatabaseId); + port = GetConfigOption("port", false, false); + username = GetUserNameFromId(GetUserId(), false); + + initStringInfo(&conninfo); + pgclone_diff_append_local_host(&conninfo); + appendStringInfo(&conninfo, " dbname=%s port=%s user=%s", + quote_literal_cstr(dbname), + port ? port : "5432", + username); + + conn = PQconnectdb(conninfo.data); + pfree(conninfo.data); + + if (PQstatus(conn) != CONNECTION_OK) + { + char *msg = pstrdup(PQerrorMessage(conn)); + PQfinish(conn); + ereport(ERROR, + (errcode(ERRCODE_CONNECTION_FAILURE), + errmsg("pgclone.diff: could not connect to local database: %s", msg))); + } + + pgclone_diff_normalize_session(conn); + pgclone_diff_begin_readonly(conn); + return conn; +} + +static PGresult * +pgclone_diff_select(PGconn *conn, const char *query) +{ + PGresult *res = PQexec(conn, query); + if (PQresultStatus(res) != PGRES_TUPLES_OK) + { + char *msg = pstrdup(PQerrorMessage(conn)); + PQclear(res); + ereport(ERROR, + (errcode(ERRCODE_EXTERNAL_ROUTINE_EXCEPTION), + errmsg("pgclone.diff: catalog query failed: %s", msg))); + } + return res; +} + +/* --------------------------------------------------------------- + * Identifier validator. Defense-in-depth: even though we always + * use quote_literal_cstr() to embed the schema name in catalog + * filters, we still reject obviously hostile inputs (NULs, long + * strings, non-printables) up front. + * --------------------------------------------------------------- */ +static void +pgclone_diff_validate_identifier(const char *ident, const char *what) +{ + size_t len; + size_t i; + + if (ident == NULL) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("pgclone.diff: %s must not be NULL", what))); + + len = strlen(ident); + if (len == 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("pgclone.diff: %s must not be empty", what))); + if (len >= NAMEDATALEN) + ereport(ERROR, + (errcode(ERRCODE_NAME_TOO_LONG), + errmsg("pgclone.diff: %s exceeds %d bytes", + what, NAMEDATALEN - 1))); + + for (i = 0; i < len; i++) + { + unsigned char c = (unsigned char) ident[i]; + if (c < 0x20 || c == 0x7F) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("pgclone.diff: %s contains a control character", what))); + } +} + +/* --------------------------------------------------------------- + * JSON helpers. We ship our own RFC 8259 string escaper to avoid + * cross-version churn around utils/jsonapi.h vs utils/json.h + * (header location moved between PostgreSQL releases). The output + * is a JSON string literal — the caller passes a bare C string, + * we wrap it in double quotes and escape per spec. + * --------------------------------------------------------------- */ +static void +pgd_escape_json(StringInfo out, const char *s) +{ + const char *p; + + appendStringInfoChar(out, '"'); + for (p = s; *p != '\0'; p++) + { + unsigned char c = (unsigned char) *p; + switch (c) + { + case '"': appendStringInfoString(out, "\\\""); break; + case '\\': appendStringInfoString(out, "\\\\"); break; + case '\b': appendStringInfoString(out, "\\b"); break; + case '\f': appendStringInfoString(out, "\\f"); break; + case '\n': appendStringInfoString(out, "\\n"); break; + case '\r': appendStringInfoString(out, "\\r"); break; + case '\t': appendStringInfoString(out, "\\t"); break; + default: + if (c < 0x20) + appendStringInfo(out, "\\u%04x", c); + else + appendStringInfoChar(out, (char) c); + break; + } + } + appendStringInfoChar(out, '"'); +} + +/* + * For NULL-able values we emit the JSON literal `null` directly. + */ +static void +emit_json_str(StringInfo out, const char *s) +{ + if (s == NULL) + appendStringInfoString(out, "null"); + else + pgd_escape_json(out, s); +} + +static void +emit_json_pair_str(StringInfo out, const char *key, const char *val, bool first) +{ + if (!first) + appendStringInfoChar(out, ','); + appendStringInfoChar(out, '"'); + appendStringInfoString(out, key); + appendStringInfoString(out, "\":"); + emit_json_str(out, val); +} + +static void +emit_json_pair_bool(StringInfo out, const char *key, bool val, bool first) +{ + if (!first) + appendStringInfoChar(out, ','); + appendStringInfo(out, "\"%s\":%s", key, val ? "true" : "false"); +} + +static void +emit_json_pair_int(StringInfo out, const char *key, int val, bool first) +{ + if (!first) + appendStringInfoChar(out, ','); + appendStringInfo(out, "\"%s\":%d", key, val); +} + +/* --------------------------------------------------------------- + * Compare PG boolean text ('t'/'f') as bool. + * --------------------------------------------------------------- */ +static bool +pg_bool_value(const char *txt) +{ + return txt != NULL && txt[0] == 't'; +} + +/* --------------------------------------------------------------- + * Find the contiguous range [start, end) of rows whose first column + * equals the value at row `start`. Catalog queries ORDER BY that + * column COLLATE "C", guaranteeing all matching rows are adjacent. + * --------------------------------------------------------------- */ +static int +range_end(PGresult *res, int start) +{ + int n = PQntuples(res); + const char *k0 = PQgetvalue(res, start, 0); + int e = start + 1; + while (e < n && strcmp(PQgetvalue(res, e, 0), k0) == 0) + e++; + return e; +} + +/* =============================================================== + * Tables + per-table column drift. + * + * Catalog query (run identically against source and target): + * SELECT relname, attname, format_type(atttypid, atttypmod), + * attnotnull, pg_get_expr(adbin, adrelid) + * FROM pg_class c + * JOIN pg_namespace n ON n.oid = c.relnamespace + * JOIN pg_attribute a ON a.attrelid = c.oid + * LEFT JOIN pg_attrdef d ON d.adrelid = a.attrelid AND d.adnum = a.attnum + * WHERE n.nspname = $schema + * AND c.relkind IN ('r','p') -- regular + partitioned + * AND a.attnum > 0 AND NOT a.attisdropped + * ORDER BY relname COLLATE "C", attname COLLATE "C" + * + * Output JSON shape: + * "tables": { + * "only_in_source": ["t1", ...], + * "only_in_target": ["t2", ...], + * "modified": [ + * { "name":"t3", + * "columns_only_in_source": [{"name":"c","type":"int","not_null":false,"default":null}], + * "columns_only_in_target": [...], + * "columns_drift": [{"name":"c","source_type":"int","target_type":"bigint", + * "source_not_null":true,"target_not_null":true, + * "source_default":null,"target_default":"0"}] + * } + * ] + * } + * =============================================================== */ +static const char * +build_columns_query(StringInfo q, const char *schema) +{ + initStringInfo(q); + appendStringInfo(q, + "SELECT c.relname, a.attname, " + " format_type(a.atttypid, a.atttypmod), " + " a.attnotnull, " + " pg_get_expr(d.adbin, d.adrelid) " + "FROM pg_catalog.pg_class c " + "JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace " + "JOIN pg_catalog.pg_attribute a ON a.attrelid = c.oid " + "LEFT JOIN pg_catalog.pg_attrdef d " + " ON d.adrelid = a.attrelid AND d.adnum = a.attnum " + "WHERE n.nspname = %s " + "AND c.relkind IN ('r','p') " + "AND a.attnum > 0 AND NOT a.attisdropped " + "ORDER BY c.relname COLLATE \"C\", a.attname COLLATE \"C\"", + quote_literal_cstr(schema)); + return q->data; +} + +static void +emit_column_object(StringInfo out, PGresult *res, int row, bool first) +{ + if (!first) + appendStringInfoChar(out, ','); + appendStringInfoChar(out, '{'); + emit_json_pair_str (out, "name", PQgetvalue(res, row, 1), true); + emit_json_pair_str (out, "type", PQgetvalue(res, row, 2), false); + emit_json_pair_bool(out, "not_null", pg_bool_value(PQgetvalue(res, row, 3)), false); + if (PQgetisnull(res, row, 4)) + appendStringInfoString(out, ",\"default\":null"); + else + emit_json_pair_str(out, "default", PQgetvalue(res, row, 4), false); + appendStringInfoChar(out, '}'); +} + +static void +emit_drift_object(StringInfo out, PGresult *src, int si, PGresult *tgt, int ti, bool first) +{ + if (!first) + appendStringInfoChar(out, ','); + appendStringInfoChar(out, '{'); + emit_json_pair_str (out, "name", PQgetvalue(src, si, 1), true); + emit_json_pair_str (out, "source_type", PQgetvalue(src, si, 2), false); + emit_json_pair_str (out, "target_type", PQgetvalue(tgt, ti, 2), false); + emit_json_pair_bool(out, "source_not_null", pg_bool_value(PQgetvalue(src, si, 3)), false); + emit_json_pair_bool(out, "target_not_null", pg_bool_value(PQgetvalue(tgt, ti, 3)), false); + appendStringInfoString(out, ",\"source_default\":"); + if (PQgetisnull(src, si, 4)) appendStringInfoString(out, "null"); + else emit_json_str(out, PQgetvalue(src, si, 4)); + appendStringInfoString(out, ",\"target_default\":"); + if (PQgetisnull(tgt, ti, 4)) appendStringInfoString(out, "null"); + else emit_json_str(out, PQgetvalue(tgt, ti, 4)); + appendStringInfoChar(out, '}'); +} + +static bool +column_attrs_equal(PGresult *a, int ai, PGresult *b, int bi) +{ + /* type, not_null, default */ + if (strcmp(PQgetvalue(a, ai, 2), PQgetvalue(b, bi, 2)) != 0) return false; + if (pg_bool_value(PQgetvalue(a, ai, 3)) != pg_bool_value(PQgetvalue(b, bi, 3))) return false; + { + bool an = PQgetisnull(a, ai, 4); + bool bn = PQgetisnull(b, bi, 4); + if (an != bn) return false; + if (!an && strcmp(PQgetvalue(a, ai, 4), PQgetvalue(b, bi, 4)) != 0) return false; + } + return true; +} + +static void +emit_only_in_table(StringInfo out, PGresult *res, int s, int e, const char *json_key) +{ + bool first_col = true; + int i; + + appendStringInfo(out, "\"%s\":[", json_key); + for (i = s; i < e; i++) + { + emit_column_object(out, res, i, first_col); + first_col = false; + } + appendStringInfoChar(out, ']'); +} + +/* + * Compare two column ranges (one table on each side). Walk-merge by + * column name. Append a "modified" table entry if any drift is found. + * Returns true when the entry was appended. + */ +static bool +emit_modified_table(StringInfo out, const char *tbl, + PGresult *src, int s_a, int s_b, + PGresult *tgt, int t_a, int t_b, + bool first_modified) +{ + /* First collect into a sub-buffer so we can decide whether to emit. */ + StringInfoData only_src, only_tgt, drift; + int i = s_a, j = t_a; + bool any_diff = false; + bool first_only_src = true, first_only_tgt = true, first_drift = true; + int cnt_only_src = 0, cnt_only_tgt = 0, cnt_drift = 0; + + initStringInfo(&only_src); + initStringInfo(&only_tgt); + initStringInfo(&drift); + + while (i < s_b || j < t_b) + { + const char *sn = (i < s_b) ? PQgetvalue(src, i, 1) : NULL; + const char *tn = (j < t_b) ? PQgetvalue(tgt, j, 1) : NULL; + int cmp; + + if (sn == NULL) cmp = 1; + else if (tn == NULL) cmp = -1; + else cmp = strcmp(sn, tn); + + if (cmp < 0) + { + emit_column_object(&only_src, src, i, first_only_src); + first_only_src = false; cnt_only_src++; any_diff = true; + i++; + } + else if (cmp > 0) + { + emit_column_object(&only_tgt, tgt, j, first_only_tgt); + first_only_tgt = false; cnt_only_tgt++; any_diff = true; + j++; + } + else + { + if (!column_attrs_equal(src, i, tgt, j)) + { + emit_drift_object(&drift, src, i, tgt, j, first_drift); + first_drift = false; cnt_drift++; any_diff = true; + } + i++; j++; + } + } + + if (any_diff) + { + if (!first_modified) + appendStringInfoChar(out, ','); + appendStringInfoChar(out, '{'); + emit_json_pair_str(out, "name", tbl, true); + appendStringInfo(out, ",\"columns_only_in_source\":[%s]", + cnt_only_src ? only_src.data : ""); + appendStringInfo(out, ",\"columns_only_in_target\":[%s]", + cnt_only_tgt ? only_tgt.data : ""); + appendStringInfo(out, ",\"columns_drift\":[%s]", + cnt_drift ? drift.data : ""); + appendStringInfoChar(out, '}'); + } + + pfree(only_src.data); + pfree(only_tgt.data); + pfree(drift.data); + return any_diff; +} + +static void +diff_tables_and_columns(PGconn *src_conn, PGconn *tgt_conn, const char *schema, + StringInfo out, int *out_diff_count, + int *only_src, int *only_tgt, int *modified) +{ + StringInfoData q; + PGresult *src_res; + PGresult *tgt_res; + int i = 0, j = 0; + int ns, nt; + bool first_src = true, first_tgt = true, first_mod = true; + StringInfoData buf_src, buf_tgt, buf_mod; + + build_columns_query(&q, schema); + + src_res = pgclone_diff_select(src_conn, q.data); + tgt_res = pgclone_diff_select(tgt_conn, q.data); + pfree(q.data); + + ns = PQntuples(src_res); + nt = PQntuples(tgt_res); + + initStringInfo(&buf_src); + initStringInfo(&buf_tgt); + initStringInfo(&buf_mod); + + while (i < ns || j < nt) + { + const char *st = (i < ns) ? PQgetvalue(src_res, i, 0) : NULL; + const char *tt = (j < nt) ? PQgetvalue(tgt_res, j, 0) : NULL; + int cmp; + + if (st == NULL) cmp = 1; + else if (tt == NULL) cmp = -1; + else cmp = strcmp(st, tt); + + if (cmp < 0) + { + int e = range_end(src_res, i); + if (!first_src) appendStringInfoChar(&buf_src, ','); + emit_json_str(&buf_src, st); + first_src = false; (*only_src)++; (*out_diff_count)++; + i = e; + } + else if (cmp > 0) + { + int e = range_end(tgt_res, j); + if (!first_tgt) appendStringInfoChar(&buf_tgt, ','); + emit_json_str(&buf_tgt, tt); + first_tgt = false; (*only_tgt)++; (*out_diff_count)++; + j = e; + } + else + { + int s_b = range_end(src_res, i); + int t_b = range_end(tgt_res, j); + if (emit_modified_table(&buf_mod, st, src_res, i, s_b, tgt_res, j, t_b, first_mod)) + { + first_mod = false; (*modified)++; (*out_diff_count)++; + } + i = s_b; + j = t_b; + } + } + + appendStringInfoString(out, "\"tables\":{"); + appendStringInfo(out, "\"only_in_source\":[%s]", buf_src.data); + appendStringInfo(out, ",\"only_in_target\":[%s]", buf_tgt.data); + appendStringInfo(out, ",\"modified\":[%s]", buf_mod.data); + appendStringInfoChar(out, '}'); + + pfree(buf_src.data); + pfree(buf_tgt.data); + pfree(buf_mod.data); + PQclear(src_res); + PQclear(tgt_res); +} + +/* =============================================================== + * Generic single-key + def diff used for indexes / constraints / + * triggers / views. + * + * Caller passes a query producing rows with a STABLE sort order on + * column 0 (the key) and column 1 holding the canonical definition + * (e.g. pg_get_indexdef). Optional column 2 may contain a parent + * table name for human-readable output. + * + * The query string is identical for source and target. + * =============================================================== */ +static void +diff_named_objects(PGconn *src_conn, PGconn *tgt_conn, + const char *category_label, const char *query, + StringInfo out, int *out_diff_count, + int *only_src, int *only_tgt, int *modified) +{ + PGresult *src_res = pgclone_diff_select(src_conn, query); + PGresult *tgt_res = pgclone_diff_select(tgt_conn, query); + int ns = PQntuples(src_res); + int nt = PQntuples(tgt_res); + int i = 0, j = 0; + bool first_src = true, first_tgt = true, first_mod = true; + bool has_parent = (PQnfields(src_res) >= 3); + StringInfoData buf_src, buf_tgt, buf_mod; + + initStringInfo(&buf_src); + initStringInfo(&buf_tgt); + initStringInfo(&buf_mod); + + while (i < ns || j < nt) + { + const char *sk = (i < ns) ? PQgetvalue(src_res, i, 0) : NULL; + const char *tk = (j < nt) ? PQgetvalue(tgt_res, j, 0) : NULL; + int cmp; + + if (sk == NULL) cmp = 1; + else if (tk == NULL) cmp = -1; + else cmp = strcmp(sk, tk); + + if (cmp < 0) + { + if (!first_src) appendStringInfoChar(&buf_src, ','); + appendStringInfoChar(&buf_src, '{'); + emit_json_pair_str(&buf_src, "name", sk, true); + if (has_parent && !PQgetisnull(src_res, i, 2)) + emit_json_pair_str(&buf_src, "table", PQgetvalue(src_res, i, 2), false); + emit_json_pair_str(&buf_src, "def", PQgetvalue(src_res, i, 1), false); + appendStringInfoChar(&buf_src, '}'); + first_src = false; (*only_src)++; (*out_diff_count)++; + i++; + } + else if (cmp > 0) + { + if (!first_tgt) appendStringInfoChar(&buf_tgt, ','); + appendStringInfoChar(&buf_tgt, '{'); + emit_json_pair_str(&buf_tgt, "name", tk, true); + if (has_parent && !PQgetisnull(tgt_res, j, 2)) + emit_json_pair_str(&buf_tgt, "table", PQgetvalue(tgt_res, j, 2), false); + emit_json_pair_str(&buf_tgt, "def", PQgetvalue(tgt_res, j, 1), false); + appendStringInfoChar(&buf_tgt, '}'); + first_tgt = false; (*only_tgt)++; (*out_diff_count)++; + j++; + } + else + { + const char *sd = PQgetvalue(src_res, i, 1); + const char *td = PQgetvalue(tgt_res, j, 1); + if (strcmp(sd, td) != 0) + { + if (!first_mod) appendStringInfoChar(&buf_mod, ','); + appendStringInfoChar(&buf_mod, '{'); + emit_json_pair_str(&buf_mod, "name", sk, true); + if (has_parent && !PQgetisnull(src_res, i, 2)) + emit_json_pair_str(&buf_mod, "table", PQgetvalue(src_res, i, 2), false); + emit_json_pair_str(&buf_mod, "source_def", sd, false); + emit_json_pair_str(&buf_mod, "target_def", td, false); + appendStringInfoChar(&buf_mod, '}'); + first_mod = false; (*modified)++; (*out_diff_count)++; + } + i++; j++; + } + } + + appendStringInfo(out, "\"%s\":{", category_label); + appendStringInfo(out, "\"only_in_source\":[%s]", buf_src.data); + appendStringInfo(out, ",\"only_in_target\":[%s]", buf_tgt.data); + appendStringInfo(out, ",\"modified\":[%s]", buf_mod.data); + appendStringInfoChar(out, '}'); + + pfree(buf_src.data); + pfree(buf_tgt.data); + pfree(buf_mod.data); + PQclear(src_res); + PQclear(tgt_res); +} + +/* =============================================================== + * Sequences — single-key only (no def comparison in v4.1.0; the + * sequence name itself is the unit of drift). + * =============================================================== */ +static void +diff_sequences(PGconn *src_conn, PGconn *tgt_conn, const char *schema, + StringInfo out, int *out_diff_count, + int *only_src, int *only_tgt) +{ + StringInfoData q; + PGresult *src_res; + PGresult *tgt_res; + int ns, nt, i = 0, j = 0; + bool first_src = true, first_tgt = true; + StringInfoData buf_src, buf_tgt; + + initStringInfo(&q); + appendStringInfo(&q, + "SELECT c.relname FROM pg_catalog.pg_class c " + "JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace " + "WHERE n.nspname = %s AND c.relkind = 'S' " + "ORDER BY c.relname COLLATE \"C\"", + quote_literal_cstr(schema)); + + src_res = pgclone_diff_select(src_conn, q.data); + tgt_res = pgclone_diff_select(tgt_conn, q.data); + pfree(q.data); + + ns = PQntuples(src_res); + nt = PQntuples(tgt_res); + + initStringInfo(&buf_src); + initStringInfo(&buf_tgt); + + while (i < ns || j < nt) + { + const char *sk = (i < ns) ? PQgetvalue(src_res, i, 0) : NULL; + const char *tk = (j < nt) ? PQgetvalue(tgt_res, j, 0) : NULL; + int cmp; + if (sk == NULL) cmp = 1; + else if (tk == NULL) cmp = -1; + else cmp = strcmp(sk, tk); + + if (cmp < 0) + { + if (!first_src) appendStringInfoChar(&buf_src, ','); + emit_json_str(&buf_src, sk); + first_src = false; (*only_src)++; (*out_diff_count)++; + i++; + } + else if (cmp > 0) + { + if (!first_tgt) appendStringInfoChar(&buf_tgt, ','); + emit_json_str(&buf_tgt, tk); + first_tgt = false; (*only_tgt)++; (*out_diff_count)++; + j++; + } + else { i++; j++; } + } + + appendStringInfoString(out, "\"sequences\":{"); + appendStringInfo(out, "\"only_in_source\":[%s]", buf_src.data); + appendStringInfo(out, ",\"only_in_target\":[%s]", buf_tgt.data); + appendStringInfoChar(out, '}'); + + pfree(buf_src.data); + pfree(buf_tgt.data); + PQclear(src_res); + PQclear(tgt_res); +} + +/* =============================================================== + * FUNCTION: pgclone_diff(source_conninfo, schema_name) RETURNS TEXT + * + * Read-only. Returns a JSON document describing DDL drift between + * the named schema on source and the same-named schema on the local + * (target) database. + * =============================================================== */ +PG_FUNCTION_INFO_V1(pgclone_diff); + +Datum +pgclone_diff(PG_FUNCTION_ARGS) +{ + text *src_conninfo_t = PG_GETARG_TEXT_PP(0); + text *schema_t = PG_GETARG_TEXT_PP(1); + char *src_conninfo = text_to_cstring(src_conninfo_t); + char *schema = text_to_cstring(schema_t); + + PGconn *src = NULL; + PGconn *tgt = NULL; + StringInfoData out; + StringInfoData body; + int diff_count = 0; + int t_only_src = 0, t_only_tgt = 0, t_modified = 0; + int i_only_src = 0, i_only_tgt = 0, i_modified = 0; + int c_only_src = 0, c_only_tgt = 0, c_modified = 0; + int g_only_src = 0, g_only_tgt = 0, g_modified = 0; + int v_only_src = 0, v_only_tgt = 0, v_modified = 0; + int s_only_src = 0, s_only_tgt = 0; + + pgclone_diff_validate_identifier(schema, "schema_name"); + + initStringInfo(&out); + initStringInfo(&body); + + PG_TRY(); + { + StringInfoData q; + + src = pgclone_diff_connect_source(src_conninfo); + tgt = pgclone_diff_connect_local(); + + /* Tables + per-table column drift ------------------------- */ + diff_tables_and_columns(src, tgt, schema, &body, &diff_count, + &t_only_src, &t_only_tgt, &t_modified); + + /* Indexes (excluding those backing constraints) ----------- */ + initStringInfo(&q); + appendStringInfo(&q, + "SELECT i.relname, pg_get_indexdef(idx.indexrelid), c.relname " + "FROM pg_catalog.pg_index idx " + "JOIN pg_catalog.pg_class i ON i.oid = idx.indexrelid " + "JOIN pg_catalog.pg_class c ON c.oid = idx.indrelid " + "JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace " + "WHERE n.nspname = %s " + "AND c.relkind IN ('r','p','m') " + "AND NOT EXISTS (SELECT 1 FROM pg_catalog.pg_constraint con " + " WHERE con.conindid = idx.indexrelid) " + "ORDER BY i.relname COLLATE \"C\"", + quote_literal_cstr(schema)); + appendStringInfoChar(&body, ','); + diff_named_objects(src, tgt, "indexes", q.data, &body, &diff_count, + &i_only_src, &i_only_tgt, &i_modified); + pfree(q.data); + + /* Constraints --------------------------------------------- */ + initStringInfo(&q); + appendStringInfo(&q, + "SELECT con.conname, pg_get_constraintdef(con.oid), cl.relname " + "FROM pg_catalog.pg_constraint con " + "JOIN pg_catalog.pg_class cl ON cl.oid = con.conrelid " + "JOIN pg_catalog.pg_namespace n ON n.oid = cl.relnamespace " + "WHERE n.nspname = %s " + "ORDER BY con.conname COLLATE \"C\"", + quote_literal_cstr(schema)); + appendStringInfoChar(&body, ','); + diff_named_objects(src, tgt, "constraints", q.data, &body, &diff_count, + &c_only_src, &c_only_tgt, &c_modified); + pfree(q.data); + + /* Triggers (user-defined only) ---------------------------- */ + initStringInfo(&q); + appendStringInfo(&q, + "SELECT tg.tgname, pg_get_triggerdef(tg.oid), cl.relname " + "FROM pg_catalog.pg_trigger tg " + "JOIN pg_catalog.pg_class cl ON cl.oid = tg.tgrelid " + "JOIN pg_catalog.pg_namespace n ON n.oid = cl.relnamespace " + "WHERE n.nspname = %s AND NOT tg.tgisinternal " + "ORDER BY tg.tgname COLLATE \"C\"", + quote_literal_cstr(schema)); + appendStringInfoChar(&body, ','); + diff_named_objects(src, tgt, "triggers", q.data, &body, &diff_count, + &g_only_src, &g_only_tgt, &g_modified); + pfree(q.data); + + /* Views + materialized views ------------------------------ */ + initStringInfo(&q); + appendStringInfo(&q, + "SELECT c.relname, pg_get_viewdef(c.oid, true) " + "FROM pg_catalog.pg_class c " + "JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace " + "WHERE n.nspname = %s AND c.relkind IN ('v','m') " + "ORDER BY c.relname COLLATE \"C\"", + quote_literal_cstr(schema)); + appendStringInfoChar(&body, ','); + diff_named_objects(src, tgt, "views", q.data, &body, &diff_count, + &v_only_src, &v_only_tgt, &v_modified); + pfree(q.data); + + /* Sequences ----------------------------------------------- */ + appendStringInfoChar(&body, ','); + diff_sequences(src, tgt, schema, &body, &diff_count, + &s_only_src, &s_only_tgt); + + pgclone_diff_rollback(src); + pgclone_diff_rollback(tgt); + PQfinish(src); src = NULL; + PQfinish(tgt); tgt = NULL; + } + PG_CATCH(); + { + if (src) { pgclone_diff_rollback(src); PQfinish(src); } + if (tgt) { pgclone_diff_rollback(tgt); PQfinish(tgt); } + PG_RE_THROW(); + } + PG_END_TRY(); + + /* Assemble the final document. ------------------------------- */ + appendStringInfoChar(&out, '{'); + emit_json_pair_str (&out, "schema", schema, true); + emit_json_pair_bool(&out, "in_sync", diff_count == 0, false); + emit_json_pair_int (&out, "diff_count", diff_count, false); + appendStringInfoString(&out, ",\"summary\":{"); + appendStringInfo(&out, + "\"tables_only_in_source\":%d,\"tables_only_in_target\":%d,\"tables_modified\":%d," + "\"indexes_only_in_source\":%d,\"indexes_only_in_target\":%d,\"indexes_modified\":%d," + "\"constraints_only_in_source\":%d,\"constraints_only_in_target\":%d,\"constraints_modified\":%d," + "\"triggers_only_in_source\":%d,\"triggers_only_in_target\":%d,\"triggers_modified\":%d," + "\"views_only_in_source\":%d,\"views_only_in_target\":%d,\"views_modified\":%d," + "\"sequences_only_in_source\":%d,\"sequences_only_in_target\":%d", + t_only_src, t_only_tgt, t_modified, + i_only_src, i_only_tgt, i_modified, + c_only_src, c_only_tgt, c_modified, + g_only_src, g_only_tgt, g_modified, + v_only_src, v_only_tgt, v_modified, + s_only_src, s_only_tgt); + appendStringInfoChar(&out, '}'); + appendStringInfoChar(&out, ','); + appendStringInfoString(&out, body.data); + appendStringInfoChar(&out, '}'); + + pfree(body.data); + PG_RETURN_TEXT_P(cstring_to_text(out.data)); +} diff --git a/test/test_loopback.sh b/test/test_loopback.sh index 73c0598..3d29dbe 100755 --- a/test/test_loopback.sh +++ b/test/test_loopback.sh @@ -111,6 +111,54 @@ run_test "drop_masking_policy runs" "[ '$RESULT' != 'ERROR' ]" pg "DROP TABLE IF EXISTS test_schema.employees_ddm CASCADE;" || true +# ---- Schema diff (v4.1.0) ---- +echo "" +echo "---- Schema diff ----" + +# Function exists under pgclone schema +DIFF_EXISTS=$(pg "SELECT 1 FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace WHERE n.nspname = 'pgclone' AND p.proname = 'diff' AND pg_catalog.pg_get_function_arguments(p.oid) = 'source_conninfo text, schema_name text';" || echo "0") +run_test "pgclone.diff(text, text) is registered" "[ '$DIFF_EXISTS' = '1' ]" + +# Returns parseable JSON with the documented top-level keys +HAS_TABLES=$(pg "SELECT pgclone.diff('${SOURCE_CONNINFO}', 'test_schema')::jsonb ? 'tables';" || echo "f") +run_test "diff JSON contains 'tables' key" "[ '$HAS_TABLES' = 't' ]" + +HAS_INDEXES=$(pg "SELECT pgclone.diff('${SOURCE_CONNINFO}', 'test_schema')::jsonb ? 'indexes';" || echo "f") +run_test "diff JSON contains 'indexes' key" "[ '$HAS_INDEXES' = 't' ]" + +HAS_SUMMARY=$(pg "SELECT pgclone.diff('${SOURCE_CONNINFO}', 'test_schema')::jsonb ? 'summary';" || echo "f") +run_test "diff JSON contains 'summary' key" "[ '$HAS_SUMMARY' = 't' ]" + +# Schema name echoed back +SCHEMA_FIELD=$(pg "SELECT pgclone.diff('${SOURCE_CONNINFO}', 'test_schema')::jsonb ->> 'schema';" || echo "") +run_test "diff echoes the requested schema" "[ '$SCHEMA_FIELD' = 'test_schema' ]" + +# in_sync is a boolean +IN_SYNC=$(pg "SELECT jsonb_typeof(pgclone.diff('${SOURCE_CONNINFO}', 'test_schema')::jsonb -> 'in_sync');" || echo "") +run_test "diff in_sync is a boolean" "[ '$IN_SYNC' = 'boolean' ]" + +# Fabricate a target-only table; diff must surface it under tables.only_in_target +pg "DROP TABLE IF EXISTS test_schema.pgclone_diff_probe CASCADE;" || true +pg "CREATE TABLE test_schema.pgclone_diff_probe (id int);" || true +ONLY_TGT=$(pg "SELECT (pgclone.diff('${SOURCE_CONNINFO}', 'test_schema')::jsonb #> '{tables,only_in_target}') ? 'pgclone_diff_probe';" || echo "f") +run_test "diff detects fabricated target-only table" "[ '$ONLY_TGT' = 't' ]" + +# diff_count must reflect the fabricated drift (>= 1) +DC=$(pg "SELECT (pgclone.diff('${SOURCE_CONNINFO}', 'test_schema')::jsonb ->> 'diff_count')::int;" || echo "0") +run_test "diff_count is positive when drift exists" "[ '$DC' -ge 1 ]" + +pg "DROP TABLE IF EXISTS test_schema.pgclone_diff_probe CASCADE;" || true + +# Read-only invariant: diff must not change relation count on either side +COUNT_BEFORE=$(pg "SELECT count(*)::int FROM information_schema.tables WHERE table_schema = 'test_schema';" || echo "0") +pg "SELECT pgclone.diff('${SOURCE_CONNINFO}', 'test_schema');" >/dev/null || true +COUNT_AFTER=$(pg "SELECT count(*)::int FROM information_schema.tables WHERE table_schema = 'test_schema';" || echo "0") +run_test "diff does not modify target catalog" "[ '$COUNT_BEFORE' = '$COUNT_AFTER' ]" + +# NULL schema_name argument must error (function is STRICT — psql returns no row) +NULL_RES=$(pg "SELECT pgclone.diff('${SOURCE_CONNINFO}', NULL);" || echo "ERROR") +run_test "diff is STRICT (NULL arg yields no result)" "[ -z '$NULL_RES' -o '$NULL_RES' = 'ERROR' ]" + echo "" echo "============================================" echo "LOOPBACK TESTS: $PASS passed, $FAIL failed"