diff --git a/README.md b/README.md index 7a4fe69..cf2118a 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ The **Schema Change** page works on a saved live connection, not on the diagram 1. **Find the tables (optional).** You do not have to choose any. The catalog (names, columns, keys, estimated rows, sizes) is read in one round trip. The words of your request are matched against table, schema and column names, weighted by how rare each word is in *your* database, and the foreign-key neighbours of the best matches are added. The scope is shown with the reason for each table and you can add or remove tables. It matches names, not meanings: a request that calls `clients` "customers" finds nothing, and the plan then goes ahead with no table in scope. That is how a request for **new tables** works: the model is told the names of the existing tables (not their contents), creates the new ones with `create_table`, and any existing table its foreign keys point at is read afterwards so the keys are checked against that table's real columns. Names already taken in the database are refused. 2. **Give the planner facts.** Only the tables in scope are sent in full (the others by name): columns, keys, existing indexes with how often each was scanned (`pg_stat_user_indexes`), row estimates, data sizes, dependent views, and two aggregate statistics per column (NULL fraction, distinct-value estimate). Never rows, default expressions, index predicates or credentials. -3. **A model proposes operations; fixed rules write the SQL.** Each operation is checked against the real catalog (names exist, types are real, a foreign key points at a unique column, a NOT NULL column has a default when the table has rows, an index name is free, a default is accepted by PostgreSQL) and compiled into a runbook: `CREATE INDEX CONCURRENTLY`, foreign keys and checks `NOT VALID` then `VALIDATE`, `NOT NULL` through a validated `CHECK`, a `lock_timeout` on every step, batched committed backfills, and for a type change that would rewrite the table an expand/contract sequence (new column, trigger, batched copy, swap in one `DO` block, the old column dropped in a step that is commented out). Every step states its lock, the work it does, the size of the table and a risk class that follows from those three by a fixed rule, plus how to undo it where that is possible. The operations are `create_table`, `rename_table`, `drop_table`, `add_column`, `alter_column`, `drop_column`, `add_relationship`, `drop_relationship`, `add_index`, `drop_index`, `add_unique`, `add_check`, `drop_constraint`, `add_primary_key`, `create_enum` and `add_enum_value`. `create_enum` makes a type (a column can use it later in the same plan, with a default that is one of its labels) and `add_enum_value` adds a label, optionally before or after another: neither locks any table, and a label can never be removed again. An existing integer column can become an identity column (`alter_column` with `autoIncrement`): the sequence is started after the column's largest value in the same transaction, which needs an index on the column unless the table is small, and a serial column's old default is dropped in the same change. A drop or a type change is refused, naming what depends on the column or table (views, triggers, policies, generated columns, SQL-body functions, publications), exactly where PostgreSQL would refuse it without `CASCADE`. Renaming an existing table keeps the old name working as a view (with the table's grants copied; from PostgreSQL 15 the view runs with the caller's own rights, and before that a table with row-level security is refused) until an optional last step drops it. The lock levels and the "rewrites the table" predictions are checked against real servers from PostgreSQL 12 to 18 (`npm run test:schema-pg:versions`). +3. **A model proposes operations; fixed rules write the SQL.** Each operation is checked against the real catalog (names exist, types are real, a foreign key points at a unique column, a NOT NULL column has a default when the table has rows, an index name is free, a default is accepted by PostgreSQL) and compiled into a runbook: `CREATE INDEX CONCURRENTLY`, foreign keys and checks `NOT VALID` then `VALIDATE`, `NOT NULL` through a validated `CHECK`, a `lock_timeout` on every step, batched committed backfills, and for a type change that would rewrite the table an expand/contract sequence (new column, trigger, batched copy, swap in one `DO` block, the old column dropped in a step that is commented out). Every step states its lock, the work it does, the size of the table and a risk class that follows from those three by a fixed rule, plus how to undo it where that is possible. The operations are `create_table`, `rename_table`, `drop_table`, `add_column`, `alter_column`, `drop_column`, `add_relationship`, `drop_relationship`, `add_index`, `drop_index`, `add_unique`, `add_check`, `drop_constraint`, `add_primary_key`, `create_enum` and `add_enum_value`. `create_enum` makes a type (a column can use it later in the same plan, with a default that is one of its labels) and `add_enum_value` adds a label, optionally before or after another: neither locks any table, and a label can never be removed again. An existing integer column can become an identity column (`alter_column` with `autoIncrement`): the sequence is started after the column's largest value in the same transaction, which needs an index on the column unless the table is small, and a serial column's old default is dropped in the same change. A drop or a type change is refused, naming what depends on the column or table (views, triggers, policies, generated columns, SQL-body functions, publications), exactly where PostgreSQL would refuse it without `CASCADE`. Renaming a column of an existing table adds a second column with the new name, keeps the two in step in both directions with a trigger, copies the data in batches and the indexes, and leaves dropping the old column and the trigger to a last step that is not run by default (a column in a key, a foreign key or a check, or one a view uses, is refused: turn `compatColumns` off to rename it directly). Renaming an existing table keeps the old name working as a view (with the table's grants copied; from PostgreSQL 15 the view runs with the caller's own rights, and before that a table with row-level security is refused) until an optional last step drops it. The lock levels and the "rewrites the table" predictions are checked against real servers from PostgreSQL 12 to 18 (`npm run test:schema-pg:versions`). 4. **Check the database, then run it.** *Check the database first* reads, without writing, who holds or waits for the tables (found through `pg_locks`, so even a role that may not read other roles' sessions sees them), transactions open for 30 seconds or more, strong locks, prepared transactions, replicas that are behind, replication slots holding WAL, vacuum running on the tables, and the timeouts the role runs with; each check says "can't tell" instead of "ok" when it cannot read something. Free disk space cannot be read from a connection, so that is for you to check. *Run it here, one step at a time* then runs each step on its own in a SQL session of its own (no transaction, because `CONCURRENTLY` and the batched backfills need that): only the next step can run, the first failure stops the run, a step can be run again or stopped, and the most recent finished step can be undone with the statement the compiler wrote for it. A step that needs a strong lock on an existing table waits at most 2 seconds per attempt and tries up to 8 times, backing off between tries; if the table never frees up it stops with the sessions holding it and changes nothing. After the last step the catalog is read again and compared, table by table, with the state the compiler said it would leave (columns, types, NOT NULL, keys, unique and check constraints, foreign keys, indexes, views); every difference is listed. Optional clean-up steps (dropping an old column, a compatibility view) are never run for you. The runbook can also be opened in the SQL Editor with the transaction switch off, or copied. 5. **Health.** A read-only report from catalog queries: tables without a primary key, foreign keys without an index, duplicate, redundant, unused and invalid indexes, constraints that were never validated, wide tables, missing or stale statistics, tables with many dead rows, sequences past half their range. A model may be asked to order and explain the findings; it can only point at findings that exist. 6. **Governance.** The model is called through the OpenAI API with your key (see above; GoForge does not store it), an allow-list of model ids, optional redaction of schema, table, column, index and constraint names into placeholders before anything leaves the server (the answer is translated back), and an audit log of every request with exactly what was sent, every plan, every runbook opened and every settings change, exportable as CSV or JSON. The audit log records that a runbook was *opened*; what you run in the SQL Editor is not tracked. @@ -119,7 +119,7 @@ Limits worth knowing: the compiler's rules assume PostgreSQL 12 or newer; the si **What has and has not been checked.** -- *Checked, on real servers:* `npm run test:schema-pg:versions` runs 15 sections against PostgreSQL 12, 13, 14, 15, 16, 17 and 18, one container each, and all pass. They check the lock level each step takes against `pg_locks`; which type changes rewrite a table, by watching the table's file; every compiled scenario executed, with the end state read back from the catalog and compared with the compiler's prediction; the drop and type-change refusals against what PostgreSQL itself refuses; the pre-flight check and the lock retry with real sessions (a transaction held open, a queued `ACCESS EXCLUSIVE`, an unread replication slot, a role timeout, a role that cannot read other sessions); the table rename with real roles and grants, row-level security, undo and the contract step; and the probes of defaults and checks. +- *Checked, on real servers:* `npm run test:schema-pg:versions` runs 16 sections against PostgreSQL 12, 13, 14, 15, 16, 17 and 18, one container each, and all pass. They check the lock level each step takes against `pg_locks`; which type changes rewrite a table, by watching the table's file; every compiled scenario executed, with the end state read back from the catalog and compared with the compiler's prediction; the drop and type-change refusals against what PostgreSQL itself refuses; the pre-flight check and the lock retry with real sessions (a transaction held open, a queued `ACCESS EXCLUSIVE`, an unread replication slot, a role timeout, a role that cannot read other sessions); the table rename with real roles and grants, row-level security, undo and the contract step; and the probes of defaults and checks. - *Checked on small data only:* the test tables hold up to a few hundred thousand rows. Whether a step stays short on a table of many gigabytes, how long a validation or index build takes, and replica lag during a large backfill have not been measured. The size classes and risk classes are fixed rules, not measurements. - *Not run against a real server:* the index build of a partitioned table (the SQL is written and unit-tested; only the catalog reading of partitioned tables is checked on a server), replicas and logical replication (the pre-flight reads them, and the rules for them are unit-tested only), prepared transactions beyond the empty case, and extensions that change how types or indexes behave. - *Models:* the planner's answers are accepted by OpenAI's structured-output check (`npm run check:openai` in `server/`), and a few real requests were planned by `gpt-5.6-luna` and their runbooks read through by hand; there is no scored evaluation of how often a model proposes the right operations. The compiler checks every operation against the catalog, so a wrong proposal is refused or shown, but a plausible wrong one that PostgreSQL accepts is not caught: read the plan. @@ -130,7 +130,7 @@ How it is tested: npm run test:schema-pg:versions # every section on PostgreSQL 12 to 18 (starts and removes one Docker container per version) docker run -d --name goforge-schema -e POSTGRES_PASSWORD=schema -p 54330:5432 postgres:16 -c max_locks_per_transaction=1024 npm run test:schema-pg # every section against that one server -npm run test:schema-pg -- model # one section: drops, dependents, lockwait, preflight, model, rename, planner, health, locks, rewrites, compile, catalog, retrieval, facts, probe +npm run test:schema-pg -- model # one section: drops, dependents, lockwait, preflight, model, rename, columnrename, planner, health, locks, rewrites, compile, catalog, retrieval, facts, probe ``` The generated-database evaluation below (`scripts/schema-change-eval.ts`, `npm run eval:schema`) is not in this repository yet. diff --git a/scripts/schema-change-pg.ts b/scripts/schema-change-pg.ts index 1f0f2e3..11c33a7 100644 --- a/scripts/schema-change-pg.ts +++ b/scripts/schema-change-pg.ts @@ -324,6 +324,16 @@ const SCENARIOS: Scenario[] = [ checks.equal((await client.query("SELECT enumlabel AS l FROM pg_enum WHERE enumtypid = 'mood'::regtype ORDER BY enumsortorder")).rows.map((row) => row.l), ["sad", "meh", "ok", "happy"], "the label is where it was asked to be"); }, }, + { + name: "rename a column and keep the old one in step (add the new column, trigger, copy, index, NOT NULL)", + setup: "CREATE TABLE accounts (id bigserial PRIMARY KEY, email text NOT NULL DEFAULT 'none', age int, note text, pad text); INSERT INTO accounts (email, age, note, pad) SELECT 'u' || g || '@example.test', g % 90, 'n' || g, repeat('x', 40) FROM generate_series(1, 300000) g; CREATE INDEX accounts_email_idx ON accounts (email); ANALYZE accounts;", + tables: ["public.accounts"], + operations: [{ type: "alter_column", tableName: "accounts", columnName: "email", changes: { newName: "contact" } }], + async verify(client, checks) { + checks.equal((await client.query("SELECT attname AS n FROM pg_attribute WHERE attrelid = 'accounts'::regclass AND attname IN ('email', 'contact') AND NOT attisdropped ORDER BY attname")).rows.map((row) => row.n), ["contact", "email"], "both columns exist"); + checks.equal(Number((await one(client, "SELECT count(*)::int AS n FROM accounts WHERE contact IS DISTINCT FROM email"))?.n), 0, "every row has the same value in both"); + }, + }, { name: "refuses NOT NULL without a default on a table with rows", tables: ["public.orders"], @@ -1150,6 +1160,44 @@ async function renameSection(checks: Checks) { } } +async function columnRenameSection(checks: Checks) { + // A renamed column with the old one kept in step: both names work, in both directions, until the last step drops the old column. + await withDatabase(async (client) => { + await client.query("CREATE TABLE accounts (id bigserial PRIMARY KEY, email text NOT NULL DEFAULT 'none', age int, note text, pad text); INSERT INTO accounts (email, age, note, pad) SELECT 'u' || g || '@example.test', g % 90, 'n' || g, repeat('x', 40) FROM generate_series(1, 300000) g; CREATE INDEX accounts_email_idx ON accounts (email); ANALYZE accounts;"); + const { runbook } = await compileOn(client, parsePlan([{ type: "alter_column", tableName: "accounts", columnName: "email", changes: { newName: "contact" } }]), ["public.accounts"]); + checks.equal(runbook.findings.filter((finding) => finding.severity === "error"), [], "the rename compiles"); + const ran = await runRunbook(client, runbook); + checks.ok(ran.ok, "the steps that run by default run", ran.failure); + const value = async (sql: string) => (await client.query(sql)).rows[0] as Record; + + checks.equal(await value("SELECT count(*)::int AS n FROM accounts WHERE contact IS DISTINCT FROM email"), { n: 0 }, "existing rows were copied"); + checks.equal(await value("SELECT attnotnull AS n FROM pg_attribute WHERE attrelid = 'accounts'::regclass AND attname = 'contact'"), { n: true }, "the new column is NOT NULL, like the old one"); + checks.equal(await value("SELECT indisvalid AS v FROM pg_index WHERE indexrelid = 'accounts_contact_idx'::regclass"), { v: true }, "the index was copied and is valid"); + + checks.equal(await value("INSERT INTO accounts (email) VALUES ('old@example.test') RETURNING contact"), { contact: "old@example.test" }, "an insert through the old name reaches the new column"); + checks.equal(await value("INSERT INTO accounts (contact) VALUES ('new@example.test') RETURNING email"), { email: "new@example.test" }, "an insert through the new name reaches the old column"); + checks.equal(await value("INSERT INTO accounts (note) VALUES ('neither') RETURNING email, contact"), { email: "none", contact: "none" }, "an insert that gives neither gets the old default in both"); + checks.equal(await value("UPDATE accounts SET email = 'changed-old@example.test' WHERE id = 1 RETURNING contact"), { contact: "changed-old@example.test" }, "an update through the old name reaches the new column"); + checks.equal(await value("UPDATE accounts SET contact = 'changed-new@example.test' WHERE id = 2 RETURNING email"), { email: "changed-new@example.test" }, "an update through the new name reaches the old column"); + checks.equal(await value("UPDATE accounts SET age = 7 WHERE id = 3 RETURNING (email = contact) AS same"), { same: true }, "an update of another column leaves them equal"); + await client.query("INSERT INTO accounts (email, contact) VALUES ('a@example.test', 'b@example.test')").then(() => undefined, () => undefined); + checks.equal(await value("SELECT contact AS c FROM accounts WHERE email = 'b@example.test' OR contact = 'b@example.test' LIMIT 1"), { c: "b@example.test" }, "an insert that gives both keeps the new name's value"); + + const actual = await loadFacts(asRunner(client), runbook.expected.tables.map((table) => table.qualified)); + const mismatches = diffExpected(runbook.expected, actual); + checks.ok(mismatches.length === 0, "the catalog is as the compiler said", mismatches.map((mismatch) => mismatch.message)); + + const contract = runbook.steps.find((step) => step.optional)!; + const done = await runRunbook(client, { ...runbook, steps: [contract] }, { includeOptional: true }); + checks.ok(done.ok, "the last step runs", done.failure); + checks.equal(await value("SELECT count(*)::int AS n FROM pg_attribute WHERE attrelid = 'accounts'::regclass AND attname = 'email' AND NOT attisdropped"), { n: 0 }, "the old column is gone"); + checks.equal(await value("SELECT count(*)::int AS n FROM pg_trigger WHERE tgrelid = 'accounts'::regclass AND NOT tgisinternal"), { n: 0 }, "and so is the trigger"); + checks.equal(await value("SELECT count(*)::int AS n FROM pg_proc WHERE proname LIKE 'accounts_email_contact__sync'"), { n: 0 }, "and its function"); + checks.equal(await value("INSERT INTO accounts (note) VALUES ('after') RETURNING contact"), { contact: "none" }, "the new column has the old default"); + checks.equal(await value("SELECT indisvalid AS v FROM pg_index WHERE indexrelid = 'accounts_contact_idx'::regclass"), { v: true }, "and the copied index stays"); + }); +} + const sections: Record = { async drops(checks) { await dropsSection(checks); @@ -1175,6 +1223,10 @@ const sections: Record = { await renameSection(checks); }, + async columnrename(checks) { + await columnRenameSection(checks); + }, + async planner(checks) { await plannerSection(checks); diff --git a/server/src/schema-change/column-rename.test.ts b/server/src/schema-change/column-rename.test.ts new file mode 100644 index 0000000..1b64c10 --- /dev/null +++ b/server/src/schema-change/column-rename.test.ts @@ -0,0 +1,83 @@ +import { describe, expect, it } from "vitest"; +import { compile, facts, MIB, plan, table } from "./test-fixtures.js"; +import type { Runbook } from "./runbook.js"; + +const orders = (overrides: Partial[0]> = {}) => + table({ + name: "orders", rows: 5_000_000, bytes: 400 * MIB, + columns: ["id:bigint:notnull", "status:character varying(20):notnull", "note:text"], + indexes: [{ name: "orders_status_idx", columns: ["status"], scans: 9 }, { name: "orders_open_idx", columns: ["status"], partial: true }], + ...overrides, + }); +const rename = (column: string, newName: string, extra: Record = {}, tables = [orders()], options: Parameters[2] = {}) => + compile(plan({ type: "alter_column", tableName: "orders", columnName: column, changes: { newName, ...extra } }), facts(...tables), options); +const errors = (runbook: Runbook) => runbook.findings.filter((finding) => finding.severity === "error").map((finding) => finding.code); + +describe("renaming a column and keeping the old name working", () => { + it("adds the new column, keeps the two in step, copies the data and the index, and leaves the old column for a last step", () => { + const runbook = rename("status", "state"); + expect(errors(runbook)).toEqual([]); + expect(runbook.steps.filter((step) => !step.optional).map((step) => step.title)).toEqual([ + "Expand: add state (character varying(20)) next to public.orders.status", + "Expand: keep state and status in step", + "Expand: copy status into state", + "Check that state matches status", + "Expand: build the copy of index orders_status_idx on state", + "Add a NOT VALID check that public.orders.state is not null", + "Validate that no row of public.orders.state is null", + "Set public.orders.state NOT NULL", + "Drop the helper check on public.orders.state", + ]); + const sync = { ...runbook.steps[1], sql: runbook.steps[1].sql.replace(/\s+/g, " ") }; + expect(sync.sql).toContain("CREATE OR REPLACE FUNCTION public.orders_status_state__sync()"); + expect(sync.sql).toContain("IF NEW.state IS NOT NULL THEN NEW.status := NEW.state; ELSE NEW.state := NEW.status; END IF;"); + expect(sync.sql).toContain("ELSIF NEW.status IS DISTINCT FROM OLD.status THEN NEW.state := NEW.status;"); + expect(sync.sql).toContain("ELSIF NEW.state IS DISTINCT FROM OLD.state THEN NEW.status := NEW.state;"); + expect(sync.sql).toContain("CREATE TRIGGER orders_status_state__sync BEFORE INSERT OR UPDATE OF status, state ON public.orders FOR EACH ROW EXECUTE FUNCTION public.orders_status_state__sync();"); + expect(runbook.steps.some((step) => step.sql.includes("CREATE INDEX CONCURRENTLY orders_state_idx ON public.orders (state);"))).toBe(true); + const contract = runbook.steps.find((step) => step.optional)!; + expect(contract).toMatchObject({ destructive: true, breaksClients: true }); + expect(contract.sql).toContain("DROP TRIGGER orders_status_state__sync ON public.orders;"); + expect(contract.sql).toContain("ALTER TABLE public.orders DROP COLUMN status;"); + expect(runbook.steps.every((step) => !step.breaksClients || step.optional)).toBe(true); + expect(runbook.findings.map((finding) => finding.code)).toContain("index-not-copied"); + }); + + it("expects both columns and the copied index after the steps that run by default", () => { + const expected = rename("status", "state").expected.tables[0]; + expect(expected.columns).toEqual(expect.arrayContaining([{ name: "status", type: "character varying(20)", notNull: true }, { name: "state", type: "character varying(20)", notNull: true }])); + expect(expected.indexes.map((index) => index.name)).toEqual(expect.arrayContaining(["orders_status_idx", "orders_state_idx"])); + }); + + it("gives the new column the old default only in the last step", () => { + const withDefault = orders(); + withDefault.columns[1].default = "'new'::character varying"; + const runbook = rename("status", "state", {}, [withDefault]); + expect(runbook.steps.filter((step) => !step.optional).some((step) => step.sql.includes("SET DEFAULT"))).toBe(false); + expect(runbook.steps.find((step) => step.optional)?.sql).toContain("ALTER TABLE public.orders ALTER COLUMN state SET DEFAULT 'new'::character varying;"); + }); + + it("renames directly when compatColumns is off, and for a table the plan creates", () => { + const direct = rename("status", "state", {}, [orders()], { settings: { compatColumns: false } }); + expect(direct.steps).toHaveLength(1); + expect(direct.steps[0]).toMatchObject({ breaksClients: true }); + expect(direct.steps[0].sql).toContain("RENAME COLUMN status TO state;"); + const created = compile(plan({ type: "create_table", table: { name: "audit", fields: [{ name: "id", dataType: "bigint", primaryKey: true }, { name: "kind", dataType: "text" }] } }, { type: "alter_column", tableName: "audit", columnName: "kind", changes: { newName: "category" } }), facts()); + expect(errors(created)).toEqual([]); + expect(created.steps.some((step) => step.sql.includes("RENAME COLUMN kind TO category;"))).toBe(true); + }); + + it("refuses a column in a key, a foreign key or a check, one a view uses, a generated one, a partitioned table, other changes in the same operation and a name that is taken", () => { + expect(errors(rename("id", "order_id"))).toEqual(["unsupported"]); + const fk = orders({ foreignKeys: [{ name: "orders_note_fk", columns: ["note"], ref: "notes", refColumns: ["id"] }] }); + expect(errors(rename("note", "memo", {}, [fk, table({ name: "notes", columns: ["id:text:notnull"] })]))).toEqual(["unsupported"]); + expect(errors(rename("status", "state", {}, [orders({ checks: [{ name: "orders_status_ck", definition: "CHECK ((status <> ''))", columns: ["status"] }] })]))).toEqual(["unsupported"]); + expect(errors(rename("status", "state", {}, [orders({ dependents: [{ column: "status", kind: "view", description: "view public.open_orders", external: true }] })]))).toEqual(["dependents"]); + const generated = orders(); + generated.columns[2].generated = "stored"; + expect(errors(rename("note", "memo", {}, [generated]))).toEqual(["unsupported"]); + expect(errors(rename("status", "state", {}, [orders({ partitions: ["public.orders_1"] })]))).toEqual(["unsupported"]); + expect(errors(rename("status", "state", { nullable: true }))).toEqual(["unsupported"]); + expect(errors(rename("status", "note"))).toEqual(["column-exists"]); + }); +}); diff --git a/server/src/schema-change/compile.ts b/server/src/schema-change/compile.ts index 6435e1f..cf2c8b0 100644 --- a/server/src/schema-change/compile.ts +++ b/server/src/schema-change/compile.ts @@ -887,24 +887,31 @@ class Compiler { if (changes.newName !== undefined && changes.newName !== column.name) { if (this.hasColumn(table, changes.newName)) this.fail("column-exists", `Column "${table.qualified}.${changes.newName}" already exists.`, table.qualified); - this.add({ - title: `Rename ${label} to ${changes.newName}`, - sql: `ALTER TABLE ${target} RENAME COLUMN ${ref} TO ${ident(changes.newName)};`, - mode: "short", - lock: "ACCESS EXCLUSIVE", - work: "metadata", - table, - breaksClients: true, - notes: ["The old name stops working immediately. Queries, functions and application code that use it will fail until they are changed.", "Indexes, constraints and views follow the column."], - undo: `ALTER TABLE ${target} RENAME COLUMN ${ident(changes.newName)} TO ${ref};`, - }); - for (const index of table.indexes) { - index.columns = index.columns.map((name) => (name === column.name ? changes.newName ?? name : name)); - index.uses = index.uses.map((name) => (name === column.name ? changes.newName ?? name : name)); + if (this.settings.compatColumns && !table.isNew) { + const others = Object.entries(changes).filter(([key, value]) => key !== "newName" && value !== undefined); + if (others.length > 0) this.fail("unsupported", `${label} is renamed with a second column kept in step, which cannot also change ${others.map(([key]) => key).join(", ")} in the same operation. Rename it in a change of its own, or turn compatColumns off to rename it directly.`, table.qualified); + this.renameColumnKeepingBoth(table, column, changes.newName); + changed = true; + } else { + this.add({ + title: `Rename ${label} to ${changes.newName}`, + sql: `ALTER TABLE ${target} RENAME COLUMN ${ref} TO ${ident(changes.newName)};`, + mode: "short", + lock: "ACCESS EXCLUSIVE", + work: "metadata", + table, + breaksClients: !table.isNew, + notes: [...(table.isNew ? [] : ["The old name stops working immediately. Queries, functions and application code that use it will fail until they are changed."]), "Indexes, constraints and views follow the column."], + undo: `ALTER TABLE ${target} RENAME COLUMN ${ident(changes.newName)} TO ${ref};`, + }); + for (const index of table.indexes) { + index.columns = index.columns.map((name) => (name === column.name ? changes.newName ?? name : name)); + index.uses = index.uses.map((name) => (name === column.name ? changes.newName ?? name : name)); + } + table.primaryKey = table.primaryKey.map((name) => (name === column.name ? changes.newName ?? name : name)); + column.name = changes.newName; + changed = true; } - table.primaryKey = table.primaryKey.map((name) => (name === column.name ? changes.newName ?? name : name)); - column.name = changes.newName; - changed = true; } if (!changed && !this.findings.some((finding) => finding.operation === this.current)) { @@ -958,6 +965,109 @@ class Compiler { column.generated = "default-identity"; } + /** + * Renaming a column for applications that cannot all change at the same moment: a second column with the new name is added next to the old + * one and kept in step with it in both directions by a trigger, filled in batches, indexed like the old one, and made NOT NULL if it is. Both names + * work from then on, applications switch when they are ready, and a last step (not run by default) drops the old column and the trigger. + */ + private renameColumnKeepingBoth(table: TableFacts, column: ColumnFacts, newName: string): void { + const label = `${table.qualified}.${column.name}`; + const target = qualifiedIdent(table.schema, table.name); + const old = ident(column.name); + const fresh = ident(newName); + this.checkNameLength(newName, "Column"); + if (table.partitioned) this.fail("unsupported", `${table.qualified} is partitioned; keeping a renamed column in step with a trigger is not compiled for it. Turn compatColumns off to rename the column directly.`, table.qualified); + if (column.generated !== null) this.fail("unsupported", `${label} is a ${column.generated.includes("identity") ? "identity" : "generated"} column; a copy of it cannot be kept in step. Turn compatColumns off to rename it directly.`, table.qualified); + const inKey = table.primaryKey.includes(column.name) || table.uniques.some((unique) => unique.columns.includes(column.name)); + if (inKey) this.fail("unsupported", `${label} is part of a key or a unique constraint. Keeping a copy in step means changing the key and every foreign key that points at it; turn compatColumns off to rename it directly (the constraint follows the column).`, table.qualified); + const checks = table.checks.filter((check) => check.columns.includes(column.name)); + const foreign = [...table.foreignKeys.filter((fk) => fk.columns.includes(column.name)), ...table.referencedBy.filter((fk) => fk.refColumns.includes(column.name))]; + if (foreign.length > 0 || checks.length > 0) { + this.fail("unsupported", `${label} appears in ${foreign.length ? `foreign key ${foreign.map((fk) => fk.name).join(", ")}` : `check ${checks.map((check) => check.name).join(", ")}`}. A copy of it would not carry that; turn compatColumns off to rename it directly (it follows the column).`, table.qualified); + } + const blockers = column.dependents.filter((dependent) => dependent.blocks); + if (blockers.length > 0) this.fail("dependents", `${label} is used by ${this.describeDependents(blockers)}, which would keep reading the old column. Change them first, or turn compatColumns off to rename the column directly (they follow it).`, table.qualified); + + const fn = limitName(`${table.name}_${column.name}_${newName}__sync`, this.names(table)); + const fnName = qualifiedIdent(table.schema, fn); + const copy = (index: IndexFacts) => { + const renamed = index.name.includes(column.name) ? index.name.replace(column.name, newName) : `${index.name}_${newName}`; + return limitName(renamed === index.name ? `${index.name}_${newName}` : renamed, this.names(table)); + }; + + this.add({ + title: `Expand: add ${newName} (${column.type}) next to ${label}`, + sql: `ALTER TABLE ${target} ADD COLUMN ${fresh} ${column.type};`, + mode: "short", + lock: "ACCESS EXCLUSIVE", + work: "metadata", + table, + notes: [ + `Instead of renaming ${column.name}, a column called ${newName} is added with the same type. Applications keep using ${column.name} until they switch.`, + "Collation, storage settings, statistics targets, comments and column-level grants of the old column are not copied.", + ], + undo: `ALTER TABLE ${target} DROP COLUMN ${fresh};`, + }); + this.add({ + title: `Expand: keep ${newName} and ${column.name} in step`, + sql: `CREATE OR REPLACE FUNCTION ${fnName}() RETURNS trigger LANGUAGE plpgsql AS $sync$\nBEGIN\n IF TG_OP = 'INSERT' THEN\n IF NEW.${fresh} IS NOT NULL THEN NEW.${old} := NEW.${fresh}; ELSE NEW.${fresh} := NEW.${old}; END IF;\n ELSIF NEW.${old} IS DISTINCT FROM OLD.${old} THEN\n NEW.${fresh} := NEW.${old};\n ELSIF NEW.${fresh} IS DISTINCT FROM OLD.${fresh} THEN\n NEW.${old} := NEW.${fresh};\n END IF;\n RETURN NEW;\nEND\n$sync$;\nCREATE TRIGGER ${ident(fn)} BEFORE INSERT OR UPDATE OF ${old}, ${fresh} ON ${target} FOR EACH ROW EXECUTE FUNCTION ${fnName}();`, + mode: "short", + lock: "SHARE ROW EXCLUSIVE", + work: "metadata", + table, + notes: [ + `Writes to ${column.name} reach ${newName} and writes to ${newName} reach ${column.name}. On an insert that gives both, ${newName} wins; on an update that changes both, ${column.name} wins.`, + "The trigger runs for every insert and update of those columns until the last step drops it.", + ], + undo: `DROP TRIGGER ${ident(fn)} ON ${target};\nDROP FUNCTION ${fnName}();`, + }); + this.backfillStep(table, `Expand: copy ${column.name} into ${newName}`, `${fresh} = ${old}`, `${fresh} IS DISTINCT FROM ${old}`); + this.add({ + title: `Check that ${newName} matches ${column.name}`, + sql: `SELECT count(*) AS rows_that_differ FROM ${target} WHERE ${fresh} IS DISTINCT FROM ${old};`, + mode: "long", + lock: "ACCESS SHARE", + work: "scan", + table, + notes: ["Reads the whole table without blocking anything. Expect 0 before you continue."], + }); + + const copied: IndexFacts[] = []; + for (const index of table.indexes.filter((candidate) => candidate.uses.includes(column.name) && !candidate.constraint)) { + if (index.partial || index.columns.some((c) => c === null) || index.method !== "btree" || !index.columns.includes(column.name)) { + this.note("warning", "index-not-copied", `Index ${index.name} on ${label} is partial, an expression index or not a btree index, so it was not copied to ${newName}. Create it on ${newName} yourself.`, table.qualified); + continue; + } + const name = copy(index); + this.claimRelation(table.schema, name, "Index"); + const columns = index.columns.map((c) => (c === column.name ? newName : (c as string))); + this.concurrentIndexStep(table, name, columns, index.unique, `Expand: build the copy of index ${index.name} on ${newName}`); + copied.push({ ...index, name, definition: "", columns, uses: index.uses.map((used) => (used === column.name ? newName : used)), constraint: null }); + } + if (column.notNull) this.notNullSteps(table, newName, { skipUpdate: true }); + + this.add({ + title: `Contract: drop the old column ${column.name} and the sync trigger`, + sql: `DROP TRIGGER ${ident(fn)} ON ${target};\nDROP FUNCTION ${fnName}();${column.default ? `\nALTER TABLE ${target} ALTER COLUMN ${fresh} SET DEFAULT ${column.default};` : ""}\nALTER TABLE ${target} DROP COLUMN ${old};`, + mode: "short", + lock: "ACCESS EXCLUSIVE", + work: "metadata", + table, + optional: true, + destructive: true, + breaksClients: true, + notes: [ + `Run this once nothing reads or writes ${column.name} any more. From then on only ${newName} exists, and anything that still uses the old name fails.`, + ...(column.default ? [`${newName} gets the default of ${column.name} here (not before: two columns with a volatile default would each draw their own value).`] : []), + `The indexes on ${column.name} are dropped with it; the copies on ${newName} stay.`, + ], + }); + + // the model after the steps that run by default: both columns, the copies of the indexes, and the old column's indexes still on it + table.columns.push({ name: newName, type: column.type, typeSchema: column.typeSchema, notNull: column.notNull, default: null, generated: null, dependents: [], nullFraction: column.nullFraction, distinct: column.distinct }); + table.indexes.push(...copied); + } + /** ALTER COLUMN ... TYPE: in place when PostgreSQL only has to change the catalog, otherwise the expand/contract sequence. */ private typeChange(table: TableFacts, column: ColumnFacts, to: string): void { const target = qualifiedIdent(table.schema, table.name); diff --git a/server/src/schema-change/runbook.ts b/server/src/schema-change/runbook.ts index 1e41ede..3b7000b 100644 --- a/server/src/schema-change/runbook.ts +++ b/server/src/schema-change/runbook.ts @@ -137,12 +137,14 @@ export interface CompileSettings { lockAttempts: number; /** Renaming an existing table leaves a view under the old name, so applications keep working until they have switched. */ compatViews: boolean; + /** Renaming a column of an existing table adds the new column next to the old one, keeps the two in step and leaves the old one for a last step, so applications keep working until they have switched. */ + compatColumns: boolean; backfillBatchRows: number; backfillPauseMs: number; defaultSchema: string; } -export const DEFAULT_COMPILE_SETTINGS: CompileSettings = { lockTimeoutSeconds: 2, lockAttempts: 8, compatViews: true, backfillBatchRows: 10_000, backfillPauseMs: 50, defaultSchema: "public" }; +export const DEFAULT_COMPILE_SETTINGS: CompileSettings = { lockTimeoutSeconds: 2, lockAttempts: 8, compatViews: true, compatColumns: true, backfillBatchRows: 10_000, backfillPauseMs: 50, defaultSchema: "public" }; const statementLines = (sql: string): string[] => sql.replace(/\s+$/, "").split("\n"); diff --git a/src/lib/schema-change-api.ts b/src/lib/schema-change-api.ts index 9b19af1..25d846d 100644 --- a/src/lib/schema-change-api.ts +++ b/src/lib/schema-change-api.ts @@ -115,7 +115,7 @@ export interface Runbook { script: string; uncompiled: number[]; server: { version: string; collectedAt: string }; - settings: { lockTimeoutSeconds: number; lockAttempts: number; compatViews: boolean; backfillBatchRows: number; backfillPauseMs: number; defaultSchema: string }; + settings: { lockTimeoutSeconds: number; lockAttempts: number; compatViews: boolean; compatColumns: boolean; backfillBatchRows: number; backfillPauseMs: number; defaultSchema: string }; /** What the tables should look like once the steps that run by default have run, for checking the database afterwards. */ expected: ExpectedState; }