From ab2dbf82c38e6e1d35fa868549cdd43a2ab97090 Mon Sep 17 00:00:00 2001 From: swsarancodes Date: Sun, 25 Oct 2026 17:32:08 +0530 Subject: [PATCH] A measured evaluation of the Schema Change pipeline on generated databases, and a stand-in model server scripts/schema-change-eval.ts builds PostgreSQL databases of 50, 300 and 1,000 tables with real foreign keys and populated tables, writes change requests in the words a person would use (typos, partial names, a column name only), and measures table selection, plan checks against the real catalog, lint, execution of the runbook statement by statement and the result, refusal of invalid requests, and the latency of every stage that does not wait for a model. The plan is the reference plan unless --live is given, so nothing in the numbers says anything about a model. scripts/lib/synthetic-schema.ts generates the schemas (with a test, which vitest now picks up from scripts/). scripts/fake-model-server.ts is a stand-in for the OpenAI Responses endpoint to try the page without a key. npm run eval:schema. --- README.md | 3 +- package.json | 1 + scripts/fake-model-server.ts | 155 +++++++ scripts/lib/synthetic-schema.test.ts | 58 +++ scripts/lib/synthetic-schema.ts | 270 +++++++++++ scripts/schema-change-eval.ts | 667 +++++++++++++++++++++++++++ vitest.config.ts | 2 +- 7 files changed, 1154 insertions(+), 2 deletions(-) create mode 100644 scripts/fake-model-server.ts create mode 100644 scripts/lib/synthetic-schema.test.ts create mode 100644 scripts/lib/synthetic-schema.ts create mode 100644 scripts/schema-change-eval.ts diff --git a/README.md b/README.md index cf2118a..f492052 100644 --- a/README.md +++ b/README.md @@ -131,9 +131,10 @@ npm run test:schema-pg:versions # every section on Post docker run -d --name goforge-schema -e POSTGRES_PASSWORD=schema -p 54330:5432 postgres:16 -c max_locks_per_transaction=1024 npm run test:schema-pg # every section against that one server npm run test:schema-pg -- model # one section: drops, dependents, lockwait, preflight, model, rename, columnrename, planner, health, locks, rewrites, compile, catalog, retrieval, facts, probe +npm run eval:schema -- --sizes 50 --per-size 14 # a quick run of the generated-database evaluation ``` -The generated-database evaluation below (`scripts/schema-change-eval.ts`, `npm run eval:schema`) is not in this repository yet. +`npm run eval:schema` measures table selection, plan checks, lint and execution on generated databases of 50, 300 and 1,000 tables (it uses the server in `SCHEMA_CHANGE_DATABASE_URL`, default the container above; `-- --sizes 50 --per-size 14` is a quick run): `npm run eval:schema` builds the databases (real foreign keys, tables with 260,000 rows), writes change requests in the words a person would use (with typos, partial names, a column name only, the same name in two schemas), runs each through retrieval, the checks and the compiler, executes the runbook on a copy of the database and verifies the result. It also feeds in requests that must be refused. Without `--live` the plans are the reference plans, so the plan-validity numbers measure the checks and the compiler, not a model; the model's own latency and token use are reported as not measured. `scripts/fake-model-server.ts` is a stand-in OpenAI endpoint for trying the page without a key. diff --git a/package.json b/package.json index f8f78e5..d8f124b 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "test:batch": "server/node_modules/.bin/tsx scripts/modernizer-batch.ts", "test:schema-pg": "server/node_modules/.bin/tsx scripts/schema-change-pg.ts", "test:schema-pg:versions": "bash scripts/schema-change-versions.sh", + "eval:schema": "server/node_modules/.bin/tsx scripts/schema-change-eval.ts", "queryshift": "server/node_modules/.bin/tsx queryshift/run.ts", "queryshift:check-model": "server/node_modules/.bin/tsx queryshift/check-model.ts" }, diff --git a/scripts/fake-model-server.ts b/scripts/fake-model-server.ts new file mode 100644 index 0000000..1e86460 --- /dev/null +++ b/scripts/fake-model-server.ts @@ -0,0 +1,155 @@ +/** + * A stand-in for the OpenAI Responses endpoint, to try the Schema Change page without an OpenAI key or a model: + * + * server/node_modules/.bin/tsx scripts/fake-model-server.ts # listens on 127.0.0.1:4010 + * then start the API with + * OPENAI_BASE_URL=http://127.0.0.1:4010/v1 OPENAI_API_KEY=fake OPENAI_MODEL=fake-model npm --prefix server run dev + * + * The API sends its real requests to it, so the whole path from the page to the wire is the one a real model would use. + * It does no thinking. For a plan request it reads the tables it was shown and proposes a few operations that exist on them (a nullable column, + * an index, NOT NULL on a nullable column, a wider integer), so the page has something real to compile. It reports no token usage, which makes + * the page show its own estimate. Nothing here says anything about what a real model would answer. + */ +import http from "node:http"; + +const PORT = Number(process.env.PORT ?? 4010); + +interface Table { + name: string; + rows: number | null; + columns: { name: string; type: string; notNull: boolean }[]; + primaryKey: string[]; + indexes: { name: string; columns: string[] }[]; +} + +/** "add a testimonials table ..." / "create a new table for reviews": a create_table, linked to the first existing table it was told about. */ +function createTablePlan(input: { request: string; tables: Table[]; otherTables?: string[] }) { + const named = input.request.match(/\b([a-z][a-z0-9_]*)\s+table\b/i)?.[1] ?? input.request.match(/\btable\s+(?:for|called|named)\s+([a-z][a-z0-9_]*)/i)?.[1] ?? "new_table"; + const name = ["a", "new", "the", "another"].includes(named.toLowerCase()) ? "new_table" : named.toLowerCase(); + const parent = input.tables[0]?.name ?? input.otherTables?.[0]; + const parentBare = parent?.split(".").pop() ?? ""; + const operations: unknown[] = [ + { + type: "create_table", + table: { + name: parent?.includes(".") ? `${parent.split(".")[0]}.${name}` : name, + fields: [ + { name: "id", dataType: "bigint", primaryKey: true, autoIncrement: true }, + ...(parent ? [{ name: `${parentBare.replace(/s$/, "")}_id`, dataType: "bigint", nullable: false }] : []), + { name: "body", dataType: "text" }, + { name: "created_at", dataType: "timestamptz", nullable: false, defaultValue: "now()" }, + ], + }, + }, + ]; + if (parent) operations.push({ type: "add_relationship", relationship: { sourceTable: parent, sourceColumn: "id", targetTable: parent.includes(".") ? `${parent.split(".")[0]}.${name}` : name, targetColumn: `${parentBare.replace(/s$/, "")}_id` } }); + return { summary: `Create ${name}${parent ? `, linked to ${parent}` : ""} (a stand-in answer, not a model's).`, assumptions: [], warnings: [], clarifyingQuestions: [], operations }; +} + +/** "remove the default from ..." : the first column the facts say has a default, with the default dropped (written as strict mode makes a model write it). */ +function dropDefaultPlan(input: { tables: (Table & { columns: { name: string; hasDefault?: boolean }[] })[] }) { + for (const table of input.tables) { + const column = table.columns.find((candidate) => candidate.hasDefault); + if (column) return { summary: `Remove the default of ${table.name}.${column.name} (a stand-in answer, not a model's).`, assumptions: [], warnings: [], clarifyingQuestions: [], operations: [{ type: "alter_column", tableName: table.name, columnName: column.name, changes: { dropDefault: true } }] }; + } + return { summary: "No column has a default.", assumptions: [], warnings: [], clarifyingQuestions: ["Which column should lose its default?"], operations: [] }; +} + +function planFor(input: { request: string; tables: Table[]; otherTables?: string[] }) { + if (/\bdefault\b/i.test(input.request) && /\b(remove|drop|delete)\b/i.test(input.request)) return dropDefaultPlan(input as never); + if (/\b(create|add|new)\b[^.]*\btable\b/i.test(input.request)) return createTablePlan(input); + // the biggest table it was shown that has an integer column, else the biggest one + const tables = [...input.tables].sort((a, b) => (b.rows ?? 0) - (a.rows ?? 0) || Number(b.columns.some((c) => c.type === "integer")) - Number(a.columns.some((c) => c.type === "integer")) || a.name.localeCompare(b.name)); + const operations: unknown[] = []; + const first = tables[0]; + if (first) { + const taken = new Set(first.columns.map((column) => column.name)); + if (!taken.has("reviewed_at")) operations.push({ type: "add_column", tableName: first.name, field: { name: "reviewed_at", dataType: "timestamptz" } }); + const candidate = first.columns.find((column) => !first.primaryKey.includes(column.name) && !first.indexes.some((index) => index.columns[0] === column.name) && !column.type.startsWith("text")); + if (candidate) operations.push({ type: "add_index", tableName: first.name, index: { name: `fake_${first.name.replace(/[^a-z0-9_]/gi, "_").slice(-30)}_${candidate.name}_idx`, columns: [candidate.name] } }); + const nullable = first.columns.find((column) => !column.notNull && !first.primaryKey.includes(column.name) && /(_id|status|code)$/.test(column.name)); + if (nullable) operations.push({ type: "alter_column", tableName: first.name, columnName: nullable.name, changes: { nullable: false } }); + const integer = first.columns.find((column) => column.type === "integer"); + if (integer) operations.push({ type: "alter_column", tableName: first.name, columnName: integer.name, changes: { dataType: "bigint" } }); + } + return { summary: `Proposed ${operations.length} change${operations.length === 1 ? "" : "s"} to ${first?.name ?? "nothing"} (a stand-in answer, not a model's).`, assumptions: [], warnings: [], clarifyingQuestions: operations.length === 0 ? ["Which table should change?"] : [], operations }; +} + +function summaryFor(input: { findings: { id: string; category: string; table: string | null; object: string | null; bytes: number | null }[] }) { + const sorted = [...input.findings].sort((a, b) => (b.bytes ?? 0) - (a.bytes ?? 0)).slice(0, 4); + return { + headline: `${input.findings.length} findings were shown to this stand-in; it ordered them by size.`, + priorities: sorted.map((finding) => ({ findingIds: [finding.id], why: `${finding.category} on ${finding.table ?? finding.object ?? "the database"}${finding.bytes ? ` involves ${finding.bytes} bytes` : ""}.` })), + }; +} + +/** A value that satisfies a JSON schema, for a prompt this stand-in has no answer for: placeholder text, true, 0, empty lists, null where allowed. */ +function placeholderFor(schema: any): unknown { + if (!schema || typeof schema !== "object") return null; + if (Array.isArray(schema.anyOf)) return placeholderFor(schema.anyOf.find((candidate: any) => candidate.type !== "null") ?? schema.anyOf[0]); + if (schema.const !== undefined) return schema.const; + if (Array.isArray(schema.enum)) return schema.enum[0]; + if (schema.type === "object") return Object.fromEntries(Object.entries(schema.properties ?? {}).map(([key, child]) => [key, (child as any)?.nullable ? null : placeholderFor(child)])); + if (schema.type === "array") return []; + if (schema.type === "string") return "check"; + if (schema.type === "boolean") return true; + return 0; +} + +/** + * What a model that was given a strict JSON schema would send: every key of every object present, the ones the answer does not use as + * null. This stand-in fills its (loose) answers out the same way, so the page is tried with the shape a real model is held to. + */ +function completeToSchema(value: unknown, schema: any): unknown { + if (!schema || typeof schema !== "object") return value; + if (Array.isArray(schema.anyOf)) { + const record = value && typeof value === "object" ? (value as Record) : undefined; + const variant = schema.anyOf.find((candidate: any) => candidate.properties?.type?.const !== undefined && candidate.properties.type.const === record?.type) ?? schema.anyOf.find((candidate: any) => candidate.type !== "null"); + return value === null || value === undefined ? null : completeToSchema(value, variant); + } + if (schema.type === "array") return Array.isArray(value) ? value.map((item) => completeToSchema(item, schema.items)) : value; + if (schema.type === "object" && schema.properties) { + const record = value && typeof value === "object" ? (value as Record) : {}; + return Object.fromEntries(Object.entries(schema.properties).map(([key, child]) => [key, key in record && record[key] !== undefined ? completeToSchema(record[key], child) : null])); + } + return value; +} + +http + .createServer((request, response) => { + const chunks: Buffer[] = []; + request.on("data", (chunk: Buffer) => chunks.push(chunk)); + request.on("end", () => { + const reply = (status: number, body: unknown) => { + response.writeHead(status, { "Content-Type": "application/json" }); + response.end(JSON.stringify(body)); + }; + if (request.method === "GET" && request.url === "/v1/models") return reply(200, { object: "list", data: [{ id: "fake-model", object: "model", owned_by: "stand-in" }] }); + if (request.method !== "POST" || request.url !== "/v1/responses") return reply(404, { error: { message: "only POST /v1/responses and GET /v1/models", type: "invalid_request_error", code: null } }); + try { + const body = JSON.parse(Buffer.concat(chunks).toString("utf8")) as { instructions?: string; input?: string; text?: { format?: { type?: string; schema?: unknown } } }; + const system = body.instructions ?? ""; + const user = body.input ?? ""; + // Anything it has no answer for is a placeholder that satisfies the schema it was given (or {"ok": true} when it was given none). + const requested = body.text?.format; + let answer: unknown = requested?.type === "json_schema" ? placeholderFor(requested.schema) : { ok: true }; + if (system.startsWith("You are GoForge's schema change planner")) answer = planFor(JSON.parse(user)); + else if (system.startsWith("You are GoForge's database schema planning engine")) { + // the diagram's AI Schema panel: the same kind of plan, from a differently shaped request + const asked = JSON.parse(user) as { userRequest: string; currentSchema: { tables: { name: string }[] } }; + answer = createTablePlan({ request: asked.userRequest, tables: asked.currentSchema.tables as never }); + } else if (system.startsWith("You summarise a PostgreSQL schema health report")) answer = summaryFor(JSON.parse(user)); + else if (system.startsWith("You are GoForge's SQL migration engine")) { + const asked = JSON.parse(user) as { originalQuery: string; ruleBasedDraft?: { sql?: string } | null }; + answer = { sql: asked.ruleBasedDraft?.sql ?? asked.originalQuery, summary: "Kept the rules' draft (a stand-in answer, not a model's).", changes: [], risks: [], unsupported: [] }; + } + // With a schema the answer is held to it, as it would be by OpenAI; without one it is left as it is. + const format = body.text?.format; + if (format?.type === "json_schema") answer = completeToSchema(answer, format.schema); + reply(200, { id: "resp_stand_in", status: "completed", output: [{ type: "message", role: "assistant", content: [{ type: "output_text", text: JSON.stringify(answer) }] }] }); + } catch (error) { + reply(400, { error: { message: error instanceof Error ? error.message : String(error), type: "invalid_request_error", code: null } }); + } + }); + }) + .listen(PORT, "127.0.0.1", () => console.log(`fake OpenAI endpoint on http://127.0.0.1:${PORT}/v1`)); diff --git a/scripts/lib/synthetic-schema.test.ts b/scripts/lib/synthetic-schema.test.ts new file mode 100644 index 0000000..821698e --- /dev/null +++ b/scripts/lib/synthetic-schema.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from "vitest"; +import { buildSearchIndex, selectScope } from "../../server/src/schema-change/retrieval"; +import type { Catalog } from "../../server/src/schema-change/catalog"; +import { BIG_TABLE_ROWS, generateSchema } from "./synthetic-schema"; + +const asCatalog = (schema: ReturnType): Catalog => ({ + serverVersionNum: 160000, + serverVersion: "16.0", + loadedAt: "2026-01-01T00:00:00Z", + tables: schema.tables.map((table) => ({ + schema: table.schema, name: table.name, qualified: table.qualified, partitioned: false, partitionCount: 0, estimatedRows: table.rows, totalBytes: 0, + columns: table.columns.map((column) => ({ name: column.name, type: column.type, notNull: column.notNull })), primaryKey: ["id"], + foreignKeys: table.foreignKeys.map((fk) => ({ name: `${table.name}_${fk.column}_fkey`, columns: [fk.column], refSchema: fk.references.split(".")[0], refTable: fk.references.split(".")[1], refColumns: ["id"], validated: true })), + })), +}); + +describe("the generated schemas the evaluation runs on", () => { + it("have exactly the number of tables asked for, with unique qualified names, and are the same every time", () => { + for (const size of [50, 300, 1000]) { + const schema = generateSchema(size, 5); + expect(schema.tables).toHaveLength(size); + expect(new Set(schema.tables.map((t) => t.qualified)).size).toBe(size); + } + expect(generateSchema(300, 5).createSql).toEqual(generateSchema(300, 5).createSql); + expect(generateSchema(300, 5).createSql).not.toEqual(generateSchema(300, 6).createSql); + }); + + it("only refer to tables and columns that exist, and populate children of populated parents", () => { + const schema = generateSchema(300, 9); + const byName = new Map(schema.tables.map((t) => [t.qualified, t])); + for (const table of schema.tables) { + const names = table.columns.map((c) => c.name); + expect(new Set(names).size).toBe(names.length); + for (const fk of table.foreignKeys) { + expect(names).toContain(fk.column); + expect(byName.has(fk.references)).toBe(true); + } + for (const index of table.indexes) for (const column of index.columns) expect(names).toContain(column); + for (const column of table.columns) if (column.refersTo) expect(byName.has(column.refersTo)).toBe(true); + } + const populated = schema.tables.filter((t) => t.rows > 0 && t.role !== "hub"); + expect(populated.length).toBeGreaterThan(0); + for (const table of populated) expect(table.rows).toBe(BIG_TABLE_ROWS); + expect(schema.populateSql.length).toBe(schema.tables.filter((t) => t.rows > 0).length); + }); + + it("give retrieval something real to find: every table is found by its own words", () => { + const schema = generateSchema(300, 3); + const index = buildSearchIndex(asCatalog(schema)); + let found = 0; + const roots = schema.tables.filter((t) => t.role === "root"); + for (const table of roots) { + const phrase = [table.module, ...table.words].map((w) => w.replace(/_/g, " ")).join(" "); + if (selectScope(index, `change the ${phrase} table`).tables.some((t) => t.qualified === table.qualified)) found++; + } + expect(found / roots.length).toBeGreaterThan(0.9); + }); +}); diff --git a/scripts/lib/synthetic-schema.ts b/scripts/lib/synthetic-schema.ts new file mode 100644 index 0000000..66475d2 --- /dev/null +++ b/scripts/lib/synthetic-schema.ts @@ -0,0 +1,270 @@ +/** + * Generates a database of N tables that looks like a grown-up business schema: modules (some as PostgreSQL schemas, some as + * name prefixes in `public`), root entities with child, history and note tables, shared lookup tables, foreign keys to them, + * domain columns, a few legacy columns that point somewhere without a foreign key, and a few extra indexes. Deterministic + * for a given (tableCount, seed). Used by the evaluation and by the demo database in the browser check. + */ + +export interface SyntheticColumn { + name: string; + type: string; + notNull: boolean; + /** SQL that fills the column when the table is populated, with `g` as the row number. Undefined: left NULL (or an identity). */ + fill?: string; + /** For a column that points at another table's id without a foreign key. */ + refersTo?: string; +} + +export interface SyntheticTable { + schema: string; + name: string; + qualified: string; + /** The words a person would use for it: ["invoice", "items"]. */ + words: string[]; + module: string; + entity: string; + role: "hub" | "root" | "child" | "history" | "notes"; + parent?: string; + columns: SyntheticColumn[]; + foreignKeys: { column: string; references: string }[]; + indexes: { name: string; columns: string[] }[]; + /** Rows to insert when the database is populated (0: left empty). */ + rows: number; +} + +export interface SyntheticSchema { + tables: SyntheticTable[]; + seed: number; + /** Creation statements in dependency order. */ + createSql: string[]; + /** Foreign keys, to be added after the data is loaded. */ + foreignKeySql: string[]; + /** Statements that load rows into the populated tables. */ + populateSql: string[]; +} + +function rng(seed: number) { + let state = seed >>> 0; + const next = () => { + state = (state + 0x6d2b79f5) >>> 0; + let t = state; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; + return { + next, + int: (max: number) => Math.floor(next() * max), + pick: (list: readonly T[]): T => list[Math.floor(next() * list.length)], + chance: (p: number) => next() < p, + shuffle: (list: readonly T[]): T[] => { + const copy = [...list]; + for (let i = copy.length - 1; i > 0; i--) { + const j = Math.floor(next() * (i + 1)); + [copy[i], copy[j]] = [copy[j], copy[i]]; + } + return copy; + }, + }; +} + +const MODULES = ["sales", "billing", "inventory", "procurement", "logistics", "support", "marketing", "finance", "hr", "payroll", "projects", "compliance", "analytics", "identity", "catalog", "pricing", "fulfillment", "returns", "contracts", "assets", "maintenance", "quality", "research", "training", "facilities", "legal", "security", "partners", "subscriptions", "notifications"]; + +const ENTITIES = ["order", "invoice", "payment", "shipment", "customer", "supplier", "product", "warehouse", "ticket", "campaign", "lead", "contract", "employee", "timesheet", "expense", "budget", "project", "task", "milestone", "audit", "policy", "incident", "asset", "vehicle", "route", "delivery", "refund", "quote", "subscription", "plan", "coupon", "review", "survey", "document", "approval", "schedule", "shift", "claim", "license", "certificate", "vendor", "requisition", "receipt", "ledger", "journal", "forecast", "report", "alert", "template", "contact", "account", "address", "session", "device", "event", "booking", "reservation", "inspection", "batch", "recipe", "component", "sample", "experiment", "trial", "filing", "dispute", "settlement", "benefit", "leave", "candidate", "interview", "offer", "course", "enrollment", "visit", "appointment", "pallet", "container", "manifest", "tariff", "rebate", "discount", "bundle", "listing", "bid", "pledge", "donation", "grant", "tenant", "lease", "meter", "reading", "outage", "permit", "recall", "warranty", "voucher", "payout", "shipping_label", "pick_list", "stock_count"]; + +const DERIVED = [ + { suffix: "items", role: "child" as const }, + { suffix: "history", role: "history" as const }, + { suffix: "notes", role: "notes" as const }, + { suffix: "attachments", role: "child" as const }, + { suffix: "tags", role: "child" as const }, + { suffix: "revisions", role: "history" as const }, +]; + +const DOMAIN_COLUMNS: { name: string; type: string; fill?: string }[] = [ + { name: "warehouse_code", type: "varchar(20)", fill: "'W' || (g % 40)" }, + { name: "shipped_at", type: "timestamptz" }, + { name: "delivered_at", type: "timestamptz" }, + { name: "invoice_no", type: "varchar(30)", fill: "'INV-' || g" }, + { name: "sku", type: "varchar(40)", fill: "'SKU-' || (g % 5000)" }, + { name: "region", type: "varchar(40)", fill: "'region-' || (g % 12)" }, + { name: "priority", type: "smallint", fill: "g % 5" }, + { name: "currency_code", type: "char(3)", fill: "'USD'" }, + { name: "tax_rate", type: "numeric(5,2)", fill: "(g % 25)::numeric" }, + { name: "discount_pct", type: "numeric(5,2)", fill: "(g % 30)::numeric" }, + { name: "quantity", type: "integer", fill: "1 + g % 50" }, + { name: "unit_price", type: "numeric(12,2)", fill: "(g % 9000)::numeric / 10" }, + { name: "total_amount", type: "numeric(14,2)", fill: "(g % 90000)::numeric / 10" }, + { name: "due_date", type: "date", fill: "current_date + (g % 90)" }, + { name: "approved_at", type: "timestamptz" }, + { name: "cost_center", type: "varchar(20)", fill: "'CC' || (g % 90)" }, + { name: "external_ref", type: "varchar(64)", fill: "md5(g::text)" }, + { name: "channel", type: "varchar(30)", fill: "'channel-' || (g % 6)" }, + { name: "weight_kg", type: "numeric(8,3)", fill: "(g % 5000)::numeric / 7" }, + { name: "language", type: "varchar(8)", fill: "'en'" }, + { name: "is_archived", type: "boolean", fill: "g % 20 = 0" }, + { name: "severity", type: "smallint", fill: "g % 4" }, + { name: "resolution", type: "text" }, + { name: "contact_email", type: "varchar(254)", fill: "'user' || g || '@example.test'" }, + { name: "phone", type: "varchar(32)", fill: "'+1555' || (1000000 + g % 8999999)" }, + { name: "latitude", type: "double precision", fill: "(g % 180) - 90" }, + { name: "longitude", type: "double precision", fill: "(g % 360) - 180" }, + { name: "batch_no", type: "varchar(24)", fill: "'B' || (g % 700)" }, + { name: "serial_no", type: "varchar(40)", fill: "'S' || g" }, + { name: "expires_on", type: "date", fill: "current_date + (g % 400)" }, + { name: "score", type: "integer", fill: "g % 100" }, + { name: "rating", type: "smallint", fill: "g % 6" }, + { name: "retry_count", type: "integer", fill: "g % 4" }, + { name: "checksum", type: "varchar(64)", fill: "md5(g::text)" }, +]; + +const SIGNATURE_WORDS = ["carrier", "courier", "depot", "dock", "lane", "yard", "bay", "gate", "pier", "terminal", "locker", "rack", "shelf", "bin", "zone", "ledger_page", "cohort", "stratum", "tranche", "cadence", "sprint", "epoch", "lot", "wave", "tier", "band", "slot", "grade", "variant", "revision_tag", "channel_key", "partner_key", "origin_code", "target_code"]; + +const HUBS: { name: string; extra: SyntheticColumn[] }[] = [ + { name: "users", extra: [{ name: "email", type: "varchar(254)", notNull: true, fill: "'u' || g || '@example.test'" }, { name: "display_name", type: "varchar(120)", notNull: false, fill: "'User ' || g" }] }, + { name: "organizations", extra: [{ name: "legal_name", type: "varchar(200)", notNull: true, fill: "'Org ' || g" }] }, + { name: "countries", extra: [{ name: "iso_code", type: "char(2)", notNull: true, fill: "'C' || chr(65 + g % 26)" }] }, + { name: "currencies", extra: [{ name: "iso_code", type: "char(3)", notNull: true, fill: "'X' || chr(65 + g % 26) || chr(65 + (g / 26) % 26)" }] }, +]; + +const HUB_ROWS = 2000; + +/** Rows loaded into the populated ("big") tables; enough that the data is above the "small" limit (16 MiB) used to size a change. */ +export const BIG_TABLE_ROWS = 260_000; + +const quote = (name: string) => (/^[a-z_][a-z0-9_]*$/.test(name) ? name : `"${name}"`); + +export function generateSchema(tableCount: number, seed: number, options: { bigTables?: number } = {}): SyntheticSchema { + const random = rng(seed); + const tables: SyntheticTable[] = []; + const taken = new Set(); + const add = (table: SyntheticTable) => { + if (taken.has(table.qualified)) return false; + taken.add(table.qualified); + tables.push(table); + return true; + }; + + for (const hub of HUBS) { + add({ + schema: "public", name: hub.name, qualified: `public.${hub.name}`, words: [hub.name], module: "core", entity: hub.name, role: "hub", + columns: [{ name: "id", type: "bigint", notNull: true }, ...hub.extra, { name: "created_at", type: "timestamptz", notNull: true, fill: "now() - (g % 1000) * interval '1 hour'" }], + foreignKeys: [], indexes: [], rows: HUB_ROWS, + }); + } + + const moduleCount = Math.max(1, Math.min(MODULES.length, Math.ceil(tableCount / 40))); + const modules = random.shuffle(MODULES).slice(0, moduleCount); + const entityPool = random.shuffle(ENTITIES); + let cursor = 0; + + // Fill module by module, root by root, until the count is reached. + let guard = 0; + while (tables.length < tableCount && guard++ < tableCount * 20) { + const module = modules[guard % modules.length]; + const separateSchema = MODULES.indexOf(module) % 3 !== 0; + const entity = entityPool[cursor++ % entityPool.length]; + const schema = separateSchema ? module : "public"; + const rootName = separateSchema ? `${entity}s` : `${module}_${entity}s`; + const root: SyntheticTable = { + schema, name: rootName, qualified: `${schema}.${rootName}`, words: separateSchema ? [entity] : [module, entity], module, entity, role: "root", + columns: [ + { name: "id", type: "bigint", notNull: true }, + { name: "code", type: "varchar(24)", notNull: true, fill: "substr(md5(g::text), 1, 12)" }, + { name: "status", type: "varchar(20)", notNull: true, fill: "'status-' || (g % 5)" }, + { name: "owner_id", type: "bigint", notNull: true, fill: `1 + g % ${HUB_ROWS}` }, + { name: "organization_id", type: "bigint", notNull: false, fill: `1 + g % ${HUB_ROWS}` }, + { name: "created_at", type: "timestamptz", notNull: true, fill: "now() - (g % 1000) * interval '1 hour'" }, + { name: "updated_at", type: "timestamptz", notNull: false }, + { name: "notes", type: "text", notNull: false }, + ], + foreignKeys: [{ column: "owner_id", references: "public.users" }, { column: "organization_id", references: "public.organizations" }], + indexes: [], rows: 0, + }; + for (const domain of random.shuffle(DOMAIN_COLUMNS).slice(0, 3 + random.int(6))) root.columns.push({ name: domain.name, type: domain.type, notNull: false, fill: domain.fill }); + if (random.chance(0.35)) { + const signature = `${entity}_${random.pick(SIGNATURE_WORDS)}_code`; + if (!root.columns.some((column) => column.name === signature)) root.columns.push({ name: signature, type: "varchar(30)", notNull: false, fill: "'sig-' || (g % 900)" }); + } + if (random.chance(0.4)) root.indexes.push({ name: `${rootName}_status_idx`, columns: ["status"] }); + if (!add(root)) continue; + + const derived = random.shuffle(DERIVED).slice(0, 1 + random.int(3)); + for (const kind of derived) { + if (tables.length >= tableCount) break; + const name = separateSchema ? `${entity}_${kind.suffix}` : `${module}_${entity}_${kind.suffix}`; + const fk = `${entity}_id`; + const child: SyntheticTable = { + schema, name, qualified: `${schema}.${name}`, words: [...root.words, kind.suffix], module, entity, role: kind.role === "history" ? "history" : kind.suffix === "notes" ? "notes" : "child", parent: root.qualified, + columns: [ + { name: "id", type: "bigint", notNull: true }, + { name: fk, type: "bigint", notNull: true }, + ...(kind.role === "history" + ? [{ name: "changed_at", type: "timestamptz", notNull: true, fill: "now() - (g % 1000) * interval '1 hour'" }, { name: "changed_by", type: "bigint", notNull: false, fill: `1 + g % ${HUB_ROWS}` }, { name: "old_value", type: "text", notNull: false }, { name: "new_value", type: "text", notNull: false }] + : kind.suffix === "notes" + ? [{ name: "body", type: "text", notNull: true, fill: "'note ' || g" }, { name: "created_at", type: "timestamptz", notNull: true, fill: "now() - (g % 1000) * interval '1 hour'" }] + : [{ name: "label", type: "varchar(120)", notNull: false, fill: "'item ' || g" }, { name: "quantity", type: "integer", notNull: true, fill: "1 + g % 50" }, { name: "created_at", type: "timestamptz", notNull: true, fill: "now() - (g % 1000) * interval '1 hour'" }]), + ], + foreignKeys: [{ column: fk, references: root.qualified }], + indexes: random.chance(0.5) ? [{ name: `${name}_${fk}_idx`, columns: [fk] }] : [], + rows: 0, + }; + // some children carry a column that points at another root without a foreign key, the way grown databases do + const other = tables.filter((t) => t.role === "root" && t.qualified !== root.qualified); + if (other.length > 0 && random.chance(0.3)) { + const target = random.pick(other); + child.columns.push({ name: `${target.entity}_ref_id`, type: "bigint", notNull: false, refersTo: target.qualified }); + } + if (random.chance(0.25)) child.indexes.push({ name: `${name}_created_idx`, columns: [kind.role === "history" ? "changed_at" : "created_at"] }); + add(child); + } + } + + // The populated tables: a few roots with their children, so a change to them is a change to real data. + const big = options.bigTables ?? Math.max(2, Math.min(10, Math.round(tableCount / 40))); + const roots = tables.filter((t) => t.role === "root"); + for (const root of random.shuffle(roots).slice(0, Math.ceil(big / 2))) { + root.rows = BIG_TABLE_ROWS; + for (const child of tables.filter((t) => t.parent === root.qualified).slice(0, 1)) child.rows = BIG_TABLE_ROWS; + } + // legacy reference columns are only filled in tables that are loaded, with ids of rows that exist + for (const table of tables) { + for (const column of table.columns) { + if (column.refersTo && table.rows > 0) { + const target = tables.find((t) => t.qualified === column.refersTo); + column.fill = target && target.rows > 0 ? `1 + g % ${target.rows}` : undefined; + } + } + } + + const createSql: string[] = []; + for (const schema of new Set(tables.map((t) => t.schema))) if (schema !== "public") createSql.push(`CREATE SCHEMA ${quote(schema)}`); + for (const table of tables) { + const columns = table.columns.map((column) => `${quote(column.name)} ${column.type}${column.name === "id" ? " GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY" : ""}${column.notNull && column.name !== "id" ? " NOT NULL" : ""}`); + createSql.push(`CREATE TABLE ${quote(table.schema)}.${quote(table.name)} (${columns.join(", ")})`); + for (const index of table.indexes) createSql.push(`CREATE INDEX ${quote(index.name)} ON ${quote(table.schema)}.${quote(table.name)} (${index.columns.map(quote).join(", ")})`); + } + + const populateSql: string[] = []; + for (const table of tables) { + if (table.rows === 0) continue; + const filled = table.columns.filter((column) => column.name !== "id" && column.fill !== undefined); + const parentRows = table.parent ? (tables.find((t) => t.qualified === table.parent)?.rows ?? 0) : 0; + const fkColumn = table.parent ? table.foreignKeys[0]?.column : undefined; + const columns = [...filled.map((column) => column.name), ...(fkColumn ? [fkColumn] : [])]; + const values = [...filled.map((column) => column.fill as string), ...(fkColumn ? [`1 + g % ${Math.max(1, parentRows)}`] : [])]; + populateSql.push(`INSERT INTO ${quote(table.schema)}.${quote(table.name)} (${columns.map(quote).join(", ")}) SELECT ${values.join(", ")} FROM generate_series(1, ${table.rows}) AS g`); + } + + const foreignKeySql = tables.flatMap((table) => + table.foreignKeys.map((fk) => { + const [schema, name] = fk.references.split("."); + return `ALTER TABLE ${quote(table.schema)}.${quote(table.name)} ADD CONSTRAINT ${quote(`${table.name}_${fk.column}_fkey`.slice(0, 63))} FOREIGN KEY (${quote(fk.column)}) REFERENCES ${quote(schema)}.${quote(name)} (id)`; + }), + ); + + return { tables, seed, createSql, foreignKeySql, populateSql }; +} + +/** `warehouse_code` -> "warehouse code" */ +export const wordsOf = (name: string): string => name.replace(/_/g, " "); diff --git a/scripts/schema-change-eval.ts b/scripts/schema-change-eval.ts new file mode 100644 index 0000000..0e34c0b --- /dev/null +++ b/scripts/schema-change-eval.ts @@ -0,0 +1,667 @@ +/** + * Measures the Schema Change pipeline on generated PostgreSQL databases of 50, 300 and 1,000 tables. + * + * docker run -d --name goforge-schema -e POSTGRES_PASSWORD=schema -e POSTGRES_DB=schema -p 54330:5432 postgres:16 -c max_locks_per_transaction=1024 + * npm run eval:schema # 50 / 300 / 1,000 tables, 25 requests each + * npm run eval:schema -- --sizes 300 --per-size 30 --seed 3 + * npm run eval:schema -- --keep # leave the generated databases for the browser (sc_eval_50, ...) + * OPENAI_API_KEY=sk-... [EVAL_MODEL=gpt-5.6-luna] npm run eval:schema -- --live # a real model plans instead of the reference plans + * (the key has to be in this command's environment, this script does not read server/.env; EVAL_MODEL empty means OPENAI_MODEL, then gpt-5.6-luna) + * + * Each size gets a database with a real schema, real foreign keys and populated tables (260,000 rows in the big ones). A request is + * a sentence in the words a person would use, with the operations a correct plan contains. For every request this measures: + * - table selection: are the tables the change touches in the scope that retrieval picks, and how big is the scope + * - plan validity: do the plan's operations pass the checks against the real catalog (with the retrieved scope, and with the right one) + * - migration lint: does the compiled runbook avoid the usual ways of locking a large table (and does a naive rendering not) + * - execution: does the runbook run, statement by statement in autocommit mode, on PostgreSQL 16, and leave the schema as asked + * - invalid requests: are they refused before any SQL is produced + * - latency of every deterministic stage, and what the request would cost to send (against sending every table) + * The planner is the reference plan unless --live is given; with the reference plan nothing here says anything about a model's quality, + * and the model's own latency and tokens are reported as not measured. + */ +import { DEFAULT_SETTINGS } from "../server/src/ai/policy"; +import { getSettingsStore } from "../server/src/ai/provider"; +import { callModelJson } from "../server/src/ai/provider"; +import { aiSchemaPlanSchema, type AiSchemaPlan } from "../server/src/ai/schema-plan"; +import { loadCatalog } from "../server/src/schema-change/catalog"; +import { compilePlan, discoverProbes } from "../server/src/schema-change/compile"; +import { loadFacts, probe, type Facts } from "../server/src/schema-change/facts"; +import { lintSteps } from "../server/src/schema-change/lint"; +import { planSchemaChange } from "../server/src/schema-change/planner"; +import { plannerInput, plannerInstructions } from "../server/src/schema-change/prompt"; +import { buildSearchIndex, selectScope } from "../server/src/schema-change/retrieval"; +import { sizeClass, type Runbook, type RunbookStep } from "../server/src/schema-change/runbook"; +import { createDatabase, dropDatabase, withDatabase, type PgClient } from "./lib/pg-scratch"; +import { asRunner, runRunbook } from "./lib/plan-runner"; +import { generateSchema, wordsOf, type SyntheticSchema, type SyntheticTable } from "./lib/synthetic-schema"; + +// ---- arguments ----------------------------------------------------------------------------------------------------------- + +const args = process.argv.slice(2); +const option = (name: string, fallback: string): string => { + const index = args.indexOf(`--${name}`); + return index >= 0 && args[index + 1] ? args[index + 1] : fallback; +}; +const SIZES = option("sizes", "50,300,1000").split(",").map(Number); +const PER_SIZE = Number(option("per-size", "20")); +const NEGATIVE_PER_SIZE = Number(option("negative", "5")); +const SEED = Number(option("seed", "11")); +const KEEP = args.includes("--keep"); +const LIVE = args.includes("--live"); +const OLD_TABLE_LIMIT = 250; + +// ---- requests ------------------------------------------------------------------------------------------------------------ + +type Style = "plain" | "with-module" | "typo" | "column-only" | "partial-name"; + +interface Case { + id: string; + kind: string; + style: Style; + text: string; + /** The tables the change touches. */ + truth: string[]; + operations: unknown[]; + /** A query that returns one row with `ok = true` when the change happened. */ + verify?: string; + /** For an invalid request: the finding code the checks must refuse it with. */ + refusedWith?: string; +} + +function lcg(seed: number) { + let state = seed >>> 0; + const next = () => { + state = (Math.imul(state, 1664525) + 1013904223) >>> 0; + return state / 4294967296; + }; + return { int: (n: number) => Math.floor(next() * n), pick: (list: readonly T[]): T => list[Math.floor(next() * list.length)], chance: (p: number) => next() < p }; +} + +const NEW_COLUMNS = [ + { name: "carrier_name", type: "varchar", words: "carrier name" }, + { name: "internal_comment", type: "text", words: "internal comment" }, + { name: "reviewed_at", type: "timestamptz", words: "reviewed at timestamp" }, + { name: "risk_score", type: "integer", words: "risk score" }, + { name: "source_system", type: "varchar", words: "source system" }, +]; + +const q = (name: string) => `"${name.replace(/"/g, '""')}"`; +const dotted = (t: SyntheticTable) => `${q(t.schema)}.${q(t.name)}`; + +function typoOf(word: string, r: ReturnType): string { + if (word.length < 6) return word; + const at = 1 + r.int(word.length - 3); + return r.chance(0.5) ? word.slice(0, at) + word.slice(at + 1) : word.slice(0, at) + word[at + 1] + word[at] + word.slice(at + 2); +} + +function buildCases(schema: SyntheticSchema, count: number, negatives: number, seed: number): Case[] { + const r = lcg(seed); + const tables = schema.tables.filter((t) => t.role !== "hub"); + const bareCount = new Map(); + for (const t of schema.tables) bareCount.set(t.name, (bareCount.get(t.name) ?? 0) + 1); + const used = new Set(); + + /** How a person would name the table: its words, the module when the name alone would be ambiguous, sometimes with a typo or only its last words. */ + const say = (t: SyntheticTable, style: Style): string => { + let words = t.words.map(wordsOf); + if (style === "partial-name" && words.length >= 3) words = words.slice(-2); + if (style === "with-module" && !words.includes(wordsOf(t.module))) words = [wordsOf(t.module), ...words]; + if (style === "typo") { + const longest = words.reduce((best, word, i) => (word.length > words[best].length ? i : best), 0); + words = words.map((word, i) => (i === longest ? typoOf(word, r) : word)); + } + return words.join(" "); + }; + const styleFor = (t: SyntheticTable, preferred: Style): Style => ((bareCount.get(t.name) ?? 0) > 1 && preferred !== "typo" ? "with-module" : preferred); + // Every request runs on its own copy of the database, so a table can be used again once the unused ones are gone. + const fresh = (list: SyntheticTable[]): SyntheticTable | undefined => { + const unused = list.filter((t) => !used.has(t.qualified)); + const options = unused.length > 0 ? unused : list; + const picked = options.length > 0 ? r.pick(options) : undefined; + if (picked) used.add(picked.qualified); + return picked; + }; + const exists = (name: string, t: SyntheticTable) => t.columns.some((c) => c.name === name); + const populated = tables.filter((t) => t.rows > 0); + const styles: Style[] = ["plain", "plain", "with-module", "typo", "partial-name"]; + const cases: Case[] = []; + + const generators: Array<() => Case | undefined> = [ + // a column nobody has yet + () => { + const t = fresh(r.chance(0.5) && populated.length ? populated : tables); + if (!t) return; + const column = r.pick(NEW_COLUMNS.filter((c) => !exists(c.name, t))); + if (!column) return; + const style = styleFor(t, r.pick(styles)); + return { + id: "", kind: "add column", style, text: `add a ${column.words} column to the ${say(t, style)} table`, truth: [t.qualified], + operations: [{ type: "add_column", tableName: t.qualified, field: { name: column.name, dataType: column.type, ...(column.type === "varchar" ? { maxLength: 120 } : {}) } }], + verify: `SELECT count(*) = 1 AS ok FROM information_schema.columns WHERE table_schema = '${t.schema}' AND table_name = '${t.name}' AND column_name = '${column.name}'`, + }; + }, + // a required flag with a default: no rewrite on a big table + () => { + const t = fresh(populated.length ? populated : tables); + if (!t) return; + const style = styleFor(t, r.pick(styles)); + return { + id: "", kind: "add required column with default", style, text: `every ${say(t, style)} record needs a required reviewed flag that is false unless someone sets it`, truth: [t.qualified], + operations: [{ type: "add_column", tableName: t.qualified, field: { name: "is_reviewed", dataType: "boolean", nullable: false, defaultValue: "false" } }], + verify: `SELECT count(*) = 1 AS ok FROM pg_attribute WHERE attrelid = '${dotted(t)}'::regclass AND attname = 'is_reviewed' AND attnotnull`, + }; + }, + // a column filled by a volatile default on existing rows + () => { + const t = fresh(populated.length ? populated : tables); + if (!t) return; + const style = styleFor(t, r.pick(styles)); + return { + id: "", kind: "add column with volatile default", style, text: `give each ${say(t, style)} a public uuid that is required`, truth: [t.qualified], + operations: [{ type: "add_column", tableName: t.qualified, field: { name: "public_uuid", dataType: "uuid", nullable: false, defaultValue: "gen_random_uuid()" } }], + verify: `SELECT count(*) = 1 AS ok FROM pg_attribute WHERE attrelid = '${dotted(t)}'::regclass AND attname = 'public_uuid' AND attnotnull`, + }; + }, + // an index on an existing column + () => { + const t = fresh(r.chance(0.6) && populated.length ? populated : tables); + if (!t) return; + const candidates = t.columns.filter((c) => !["id", "notes"].includes(c.name) && !t.indexes.some((i) => i.columns[0] === c.name) && !c.type.startsWith("text")); + const column = candidates.length ? r.pick(candidates) : undefined; + if (!column) return; + const style = styleFor(t, r.pick(styles)); + const name = `${t.name}_${column.name}_eval_idx`.slice(0, 60); + return { + id: "", kind: "add index", style, text: `queries on ${say(t, style)} filtered by ${wordsOf(column.name)} are slow, add an index`, truth: [t.qualified], + operations: [{ type: "add_index", tableName: t.qualified, index: { name, columns: [column.name] } }], + verify: `SELECT count(*) = 1 AS ok FROM pg_index WHERE indexrelid = '${q(t.schema)}.${q(name)}'::regclass AND indisvalid`, + }; + }, + // an index found through the column name alone + () => { + const counts = new Map(); + for (const t of tables) for (const c of t.columns) counts.set(c.name, (counts.get(c.name) ?? 0) + 1); + const t = fresh(tables.filter((x) => x.columns.some((c) => c.name.endsWith("_code") && c.name !== "code" && counts.get(c.name) === 1))); + if (!t) return; + const column = t.columns.find((c) => c.name.endsWith("_code") && c.name !== "code" && counts.get(c.name) === 1) ?? t.columns[0]; + const name = `${t.name}_${column.name}_eval_idx`.slice(0, 60); + return { + id: "", kind: "add index (column only)", style: "column-only", text: `lookups by ${wordsOf(column.name)} need an index`, truth: [t.qualified], + operations: [{ type: "add_index", tableName: t.qualified, index: { name, columns: [column.name] } }], + verify: `SELECT count(*) = 1 AS ok FROM pg_index WHERE indexrelid = '${q(t.schema)}.${q(name)}'::regclass AND indisvalid`, + }; + }, + // unique codes + () => { + const t = fresh(schema.tables.filter((x) => x.role === "root" && (x.rows > 0 || r.chance(0.5)))); + if (!t) return; + const style = styleFor(t, r.pick(styles)); + return { + id: "", kind: "make unique", style, text: `the code of a ${say(t, style)} must be unique`, truth: [t.qualified], + operations: [{ type: "alter_column", tableName: t.qualified, columnName: "code", changes: { unique: true } }], + verify: `SELECT count(*) = 1 AS ok FROM pg_constraint WHERE conrelid = '${dotted(t)}'::regclass AND contype = 'u'`, + }; + }, + // required column that already has values + () => { + const t = fresh(schema.tables.filter((x) => x.role === "root" && x.columns.some((c) => c.name === "organization_id") && (x.rows > 0 || r.chance(0.5)))); + if (!t) return; + const style = styleFor(t, r.pick(styles)); + return { + id: "", kind: "make column required", style, text: `a ${say(t, style)} must always belong to an organization`, truth: [t.qualified], + operations: [{ type: "alter_column", tableName: t.qualified, columnName: "organization_id", changes: { nullable: false } }], + verify: `SELECT count(*) = 1 AS ok FROM pg_attribute WHERE attrelid = '${dotted(t)}'::regclass AND attname = 'organization_id' AND attnotnull`, + }; + }, + // longer text: only the catalog changes + () => { + const t = fresh(populated.length ? populated : tables); + if (!t) return; + const column = t.columns.find((c) => /^varchar\(\d+\)$/.test(c.type) && c.name !== "code"); + if (!column) return; + const length = Number(/\((\d+)\)/.exec(column.type)?.[1]) + 100; + const style = styleFor(t, r.pick(styles)); + return { + id: "", kind: "lengthen a varchar", style, text: `${wordsOf(column.name)} on ${say(t, style)} is too short, allow longer values`, truth: [t.qualified], + operations: [{ type: "alter_column", tableName: t.qualified, columnName: column.name, changes: { dataType: "varchar", maxLength: length } }], + verify: `SELECT format_type(atttypid, atttypmod) = 'character varying(${length})' AS ok FROM pg_attribute WHERE attrelid = '${dotted(t)}'::regclass AND attname = '${column.name}'`, + }; + }, + // integer to bigint: a rewrite unless it is done in steps + () => { + const t = fresh(populated.filter((x) => x.columns.some((c) => c.type === "integer")).length ? populated.filter((x) => x.columns.some((c) => c.type === "integer")) : tables.filter((x) => x.columns.some((c) => c.type === "integer"))); + if (!t) return; + const column = t.columns.find((c) => c.type === "integer"); + if (!column) return; + const style = styleFor(t, r.pick(styles)); + return { + id: "", kind: "change integer to bigint", style, text: `${wordsOf(column.name)} on ${say(t, style)} is about to overflow, make it a bigint`, truth: [t.qualified], + operations: [{ type: "alter_column", tableName: t.qualified, columnName: column.name, changes: { dataType: "bigint" } }], + verify: `SELECT format_type(atttypid, atttypmod) = 'bigint' AS ok FROM pg_attribute WHERE attrelid = '${dotted(t)}'::regclass AND attname = '${column.name}'`, + }; + }, + // a foreign key for a column that points somewhere without one + () => { + const child = fresh(tables.filter((t) => t.columns.some((c) => c.refersTo))); + if (!child) return; + const column = child.columns.find((c) => c.refersTo)!; + const parent = schema.tables.find((t) => t.qualified === column.refersTo)!; + const childStyle = styleFor(child, r.pick(["plain", "with-module"])); + const parentStyle = styleFor(parent, "plain"); + return { + id: "", kind: "add foreign key", style: childStyle, text: `${wordsOf(column.name)} on ${say(child, childStyle)} should be a real foreign key to ${say(parent, parentStyle)}`, truth: [child.qualified, parent.qualified], + operations: [{ type: "add_relationship", relationship: { sourceTable: parent.qualified, sourceColumn: "id", targetTable: child.qualified, targetColumn: column.name } }], + verify: `SELECT count(*) = 1 AS ok FROM pg_constraint WHERE conrelid = '${dotted(child)}'::regclass AND contype = 'f' AND convalidated AND conkey = ARRAY[(SELECT attnum FROM pg_attribute WHERE attrelid = '${dotted(child)}'::regclass AND attname = '${column.name}')]`, + }; + }, + // drop an index nobody needs + () => { + const t = fresh(tables.filter((x) => x.indexes.length > 0)); + if (!t) return; + const index = t.indexes[0]; + const style = styleFor(t, r.pick(styles)); + return { + id: "", kind: "drop index", style, text: `the index on ${index.columns.map(wordsOf).join(" and ")} of ${say(t, style)} is never used, drop it`, truth: [t.qualified], + operations: [{ type: "drop_index", tableName: t.qualified, indexName: index.name }], + verify: `SELECT to_regclass('${q(t.schema)}.${q(index.name)}') IS NULL AS ok`, + }; + }, + // rename a column + () => { + const t = fresh(tables.filter((x) => x.role === "root")); + if (!t) return; + const style = styleFor(t, r.pick(styles)); + return { + id: "", kind: "rename column", style, text: `rename the notes column of ${say(t, style)} to remarks`, truth: [t.qualified], + operations: [{ type: "alter_column", tableName: t.qualified, columnName: "notes", changes: { newName: "remarks" } }], + verify: `SELECT count(*) = 1 AS ok FROM pg_attribute WHERE attrelid = '${dotted(t)}'::regclass AND attname = 'remarks'`, + }; + }, + // a new table that points at an existing one + () => { + const t = fresh(schema.tables.filter((x) => x.role === "root")); + if (!t) return; + const style = styleFor(t, r.pick(styles)); + const name = `${t.name}_eval_audit`.slice(0, 60); + return { + id: "", kind: "create table with foreign key", style, text: `create an audit trail table for ${say(t, style)} that records who changed what and when`, truth: [t.qualified], + operations: [ + { type: "create_table", table: { name: `${t.schema}.${name}`, fields: [{ name: "id", dataType: "bigint", primaryKey: true, autoIncrement: true }, { name: `${t.entity}_id`, dataType: "bigint", nullable: false }, { name: "changed_by", dataType: "bigint" }, { name: "changed_at", dataType: "timestamptz", nullable: false, defaultValue: "now()" }], indexes: [{ name: `${name}_changed_idx`, columns: ["changed_at"] }] } }, + { type: "add_relationship", relationship: { sourceTable: t.qualified, sourceColumn: "id", targetTable: `${t.schema}.${name}`, targetColumn: `${t.entity}_id` } }, + ], + verify: `SELECT count(*) = 1 AS ok FROM pg_constraint WHERE conrelid = '${q(t.schema)}.${q(name)}'::regclass AND contype = 'f' AND convalidated`, + }; + }, + // one change in two tables + () => { + const a = fresh(populated.length ? populated : tables); + const b = fresh(tables); + if (!a || !b) return; + const sa = styleFor(a, "plain"); + const sb = styleFor(b, "plain"); + return { + id: "", kind: "same change in two tables", style: sa, text: `add an archived at timestamp to ${say(a, sa)} and to ${say(b, sb)}`, truth: [a.qualified, b.qualified], + operations: [a, b].map((t) => ({ type: "add_column", tableName: t.qualified, field: { name: "archived_at", dataType: "timestamptz" } })), + verify: `SELECT count(*) = 2 AS ok FROM pg_attribute WHERE attname = 'archived_at' AND attrelid IN ('${dotted(a)}'::regclass, '${dotted(b)}'::regclass)`, + }; + }, + ]; + + for (let attempt = 0; cases.length < count && attempt < count * 30; attempt++) { + const made = generators[attempt % generators.length](); + if (made) cases.push({ ...made, id: `v${cases.length + 1}` }); + } + + // requests that must be refused before any SQL is produced + const refusals: Array<() => Case | undefined> = [ + () => { + const t = r.pick(tables); + return { id: "", kind: "index on a column that does not exist", style: "plain", text: `index the ${say(t, "plain")} table by its fiscal period`, truth: [t.qualified], operations: [{ type: "add_index", tableName: t.qualified, index: { name: `${t.name}_fiscal_idx`.slice(0, 60), columns: ["fiscal_period"] } }], refusedWith: "unknown-column" }; + }, + () => { + const t = r.pick(populated.length ? populated : tables); + return { id: "", kind: "NOT NULL column without a default on a table with rows", style: "plain", text: `add a required approver to ${say(t, "plain")}`, truth: [t.qualified], operations: [{ type: "add_column", tableName: t.qualified, field: { name: "approver", dataType: "text", nullable: false } }], refusedWith: "not-null-needs-default" }; + }, + () => { + const t = r.pick(tables.filter((x) => x.role === "root")); + const column = t.columns.find((c) => c.type.startsWith("varchar")) ?? t.columns[1]; + return { id: "", kind: "foreign key between different types", style: "plain", text: `${wordsOf(column.name)} of ${say(t, "plain")} should reference users`, truth: [t.qualified, "public.users"], operations: [{ type: "add_relationship", relationship: { sourceTable: "public.users", sourceColumn: "id", targetTable: t.qualified, targetColumn: column.name } }], refusedWith: "type-mismatch" }; + }, + () => { + const roots = tables.filter((x) => x.role === "root" && schema.tables.some((c) => c.parent === x.qualified)); + const t = r.pick(roots); + return { id: "", kind: "drop a table other tables point at", style: "plain", text: `drop the ${say(t, "plain")} table, we do not need it any more`, truth: [t.qualified, ...schema.tables.filter((c) => c.parent === t.qualified).map((c) => c.qualified)], operations: [{ type: "drop_table", tableName: t.qualified }], refusedWith: "referenced" }; + }, + () => { + const t = r.pick(tables); + return { id: "", kind: "default that carries a second statement", style: "plain", text: `${say(t, "plain")} gets a status note defaulting to nothing`, truth: [t.qualified], operations: [{ type: "add_column", tableName: t.qualified, field: { name: "status_note", dataType: "text", defaultValue: "'x'; DROP TABLE users" } }], refusedWith: "bad-default" }; + }, + () => { + const t = r.pick(tables.filter((x) => x.role === "root")); + return { id: "", kind: "make the primary key nullable", style: "plain", text: `${say(t, "plain")} ids may be empty`, truth: [t.qualified], operations: [{ type: "alter_column", tableName: t.qualified, columnName: "id", changes: { nullable: true } }], refusedWith: "primary-key-column" }; + }, + ]; + for (let i = 0; i < negatives; i++) { + const made = refusals[i % refusals.length](); + if (made) cases.push({ ...made, id: `n${i + 1}` }); + } + return cases; +} + +// ---- measuring ----------------------------------------------------------------------------------------------------------- + +const median = (values: number[]): number => (values.length === 0 ? 0 : [...values].sort((a, b) => a - b)[Math.floor(values.length / 2)]); +const percentile = (values: number[], p: number): number => (values.length === 0 ? 0 : [...values].sort((a, b) => a - b)[Math.min(values.length - 1, Math.floor(values.length * p))]); +const pct = (n: number, d: number): string => (d === 0 ? "n/a" : `${((n / d) * 100).toFixed(0)}%`.padStart(4)); +const ms = (value: number): string => `${value < 10 ? value.toFixed(1) : Math.round(value)} ms`; +const time = async (fn: () => Promise): Promise<[T, number]> => { + const start = performance.now(); + const result = await fn(); + return [result, performance.now() - start]; +}; + +/** What a plain rendering of the same operations would be: one statement each, the way a schema diff tool writes them. */ +function naiveSteps(operations: AiSchemaPlan["operations"], facts: Facts): RunbookStep[] { + const base = { operation: 0, lock: "ACCESS EXCLUSIVE" as const, alsoLocks: [], work: "metadata" as const, risk: "low" as const, reason: "", notes: [], destructive: false, breaksClients: false, optional: false }; + const steps: RunbookStep[] = []; + const push = (tableName: string, sql: string, work: RunbookStep["work"] = "metadata") => { + const table = facts.tables.get(tableName); + steps.push({ ...base, id: steps.length + 1, title: sql.slice(0, 40), sql, table: tableName, work, sizeClass: sizeClass(table), tableBytes: table?.heapBytes ?? null, tableRows: table?.estimatedRows ?? null }); + }; + for (const op of operations) { + const t = "tableName" in op ? (op.tableName as string) : ""; + switch (op.type) { + case "add_column": + push(t, `ALTER TABLE ${t} ADD COLUMN ${op.field.name} ${op.field.dataType}${op.field.nullable === false ? " NOT NULL" : ""}${op.field.defaultValue ? ` DEFAULT ${op.field.defaultValue}` : ""};`, op.field.defaultValue?.includes("(") ? "rewrite" : "metadata"); + break; + case "add_index": + push(t, `CREATE ${op.index.unique ? "UNIQUE " : ""}INDEX ${op.index.name} ON ${t} (${op.index.columns.join(", ")});`, "index-build"); + break; + case "drop_index": + push(t, `DROP INDEX ${op.indexName};`); + break; + case "alter_column": + if (op.changes.nullable === false) push(t, `ALTER TABLE ${t} ALTER COLUMN ${op.columnName} SET NOT NULL;`, "scan"); + if (op.changes.dataType) push(t, `ALTER TABLE ${t} ALTER COLUMN ${op.columnName} TYPE ${op.changes.dataType};`, "rewrite"); + if (op.changes.unique) push(t, `ALTER TABLE ${t} ADD CONSTRAINT ${t.split(".")[1]}_${op.columnName}_key UNIQUE (${op.columnName});`, "index-build"); + if (op.changes.newName) push(t, `ALTER TABLE ${t} RENAME COLUMN ${op.columnName} TO ${op.changes.newName};`); + break; + case "add_relationship": + push(op.relationship.targetTable, `ALTER TABLE ${op.relationship.targetTable} ADD FOREIGN KEY (${op.relationship.targetColumn}) REFERENCES ${op.relationship.sourceTable} (${op.relationship.sourceColumn});`, "scan"); + break; + default: + break; + } + } + return steps; +} + +interface Row { + kase: Case; + style: Style; + scopeSize: number; + full: boolean; + recall: number; + searchMs: number; + e2eOk: boolean; + oracleOk?: boolean; + lintOk?: boolean; + executed?: boolean; + verified?: boolean; + refused?: boolean; + refusedRight?: boolean; + compileMs: number; + factsMs: number; + stageMs: number; + scopedChars: number; + steps: number; + risk: Record; + naiveFlagged?: boolean; + sizeClasses: string[]; + problem?: string; +} + +async function evaluate(size: number): Promise<{ rows: Row[]; header: Record; failures: string[] }> { + const schema = generateSchema(size, SEED + size); + const template = `sc_eval_${size}`; + const header: Record = { tables: schema.tables.length, populated: schema.tables.filter((t) => t.rows > 0).length }; + const failures: string[] = []; + + console.log(`\n=== ${size} tables ===`); + const [, buildMs] = await time(async () => { + await createDatabase(template, async (client) => { + await client.query(schema.createSql.join(";\n")); + for (const statement of schema.populateSql) await client.query(statement); + for (const statement of schema.foreignKeySql) await client.query(statement); + await client.query("ANALYZE"); + }); + }); + console.log(` built and populated in ${(buildMs / 1000).toFixed(1)} s`); + header.buildSeconds = Math.round(buildMs / 100) / 10; + + const rows: Row[] = []; + try { + // the catalog the page reads once and searches many times + const catalogClient = await (await import("./lib/pg-scratch")).connect(template); + const [catalog, catalogMs] = await time(() => loadCatalog(asRunner(catalogClient))); + const [index, indexMs] = await time(async () => buildSearchIndex(catalog)); + header.catalogMs = Math.round(catalogMs); + header.indexMs = Math.round(indexMs * 10) / 10; + + // what sending every table would cost, for comparison + const everyone = catalog.tables.map((t) => t.qualified); + const [allFacts, allFactsMs] = await time(() => loadFacts(asRunner(catalogClient), everyone)); + const fullChars = JSON.stringify(plannerInput({ prompt: "x", facts: allFacts, omittedTables: 0 })).length + plannerInstructions().length; + header.allFactsMs = Math.round(allFactsMs); + header.fullChars = fullChars; + const sampleFacts = allFacts; + await catalogClient.end(); + console.log(` catalog ${ms(catalogMs)}, search index ${ms(indexMs)}, facts for all ${catalog.tables.length} tables ${ms(allFactsMs)} (${(fullChars / 1000).toFixed(0)}k characters if every table were sent)`); + + const cases = buildCases(schema, PER_SIZE, NEGATIVE_PER_SIZE, SEED * 1000 + size); + console.log(` ${cases.filter((c) => !c.refusedWith).length} change requests and ${cases.filter((c) => c.refusedWith).length} that should be refused`); + + for (const kase of cases) { + const plan = aiSchemaPlanSchema.parse({ summary: kase.text, operations: kase.operations }); + + // 1. which tables + const [scope, searchMs] = await time(async () => selectScope(index, kase.text)); + const picked = new Set(scope.tables.map((t) => t.qualified)); + const hit = kase.truth.filter((name) => picked.has(name)).length; + + const row: Row = { + kase, style: kase.style, scopeSize: scope.tables.length, full: hit === kase.truth.length, recall: kase.truth.length ? hit / kase.truth.length : 1, searchMs, e2eOk: false, + compileMs: 0, factsMs: 0, stageMs: 0, scopedChars: 0, steps: 0, risk: { low: 0, medium: 0, high: 0 }, sizeClasses: [], + }; + + await withDatabase(async (client) => { + const db = asRunner(client); + const compileFor = async (scopeTables: string[], operations = plan) => { + const [facts, factsMs] = await time(() => loadFacts(db, scopeTables)); + const [runbook, compileMs] = await time(async () => { + const probes = await probe(db, discoverProbes({ plan: operations, facts })); + return compilePlan({ plan: operations, facts, probes }); + }); + return { facts, runbook, factsMs, compileMs }; + }; + + // 2. the plan against the tables retrieval chose (when it chose the right ones, this is what a person would get) + const retrieved = await compileFor([...picked]); + row.factsMs = retrieved.factsMs; + row.compileMs = retrieved.compileMs; + row.scopedChars = JSON.stringify(plannerInput({ prompt: kase.text, facts: retrieved.facts, omittedTables: size - picked.size })).length + plannerInstructions().length; + const retrievedErrors = retrieved.runbook.findings.filter((f) => f.severity === "error"); + row.e2eOk = retrievedErrors.length === 0; + + // 3. the plan against the right tables + const oracle = await compileFor(kase.truth); + const errors = oracle.runbook.findings.filter((f) => f.severity === "error"); + row.stageMs = row.searchMs + oracle.factsMs + oracle.compileMs; + + if (kase.refusedWith) { + row.refused = errors.length > 0; + row.refusedRight = errors.some((f) => f.code === kase.refusedWith); + row.problem = row.refusedRight ? undefined : `expected ${kase.refusedWith}, got ${errors.map((f) => f.code).join(", ") || "no error"}`; + return; + } + + row.oracleOk = errors.length === 0; + if (!row.oracleOk) { + row.problem = errors.map((f) => f.message).join(" | ").slice(0, 300); + return; + } + const lint = lintSteps(oracle.runbook.steps); + row.lintOk = lint.violations.length === 0; + if (!row.lintOk) row.problem = lint.violations.map((v) => `${v.rule}: ${v.statement}`).join(" | ").slice(0, 300); + row.steps = oracle.runbook.steps.length; + for (const step of oracle.runbook.steps) if (!step.optional) row.risk[step.risk]++; + row.sizeClasses = [...new Set(oracle.runbook.steps.map((s) => s.sizeClass))]; + + const naive = lintSteps(naiveSteps(plan.operations, oracle.facts)); + row.naiveFlagged = naive.violations.length > 0; + + const result = await runRunbook(client, oracle.runbook, { includeOptional: true }); + row.executed = result.ok; + if (!result.ok) { + row.problem = `step ${result.failure?.step} (${result.failure?.title}): ${result.failure?.message}`.slice(0, 300); + return; + } + if (kase.verify) row.verified = Boolean((await client.query(kase.verify)).rows[0]?.ok); + if (row.verified === false) row.problem = "the schema is not what the request asked for after the runbook ran"; + }, { template }); + + rows.push(row); + const flag = kase.refusedWith ? (row.refusedRight ? "refused" : "MISSED") : row.verified ? "ok" : "FAILED"; + process.stdout.write(` ${kase.id.padEnd(4)} ${flag.padEnd(8)} recall ${row.full ? "full" : "MISS"} scope ${String(row.scopeSize).padStart(2)} ${kase.kind}\n`); + if (flag === "FAILED" || flag === "MISSED") failures.push(`${size} tables, ${kase.id} (${kase.kind}): ${row.problem}`); + } + + // 4. the same checks when a real model plans (only with --live) + if (LIVE) { + console.log(" live model: planning the valid requests through the full pipeline"); + const live = new LiveStats(); + await live.run(template, cases.filter((c) => !c.refusedWith), index, catalog.tables.length, sampleFacts); + header.liveValid = live.valid; + header.liveTotal = live.total; + header.liveLatencyMedianMs = Math.round(median(live.latencies)); + header.liveInputTokens = live.inputTokens; + header.liveOutputTokens = live.outputTokens; + } + } finally { + if (!KEEP) await dropDatabase(template); + } + return { rows, header, failures }; +} + +class LiveStats { + valid = 0; + total = 0; + latencies: number[] = []; + inputTokens = 0; + outputTokens = 0; + + async run(template: string, cases: Case[], _index: unknown, _tables: number, _facts: Facts) { + const settings = getSettingsStore(); + settings.write({ ...DEFAULT_SETTINGS, model: process.env.EVAL_MODEL ?? "" }); + const connect = (await import("./lib/pg-scratch")).connect; + const client = await connect(template); + try { + const db = asRunner(client); + const catalog = await loadCatalog(db); + const index = buildSearchIndex(catalog); + for (const kase of cases) { + this.total++; + const scope = selectScope(index, kase.text).tables.map((t) => t.qualified); + try { + const result = await planSchemaChange( + { loadFacts: (names) => loadFacts(db, names), probe: (request) => probe(db, request), wholeDatabase: async () => ({ tables: catalog.tables }), callModel: callModelJson }, + { prompt: kase.text, tables: scope, redact: false, connection: { id: "eval", name: template } }, + ); + this.latencies.push(result.modelLatencyMs); + this.inputTokens += result.usage.inputTokens ?? 0; + this.outputTokens += result.usage.outputTokens ?? 0; + if (result.runbook && !result.findings.some((f) => f.severity === "error")) this.valid++; + } catch (error) { + console.log(` ${kase.id}: ${error instanceof Error ? error.message : error}`); + } + } + } finally { + await client.end(); + } + } +} + +// ---- report ---------------------------------------------------------------------------------------------------------------- + +function report(size: number, rows: Row[], header: Record) { + const valid = rows.filter((r) => !r.kase.refusedWith); + const refused = rows.filter((r) => r.kase.refusedWith); + console.log(`\n--- ${size} tables: results ---`); + + console.log("table selection (the tables the change touches, found from the request's words, columns and foreign keys)"); + console.log(` every table found ${pct(valid.filter((r) => r.full).length, valid.length)} (${valid.filter((r) => r.full).length}/${valid.length})`); + console.log(` mean share found ${pct(valid.reduce((s, r) => s + r.recall, 0), valid.length)}`); + console.log(` scope size median ${median(valid.map((r) => r.scopeSize))}, max ${Math.max(0, ...valid.map((r) => r.scopeSize))} of ${size} tables`); + for (const style of ["plain", "with-module", "typo", "partial-name", "column-only"] as Style[]) { + const group = valid.filter((r) => r.style === style); + if (group.length > 0) console.log(` ${style.padEnd(13)} every table found ${pct(group.filter((r) => r.full).length, group.length)} (${group.filter((r) => r.full).length}/${group.length})`); + } + const misses = valid.filter((r) => !r.full); + for (const miss of misses.slice(0, 4)) console.log(` missed: "${miss.kase.text}" -> wanted ${miss.kase.truth.join(", ")}`); + + console.log("what is sent to the model"); + const scoped = median(valid.map((r) => r.scopedChars)); + const full = Number(header.fullChars); + console.log(` every table ${(full / 1000).toFixed(0)}k characters (about ${Math.round(full / 4000)}k tokens, estimated)${size > OLD_TABLE_LIMIT ? `, and the old request refused anything over ${OLD_TABLE_LIMIT} tables` : ""}`); + console.log(` chosen scope ${(scoped / 1000).toFixed(0)}k characters median (about ${Math.round(scoped / 4000)}k tokens), ${(full / Math.max(1, scoped)).toFixed(0)}x smaller`); + + console.log("plan checks against the real catalog (reference plans: these measure the checks and the compiler, not a model)"); + console.log(` with the retrieved tables ${pct(valid.filter((r) => r.e2eOk).length, valid.length)} (a missed table makes its change unplannable)`); + console.log(` with the right tables ${pct(valid.filter((r) => r.oracleOk).length, valid.length)} (${valid.filter((r) => r.oracleOk).length}/${valid.length})`); + console.log(` invalid requests refused ${pct(refused.filter((r) => r.refusedRight).length, refused.length)} (${refused.filter((r) => r.refusedRight).length}/${refused.length}, each with the expected reason)`); + + console.log("migration lint and execution"); + const compiled = valid.filter((r) => r.oracleOk); + console.log(` lint passes ${pct(compiled.filter((r) => r.lintOk).length, compiled.length)} (${compiled.filter((r) => r.lintOk).length}/${compiled.length})`); + console.log(` a plain rendering is flagged ${pct(compiled.filter((r) => r.naiveFlagged).length, compiled.length)} (${compiled.filter((r) => r.naiveFlagged).length}/${compiled.length}: the same lint on one statement per operation)`); + console.log(` runs on PostgreSQL 16 ${pct(compiled.filter((r) => r.executed).length, compiled.length)} (${compiled.filter((r) => r.executed).length}/${compiled.length}, every statement on its own)`); + console.log(` leaves the schema as asked ${pct(compiled.filter((r) => r.verified).length, compiled.length)} (${compiled.filter((r) => r.verified).length}/${compiled.length})`); + const touched = compiled.filter((r) => r.sizeClasses.some((c) => c === "medium" || c === "large")); + console.log(` of those, on a table with data (16 MiB or more): ${touched.length}; steps by risk ${JSON.stringify(compiled.reduce((s, r) => ({ low: s.low + r.risk.low, medium: s.medium + r.risk.medium, high: s.high + r.risk.high }), { low: 0, medium: 0, high: 0 }))}`); + + console.log("latency (every stage that doesn't wait for a model)"); + console.log(` read the catalog ${ms(Number(header.catalogMs))} once, search index ${ms(Number(header.indexMs))}`); + console.log(` find the tables median ${ms(median(valid.map((r) => r.searchMs)))}, p95 ${ms(percentile(valid.map((r) => r.searchMs), 0.95))}`); + console.log(` read facts of the scope median ${ms(median(valid.map((r) => r.factsMs)))}`); + console.log(` check and compile median ${ms(median(valid.map((r) => r.compileMs)))}, p95 ${ms(percentile(valid.map((r) => r.compileMs), 0.95))}`); + console.log(` whole deterministic path median ${ms(median(rows.map((r) => r.stageMs)))}, p95 ${ms(percentile(rows.map((r) => r.stageMs), 0.95))}`); + if (LIVE) console.log(` model median ${ms(Number(header.liveLatencyMedianMs))}, ${header.liveValid}/${header.liveTotal} plans without errors, ${header.liveInputTokens} in / ${header.liveOutputTokens} out tokens (reported by the endpoint)`); + else console.log(" model not measured (no live model: have OPENAI_API_KEY available and pass --live)"); +} + +async function main() { + console.log(`Schema Change evaluation: sizes ${SIZES.join(", ")}, ${PER_SIZE} valid + ${NEGATIVE_PER_SIZE} invalid requests each, seed ${SEED}${LIVE ? ", live model" : ", reference plans"}`); + const all: Row[] = []; + const failures: string[] = []; + for (const size of SIZES) { + const { rows, header, failures: failed } = await evaluate(size); + report(size, rows, header); + all.push(...rows); + failures.push(...failed); + } + + const valid = all.filter((r) => !r.kase.refusedWith); + const refused = all.filter((r) => r.kase.refusedWith); + console.log(`\n=== all sizes: ${all.length} requests (${valid.length} valid, ${refused.length} invalid) ===`); + console.log(` every table found ${pct(valid.filter((r) => r.full).length, valid.length)} · plan valid (right tables) ${pct(valid.filter((r) => r.oracleOk).length, valid.length)} · lint ${pct(valid.filter((r) => r.lintOk).length, valid.filter((r) => r.oracleOk).length)} · executed ${pct(valid.filter((r) => r.executed).length, valid.filter((r) => r.oracleOk).length)} · verified ${pct(valid.filter((r) => r.verified).length, valid.filter((r) => r.oracleOk).length)} · invalid refused ${pct(refused.filter((r) => r.refusedRight).length, refused.length)}`); + if (failures.length > 0) { + console.log(`\n${failures.length} failure(s) in the checks or the compiler:`); + for (const failure of failures) console.log(` ${failure}`); + process.exit(1); + } +} + +main().catch((error) => { + console.error(error); + process.exit(2); +}); diff --git a/vitest.config.ts b/vitest.config.ts index c57a4f1..3847026 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -8,6 +8,6 @@ export default defineConfig({ plugins: [tsconfigPaths()], test: { environment: "node", - include: ["src/**/*.test.ts", "server/src/**/*.test.ts", "queryshift/**/*.test.ts"], + include: ["src/**/*.test.ts", "server/src/**/*.test.ts", "scripts/**/*.test.ts", "queryshift/**/*.test.ts"], }, });