diff --git a/README.md b/README.md index cf2118a..f492052 100644 --- a/README.md +++ b/README.md @@ -131,9 +131,10 @@ npm run test:schema-pg:versions # every section on Post docker run -d --name goforge-schema -e POSTGRES_PASSWORD=schema -p 54330:5432 postgres:16 -c max_locks_per_transaction=1024 npm run test:schema-pg # every section against that one server npm run test:schema-pg -- model # one section: drops, dependents, lockwait, preflight, model, rename, columnrename, planner, health, locks, rewrites, compile, catalog, retrieval, facts, probe +npm run eval:schema -- --sizes 50 --per-size 14 # a quick run of the generated-database evaluation ``` -The generated-database evaluation below (`scripts/schema-change-eval.ts`, `npm run eval:schema`) is not in this repository yet. +`npm run eval:schema` measures table selection, plan checks, lint and execution on generated databases of 50, 300 and 1,000 tables (it uses the server in `SCHEMA_CHANGE_DATABASE_URL`, default the container above; `-- --sizes 50 --per-size 14` is a quick run): `npm run eval:schema` builds the databases (real foreign keys, tables with 260,000 rows), writes change requests in the words a person would use (with typos, partial names, a column name only, the same name in two schemas), runs each through retrieval, the checks and the compiler, executes the runbook on a copy of the database and verifies the result. It also feeds in requests that must be refused. Without `--live` the plans are the reference plans, so the plan-validity numbers measure the checks and the compiler, not a model; the model's own latency and token use are reported as not measured. `scripts/fake-model-server.ts` is a stand-in OpenAI endpoint for trying the page without a key. diff --git a/package.json b/package.json index f8f78e5..d8f124b 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "test:batch": "server/node_modules/.bin/tsx scripts/modernizer-batch.ts", "test:schema-pg": "server/node_modules/.bin/tsx scripts/schema-change-pg.ts", "test:schema-pg:versions": "bash scripts/schema-change-versions.sh", + "eval:schema": "server/node_modules/.bin/tsx scripts/schema-change-eval.ts", "queryshift": "server/node_modules/.bin/tsx queryshift/run.ts", "queryshift:check-model": "server/node_modules/.bin/tsx queryshift/check-model.ts" }, diff --git a/scripts/fake-model-server.ts b/scripts/fake-model-server.ts new file mode 100644 index 0000000..1e86460 --- /dev/null +++ b/scripts/fake-model-server.ts @@ -0,0 +1,155 @@ +/** + * A stand-in for the OpenAI Responses endpoint, to try the Schema Change page without an OpenAI key or a model: + * + * server/node_modules/.bin/tsx scripts/fake-model-server.ts # listens on 127.0.0.1:4010 + * then start the API with + * OPENAI_BASE_URL=http://127.0.0.1:4010/v1 OPENAI_API_KEY=fake OPENAI_MODEL=fake-model npm --prefix server run dev + * + * The API sends its real requests to it, so the whole path from the page to the wire is the one a real model would use. + * It does no thinking. For a plan request it reads the tables it was shown and proposes a few operations that exist on them (a nullable column, + * an index, NOT NULL on a nullable column, a wider integer), so the page has something real to compile. It reports no token usage, which makes + * the page show its own estimate. Nothing here says anything about what a real model would answer. + */ +import http from "node:http"; + +const PORT = Number(process.env.PORT ?? 4010); + +interface Table { + name: string; + rows: number | null; + columns: { name: string; type: string; notNull: boolean }[]; + primaryKey: string[]; + indexes: { name: string; columns: string[] }[]; +} + +/** "add a testimonials table ..." / "create a new table for reviews": a create_table, linked to the first existing table it was told about. */ +function createTablePlan(input: { request: string; tables: Table[]; otherTables?: string[] }) { + const named = input.request.match(/\b([a-z][a-z0-9_]*)\s+table\b/i)?.[1] ?? input.request.match(/\btable\s+(?:for|called|named)\s+([a-z][a-z0-9_]*)/i)?.[1] ?? "new_table"; + const name = ["a", "new", "the", "another"].includes(named.toLowerCase()) ? "new_table" : named.toLowerCase(); + const parent = input.tables[0]?.name ?? input.otherTables?.[0]; + const parentBare = parent?.split(".").pop() ?? ""; + const operations: unknown[] = [ + { + type: "create_table", + table: { + name: parent?.includes(".") ? `${parent.split(".")[0]}.${name}` : name, + fields: [ + { name: "id", dataType: "bigint", primaryKey: true, autoIncrement: true }, + ...(parent ? [{ name: `${parentBare.replace(/s$/, "")}_id`, dataType: "bigint", nullable: false }] : []), + { name: "body", dataType: "text" }, + { name: "created_at", dataType: "timestamptz", nullable: false, defaultValue: "now()" }, + ], + }, + }, + ]; + if (parent) operations.push({ type: "add_relationship", relationship: { sourceTable: parent, sourceColumn: "id", targetTable: parent.includes(".") ? `${parent.split(".")[0]}.${name}` : name, targetColumn: `${parentBare.replace(/s$/, "")}_id` } }); + return { summary: `Create ${name}${parent ? `, linked to ${parent}` : ""} (a stand-in answer, not a model's).`, assumptions: [], warnings: [], clarifyingQuestions: [], operations }; +} + +/** "remove the default from ..." : the first column the facts say has a default, with the default dropped (written as strict mode makes a model write it). */ +function dropDefaultPlan(input: { tables: (Table & { columns: { name: string; hasDefault?: boolean }[] })[] }) { + for (const table of input.tables) { + const column = table.columns.find((candidate) => candidate.hasDefault); + if (column) return { summary: `Remove the default of ${table.name}.${column.name} (a stand-in answer, not a model's).`, assumptions: [], warnings: [], clarifyingQuestions: [], operations: [{ type: "alter_column", tableName: table.name, columnName: column.name, changes: { dropDefault: true } }] }; + } + return { summary: "No column has a default.", assumptions: [], warnings: [], clarifyingQuestions: ["Which column should lose its default?"], operations: [] }; +} + +function planFor(input: { request: string; tables: Table[]; otherTables?: string[] }) { + if (/\bdefault\b/i.test(input.request) && /\b(remove|drop|delete)\b/i.test(input.request)) return dropDefaultPlan(input as never); + if (/\b(create|add|new)\b[^.]*\btable\b/i.test(input.request)) return createTablePlan(input); + // the biggest table it was shown that has an integer column, else the biggest one + const tables = [...input.tables].sort((a, b) => (b.rows ?? 0) - (a.rows ?? 0) || Number(b.columns.some((c) => c.type === "integer")) - Number(a.columns.some((c) => c.type === "integer")) || a.name.localeCompare(b.name)); + const operations: unknown[] = []; + const first = tables[0]; + if (first) { + const taken = new Set(first.columns.map((column) => column.name)); + if (!taken.has("reviewed_at")) operations.push({ type: "add_column", tableName: first.name, field: { name: "reviewed_at", dataType: "timestamptz" } }); + const candidate = first.columns.find((column) => !first.primaryKey.includes(column.name) && !first.indexes.some((index) => index.columns[0] === column.name) && !column.type.startsWith("text")); + if (candidate) operations.push({ type: "add_index", tableName: first.name, index: { name: `fake_${first.name.replace(/[^a-z0-9_]/gi, "_").slice(-30)}_${candidate.name}_idx`, columns: [candidate.name] } }); + const nullable = first.columns.find((column) => !column.notNull && !first.primaryKey.includes(column.name) && /(_id|status|code)$/.test(column.name)); + if (nullable) operations.push({ type: "alter_column", tableName: first.name, columnName: nullable.name, changes: { nullable: false } }); + const integer = first.columns.find((column) => column.type === "integer"); + if (integer) operations.push({ type: "alter_column", tableName: first.name, columnName: integer.name, changes: { dataType: "bigint" } }); + } + return { summary: `Proposed ${operations.length} change${operations.length === 1 ? "" : "s"} to ${first?.name ?? "nothing"} (a stand-in answer, not a model's).`, assumptions: [], warnings: [], clarifyingQuestions: operations.length === 0 ? ["Which table should change?"] : [], operations }; +} + +function summaryFor(input: { findings: { id: string; category: string; table: string | null; object: string | null; bytes: number | null }[] }) { + const sorted = [...input.findings].sort((a, b) => (b.bytes ?? 0) - (a.bytes ?? 0)).slice(0, 4); + return { + headline: `${input.findings.length} findings were shown to this stand-in; it ordered them by size.`, + priorities: sorted.map((finding) => ({ findingIds: [finding.id], why: `${finding.category} on ${finding.table ?? finding.object ?? "the database"}${finding.bytes ? ` involves ${finding.bytes} bytes` : ""}.` })), + }; +} + +/** A value that satisfies a JSON schema, for a prompt this stand-in has no answer for: placeholder text, true, 0, empty lists, null where allowed. */ +function placeholderFor(schema: any): unknown { + if (!schema || typeof schema !== "object") return null; + if (Array.isArray(schema.anyOf)) return placeholderFor(schema.anyOf.find((candidate: any) => candidate.type !== "null") ?? schema.anyOf[0]); + if (schema.const !== undefined) return schema.const; + if (Array.isArray(schema.enum)) return schema.enum[0]; + if (schema.type === "object") return Object.fromEntries(Object.entries(schema.properties ?? {}).map(([key, child]) => [key, (child as any)?.nullable ? null : placeholderFor(child)])); + if (schema.type === "array") return []; + if (schema.type === "string") return "check"; + if (schema.type === "boolean") return true; + return 0; +} + +/** + * What a model that was given a strict JSON schema would send: every key of every object present, the ones the answer does not use as + * null. This stand-in fills its (loose) answers out the same way, so the page is tried with the shape a real model is held to. + */ +function completeToSchema(value: unknown, schema: any): unknown { + if (!schema || typeof schema !== "object") return value; + if (Array.isArray(schema.anyOf)) { + const record = value && typeof value === "object" ? (value as Record) : undefined; + const variant = schema.anyOf.find((candidate: any) => candidate.properties?.type?.const !== undefined && candidate.properties.type.const === record?.type) ?? schema.anyOf.find((candidate: any) => candidate.type !== "null"); + return value === null || value === undefined ? null : completeToSchema(value, variant); + } + if (schema.type === "array") return Array.isArray(value) ? value.map((item) => completeToSchema(item, schema.items)) : value; + if (schema.type === "object" && schema.properties) { + const record = value && typeof value === "object" ? (value as Record) : {}; + return Object.fromEntries(Object.entries(schema.properties).map(([key, child]) => [key, key in record && record[key] !== undefined ? completeToSchema(record[key], child) : null])); + } + return value; +} + +http + .createServer((request, response) => { + const chunks: Buffer[] = []; + request.on("data", (chunk: Buffer) => chunks.push(chunk)); + request.on("end", () => { + const reply = (status: number, body: unknown) => { + response.writeHead(status, { "Content-Type": "application/json" }); + response.end(JSON.stringify(body)); + }; + if (request.method === "GET" && request.url === "/v1/models") return reply(200, { object: "list", data: [{ id: "fake-model", object: "model", owned_by: "stand-in" }] }); + if (request.method !== "POST" || request.url !== "/v1/responses") return reply(404, { error: { message: "only POST /v1/responses and GET /v1/models", type: "invalid_request_error", code: null } }); + try { + const body = JSON.parse(Buffer.concat(chunks).toString("utf8")) as { instructions?: string; input?: string; text?: { format?: { type?: string; schema?: unknown } } }; + const system = body.instructions ?? ""; + const user = body.input ?? ""; + // Anything it has no answer for is a placeholder that satisfies the schema it was given (or {"ok": true} when it was given none). + const requested = body.text?.format; + let answer: unknown = requested?.type === "json_schema" ? placeholderFor(requested.schema) : { ok: true }; + if (system.startsWith("You are GoForge's schema change planner")) answer = planFor(JSON.parse(user)); + else if (system.startsWith("You are GoForge's database schema planning engine")) { + // the diagram's AI Schema panel: the same kind of plan, from a differently shaped request + const asked = JSON.parse(user) as { userRequest: string; currentSchema: { tables: { name: string }[] } }; + answer = createTablePlan({ request: asked.userRequest, tables: asked.currentSchema.tables as never }); + } else if (system.startsWith("You summarise a PostgreSQL schema health report")) answer = summaryFor(JSON.parse(user)); + else if (system.startsWith("You are GoForge's SQL migration engine")) { + const asked = JSON.parse(user) as { originalQuery: string; ruleBasedDraft?: { sql?: string } | null }; + answer = { sql: asked.ruleBasedDraft?.sql ?? asked.originalQuery, summary: "Kept the rules' draft (a stand-in answer, not a model's).", changes: [], risks: [], unsupported: [] }; + } + // With a schema the answer is held to it, as it would be by OpenAI; without one it is left as it is. + const format = body.text?.format; + if (format?.type === "json_schema") answer = completeToSchema(answer, format.schema); + reply(200, { id: "resp_stand_in", status: "completed", output: [{ type: "message", role: "assistant", content: [{ type: "output_text", text: JSON.stringify(answer) }] }] }); + } catch (error) { + reply(400, { error: { message: error instanceof Error ? error.message : String(error), type: "invalid_request_error", code: null } }); + } + }); + }) + .listen(PORT, "127.0.0.1", () => console.log(`fake OpenAI endpoint on http://127.0.0.1:${PORT}/v1`)); diff --git a/scripts/lib/synthetic-schema.test.ts b/scripts/lib/synthetic-schema.test.ts new file mode 100644 index 0000000..821698e --- /dev/null +++ b/scripts/lib/synthetic-schema.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from "vitest"; +import { buildSearchIndex, selectScope } from "../../server/src/schema-change/retrieval"; +import type { Catalog } from "../../server/src/schema-change/catalog"; +import { BIG_TABLE_ROWS, generateSchema } from "./synthetic-schema"; + +const asCatalog = (schema: ReturnType): Catalog => ({ + serverVersionNum: 160000, + serverVersion: "16.0", + loadedAt: "2026-01-01T00:00:00Z", + tables: schema.tables.map((table) => ({ + schema: table.schema, name: table.name, qualified: table.qualified, partitioned: false, partitionCount: 0, estimatedRows: table.rows, totalBytes: 0, + columns: table.columns.map((column) => ({ name: column.name, type: column.type, notNull: column.notNull })), primaryKey: ["id"], + foreignKeys: table.foreignKeys.map((fk) => ({ name: `${table.name}_${fk.column}_fkey`, columns: [fk.column], refSchema: fk.references.split(".")[0], refTable: fk.references.split(".")[1], refColumns: ["id"], validated: true })), + })), +}); + +describe("the generated schemas the evaluation runs on", () => { + it("have exactly the number of tables asked for, with unique qualified names, and are the same every time", () => { + for (const size of [50, 300, 1000]) { + const schema = generateSchema(size, 5); + expect(schema.tables).toHaveLength(size); + expect(new Set(schema.tables.map((t) => t.qualified)).size).toBe(size); + } + expect(generateSchema(300, 5).createSql).toEqual(generateSchema(300, 5).createSql); + expect(generateSchema(300, 5).createSql).not.toEqual(generateSchema(300, 6).createSql); + }); + + it("only refer to tables and columns that exist, and populate children of populated parents", () => { + const schema = generateSchema(300, 9); + const byName = new Map(schema.tables.map((t) => [t.qualified, t])); + for (const table of schema.tables) { + const names = table.columns.map((c) => c.name); + expect(new Set(names).size).toBe(names.length); + for (const fk of table.foreignKeys) { + expect(names).toContain(fk.column); + expect(byName.has(fk.references)).toBe(true); + } + for (const index of table.indexes) for (const column of index.columns) expect(names).toContain(column); + for (const column of table.columns) if (column.refersTo) expect(byName.has(column.refersTo)).toBe(true); + } + const populated = schema.tables.filter((t) => t.rows > 0 && t.role !== "hub"); + expect(populated.length).toBeGreaterThan(0); + for (const table of populated) expect(table.rows).toBe(BIG_TABLE_ROWS); + expect(schema.populateSql.length).toBe(schema.tables.filter((t) => t.rows > 0).length); + }); + + it("give retrieval something real to find: every table is found by its own words", () => { + const schema = generateSchema(300, 3); + const index = buildSearchIndex(asCatalog(schema)); + let found = 0; + const roots = schema.tables.filter((t) => t.role === "root"); + for (const table of roots) { + const phrase = [table.module, ...table.words].map((w) => w.replace(/_/g, " ")).join(" "); + if (selectScope(index, `change the ${phrase} table`).tables.some((t) => t.qualified === table.qualified)) found++; + } + expect(found / roots.length).toBeGreaterThan(0.9); + }); +}); diff --git a/scripts/lib/synthetic-schema.ts b/scripts/lib/synthetic-schema.ts new file mode 100644 index 0000000..66475d2 --- /dev/null +++ b/scripts/lib/synthetic-schema.ts @@ -0,0 +1,270 @@ +/** + * Generates a database of N tables that looks like a grown-up business schema: modules (some as PostgreSQL schemas, some as + * name prefixes in `public`), root entities with child, history and note tables, shared lookup tables, foreign keys to them, + * domain columns, a few legacy columns that point somewhere without a foreign key, and a few extra indexes. Deterministic + * for a given (tableCount, seed). Used by the evaluation and by the demo database in the browser check. + */ + +export interface SyntheticColumn { + name: string; + type: string; + notNull: boolean; + /** SQL that fills the column when the table is populated, with `g` as the row number. Undefined: left NULL (or an identity). */ + fill?: string; + /** For a column that points at another table's id without a foreign key. */ + refersTo?: string; +} + +export interface SyntheticTable { + schema: string; + name: string; + qualified: string; + /** The words a person would use for it: ["invoice", "items"]. */ + words: string[]; + module: string; + entity: string; + role: "hub" | "root" | "child" | "history" | "notes"; + parent?: string; + columns: SyntheticColumn[]; + foreignKeys: { column: string; references: string }[]; + indexes: { name: string; columns: string[] }[]; + /** Rows to insert when the database is populated (0: left empty). */ + rows: number; +} + +export interface SyntheticSchema { + tables: SyntheticTable[]; + seed: number; + /** Creation statements in dependency order. */ + createSql: string[]; + /** Foreign keys, to be added after the data is loaded. */ + foreignKeySql: string[]; + /** Statements that load rows into the populated tables. */ + populateSql: string[]; +} + +function rng(seed: number) { + let state = seed >>> 0; + const next = () => { + state = (state + 0x6d2b79f5) >>> 0; + let t = state; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; + return { + next, + int: (max: number) => Math.floor(next() * max), + pick: (list: readonly T[]): T => list[Math.floor(next() * list.length)], + chance: (p: number) => next() < p, + shuffle: (list: readonly T[]): T[] => { + const copy = [...list]; + for (let i = copy.length - 1; i > 0; i--) { + const j = Math.floor(next() * (i + 1)); + [copy[i], copy[j]] = [copy[j], copy[i]]; + } + return copy; + }, + }; +} + +const MODULES = ["sales", "billing", "inventory", "procurement", "logistics", "support", "marketing", "finance", "hr", "payroll", "projects", "compliance", "analytics", "identity", "catalog", "pricing", "fulfillment", "returns", "contracts", "assets", "maintenance", "quality", "research", "training", "facilities", "legal", "security", "partners", "subscriptions", "notifications"]; + +const ENTITIES = ["order", "invoice", "payment", "shipment", "customer", "supplier", "product", "warehouse", "ticket", "campaign", "lead", "contract", "employee", "timesheet", "expense", "budget", "project", "task", "milestone", "audit", "policy", "incident", "asset", "vehicle", "route", "delivery", "refund", "quote", "subscription", "plan", "coupon", "review", "survey", "document", "approval", "schedule", "shift", "claim", "license", "certificate", "vendor", "requisition", "receipt", "ledger", "journal", "forecast", "report", "alert", "template", "contact", "account", "address", "session", "device", "event", "booking", "reservation", "inspection", "batch", "recipe", "component", "sample", "experiment", "trial", "filing", "dispute", "settlement", "benefit", "leave", "candidate", "interview", "offer", "course", "enrollment", "visit", "appointment", "pallet", "container", "manifest", "tariff", "rebate", "discount", "bundle", "listing", "bid", "pledge", "donation", "grant", "tenant", "lease", "meter", "reading", "outage", "permit", "recall", "warranty", "voucher", "payout", "shipping_label", "pick_list", "stock_count"]; + +const DERIVED = [ + { suffix: "items", role: "child" as const }, + { suffix: "history", role: "history" as const }, + { suffix: "notes", role: "notes" as const }, + { suffix: "attachments", role: "child" as const }, + { suffix: "tags", role: "child" as const }, + { suffix: "revisions", role: "history" as const }, +]; + +const DOMAIN_COLUMNS: { name: string; type: string; fill?: string }[] = [ + { name: "warehouse_code", type: "varchar(20)", fill: "'W' || (g % 40)" }, + { name: "shipped_at", type: "timestamptz" }, + { name: "delivered_at", type: "timestamptz" }, + { name: "invoice_no", type: "varchar(30)", fill: "'INV-' || g" }, + { name: "sku", type: "varchar(40)", fill: "'SKU-' || (g % 5000)" }, + { name: "region", type: "varchar(40)", fill: "'region-' || (g % 12)" }, + { name: "priority", type: "smallint", fill: "g % 5" }, + { name: "currency_code", type: "char(3)", fill: "'USD'" }, + { name: "tax_rate", type: "numeric(5,2)", fill: "(g % 25)::numeric" }, + { name: "discount_pct", type: "numeric(5,2)", fill: "(g % 30)::numeric" }, + { name: "quantity", type: "integer", fill: "1 + g % 50" }, + { name: "unit_price", type: "numeric(12,2)", fill: "(g % 9000)::numeric / 10" }, + { name: "total_amount", type: "numeric(14,2)", fill: "(g % 90000)::numeric / 10" }, + { name: "due_date", type: "date", fill: "current_date + (g % 90)" }, + { name: "approved_at", type: "timestamptz" }, + { name: "cost_center", type: "varchar(20)", fill: "'CC' || (g % 90)" }, + { name: "external_ref", type: "varchar(64)", fill: "md5(g::text)" }, + { name: "channel", type: "varchar(30)", fill: "'channel-' || (g % 6)" }, + { name: "weight_kg", type: "numeric(8,3)", fill: "(g % 5000)::numeric / 7" }, + { name: "language", type: "varchar(8)", fill: "'en'" }, + { name: "is_archived", type: "boolean", fill: "g % 20 = 0" }, + { name: "severity", type: "smallint", fill: "g % 4" }, + { name: "resolution", type: "text" }, + { name: "contact_email", type: "varchar(254)", fill: "'user' || g || '@example.test'" }, + { name: "phone", type: "varchar(32)", fill: "'+1555' || (1000000 + g % 8999999)" }, + { name: "latitude", type: "double precision", fill: "(g % 180) - 90" }, + { name: "longitude", type: "double precision", fill: "(g % 360) - 180" }, + { name: "batch_no", type: "varchar(24)", fill: "'B' || (g % 700)" }, + { name: "serial_no", type: "varchar(40)", fill: "'S' || g" }, + { name: "expires_on", type: "date", fill: "current_date + (g % 400)" }, + { name: "score", type: "integer", fill: "g % 100" }, + { name: "rating", type: "smallint", fill: "g % 6" }, + { name: "retry_count", type: "integer", fill: "g % 4" }, + { name: "checksum", type: "varchar(64)", fill: "md5(g::text)" }, +]; + +const SIGNATURE_WORDS = ["carrier", "courier", "depot", "dock", "lane", "yard", "bay", "gate", "pier", "terminal", "locker", "rack", "shelf", "bin", "zone", "ledger_page", "cohort", "stratum", "tranche", "cadence", "sprint", "epoch", "lot", "wave", "tier", "band", "slot", "grade", "variant", "revision_tag", "channel_key", "partner_key", "origin_code", "target_code"]; + +const HUBS: { name: string; extra: SyntheticColumn[] }[] = [ + { name: "users", extra: [{ name: "email", type: "varchar(254)", notNull: true, fill: "'u' || g || '@example.test'" }, { name: "display_name", type: "varchar(120)", notNull: false, fill: "'User ' || g" }] }, + { name: "organizations", extra: [{ name: "legal_name", type: "varchar(200)", notNull: true, fill: "'Org ' || g" }] }, + { name: "countries", extra: [{ name: "iso_code", type: "char(2)", notNull: true, fill: "'C' || chr(65 + g % 26)" }] }, + { name: "currencies", extra: [{ name: "iso_code", type: "char(3)", notNull: true, fill: "'X' || chr(65 + g % 26) || chr(65 + (g / 26) % 26)" }] }, +]; + +const HUB_ROWS = 2000; + +/** Rows loaded into the populated ("big") tables; enough that the data is above the "small" limit (16 MiB) used to size a change. */ +export const BIG_TABLE_ROWS = 260_000; + +const quote = (name: string) => (/^[a-z_][a-z0-9_]*$/.test(name) ? name : `"${name}"`); + +export function generateSchema(tableCount: number, seed: number, options: { bigTables?: number } = {}): SyntheticSchema { + const random = rng(seed); + const tables: SyntheticTable[] = []; + const taken = new Set(); + const add = (table: SyntheticTable) => { + if (taken.has(table.qualified)) return false; + taken.add(table.qualified); + tables.push(table); + return true; + }; + + for (const hub of HUBS) { + add({ + schema: "public", name: hub.name, qualified: `public.${hub.name}`, words: [hub.name], module: "core", entity: hub.name, role: "hub", + columns: [{ name: "id", type: "bigint", notNull: true }, ...hub.extra, { name: "created_at", type: "timestamptz", notNull: true, fill: "now() - (g % 1000) * interval '1 hour'" }], + foreignKeys: [], indexes: [], rows: HUB_ROWS, + }); + } + + const moduleCount = Math.max(1, Math.min(MODULES.length, Math.ceil(tableCount / 40))); + const modules = random.shuffle(MODULES).slice(0, moduleCount); + const entityPool = random.shuffle(ENTITIES); + let cursor = 0; + + // Fill module by module, root by root, until the count is reached. + let guard = 0; + while (tables.length < tableCount && guard++ < tableCount * 20) { + const module = modules[guard % modules.length]; + const separateSchema = MODULES.indexOf(module) % 3 !== 0; + const entity = entityPool[cursor++ % entityPool.length]; + const schema = separateSchema ? module : "public"; + const rootName = separateSchema ? `${entity}s` : `${module}_${entity}s`; + const root: SyntheticTable = { + schema, name: rootName, qualified: `${schema}.${rootName}`, words: separateSchema ? [entity] : [module, entity], module, entity, role: "root", + columns: [ + { name: "id", type: "bigint", notNull: true }, + { name: "code", type: "varchar(24)", notNull: true, fill: "substr(md5(g::text), 1, 12)" }, + { name: "status", type: "varchar(20)", notNull: true, fill: "'status-' || (g % 5)" }, + { name: "owner_id", type: "bigint", notNull: true, fill: `1 + g % ${HUB_ROWS}` }, + { name: "organization_id", type: "bigint", notNull: false, fill: `1 + g % ${HUB_ROWS}` }, + { name: "created_at", type: "timestamptz", notNull: true, fill: "now() - (g % 1000) * interval '1 hour'" }, + { name: "updated_at", type: "timestamptz", notNull: false }, + { name: "notes", type: "text", notNull: false }, + ], + foreignKeys: [{ column: "owner_id", references: "public.users" }, { column: "organization_id", references: "public.organizations" }], + indexes: [], rows: 0, + }; + for (const domain of random.shuffle(DOMAIN_COLUMNS).slice(0, 3 + random.int(6))) root.columns.push({ name: domain.name, type: domain.type, notNull: false, fill: domain.fill }); + if (random.chance(0.35)) { + const signature = `${entity}_${random.pick(SIGNATURE_WORDS)}_code`; + if (!root.columns.some((column) => column.name === signature)) root.columns.push({ name: signature, type: "varchar(30)", notNull: false, fill: "'sig-' || (g % 900)" }); + } + if (random.chance(0.4)) root.indexes.push({ name: `${rootName}_status_idx`, columns: ["status"] }); + if (!add(root)) continue; + + const derived = random.shuffle(DERIVED).slice(0, 1 + random.int(3)); + for (const kind of derived) { + if (tables.length >= tableCount) break; + const name = separateSchema ? `${entity}_${kind.suffix}` : `${module}_${entity}_${kind.suffix}`; + const fk = `${entity}_id`; + const child: SyntheticTable = { + schema, name, qualified: `${schema}.${name}`, words: [...root.words, kind.suffix], module, entity, role: kind.role === "history" ? "history" : kind.suffix === "notes" ? "notes" : "child", parent: root.qualified, + columns: [ + { name: "id", type: "bigint", notNull: true }, + { name: fk, type: "bigint", notNull: true }, + ...(kind.role === "history" + ? [{ name: "changed_at", type: "timestamptz", notNull: true, fill: "now() - (g % 1000) * interval '1 hour'" }, { name: "changed_by", type: "bigint", notNull: false, fill: `1 + g % ${HUB_ROWS}` }, { name: "old_value", type: "text", notNull: false }, { name: "new_value", type: "text", notNull: false }] + : kind.suffix === "notes" + ? [{ name: "body", type: "text", notNull: true, fill: "'note ' || g" }, { name: "created_at", type: "timestamptz", notNull: true, fill: "now() - (g % 1000) * interval '1 hour'" }] + : [{ name: "label", type: "varchar(120)", notNull: false, fill: "'item ' || g" }, { name: "quantity", type: "integer", notNull: true, fill: "1 + g % 50" }, { name: "created_at", type: "timestamptz", notNull: true, fill: "now() - (g % 1000) * interval '1 hour'" }]), + ], + foreignKeys: [{ column: fk, references: root.qualified }], + indexes: random.chance(0.5) ? [{ name: `${name}_${fk}_idx`, columns: [fk] }] : [], + rows: 0, + }; + // some children carry a column that points at another root without a foreign key, the way grown databases do + const other = tables.filter((t) => t.role === "root" && t.qualified !== root.qualified); + if (other.length > 0 && random.chance(0.3)) { + const target = random.pick(other); + child.columns.push({ name: `${target.entity}_ref_id`, type: "bigint", notNull: false, refersTo: target.qualified }); + } + if (random.chance(0.25)) child.indexes.push({ name: `${name}_created_idx`, columns: [kind.role === "history" ? "changed_at" : "created_at"] }); + add(child); + } + } + + // The populated tables: a few roots with their children, so a change to them is a change to real data. + const big = options.bigTables ?? Math.max(2, Math.min(10, Math.round(tableCount / 40))); + const roots = tables.filter((t) => t.role === "root"); + for (const root of random.shuffle(roots).slice(0, Math.ceil(big / 2))) { + root.rows = BIG_TABLE_ROWS; + for (const child of tables.filter((t) => t.parent === root.qualified).slice(0, 1)) child.rows = BIG_TABLE_ROWS; + } + // legacy reference columns are only filled in tables that are loaded, with ids of rows that exist + for (const table of tables) { + for (const column of table.columns) { + if (column.refersTo && table.rows > 0) { + const target = tables.find((t) => t.qualified === column.refersTo); + column.fill = target && target.rows > 0 ? `1 + g % ${target.rows}` : undefined; + } + } + } + + const createSql: string[] = []; + for (const schema of new Set(tables.map((t) => t.schema))) if (schema !== "public") createSql.push(`CREATE SCHEMA ${quote(schema)}`); + for (const table of tables) { + const columns = table.columns.map((column) => `${quote(column.name)} ${column.type}${column.name === "id" ? " GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY" : ""}${column.notNull && column.name !== "id" ? " NOT NULL" : ""}`); + createSql.push(`CREATE TABLE ${quote(table.schema)}.${quote(table.name)} (${columns.join(", ")})`); + for (const index of table.indexes) createSql.push(`CREATE INDEX ${quote(index.name)} ON ${quote(table.schema)}.${quote(table.name)} (${index.columns.map(quote).join(", ")})`); + } + + const populateSql: string[] = []; + for (const table of tables) { + if (table.rows === 0) continue; + const filled = table.columns.filter((column) => column.name !== "id" && column.fill !== undefined); + const parentRows = table.parent ? (tables.find((t) => t.qualified === table.parent)?.rows ?? 0) : 0; + const fkColumn = table.parent ? table.foreignKeys[0]?.column : undefined; + const columns = [...filled.map((column) => column.name), ...(fkColumn ? [fkColumn] : [])]; + const values = [...filled.map((column) => column.fill as string), ...(fkColumn ? [`1 + g % ${Math.max(1, parentRows)}`] : [])]; + populateSql.push(`INSERT INTO ${quote(table.schema)}.${quote(table.name)} (${columns.map(quote).join(", ")}) SELECT ${values.join(", ")} FROM generate_series(1, ${table.rows}) AS g`); + } + + const foreignKeySql = tables.flatMap((table) => + table.foreignKeys.map((fk) => { + const [schema, name] = fk.references.split("."); + return `ALTER TABLE ${quote(table.schema)}.${quote(table.name)} ADD CONSTRAINT ${quote(`${table.name}_${fk.column}_fkey`.slice(0, 63))} FOREIGN KEY (${quote(fk.column)}) REFERENCES ${quote(schema)}.${quote(name)} (id)`; + }), + ); + + return { tables, seed, createSql, foreignKeySql, populateSql }; +} + +/** `warehouse_code` -> "warehouse code" */ +export const wordsOf = (name: string): string => name.replace(/_/g, " "); diff --git a/scripts/schema-change-eval.ts b/scripts/schema-change-eval.ts new file mode 100644 index 0000000..0e34c0b --- /dev/null +++ b/scripts/schema-change-eval.ts @@ -0,0 +1,667 @@ +/** + * Measures the Schema Change pipeline on generated PostgreSQL databases of 50, 300 and 1,000 tables. + * + * docker run -d --name goforge-schema -e POSTGRES_PASSWORD=schema -e POSTGRES_DB=schema -p 54330:5432 postgres:16 -c max_locks_per_transaction=1024 + * npm run eval:schema # 50 / 300 / 1,000 tables, 25 requests each + * npm run eval:schema -- --sizes 300 --per-size 30 --seed 3 + * npm run eval:schema -- --keep # leave the generated databases for the browser (sc_eval_50, ...) + * OPENAI_API_KEY=sk-... [EVAL_MODEL=gpt-5.6-luna] npm run eval:schema -- --live # a real model plans instead of the reference plans + * (the key has to be in this command's environment, this script does not read server/.env; EVAL_MODEL empty means OPENAI_MODEL, then gpt-5.6-luna) + * + * Each size gets a database with a real schema, real foreign keys and populated tables (260,000 rows in the big ones). A request is + * a sentence in the words a person would use, with the operations a correct plan contains. For every request this measures: + * - table selection: are the tables the change touches in the scope that retrieval picks, and how big is the scope + * - plan validity: do the plan's operations pass the checks against the real catalog (with the retrieved scope, and with the right one) + * - migration lint: does the compiled runbook avoid the usual ways of locking a large table (and does a naive rendering not) + * - execution: does the runbook run, statement by statement in autocommit mode, on PostgreSQL 16, and leave the schema as asked + * - invalid requests: are they refused before any SQL is produced + * - latency of every deterministic stage, and what the request would cost to send (against sending every table) + * The planner is the reference plan unless --live is given; with the reference plan nothing here says anything about a model's quality, + * and the model's own latency and tokens are reported as not measured. + */ +import { DEFAULT_SETTINGS } from "../server/src/ai/policy"; +import { getSettingsStore } from "../server/src/ai/provider"; +import { callModelJson } from "../server/src/ai/provider"; +import { aiSchemaPlanSchema, type AiSchemaPlan } from "../server/src/ai/schema-plan"; +import { loadCatalog } from "../server/src/schema-change/catalog"; +import { compilePlan, discoverProbes } from "../server/src/schema-change/compile"; +import { loadFacts, probe, type Facts } from "../server/src/schema-change/facts"; +import { lintSteps } from "../server/src/schema-change/lint"; +import { planSchemaChange } from "../server/src/schema-change/planner"; +import { plannerInput, plannerInstructions } from "../server/src/schema-change/prompt"; +import { buildSearchIndex, selectScope } from "../server/src/schema-change/retrieval"; +import { sizeClass, type Runbook, type RunbookStep } from "../server/src/schema-change/runbook"; +import { createDatabase, dropDatabase, withDatabase, type PgClient } from "./lib/pg-scratch"; +import { asRunner, runRunbook } from "./lib/plan-runner"; +import { generateSchema, wordsOf, type SyntheticSchema, type SyntheticTable } from "./lib/synthetic-schema"; + +// ---- arguments ----------------------------------------------------------------------------------------------------------- + +const args = process.argv.slice(2); +const option = (name: string, fallback: string): string => { + const index = args.indexOf(`--${name}`); + return index >= 0 && args[index + 1] ? args[index + 1] : fallback; +}; +const SIZES = option("sizes", "50,300,1000").split(",").map(Number); +const PER_SIZE = Number(option("per-size", "20")); +const NEGATIVE_PER_SIZE = Number(option("negative", "5")); +const SEED = Number(option("seed", "11")); +const KEEP = args.includes("--keep"); +const LIVE = args.includes("--live"); +const OLD_TABLE_LIMIT = 250; + +// ---- requests ------------------------------------------------------------------------------------------------------------ + +type Style = "plain" | "with-module" | "typo" | "column-only" | "partial-name"; + +interface Case { + id: string; + kind: string; + style: Style; + text: string; + /** The tables the change touches. */ + truth: string[]; + operations: unknown[]; + /** A query that returns one row with `ok = true` when the change happened. */ + verify?: string; + /** For an invalid request: the finding code the checks must refuse it with. */ + refusedWith?: string; +} + +function lcg(seed: number) { + let state = seed >>> 0; + const next = () => { + state = (Math.imul(state, 1664525) + 1013904223) >>> 0; + return state / 4294967296; + }; + return { int: (n: number) => Math.floor(next() * n), pick: (list: readonly T[]): T => list[Math.floor(next() * list.length)], chance: (p: number) => next() < p }; +} + +const NEW_COLUMNS = [ + { name: "carrier_name", type: "varchar", words: "carrier name" }, + { name: "internal_comment", type: "text", words: "internal comment" }, + { name: "reviewed_at", type: "timestamptz", words: "reviewed at timestamp" }, + { name: "risk_score", type: "integer", words: "risk score" }, + { name: "source_system", type: "varchar", words: "source system" }, +]; + +const q = (name: string) => `"${name.replace(/"/g, '""')}"`; +const dotted = (t: SyntheticTable) => `${q(t.schema)}.${q(t.name)}`; + +function typoOf(word: string, r: ReturnType): string { + if (word.length < 6) return word; + const at = 1 + r.int(word.length - 3); + return r.chance(0.5) ? word.slice(0, at) + word.slice(at + 1) : word.slice(0, at) + word[at + 1] + word[at] + word.slice(at + 2); +} + +function buildCases(schema: SyntheticSchema, count: number, negatives: number, seed: number): Case[] { + const r = lcg(seed); + const tables = schema.tables.filter((t) => t.role !== "hub"); + const bareCount = new Map(); + for (const t of schema.tables) bareCount.set(t.name, (bareCount.get(t.name) ?? 0) + 1); + const used = new Set(); + + /** How a person would name the table: its words, the module when the name alone would be ambiguous, sometimes with a typo or only its last words. */ + const say = (t: SyntheticTable, style: Style): string => { + let words = t.words.map(wordsOf); + if (style === "partial-name" && words.length >= 3) words = words.slice(-2); + if (style === "with-module" && !words.includes(wordsOf(t.module))) words = [wordsOf(t.module), ...words]; + if (style === "typo") { + const longest = words.reduce((best, word, i) => (word.length > words[best].length ? i : best), 0); + words = words.map((word, i) => (i === longest ? typoOf(word, r) : word)); + } + return words.join(" "); + }; + const styleFor = (t: SyntheticTable, preferred: Style): Style => ((bareCount.get(t.name) ?? 0) > 1 && preferred !== "typo" ? "with-module" : preferred); + // Every request runs on its own copy of the database, so a table can be used again once the unused ones are gone. + const fresh = (list: SyntheticTable[]): SyntheticTable | undefined => { + const unused = list.filter((t) => !used.has(t.qualified)); + const options = unused.length > 0 ? unused : list; + const picked = options.length > 0 ? r.pick(options) : undefined; + if (picked) used.add(picked.qualified); + return picked; + }; + const exists = (name: string, t: SyntheticTable) => t.columns.some((c) => c.name === name); + const populated = tables.filter((t) => t.rows > 0); + const styles: Style[] = ["plain", "plain", "with-module", "typo", "partial-name"]; + const cases: Case[] = []; + + const generators: Array<() => Case | undefined> = [ + // a column nobody has yet + () => { + const t = fresh(r.chance(0.5) && populated.length ? populated : tables); + if (!t) return; + const column = r.pick(NEW_COLUMNS.filter((c) => !exists(c.name, t))); + if (!column) return; + const style = styleFor(t, r.pick(styles)); + return { + id: "", kind: "add column", style, text: `add a ${column.words} column to the ${say(t, style)} table`, truth: [t.qualified], + operations: [{ type: "add_column", tableName: t.qualified, field: { name: column.name, dataType: column.type, ...(column.type === "varchar" ? { maxLength: 120 } : {}) } }], + verify: `SELECT count(*) = 1 AS ok FROM information_schema.columns WHERE table_schema = '${t.schema}' AND table_name = '${t.name}' AND column_name = '${column.name}'`, + }; + }, + // a required flag with a default: no rewrite on a big table + () => { + const t = fresh(populated.length ? populated : tables); + if (!t) return; + const style = styleFor(t, r.pick(styles)); + return { + id: "", kind: "add required column with default", style, text: `every ${say(t, style)} record needs a required reviewed flag that is false unless someone sets it`, truth: [t.qualified], + operations: [{ type: "add_column", tableName: t.qualified, field: { name: "is_reviewed", dataType: "boolean", nullable: false, defaultValue: "false" } }], + verify: `SELECT count(*) = 1 AS ok FROM pg_attribute WHERE attrelid = '${dotted(t)}'::regclass AND attname = 'is_reviewed' AND attnotnull`, + }; + }, + // a column filled by a volatile default on existing rows + () => { + const t = fresh(populated.length ? populated : tables); + if (!t) return; + const style = styleFor(t, r.pick(styles)); + return { + id: "", kind: "add column with volatile default", style, text: `give each ${say(t, style)} a public uuid that is required`, truth: [t.qualified], + operations: [{ type: "add_column", tableName: t.qualified, field: { name: "public_uuid", dataType: "uuid", nullable: false, defaultValue: "gen_random_uuid()" } }], + verify: `SELECT count(*) = 1 AS ok FROM pg_attribute WHERE attrelid = '${dotted(t)}'::regclass AND attname = 'public_uuid' AND attnotnull`, + }; + }, + // an index on an existing column + () => { + const t = fresh(r.chance(0.6) && populated.length ? populated : tables); + if (!t) return; + const candidates = t.columns.filter((c) => !["id", "notes"].includes(c.name) && !t.indexes.some((i) => i.columns[0] === c.name) && !c.type.startsWith("text")); + const column = candidates.length ? r.pick(candidates) : undefined; + if (!column) return; + const style = styleFor(t, r.pick(styles)); + const name = `${t.name}_${column.name}_eval_idx`.slice(0, 60); + return { + id: "", kind: "add index", style, text: `queries on ${say(t, style)} filtered by ${wordsOf(column.name)} are slow, add an index`, truth: [t.qualified], + operations: [{ type: "add_index", tableName: t.qualified, index: { name, columns: [column.name] } }], + verify: `SELECT count(*) = 1 AS ok FROM pg_index WHERE indexrelid = '${q(t.schema)}.${q(name)}'::regclass AND indisvalid`, + }; + }, + // an index found through the column name alone + () => { + const counts = new Map(); + for (const t of tables) for (const c of t.columns) counts.set(c.name, (counts.get(c.name) ?? 0) + 1); + const t = fresh(tables.filter((x) => x.columns.some((c) => c.name.endsWith("_code") && c.name !== "code" && counts.get(c.name) === 1))); + if (!t) return; + const column = t.columns.find((c) => c.name.endsWith("_code") && c.name !== "code" && counts.get(c.name) === 1) ?? t.columns[0]; + const name = `${t.name}_${column.name}_eval_idx`.slice(0, 60); + return { + id: "", kind: "add index (column only)", style: "column-only", text: `lookups by ${wordsOf(column.name)} need an index`, truth: [t.qualified], + operations: [{ type: "add_index", tableName: t.qualified, index: { name, columns: [column.name] } }], + verify: `SELECT count(*) = 1 AS ok FROM pg_index WHERE indexrelid = '${q(t.schema)}.${q(name)}'::regclass AND indisvalid`, + }; + }, + // unique codes + () => { + const t = fresh(schema.tables.filter((x) => x.role === "root" && (x.rows > 0 || r.chance(0.5)))); + if (!t) return; + const style = styleFor(t, r.pick(styles)); + return { + id: "", kind: "make unique", style, text: `the code of a ${say(t, style)} must be unique`, truth: [t.qualified], + operations: [{ type: "alter_column", tableName: t.qualified, columnName: "code", changes: { unique: true } }], + verify: `SELECT count(*) = 1 AS ok FROM pg_constraint WHERE conrelid = '${dotted(t)}'::regclass AND contype = 'u'`, + }; + }, + // required column that already has values + () => { + const t = fresh(schema.tables.filter((x) => x.role === "root" && x.columns.some((c) => c.name === "organization_id") && (x.rows > 0 || r.chance(0.5)))); + if (!t) return; + const style = styleFor(t, r.pick(styles)); + return { + id: "", kind: "make column required", style, text: `a ${say(t, style)} must always belong to an organization`, truth: [t.qualified], + operations: [{ type: "alter_column", tableName: t.qualified, columnName: "organization_id", changes: { nullable: false } }], + verify: `SELECT count(*) = 1 AS ok FROM pg_attribute WHERE attrelid = '${dotted(t)}'::regclass AND attname = 'organization_id' AND attnotnull`, + }; + }, + // longer text: only the catalog changes + () => { + const t = fresh(populated.length ? populated : tables); + if (!t) return; + const column = t.columns.find((c) => /^varchar\(\d+\)$/.test(c.type) && c.name !== "code"); + if (!column) return; + const length = Number(/\((\d+)\)/.exec(column.type)?.[1]) + 100; + const style = styleFor(t, r.pick(styles)); + return { + id: "", kind: "lengthen a varchar", style, text: `${wordsOf(column.name)} on ${say(t, style)} is too short, allow longer values`, truth: [t.qualified], + operations: [{ type: "alter_column", tableName: t.qualified, columnName: column.name, changes: { dataType: "varchar", maxLength: length } }], + verify: `SELECT format_type(atttypid, atttypmod) = 'character varying(${length})' AS ok FROM pg_attribute WHERE attrelid = '${dotted(t)}'::regclass AND attname = '${column.name}'`, + }; + }, + // integer to bigint: a rewrite unless it is done in steps + () => { + const t = fresh(populated.filter((x) => x.columns.some((c) => c.type === "integer")).length ? populated.filter((x) => x.columns.some((c) => c.type === "integer")) : tables.filter((x) => x.columns.some((c) => c.type === "integer"))); + if (!t) return; + const column = t.columns.find((c) => c.type === "integer"); + if (!column) return; + const style = styleFor(t, r.pick(styles)); + return { + id: "", kind: "change integer to bigint", style, text: `${wordsOf(column.name)} on ${say(t, style)} is about to overflow, make it a bigint`, truth: [t.qualified], + operations: [{ type: "alter_column", tableName: t.qualified, columnName: column.name, changes: { dataType: "bigint" } }], + verify: `SELECT format_type(atttypid, atttypmod) = 'bigint' AS ok FROM pg_attribute WHERE attrelid = '${dotted(t)}'::regclass AND attname = '${column.name}'`, + }; + }, + // a foreign key for a column that points somewhere without one + () => { + const child = fresh(tables.filter((t) => t.columns.some((c) => c.refersTo))); + if (!child) return; + const column = child.columns.find((c) => c.refersTo)!; + const parent = schema.tables.find((t) => t.qualified === column.refersTo)!; + const childStyle = styleFor(child, r.pick(["plain", "with-module"])); + const parentStyle = styleFor(parent, "plain"); + return { + id: "", kind: "add foreign key", style: childStyle, text: `${wordsOf(column.name)} on ${say(child, childStyle)} should be a real foreign key to ${say(parent, parentStyle)}`, truth: [child.qualified, parent.qualified], + operations: [{ type: "add_relationship", relationship: { sourceTable: parent.qualified, sourceColumn: "id", targetTable: child.qualified, targetColumn: column.name } }], + verify: `SELECT count(*) = 1 AS ok FROM pg_constraint WHERE conrelid = '${dotted(child)}'::regclass AND contype = 'f' AND convalidated AND conkey = ARRAY[(SELECT attnum FROM pg_attribute WHERE attrelid = '${dotted(child)}'::regclass AND attname = '${column.name}')]`, + }; + }, + // drop an index nobody needs + () => { + const t = fresh(tables.filter((x) => x.indexes.length > 0)); + if (!t) return; + const index = t.indexes[0]; + const style = styleFor(t, r.pick(styles)); + return { + id: "", kind: "drop index", style, text: `the index on ${index.columns.map(wordsOf).join(" and ")} of ${say(t, style)} is never used, drop it`, truth: [t.qualified], + operations: [{ type: "drop_index", tableName: t.qualified, indexName: index.name }], + verify: `SELECT to_regclass('${q(t.schema)}.${q(index.name)}') IS NULL AS ok`, + }; + }, + // rename a column + () => { + const t = fresh(tables.filter((x) => x.role === "root")); + if (!t) return; + const style = styleFor(t, r.pick(styles)); + return { + id: "", kind: "rename column", style, text: `rename the notes column of ${say(t, style)} to remarks`, truth: [t.qualified], + operations: [{ type: "alter_column", tableName: t.qualified, columnName: "notes", changes: { newName: "remarks" } }], + verify: `SELECT count(*) = 1 AS ok FROM pg_attribute WHERE attrelid = '${dotted(t)}'::regclass AND attname = 'remarks'`, + }; + }, + // a new table that points at an existing one + () => { + const t = fresh(schema.tables.filter((x) => x.role === "root")); + if (!t) return; + const style = styleFor(t, r.pick(styles)); + const name = `${t.name}_eval_audit`.slice(0, 60); + return { + id: "", kind: "create table with foreign key", style, text: `create an audit trail table for ${say(t, style)} that records who changed what and when`, truth: [t.qualified], + operations: [ + { type: "create_table", table: { name: `${t.schema}.${name}`, fields: [{ name: "id", dataType: "bigint", primaryKey: true, autoIncrement: true }, { name: `${t.entity}_id`, dataType: "bigint", nullable: false }, { name: "changed_by", dataType: "bigint" }, { name: "changed_at", dataType: "timestamptz", nullable: false, defaultValue: "now()" }], indexes: [{ name: `${name}_changed_idx`, columns: ["changed_at"] }] } }, + { type: "add_relationship", relationship: { sourceTable: t.qualified, sourceColumn: "id", targetTable: `${t.schema}.${name}`, targetColumn: `${t.entity}_id` } }, + ], + verify: `SELECT count(*) = 1 AS ok FROM pg_constraint WHERE conrelid = '${q(t.schema)}.${q(name)}'::regclass AND contype = 'f' AND convalidated`, + }; + }, + // one change in two tables + () => { + const a = fresh(populated.length ? populated : tables); + const b = fresh(tables); + if (!a || !b) return; + const sa = styleFor(a, "plain"); + const sb = styleFor(b, "plain"); + return { + id: "", kind: "same change in two tables", style: sa, text: `add an archived at timestamp to ${say(a, sa)} and to ${say(b, sb)}`, truth: [a.qualified, b.qualified], + operations: [a, b].map((t) => ({ type: "add_column", tableName: t.qualified, field: { name: "archived_at", dataType: "timestamptz" } })), + verify: `SELECT count(*) = 2 AS ok FROM pg_attribute WHERE attname = 'archived_at' AND attrelid IN ('${dotted(a)}'::regclass, '${dotted(b)}'::regclass)`, + }; + }, + ]; + + for (let attempt = 0; cases.length < count && attempt < count * 30; attempt++) { + const made = generators[attempt % generators.length](); + if (made) cases.push({ ...made, id: `v${cases.length + 1}` }); + } + + // requests that must be refused before any SQL is produced + const refusals: Array<() => Case | undefined> = [ + () => { + const t = r.pick(tables); + return { id: "", kind: "index on a column that does not exist", style: "plain", text: `index the ${say(t, "plain")} table by its fiscal period`, truth: [t.qualified], operations: [{ type: "add_index", tableName: t.qualified, index: { name: `${t.name}_fiscal_idx`.slice(0, 60), columns: ["fiscal_period"] } }], refusedWith: "unknown-column" }; + }, + () => { + const t = r.pick(populated.length ? populated : tables); + return { id: "", kind: "NOT NULL column without a default on a table with rows", style: "plain", text: `add a required approver to ${say(t, "plain")}`, truth: [t.qualified], operations: [{ type: "add_column", tableName: t.qualified, field: { name: "approver", dataType: "text", nullable: false } }], refusedWith: "not-null-needs-default" }; + }, + () => { + const t = r.pick(tables.filter((x) => x.role === "root")); + const column = t.columns.find((c) => c.type.startsWith("varchar")) ?? t.columns[1]; + return { id: "", kind: "foreign key between different types", style: "plain", text: `${wordsOf(column.name)} of ${say(t, "plain")} should reference users`, truth: [t.qualified, "public.users"], operations: [{ type: "add_relationship", relationship: { sourceTable: "public.users", sourceColumn: "id", targetTable: t.qualified, targetColumn: column.name } }], refusedWith: "type-mismatch" }; + }, + () => { + const roots = tables.filter((x) => x.role === "root" && schema.tables.some((c) => c.parent === x.qualified)); + const t = r.pick(roots); + return { id: "", kind: "drop a table other tables point at", style: "plain", text: `drop the ${say(t, "plain")} table, we do not need it any more`, truth: [t.qualified, ...schema.tables.filter((c) => c.parent === t.qualified).map((c) => c.qualified)], operations: [{ type: "drop_table", tableName: t.qualified }], refusedWith: "referenced" }; + }, + () => { + const t = r.pick(tables); + return { id: "", kind: "default that carries a second statement", style: "plain", text: `${say(t, "plain")} gets a status note defaulting to nothing`, truth: [t.qualified], operations: [{ type: "add_column", tableName: t.qualified, field: { name: "status_note", dataType: "text", defaultValue: "'x'; DROP TABLE users" } }], refusedWith: "bad-default" }; + }, + () => { + const t = r.pick(tables.filter((x) => x.role === "root")); + return { id: "", kind: "make the primary key nullable", style: "plain", text: `${say(t, "plain")} ids may be empty`, truth: [t.qualified], operations: [{ type: "alter_column", tableName: t.qualified, columnName: "id", changes: { nullable: true } }], refusedWith: "primary-key-column" }; + }, + ]; + for (let i = 0; i < negatives; i++) { + const made = refusals[i % refusals.length](); + if (made) cases.push({ ...made, id: `n${i + 1}` }); + } + return cases; +} + +// ---- measuring ----------------------------------------------------------------------------------------------------------- + +const median = (values: number[]): number => (values.length === 0 ? 0 : [...values].sort((a, b) => a - b)[Math.floor(values.length / 2)]); +const percentile = (values: number[], p: number): number => (values.length === 0 ? 0 : [...values].sort((a, b) => a - b)[Math.min(values.length - 1, Math.floor(values.length * p))]); +const pct = (n: number, d: number): string => (d === 0 ? "n/a" : `${((n / d) * 100).toFixed(0)}%`.padStart(4)); +const ms = (value: number): string => `${value < 10 ? value.toFixed(1) : Math.round(value)} ms`; +const time = async (fn: () => Promise): Promise<[T, number]> => { + const start = performance.now(); + const result = await fn(); + return [result, performance.now() - start]; +}; + +/** What a plain rendering of the same operations would be: one statement each, the way a schema diff tool writes them. */ +function naiveSteps(operations: AiSchemaPlan["operations"], facts: Facts): RunbookStep[] { + const base = { operation: 0, lock: "ACCESS EXCLUSIVE" as const, alsoLocks: [], work: "metadata" as const, risk: "low" as const, reason: "", notes: [], destructive: false, breaksClients: false, optional: false }; + const steps: RunbookStep[] = []; + const push = (tableName: string, sql: string, work: RunbookStep["work"] = "metadata") => { + const table = facts.tables.get(tableName); + steps.push({ ...base, id: steps.length + 1, title: sql.slice(0, 40), sql, table: tableName, work, sizeClass: sizeClass(table), tableBytes: table?.heapBytes ?? null, tableRows: table?.estimatedRows ?? null }); + }; + for (const op of operations) { + const t = "tableName" in op ? (op.tableName as string) : ""; + switch (op.type) { + case "add_column": + push(t, `ALTER TABLE ${t} ADD COLUMN ${op.field.name} ${op.field.dataType}${op.field.nullable === false ? " NOT NULL" : ""}${op.field.defaultValue ? ` DEFAULT ${op.field.defaultValue}` : ""};`, op.field.defaultValue?.includes("(") ? "rewrite" : "metadata"); + break; + case "add_index": + push(t, `CREATE ${op.index.unique ? "UNIQUE " : ""}INDEX ${op.index.name} ON ${t} (${op.index.columns.join(", ")});`, "index-build"); + break; + case "drop_index": + push(t, `DROP INDEX ${op.indexName};`); + break; + case "alter_column": + if (op.changes.nullable === false) push(t, `ALTER TABLE ${t} ALTER COLUMN ${op.columnName} SET NOT NULL;`, "scan"); + if (op.changes.dataType) push(t, `ALTER TABLE ${t} ALTER COLUMN ${op.columnName} TYPE ${op.changes.dataType};`, "rewrite"); + if (op.changes.unique) push(t, `ALTER TABLE ${t} ADD CONSTRAINT ${t.split(".")[1]}_${op.columnName}_key UNIQUE (${op.columnName});`, "index-build"); + if (op.changes.newName) push(t, `ALTER TABLE ${t} RENAME COLUMN ${op.columnName} TO ${op.changes.newName};`); + break; + case "add_relationship": + push(op.relationship.targetTable, `ALTER TABLE ${op.relationship.targetTable} ADD FOREIGN KEY (${op.relationship.targetColumn}) REFERENCES ${op.relationship.sourceTable} (${op.relationship.sourceColumn});`, "scan"); + break; + default: + break; + } + } + return steps; +} + +interface Row { + kase: Case; + style: Style; + scopeSize: number; + full: boolean; + recall: number; + searchMs: number; + e2eOk: boolean; + oracleOk?: boolean; + lintOk?: boolean; + executed?: boolean; + verified?: boolean; + refused?: boolean; + refusedRight?: boolean; + compileMs: number; + factsMs: number; + stageMs: number; + scopedChars: number; + steps: number; + risk: Record; + naiveFlagged?: boolean; + sizeClasses: string[]; + problem?: string; +} + +async function evaluate(size: number): Promise<{ rows: Row[]; header: Record; failures: string[] }> { + const schema = generateSchema(size, SEED + size); + const template = `sc_eval_${size}`; + const header: Record = { tables: schema.tables.length, populated: schema.tables.filter((t) => t.rows > 0).length }; + const failures: string[] = []; + + console.log(`\n=== ${size} tables ===`); + const [, buildMs] = await time(async () => { + await createDatabase(template, async (client) => { + await client.query(schema.createSql.join(";\n")); + for (const statement of schema.populateSql) await client.query(statement); + for (const statement of schema.foreignKeySql) await client.query(statement); + await client.query("ANALYZE"); + }); + }); + console.log(` built and populated in ${(buildMs / 1000).toFixed(1)} s`); + header.buildSeconds = Math.round(buildMs / 100) / 10; + + const rows: Row[] = []; + try { + // the catalog the page reads once and searches many times + const catalogClient = await (await import("./lib/pg-scratch")).connect(template); + const [catalog, catalogMs] = await time(() => loadCatalog(asRunner(catalogClient))); + const [index, indexMs] = await time(async () => buildSearchIndex(catalog)); + header.catalogMs = Math.round(catalogMs); + header.indexMs = Math.round(indexMs * 10) / 10; + + // what sending every table would cost, for comparison + const everyone = catalog.tables.map((t) => t.qualified); + const [allFacts, allFactsMs] = await time(() => loadFacts(asRunner(catalogClient), everyone)); + const fullChars = JSON.stringify(plannerInput({ prompt: "x", facts: allFacts, omittedTables: 0 })).length + plannerInstructions().length; + header.allFactsMs = Math.round(allFactsMs); + header.fullChars = fullChars; + const sampleFacts = allFacts; + await catalogClient.end(); + console.log(` catalog ${ms(catalogMs)}, search index ${ms(indexMs)}, facts for all ${catalog.tables.length} tables ${ms(allFactsMs)} (${(fullChars / 1000).toFixed(0)}k characters if every table were sent)`); + + const cases = buildCases(schema, PER_SIZE, NEGATIVE_PER_SIZE, SEED * 1000 + size); + console.log(` ${cases.filter((c) => !c.refusedWith).length} change requests and ${cases.filter((c) => c.refusedWith).length} that should be refused`); + + for (const kase of cases) { + const plan = aiSchemaPlanSchema.parse({ summary: kase.text, operations: kase.operations }); + + // 1. which tables + const [scope, searchMs] = await time(async () => selectScope(index, kase.text)); + const picked = new Set(scope.tables.map((t) => t.qualified)); + const hit = kase.truth.filter((name) => picked.has(name)).length; + + const row: Row = { + kase, style: kase.style, scopeSize: scope.tables.length, full: hit === kase.truth.length, recall: kase.truth.length ? hit / kase.truth.length : 1, searchMs, e2eOk: false, + compileMs: 0, factsMs: 0, stageMs: 0, scopedChars: 0, steps: 0, risk: { low: 0, medium: 0, high: 0 }, sizeClasses: [], + }; + + await withDatabase(async (client) => { + const db = asRunner(client); + const compileFor = async (scopeTables: string[], operations = plan) => { + const [facts, factsMs] = await time(() => loadFacts(db, scopeTables)); + const [runbook, compileMs] = await time(async () => { + const probes = await probe(db, discoverProbes({ plan: operations, facts })); + return compilePlan({ plan: operations, facts, probes }); + }); + return { facts, runbook, factsMs, compileMs }; + }; + + // 2. the plan against the tables retrieval chose (when it chose the right ones, this is what a person would get) + const retrieved = await compileFor([...picked]); + row.factsMs = retrieved.factsMs; + row.compileMs = retrieved.compileMs; + row.scopedChars = JSON.stringify(plannerInput({ prompt: kase.text, facts: retrieved.facts, omittedTables: size - picked.size })).length + plannerInstructions().length; + const retrievedErrors = retrieved.runbook.findings.filter((f) => f.severity === "error"); + row.e2eOk = retrievedErrors.length === 0; + + // 3. the plan against the right tables + const oracle = await compileFor(kase.truth); + const errors = oracle.runbook.findings.filter((f) => f.severity === "error"); + row.stageMs = row.searchMs + oracle.factsMs + oracle.compileMs; + + if (kase.refusedWith) { + row.refused = errors.length > 0; + row.refusedRight = errors.some((f) => f.code === kase.refusedWith); + row.problem = row.refusedRight ? undefined : `expected ${kase.refusedWith}, got ${errors.map((f) => f.code).join(", ") || "no error"}`; + return; + } + + row.oracleOk = errors.length === 0; + if (!row.oracleOk) { + row.problem = errors.map((f) => f.message).join(" | ").slice(0, 300); + return; + } + const lint = lintSteps(oracle.runbook.steps); + row.lintOk = lint.violations.length === 0; + if (!row.lintOk) row.problem = lint.violations.map((v) => `${v.rule}: ${v.statement}`).join(" | ").slice(0, 300); + row.steps = oracle.runbook.steps.length; + for (const step of oracle.runbook.steps) if (!step.optional) row.risk[step.risk]++; + row.sizeClasses = [...new Set(oracle.runbook.steps.map((s) => s.sizeClass))]; + + const naive = lintSteps(naiveSteps(plan.operations, oracle.facts)); + row.naiveFlagged = naive.violations.length > 0; + + const result = await runRunbook(client, oracle.runbook, { includeOptional: true }); + row.executed = result.ok; + if (!result.ok) { + row.problem = `step ${result.failure?.step} (${result.failure?.title}): ${result.failure?.message}`.slice(0, 300); + return; + } + if (kase.verify) row.verified = Boolean((await client.query(kase.verify)).rows[0]?.ok); + if (row.verified === false) row.problem = "the schema is not what the request asked for after the runbook ran"; + }, { template }); + + rows.push(row); + const flag = kase.refusedWith ? (row.refusedRight ? "refused" : "MISSED") : row.verified ? "ok" : "FAILED"; + process.stdout.write(` ${kase.id.padEnd(4)} ${flag.padEnd(8)} recall ${row.full ? "full" : "MISS"} scope ${String(row.scopeSize).padStart(2)} ${kase.kind}\n`); + if (flag === "FAILED" || flag === "MISSED") failures.push(`${size} tables, ${kase.id} (${kase.kind}): ${row.problem}`); + } + + // 4. the same checks when a real model plans (only with --live) + if (LIVE) { + console.log(" live model: planning the valid requests through the full pipeline"); + const live = new LiveStats(); + await live.run(template, cases.filter((c) => !c.refusedWith), index, catalog.tables.length, sampleFacts); + header.liveValid = live.valid; + header.liveTotal = live.total; + header.liveLatencyMedianMs = Math.round(median(live.latencies)); + header.liveInputTokens = live.inputTokens; + header.liveOutputTokens = live.outputTokens; + } + } finally { + if (!KEEP) await dropDatabase(template); + } + return { rows, header, failures }; +} + +class LiveStats { + valid = 0; + total = 0; + latencies: number[] = []; + inputTokens = 0; + outputTokens = 0; + + async run(template: string, cases: Case[], _index: unknown, _tables: number, _facts: Facts) { + const settings = getSettingsStore(); + settings.write({ ...DEFAULT_SETTINGS, model: process.env.EVAL_MODEL ?? "" }); + const connect = (await import("./lib/pg-scratch")).connect; + const client = await connect(template); + try { + const db = asRunner(client); + const catalog = await loadCatalog(db); + const index = buildSearchIndex(catalog); + for (const kase of cases) { + this.total++; + const scope = selectScope(index, kase.text).tables.map((t) => t.qualified); + try { + const result = await planSchemaChange( + { loadFacts: (names) => loadFacts(db, names), probe: (request) => probe(db, request), wholeDatabase: async () => ({ tables: catalog.tables }), callModel: callModelJson }, + { prompt: kase.text, tables: scope, redact: false, connection: { id: "eval", name: template } }, + ); + this.latencies.push(result.modelLatencyMs); + this.inputTokens += result.usage.inputTokens ?? 0; + this.outputTokens += result.usage.outputTokens ?? 0; + if (result.runbook && !result.findings.some((f) => f.severity === "error")) this.valid++; + } catch (error) { + console.log(` ${kase.id}: ${error instanceof Error ? error.message : error}`); + } + } + } finally { + await client.end(); + } + } +} + +// ---- report ---------------------------------------------------------------------------------------------------------------- + +function report(size: number, rows: Row[], header: Record) { + const valid = rows.filter((r) => !r.kase.refusedWith); + const refused = rows.filter((r) => r.kase.refusedWith); + console.log(`\n--- ${size} tables: results ---`); + + console.log("table selection (the tables the change touches, found from the request's words, columns and foreign keys)"); + console.log(` every table found ${pct(valid.filter((r) => r.full).length, valid.length)} (${valid.filter((r) => r.full).length}/${valid.length})`); + console.log(` mean share found ${pct(valid.reduce((s, r) => s + r.recall, 0), valid.length)}`); + console.log(` scope size median ${median(valid.map((r) => r.scopeSize))}, max ${Math.max(0, ...valid.map((r) => r.scopeSize))} of ${size} tables`); + for (const style of ["plain", "with-module", "typo", "partial-name", "column-only"] as Style[]) { + const group = valid.filter((r) => r.style === style); + if (group.length > 0) console.log(` ${style.padEnd(13)} every table found ${pct(group.filter((r) => r.full).length, group.length)} (${group.filter((r) => r.full).length}/${group.length})`); + } + const misses = valid.filter((r) => !r.full); + for (const miss of misses.slice(0, 4)) console.log(` missed: "${miss.kase.text}" -> wanted ${miss.kase.truth.join(", ")}`); + + console.log("what is sent to the model"); + const scoped = median(valid.map((r) => r.scopedChars)); + const full = Number(header.fullChars); + console.log(` every table ${(full / 1000).toFixed(0)}k characters (about ${Math.round(full / 4000)}k tokens, estimated)${size > OLD_TABLE_LIMIT ? `, and the old request refused anything over ${OLD_TABLE_LIMIT} tables` : ""}`); + console.log(` chosen scope ${(scoped / 1000).toFixed(0)}k characters median (about ${Math.round(scoped / 4000)}k tokens), ${(full / Math.max(1, scoped)).toFixed(0)}x smaller`); + + console.log("plan checks against the real catalog (reference plans: these measure the checks and the compiler, not a model)"); + console.log(` with the retrieved tables ${pct(valid.filter((r) => r.e2eOk).length, valid.length)} (a missed table makes its change unplannable)`); + console.log(` with the right tables ${pct(valid.filter((r) => r.oracleOk).length, valid.length)} (${valid.filter((r) => r.oracleOk).length}/${valid.length})`); + console.log(` invalid requests refused ${pct(refused.filter((r) => r.refusedRight).length, refused.length)} (${refused.filter((r) => r.refusedRight).length}/${refused.length}, each with the expected reason)`); + + console.log("migration lint and execution"); + const compiled = valid.filter((r) => r.oracleOk); + console.log(` lint passes ${pct(compiled.filter((r) => r.lintOk).length, compiled.length)} (${compiled.filter((r) => r.lintOk).length}/${compiled.length})`); + console.log(` a plain rendering is flagged ${pct(compiled.filter((r) => r.naiveFlagged).length, compiled.length)} (${compiled.filter((r) => r.naiveFlagged).length}/${compiled.length}: the same lint on one statement per operation)`); + console.log(` runs on PostgreSQL 16 ${pct(compiled.filter((r) => r.executed).length, compiled.length)} (${compiled.filter((r) => r.executed).length}/${compiled.length}, every statement on its own)`); + console.log(` leaves the schema as asked ${pct(compiled.filter((r) => r.verified).length, compiled.length)} (${compiled.filter((r) => r.verified).length}/${compiled.length})`); + const touched = compiled.filter((r) => r.sizeClasses.some((c) => c === "medium" || c === "large")); + console.log(` of those, on a table with data (16 MiB or more): ${touched.length}; steps by risk ${JSON.stringify(compiled.reduce((s, r) => ({ low: s.low + r.risk.low, medium: s.medium + r.risk.medium, high: s.high + r.risk.high }), { low: 0, medium: 0, high: 0 }))}`); + + console.log("latency (every stage that doesn't wait for a model)"); + console.log(` read the catalog ${ms(Number(header.catalogMs))} once, search index ${ms(Number(header.indexMs))}`); + console.log(` find the tables median ${ms(median(valid.map((r) => r.searchMs)))}, p95 ${ms(percentile(valid.map((r) => r.searchMs), 0.95))}`); + console.log(` read facts of the scope median ${ms(median(valid.map((r) => r.factsMs)))}`); + console.log(` check and compile median ${ms(median(valid.map((r) => r.compileMs)))}, p95 ${ms(percentile(valid.map((r) => r.compileMs), 0.95))}`); + console.log(` whole deterministic path median ${ms(median(rows.map((r) => r.stageMs)))}, p95 ${ms(percentile(rows.map((r) => r.stageMs), 0.95))}`); + if (LIVE) console.log(` model median ${ms(Number(header.liveLatencyMedianMs))}, ${header.liveValid}/${header.liveTotal} plans without errors, ${header.liveInputTokens} in / ${header.liveOutputTokens} out tokens (reported by the endpoint)`); + else console.log(" model not measured (no live model: have OPENAI_API_KEY available and pass --live)"); +} + +async function main() { + console.log(`Schema Change evaluation: sizes ${SIZES.join(", ")}, ${PER_SIZE} valid + ${NEGATIVE_PER_SIZE} invalid requests each, seed ${SEED}${LIVE ? ", live model" : ", reference plans"}`); + const all: Row[] = []; + const failures: string[] = []; + for (const size of SIZES) { + const { rows, header, failures: failed } = await evaluate(size); + report(size, rows, header); + all.push(...rows); + failures.push(...failed); + } + + const valid = all.filter((r) => !r.kase.refusedWith); + const refused = all.filter((r) => r.kase.refusedWith); + console.log(`\n=== all sizes: ${all.length} requests (${valid.length} valid, ${refused.length} invalid) ===`); + console.log(` every table found ${pct(valid.filter((r) => r.full).length, valid.length)} · plan valid (right tables) ${pct(valid.filter((r) => r.oracleOk).length, valid.length)} · lint ${pct(valid.filter((r) => r.lintOk).length, valid.filter((r) => r.oracleOk).length)} · executed ${pct(valid.filter((r) => r.executed).length, valid.filter((r) => r.oracleOk).length)} · verified ${pct(valid.filter((r) => r.verified).length, valid.filter((r) => r.oracleOk).length)} · invalid refused ${pct(refused.filter((r) => r.refusedRight).length, refused.length)}`); + if (failures.length > 0) { + console.log(`\n${failures.length} failure(s) in the checks or the compiler:`); + for (const failure of failures) console.log(` ${failure}`); + process.exit(1); + } +} + +main().catch((error) => { + console.error(error); + process.exit(2); +}); diff --git a/vitest.config.ts b/vitest.config.ts index c57a4f1..3847026 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -8,6 +8,6 @@ export default defineConfig({ plugins: [tsconfigPaths()], test: { environment: "node", - include: ["src/**/*.test.ts", "server/src/**/*.test.ts", "queryshift/**/*.test.ts"], + include: ["src/**/*.test.ts", "server/src/**/*.test.ts", "scripts/**/*.test.ts", "queryshift/**/*.test.ts"], }, });