diff --git a/packages/opencode/src/altimate/native/altimate-core.ts b/packages/opencode/src/altimate/native/altimate-core.ts index 84217553b5..1e8a22b7ef 100644 --- a/packages/opencode/src/altimate/native/altimate-core.ts +++ b/packages/opencode/src/altimate/native/altimate-core.ts @@ -11,7 +11,7 @@ import * as core from "@altimateai/altimate-core" import { EngineCoerce } from "./engine-coerce" import { register } from "./dispatcher" -import { schemaOrEmpty, resolveSchema, SchemaResolver } from "./schema-resolver" +import { schemaOrEmpty, SchemaResolver, prepareSql } from "./schema-resolver" import type { AltimateCoreResult } from "./types" // --------------------------------------------------------------------------- @@ -19,6 +19,15 @@ import type { AltimateCoreResult } from "./types" // --------------------------------------------------------------------------- /** Spread a rich TypeScript object into a plain Record for the data field. */ +// altimate_change start — the engine's existence findings, which mean nothing without a schema +const EXISTENCE_CODES = new Set(["E001", "E002"]) +const EXISTENCE_KINDS = new Set(["TableNotFound", "ColumnNotFound"]) +export function isExistenceError(err: unknown): boolean { + const e = err as { code?: unknown; kind?: { type?: unknown } } | null + return EXISTENCE_CODES.has(String(e?.code)) || EXISTENCE_KINDS.has(String(e?.kind?.type)) +} +// altimate_change end + function toData(obj: unknown): Record { if (obj === null || obj === undefined) return {} if (typeof obj !== "object") return { value: obj } @@ -89,9 +98,22 @@ export function registerAll(): void { // 1. altimate_core.validate register("altimate_core.validate", async (params) => { try { - const schema = schemaOrEmpty(params.schema_path, params.schema_context) - const raw = await core.validate(params.sql, schema) + const { sql, schema, hasSchema } = prepareSql(params.sql, params.schema_path, params.schema_context) + const raw = await core.validate(sql, schema) const data = toData(raw) + // altimate_change start — without a schema the engine still runs against the + // `_empty_` placeholder and reports every table as missing. The tool promises that + // existence checks are skipped when no schema is given, so those findings are + // dropped here and `valid` is recomputed from what remains (syntax, dialect). + // Said back to the tool, which used to decide it separately (and differently). + ;(data as Record).has_schema = hasSchema + const errors = (data as { errors?: unknown }).errors + if (!hasSchema && Array.isArray(errors)) { + const kept = errors.filter((err) => !isExistenceError(err)) + ;(data as Record).errors = kept + ;(data as Record).valid = kept.length === 0 + } + // altimate_change end return ok(true, data) } catch (e) { return fail(e) @@ -101,8 +123,8 @@ export function registerAll(): void { // 2. altimate_core.lint register("altimate_core.lint", async (params) => { try { - const schema = schemaOrEmpty(params.schema_path, params.schema_context) - const raw = core.lint(params.sql, schema) + const { sql, schema } = prepareSql(params.sql, params.schema_path, params.schema_context) + const raw = core.lint(sql, schema) const data = toData(raw) return ok(true, data) } catch (e) { @@ -156,8 +178,8 @@ export function registerAll(): void { // 5. altimate_core.explain register("altimate_core.explain", async (params) => { try { - const schema = schemaOrEmpty(params.schema_path, params.schema_context) - const raw = await core.explain(params.sql, schema) + const { sql, schema } = prepareSql(params.sql, params.schema_path, params.schema_context) + const raw = await core.explain(sql, schema) const data = toData(raw) return ok(true, data) } catch (e) { @@ -168,32 +190,37 @@ export function registerAll(): void { // 6. altimate_core.check — composite: validate + lint + scan_sql register("altimate_core.check", async (params) => { try { - const schema = schemaOrEmpty(params.schema_path, params.schema_context) + const { sql, schema, foldSql } = prepareSql(params.sql, params.schema_path, params.schema_context) + // The base SQL is matched against the same schema (lintDiff, the PII subtraction), + // so it gets exactly the preparation the head SQL got. + const baseSql = params.base_sql ? foldSql(params.base_sql) : params.base_sql // NOTE: validation is deliberately NOT diff-scoped against base_sql. // The engine validates fail-fast (only the FIRST error is reported), so // subtracting base errors can hide genuinely new breakage behind a // pre-existing one. Re-reporting a pre-existing error is the safe mode. - const validation: Record = toData(await core.validate(params.sql, schema)) + const validation: Record = toData(await core.validate(sql, schema)) // Diff-scoped lint: when a base SQL is supplied, core returns only the // findings the change INTRODUCED (pre-existing issues in the file are // dropped) — the structural comparison stays in the AST engine. const lintResult = - params.base_sql && typeof core.lintDiff === "function" + baseSql && typeof core.lintDiff === "function" ? core.lintDiff( - params.sql, - params.base_sql, + sql, + baseSql, // lintDiff takes SchemaDefinition JSON — normalize flat agent // schemas too, or the whole composite throws "missing field tables". - params.schema_context ? SchemaResolver.normalizeSchemaContext(params.schema_context) : undefined, + params.schema_context + ? SchemaResolver.normalizeSchemaContext(params.schema_context, { fold: true }) + : undefined, ) - : core.lint(params.sql, schema) + : core.lint(sql, schema) // Diff-scope safety like lint: threats present in the base SQL are // pre-existing, not introduced by this change. Subtract as a MULTISET on // (rule, matched_pattern) — one base occurrence consumes one head // occurrence, so a PR that ADDS a second identical injection still // reports it. Recompute safe/risk_score from the surviving threats so a // fully pre-existing threat set doesn't leave a stale unsafe verdict. - let safety: Record = toData(core.scanSql(params.sql)) + let safety: Record = toData(core.scanSql(sql)) if (params.base_sql) { try { const baseCounts = new Map() @@ -238,8 +265,8 @@ export function registerAll(): void { // must not fail the whole composite. let pii: Record try { - pii = toData(core.checkQueryPii(params.sql, schema)) - if (params.base_sql && Array.isArray(pii.pii_columns) && (pii.pii_columns as any[]).length) { + pii = toData(core.checkQueryPii(sql, schema)) + if (baseSql && Array.isArray(pii.pii_columns) && (pii.pii_columns as any[]).length) { try { // Pre-existing exposures are not introduced by this change. The // identity INCLUDES the sorted output aliases — adding or renaming @@ -247,7 +274,7 @@ export function registerAll(): void { // output exposure and must still surface. const exposureKey = (c: any) => `${c.table}|${c.column}|${[...(c.query_targets ?? [])].sort().join(",")}` - const baseExposed = new Set(core.checkQueryPii(params.base_sql, schema).pii_columns.map(exposureKey)) + const baseExposed = new Set(core.checkQueryPii(baseSql, schema).pii_columns.map(exposureKey)) const remaining = (pii.pii_columns as any[]).filter((c: any) => !baseExposed.has(exposureKey(c))) pii = { ...pii, @@ -358,9 +385,10 @@ export function registerAll(): void { // 7. altimate_core.fix register("altimate_core.fix", async (params) => { try { - const schema = schemaOrEmpty(params.schema_path, params.schema_context) - const raw = await core.fix(params.sql, schema, params.max_iterations ?? undefined) - const data = toData(raw) + const { sql, schema, unfold } = prepareSql(params.sql, params.schema_path, params.schema_context) + const raw = await core.fix(sql, schema, params.max_iterations ?? undefined) + // Generated SQL goes back in the caller's spelling (see `PreparedSql.unfold`). + const data = unfold(toData(raw)) return ok(true, data) } catch (e) { return fail(e) @@ -370,8 +398,8 @@ export function registerAll(): void { // 8. altimate_core.policy register("altimate_core.policy", async (params) => { try { - const schema = schemaOrEmpty(params.schema_path, params.schema_context) - const raw = await core.checkPolicy(params.sql, schema, params.policy_json) + const { sql, schema } = prepareSql(params.sql, params.schema_path, params.schema_context) + const raw = await core.checkPolicy(sql, schema, params.policy_json) const data = toData(raw) return ok(true, data) } catch (e) { @@ -382,8 +410,8 @@ export function registerAll(): void { // 9. altimate_core.semantics register("altimate_core.semantics", async (params) => { try { - const schema = schemaOrEmpty(params.schema_path, params.schema_context) - const raw = await core.checkSemantics(params.sql, schema) + const { sql, schema } = prepareSql(params.sql, params.schema_path, params.schema_context) + const raw = await core.checkSemantics(sql, schema) const data = toData(raw) return ok(true, data) } catch (e) { @@ -394,9 +422,9 @@ export function registerAll(): void { // 10. altimate_core.testgen register("altimate_core.testgen", async (params) => { try { - const schema = schemaOrEmpty(params.schema_path, params.schema_context) - const raw = core.generateTests(params.sql, schema) - return ok(true, toData(raw)) + const { sql, schema, unfold } = prepareSql(params.sql, params.schema_path, params.schema_context) + const raw = core.generateTests(sql, schema) + return ok(true, unfold(toData(raw))) } catch (e) { return fail(e) } @@ -405,14 +433,14 @@ export function registerAll(): void { // 11. altimate_core.equivalence register("altimate_core.equivalence", async (params) => { try { - const schema = schemaOrEmpty(params.schema_path, params.schema_context) + const one = prepareSql(params.sql1, params.schema_path, params.schema_context) // Pass the optional dialect hint so dialect-specific compiled warehouse SQL // (e.g. Snowflake semi-structured `col:field`) parses and the pair is // decidable instead of abstaining on a syntax error. Supported since // altimate-core@0.5.1. dialectHint coerces "" (the ReviewConfig default) // to undefined: the engine throws on an unknown dialect "", and "" must // mean auto-detect, not a real dialect. - const raw = await core.checkEquivalence(params.sql1, params.sql2, schema, EngineCoerce.dialectHint(params.dialect)) + const raw = await core.checkEquivalence(one.sql, one.foldSql(params.sql2), one.schema, EngineCoerce.dialectHint(params.dialect)) const data = toData(raw) return ok(true, data) } catch (e) { @@ -449,9 +477,9 @@ export function registerAll(): void { // 14. altimate_core.rewrite register("altimate_core.rewrite", async (params) => { try { - const schema = schemaOrEmpty(params.schema_path, params.schema_context) - const raw = core.rewrite(params.sql, schema) - return ok(true, toData(raw)) + const { sql, schema, unfold } = prepareSql(params.sql, params.schema_path, params.schema_context) + const raw = core.rewrite(sql, schema) + return ok(true, unfold(toData(raw))) } catch (e) { return fail(e) } @@ -460,9 +488,9 @@ export function registerAll(): void { // 15. altimate_core.correct register("altimate_core.correct", async (params) => { try { - const schema = schemaOrEmpty(params.schema_path, params.schema_context) - const raw = await core.correct(params.sql, schema) - const data = toData(raw) + const { sql, schema, unfold } = prepareSql(params.sql, params.schema_path, params.schema_context) + const raw = await core.correct(sql, schema) + const data = unfold(toData(raw)) return ok(true, data) } catch (e) { return fail(e) @@ -472,8 +500,8 @@ export function registerAll(): void { // 16. altimate_core.grade register("altimate_core.grade", async (params) => { try { - const schema = schemaOrEmpty(params.schema_path, params.schema_context) - const raw = await core.evaluate(params.sql, schema) + const { sql, schema } = prepareSql(params.sql, params.schema_path, params.schema_context) + const raw = await core.evaluate(sql, schema) const data = toData(raw) // EvalResult embeds a full safety scan — redact its threat echoes too // (the CLI grade check renders nested threat messages). @@ -500,8 +528,8 @@ export function registerAll(): void { // 18. altimate_core.query_pii register("altimate_core.query_pii", async (params) => { try { - const schema = schemaOrEmpty(params.schema_path, params.schema_context) - const raw = core.checkQueryPii(params.sql, schema) + const { sql, schema } = prepareSql(params.sql, params.schema_path, params.schema_context) + const raw = core.checkQueryPii(sql, schema) return ok(true, toData(raw)) } catch (e) { return fail(e) @@ -524,8 +552,8 @@ export function registerAll(): void { // 20. altimate_core.column_lineage register("altimate_core.column_lineage", async (params) => { try { - const schema = resolveSchema(params.schema_path, params.schema_context) - const raw = core.columnLineage(params.sql, EngineCoerce.dialectHint(params.dialect), schema ?? undefined) + const { sql, schema, hasSchema } = prepareSql(params.sql, params.schema_path, params.schema_context) + const raw = core.columnLineage(sql, EngineCoerce.dialectHint(params.dialect), hasSchema ? schema : undefined) return ok(true, toData(raw)) } catch (e) { return fail(e) @@ -535,8 +563,8 @@ export function registerAll(): void { // 21. altimate_core.track_lineage register("altimate_core.track_lineage", async (params) => { try { - const schema = schemaOrEmpty(params.schema_path, params.schema_context) - const raw = core.trackLineage(params.queries, schema) + const { schema, foldSql } = prepareSql("", params.schema_path, params.schema_context) + const raw = core.trackLineage(params.queries.map(foldSql), schema) return ok(true, toData(raw)) } catch (e) { return fail(e) @@ -577,9 +605,9 @@ export function registerAll(): void { // 25. altimate_core.complete register("altimate_core.complete", async (params) => { try { - const schema = schemaOrEmpty(params.schema_path, params.schema_context) - const raw = core.complete(params.sql, params.cursor_pos, schema) - return ok(true, toData(raw)) + const { sql, schema, unfold } = prepareSql(params.sql, params.schema_path, params.schema_context) + const raw = core.complete(sql, params.cursor_pos, schema) + return ok(true, unfold(toData(raw))) } catch (e) { return fail(e) } @@ -599,9 +627,9 @@ export function registerAll(): void { // 27. altimate_core.optimize_for_query register("altimate_core.optimize_for_query", async (params) => { try { - const schema = schemaOrEmpty(params.schema_path, params.schema_context) - const raw = core.optimizeForQuery(params.sql, schema) - return ok(true, toData(raw)) + const { sql, schema, unfold } = prepareSql(params.sql, params.schema_path, params.schema_context) + const raw = core.optimizeForQuery(sql, schema) + return ok(true, unfold(toData(raw))) } catch (e) { return fail(e) } @@ -610,8 +638,8 @@ export function registerAll(): void { // 28. altimate_core.prune_schema register("altimate_core.prune_schema", async (params) => { try { - const schema = schemaOrEmpty(params.schema_path, params.schema_context) - const raw = core.pruneSchema(params.sql, schema) + const { sql, schema } = prepareSql(params.sql, params.schema_path, params.schema_context) + const raw = core.pruneSchema(sql, schema) return ok(true, toData(raw)) } catch (e) { return fail(e) diff --git a/packages/opencode/src/altimate/native/schema-resolver.ts b/packages/opencode/src/altimate/native/schema-resolver.ts index 0724ffe303..4688feb70e 100644 --- a/packages/opencode/src/altimate/native/schema-resolver.ts +++ b/packages/opencode/src/altimate/native/schema-resolver.ts @@ -17,6 +17,9 @@ */ import { Schema } from "@altimateai/altimate-core" +import fs from "node:fs" +import path from "node:path" +import YAML from "yaml" /** * Detect whether a schema_context object is in flat format or SchemaDefinition format. @@ -48,7 +51,7 @@ function isSchemaDefinitionFormat(ctx: Record): boolean { * Output: `{ "tables": { "customers": { "columns": [{ "name": "id", "type": "INTEGER" }, ...] } } }` */ function flatToSchemaDefinition(flat: Record): Record { - const tables: Record = {} + const tables: Record = Object.create(null) for (const [tableName, colsOrDef] of Object.entries(flat)) { if (colsOrDef === null || colsOrDef === undefined) continue @@ -80,15 +83,131 @@ function flatToSchemaDefinition(flat: Record): Record return { tables } } +/** + * The engine compares an UNQUOTED identifier from the SQL in lowercase and a QUOTED one + * exactly, while warehouse metadata (`schema_inspect`, `snowflake_get_table_stats`) comes + * back in the warehouse's storage case — uppercase on Snowflake. A correct query validated + * against real metadata therefore failed with ColumnNotFound, and uppercasing the SQL did + * not help because the engine lowercased it again (#1333). + * + * An all-uppercase name is the storage form of an identifier that was created unquoted, so + * it is stored lowercase here to meet the engine's unquoted comparison. A mixed-case name + * was created quoted and must be referenced quoted, which the engine matches exactly, so it + * is left alone. Lowercase names are already in the engine's form. + */ +export function foldIdentifierCase(name: string): string { + return name !== name.toLowerCase() && name === name.toUpperCase() ? name.toLowerCase() : name +} + +/** + * Fold every table key and column name. A fold that would land on a name the schema + * already carries (metadata with both `ORDERS` and `orders`, or `FOO` and `"foo"` columns) + * keeps the entry as written instead: the two are distinct objects in the warehouse, the + * schema shape has no quote identity to tell them apart, and overwriting one would drop + * its columns from validation. The engine resolves such a pair by its own rule (tables by + * lowercase key, columns first-wins case-insensitively), which is what it did before the + * fold existed. Null-prototype maps, so a table named `__PROTO__` is an entry, not a + * prototype write. + */ +export function foldSchemaCase(def: { tables: Record }): { tables: Record } { + const source: Record = def.tables ?? {} + const tables: Record = Object.create(null) + const target = (name: string) => { + const folded = foldIdentifierCase(name) + return folded !== name && Object.hasOwn(source, folded) ? name : folded + } + for (const [tableName, table] of Object.entries(source)) { + const present = new Set( + Array.isArray(table?.columns) ? table.columns.map((c: any) => c?.name).filter((n: unknown) => typeof n === "string") : [], + ) + const columns = Array.isArray(table?.columns) + ? table.columns.map((c: any) => { + if (typeof c?.name !== "string") return c + const folded = foldIdentifierCase(c.name) + return { ...c, name: folded !== c.name && present.has(folded) ? c.name : folded } + }) + : table?.columns + tables[target(tableName)] = { ...table, columns } + } + return { ...def, tables } +} + +/** + * The other half of the fold, on the SQL. The engine matches a quoted identifier exactly, + * so once metadata `ORDER_MONTH` is stored as `order_month` a query that writes + * `"ORDER_MONTH"` — dbt with `quote_columns`, most BI tools on Snowflake — would miss it. + * On the warehouses whose metadata comes back uppercase, `"ORDER_MONTH"` names the same + * column as `order_month` unquoted, so the quoted all-uppercase spelling is lowercased + * inside its quotes. Same length, so positions in findings do not move; still quoted, so a + * reserved word (`"ORDER"`) stays an identifier. String literals and comments are stepped + * over, not searched. Mixed-case quoted names are the exact-match case and are untouched. + */ +export function foldQuotedIdentifierCase(sql: string, names?: ReadonlySet, folded?: Set): string { + // Skipped spans, in order: dollar-quoted strings ($$…$$, $tag$…$tag$), E'…' strings + // with backslash escapes, ordinary '…' strings ('' doubles), line and block comments. + // Then the two quoted-identifier forms: "…" (SQL) and `…` (BigQuery/Databricks), a + // doubled quote inside either kept as written — that name is not a plain identifier. + // With `names`, only a token whose lowercase form (or last dotted segment) is a name + // the folded schema holds is touched: on MySQL/BigQuery/SQLite `"SHIPPED"` is a + // string literal, and a query's values must not change under validation. Every + // token folded is recorded in `folded`, so generated SQL can be given back in the + // caller's spelling (see `PreparedSql.unfold`). The dollar-quote branch needs an + // identifier boundary before it: `foo$t$` can be an identifier, not a string start. + return sql.replace( + /(? { + const quoted = dq ?? bq + if (quoted === undefined) return match + // A doubled quote inside the name (`"A""B"`) is not a plain identifier: kept. + if (!/^[A-Z_][A-Z0-9_$.]*$/.test(quoted)) return match + const lower = quoted.toLowerCase() + if (names && !names.has(lower) && !names.has(lower.slice(lower.lastIndexOf(".") + 1))) return match + folded?.add(lower) + const mark = match[0] + return `${mark}${lower}${mark}` + }, + ) +} + +/** Every folded table key and column name in a definition, plus each dotted segment + * of a table key, so `"DB"."SCHEMA"."ORDERS"` can meet a `db.schema.orders` key. */ +function schemaNames(def: { tables: Record }): Set { + const names = new Set() + for (const [table, value] of Object.entries(def.tables ?? {})) { + names.add(table) + for (const segment of table.split(".")) names.add(segment) + if (Array.isArray(value?.columns)) for (const c of value.columns) if (typeof c?.name === "string") names.add(c.name) + } + return names +} + /** * Normalize a schema_context into SchemaDefinition JSON format. * Accepts both flat and SchemaDefinition formats. */ -export function normalizeSchemaContext(ctx: Record): string { - if (isSchemaDefinitionFormat(ctx)) { - return JSON.stringify(ctx) - } - return JSON.stringify(flatToSchemaDefinition(ctx)) +export function normalizeSchemaContext(ctx: Record, opts: { fold?: boolean } = {}): string { + return JSON.stringify(normalizedSchemaDefinition(ctx, opts)) +} + +/** `fold` applies the identifier-case fold (see `foldIdentifierCase`). It is for + * operations that match SQL against the schema — the engine's comparison rules are + * what the fold answers to. Schema-only operations (diff, export, fingerprint) get + * the names as the caller wrote them. */ +function normalizedSchemaDefinition(ctx: Record, opts: { fold?: boolean } = {}): { tables: Record } { + const def = (isSchemaDefinitionFormat(ctx) ? ctx : flatToSchemaDefinition(ctx)) as { tables: Record } + return opts.fold ? foldSchemaCase(def) : def +} + +/** + * Whether the caller really supplied a schema to check existence against. A `schema_path` + * counts as one. A `schema_context` counts only if it normalises to at least one table: + * `{ tables: {} }` and `{ users: {} }` are reachable inputs that carry no table; the + * engine refuses them, and the no-schema path is what the caller meant. + */ +export function schemaProvided(schemaPath?: string, schemaContext?: Record): boolean { + if (schemaPath) return true + if (!schemaContext || Object.keys(schemaContext).length === 0) return false + return Object.keys(normalizedSchemaDefinition(schemaContext).tables).length > 0 } /** @@ -102,12 +221,108 @@ export function resolveSchema( if (schemaPath) { return Schema.fromFile(schemaPath) } - if (schemaContext && Object.keys(schemaContext).length > 0) { - return Schema.fromJson(normalizeSchemaContext(schemaContext)) + // A context that normalises to no table is no schema: the engine refuses an empty + // definition outright ("Schema must define at least one table"), so `{ tables: {} }` + // used to fail the call instead of running the no-schema path. + if (schemaProvided(undefined, schemaContext)) { + return Schema.fromJson(normalizeSchemaContext(schemaContext!)) } return null } +/** What an operation that matches SQL against a schema runs on. */ +export interface PreparedSql { + sql: string + schema: Schema + /** A schema was really supplied (see `schemaProvided`). */ + hasSchema: boolean + /** The same preparation for another SQL text matched against this schema (a base + * query, a lineage batch): folded exactly when `sql` was, untouched otherwise. */ + foldSql: (other: string) => string + /** The caller's spelling restored in engine output: every quoted token this + * preparation lowercased is put back in uppercase wherever it appears in a string + * — a rewritten query, a fix, a generated test — walking objects and arrays. The + * fold is a comparison form for matching against folded metadata, not a spelling + * a case-sensitive warehouse would accept, so generated SQL must not carry it. */ + unfold: (value: T) => T +} + +/** + * The SQL and schema for an operation that matches one against the other: the schema + * with its identifier case folded and the SQL's quoted all-uppercase identifiers folded + * to meet it (#1333) — both halves or neither, since a folded schema against unfolded + * SQL, or the reverse, is the mismatch the fold exists to remove. Only quoted names the + * folded schema holds are touched. A `schema_path` in JSON or YAML goes through the + * same normalisation as an inline context; a DDL file carries its own case semantics + * and is loaded as-is, with the SQL left alone. + */ +export function prepareSql(sql: string, schemaPath?: string, schemaContext?: Record): PreparedSql { + const asIs = (other: string) => other + const identity = (value: T) => value + const folding = (def: { tables: Record }, schema: Schema): PreparedSql => { + const names = schemaNames(def) + const folded = new Set() + const foldSql = (other: string) => foldQuotedIdentifierCase(other, names, folded) + return { sql: foldSql(sql), schema, hasSchema: true, foldSql, unfold: (value) => unfoldValue(value, folded) } + } + if (schemaPath) { + const loaded = loadSchemaFile(schemaPath) + if (loaded.folded && loaded.schema) return folding(loaded.folded, loaded.schema) + if (!loaded.schema) return { sql, schema: EMPTY_SCHEMA(), hasSchema: false, foldSql: asIs, unfold: identity } + return { sql, schema: loaded.schema, hasSchema: true, foldSql: asIs, unfold: identity } + } + if (schemaProvided(undefined, schemaContext)) { + const def = normalizedSchemaDefinition(schemaContext!, { fold: true }) + return folding(def, Schema.fromJson(JSON.stringify(def))) + } + return { sql, schema: EMPTY_SCHEMA(), hasSchema: false, foldSql: asIs, unfold: identity } +} + +function unfoldValue(value: T, folded: ReadonlySet): T { + if (folded.size === 0) return value + if (typeof value === "string") return unfoldText(value, folded) as T + if (Array.isArray(value)) return value.map((item) => unfoldValue(item, folded)) as T + if (value && typeof value === "object") { + const out: Record = {} + for (const [k, v] of Object.entries(value)) out[k] = unfoldValue(v, folded) + return out as T + } + return value +} + +/** Quoted tokens (`"…"` or `` `…` ``) whose content is one this preparation folded go + * back to uppercase — the spelling the caller wrote, since the fold only ever took an + * all-uppercase name down. A lowercase quoted name the caller wrote themselves was + * never recorded and is left alone. */ +function unfoldText(text: string, folded: ReadonlySet): string { + return text.replace(/"([^"]*)"|`([^`]*)`/g, (match, dq?: string, bq?: string) => { + const quoted = dq ?? bq + if (quoted === undefined || !folded.has(quoted)) return match + const mark = match[0] + return `${mark}${quoted.toUpperCase()}${mark}` + }) +} + +/** `folded` carries the folded definition when the file was normalised here; no + * `schema` at all means the file parsed to zero tables — no schema, not an error + * (the engine would refuse an empty definition outright). An unreadable or + * malformed file still throws. */ +function loadSchemaFile(schemaPath: string): { schema?: Schema; folded?: { tables: Record } } { + const ext = path.extname(schemaPath).toLowerCase() + if (ext === ".json" || ext === ".yaml" || ext === ".yml") { + const text = fs.readFileSync(schemaPath, "utf8") + const parsed = ext === ".json" ? JSON.parse(text) : YAML.parse(text) + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { + const folded = normalizedSchemaDefinition(parsed, { fold: true }) + if (Object.keys(folded.tables).length === 0) return {} + return { schema: Schema.fromJson(JSON.stringify(folded)), folded } + } + } + return { schema: Schema.fromFile(schemaPath) } +} + +const EMPTY_SCHEMA = () => Schema.fromDdl("CREATE TABLE _empty_ (id INT);") + /** * Resolve a Schema, falling back to a minimal empty schema when none is provided. * Use this for functions that require a non-null Schema argument. @@ -118,7 +333,7 @@ export function schemaOrEmpty( ): Schema { const s = resolveSchema(schemaPath, schemaContext) if (s !== null) return s - return Schema.fromDdl("CREATE TABLE _empty_ (id INT);") + return EMPTY_SCHEMA() } export * as SchemaResolver from "./schema-resolver" diff --git a/packages/opencode/src/altimate/native/sql/register.ts b/packages/opencode/src/altimate/native/sql/register.ts index 0d3a76c242..8eab811850 100644 --- a/packages/opencode/src/altimate/native/sql/register.ts +++ b/packages/opencode/src/altimate/native/sql/register.ts @@ -8,7 +8,7 @@ import * as core from "@altimateai/altimate-core" import { register } from "../dispatcher" -import { schemaOrEmpty, resolveSchema } from "../schema-resolver" +import { prepareSql } from "../schema-resolver" import { preprocessIff, postprocessQualify } from "../altimate-core" import { EngineCoerce } from "../engine-coerce" import type { @@ -29,11 +29,11 @@ export function registerAllSql(): void { // --------------------------------------------------------------------------- register("sql.analyze", async (params) => { try { - const schema = schemaOrEmpty(params.schema_path, params.schema_context) + const { sql, schema } = prepareSql(params.sql, params.schema_path, params.schema_context) const [lintRaw, semanticsRaw, safetyRaw] = await Promise.all([ - core.lint(params.sql, schema), - core.checkSemantics(params.sql, schema), - core.scanSql(params.sql), + core.lint(sql, schema), + core.checkSemantics(sql, schema), + core.scanSql(sql), ]) const lint = JSON.parse(JSON.stringify(lintRaw)) @@ -135,10 +135,11 @@ export function registerAllSql(): void { // --------------------------------------------------------------------------- register("sql.optimize", async (params) => { try { - const schema = schemaOrEmpty(params.schema_path, params.schema_context) - const [rewriteRaw, lintRaw] = await Promise.all([core.rewrite(params.sql, schema), core.lint(params.sql, schema)]) + const { sql, schema, unfold } = prepareSql(params.sql, params.schema_path, params.schema_context) + // Generated SQL comes back in the caller's spelling (see `PreparedSql.unfold`). + const [rewriteRaw, lintRaw] = await Promise.all([core.rewrite(sql, schema), core.lint(sql, schema)]) - const rewrite = JSON.parse(JSON.stringify(rewriteRaw)) + const rewrite = unfold(JSON.parse(JSON.stringify(rewriteRaw))) const lint = JSON.parse(JSON.stringify(lintRaw)) const suggestions: SqlOptimizeSuggestion[] = (rewrite.suggestions ?? []).map((s: any) => ({ @@ -203,9 +204,9 @@ export function registerAllSql(): void { // --------------------------------------------------------------------------- register("sql.fix", async (params) => { try { - const schema = schemaOrEmpty(params.schema_path, params.schema_context) - const raw = await core.fix(params.sql, schema) - const result = JSON.parse(JSON.stringify(raw)) + const { sql, schema, unfold } = prepareSql(params.sql, params.schema_path, params.schema_context) + const raw = await core.fix(sql, schema) + const result = unfold(JSON.parse(JSON.stringify(raw))) const suggestions = (result.fixes_applied ?? []).map((f: any) => ({ type: f.type ?? f.rule ?? "fix", @@ -356,14 +357,18 @@ export function registerAllSql(): void { // --------------------------------------------------------------------------- register("sql.diff", async (params) => { try { - const schema = params.schema_context ? (resolveSchema(undefined, params.schema_context) ?? undefined) : undefined - const sqlA = params.original ?? params.sql_a const sqlB = params.modified ?? params.sql_b + // The folded copies are for the equivalence check only; the text diff below is + // rendered from what the caller sent, so a case-only edit is not hidden. + const prepared = prepareSql(sqlA, undefined, params.schema_context) + const schema = prepared.hasSchema ? prepared.schema : undefined // `|| undefined`: coerce a default empty-string dialect to "no hint" — the // engine throws on an unknown dialect "". - const compareRaw = schema ? await core.checkEquivalence(sqlA, sqlB, schema, params.dialect || undefined) : null + const compareRaw = schema + ? await core.checkEquivalence(prepared.sql, prepared.foldSql(sqlB), schema, params.dialect || undefined) + : null const compare = compareRaw ? JSON.parse(JSON.stringify(compareRaw)) : null // Simple line-based diff @@ -404,9 +409,9 @@ export function registerAllSql(): void { // --------------------------------------------------------------------------- register("sql.rewrite", async (params) => { try { - const schema = schemaOrEmpty(params.schema_path, params.schema_context) - const raw = core.rewrite(params.sql, schema) - const result = JSON.parse(JSON.stringify(raw)) + const { sql, schema, unfold } = prepareSql(params.sql, params.schema_path, params.schema_context) + const raw = core.rewrite(sql, schema) + const result = unfold(JSON.parse(JSON.stringify(raw))) return { success: true, original_sql: params.sql, @@ -463,8 +468,8 @@ export function registerAllSql(): void { // --------------------------------------------------------------------------- register("lineage.check", async (params) => { try { - const schema = params.schema_context ? (resolveSchema(undefined, params.schema_context) ?? undefined) : undefined - const raw = core.columnLineage(params.sql, EngineCoerce.dialectHint(params.dialect), schema ?? undefined) + const prepared = prepareSql(params.sql, undefined, params.schema_context) + const raw = core.columnLineage(prepared.sql, EngineCoerce.dialectHint(params.dialect), prepared.hasSchema ? prepared.schema : undefined) const result = JSON.parse(JSON.stringify(raw)) return { success: true, diff --git a/packages/opencode/src/altimate/tools/altimate-core-validate.ts b/packages/opencode/src/altimate/tools/altimate-core-validate.ts index cddbc659d9..7129cf7bcc 100644 --- a/packages/opencode/src/altimate/tools/altimate-core-validate.ts +++ b/packages/opencode/src/altimate/tools/altimate-core-validate.ts @@ -12,7 +12,7 @@ export const AltimateCoreValidateTool = Tool.define("altimate_core_validate", { schema_context: z.record(z.string(), z.any()).optional().describe("Inline schema definition"), }), async execute(args, _ctx) { - const hasSchema = !!(args.schema_path || (args.schema_context && Object.keys(args.schema_context).length > 0)) + let hasSchema = !!(args.schema_path || (args.schema_context && Object.keys(args.schema_context).length > 0)) try { const result = await Dispatcher.call("altimate_core.validate", { sql: args.sql, @@ -20,6 +20,19 @@ export const AltimateCoreValidateTool = Tool.define("altimate_core_validate", { schema_context: args.schema_context, }) const data = (result.data ?? {}) as Record + // altimate_change start — the handler decides whether a schema was really + // supplied (a context with no tables is none) and whether the engine ran at + // all; a failed load is an engine failure, not a validation result. + if (typeof data.has_schema === "boolean") hasSchema = data.has_schema + if (result.success === false) { + const msg = result.error ?? "altimate-core validate failed" + return { + title: "Validate: ERROR", + metadata: { success: false, valid: false, has_schema: hasSchema, error: msg, error_class: "engine_failure" }, + output: `Validation could not run: ${msg}`, + } + } + // altimate_change end const error = result.error ?? data.error ?? extractValidationErrors(data) // altimate_change start — sql quality findings for telemetry const errors = Array.isArray(data.errors) ? data.errors : [] diff --git a/packages/opencode/test/altimate/validate-identifier-case.test.ts b/packages/opencode/test/altimate/validate-identifier-case.test.ts new file mode 100644 index 0000000000..170f90ddfd --- /dev/null +++ b/packages/opencode/test/altimate/validate-identifier-case.test.ts @@ -0,0 +1,387 @@ +// Regression for #1333: `altimate_core_validate` against warehouse metadata. +// +// The engine compares an UNQUOTED identifier in lowercase and a QUOTED one exactly; +// Snowflake metadata comes back UPPERCASE. A correct query therefore failed with +// ColumnNotFound (the engine's own DidYouMean pointed at the same column in uppercase, +// confidence 1), and with no schema at all the tool reported TableNotFound although its +// description promises existence checks are skipped. +import { afterAll, beforeAll, describe, expect, test } from "bun:test" + +// Both source modules import `@altimateai/altimate-core` statically, so they are loaded +// lazily: a static import here would fail the whole file where the napi binding cannot +// load, and the skip guard below would never run. (bot review) +const hasCore = (() => { + try { + // `require`, not `require.resolve`: index.js loads the platform binding and throws + // when none loads, which is the case the guard exists for. (bot review) + require("@altimateai/altimate-core") + return true + } catch { + return false + } +})() +const describeIf = hasCore ? describe : describe.skip + +const SQL = `select customer_region, sum(net_revenue) as net_revenue +from TPCH_ANALYTICS.PUBLIC_REPORTING.RPT_MONTHLY_SALES_BY_REGION +where order_month >= '1997-01-01' group by 1` + +// Exactly what `schema_inspect` / `snowflake_get_table_stats` return on Snowflake. +const UPPER = { + "TPCH_ANALYTICS.PUBLIC_REPORTING.RPT_MONTHLY_SALES_BY_REGION": { + columns: [ + { name: "ORDER_MONTH", type: "DATE" }, + { name: "CUSTOMER_REGION", type: "VARCHAR" }, + { name: "NET_REVENUE", type: "NUMBER" }, + ], + }, +} + +describeIf("foldIdentifierCase", () => { + let foldIdentifierCase: typeof import("../../src/altimate/native/schema-resolver").foldIdentifierCase + let foldQuotedIdentifierCase: typeof import("../../src/altimate/native/schema-resolver").foldQuotedIdentifierCase + let normalizeSchemaContext: typeof import("../../src/altimate/native/schema-resolver").normalizeSchemaContext + let schemaProvided: typeof import("../../src/altimate/native/schema-resolver").schemaProvided + let prepareSql: typeof import("../../src/altimate/native/schema-resolver").prepareSql + beforeAll(async () => { + ;({ foldIdentifierCase, foldQuotedIdentifierCase, normalizeSchemaContext, schemaProvided, prepareSql } = await import( + "../../src/altimate/native/schema-resolver" + )) + }) + + test("an all-uppercase name (created unquoted) folds to the engine's lowercase form", () => { + expect(foldIdentifierCase("ORDER_MONTH")).toBe("order_month") + expect(foldIdentifierCase("TPCH_ANALYTICS.PUBLIC_REPORTING.RPT_MONTHLY_SALES_BY_REGION")).toBe( + "tpch_analytics.public_reporting.rpt_monthly_sales_by_region", + ) + }) + + test("a mixed-case name (created quoted, referenced quoted) is kept exact; lowercase is untouched", () => { + expect(foldIdentifierCase("Customer_Region")).toBe("Customer_Region") + expect(foldIdentifierCase("customer_region")).toBe("customer_region") + expect(foldIdentifierCase("ORDER_2024")).toBe("order_2024") // digits do not make it mixed-case + expect(foldIdentifierCase("")).toBe("") + }) + + test("normalizeSchemaContext folds table keys and column names in both input shapes when asked, and not otherwise", () => { + expect(Object.keys(JSON.parse(normalizeSchemaContext(UPPER)).tables)).toEqual(Object.keys(UPPER)) + const fromDefinition = JSON.parse(normalizeSchemaContext(UPPER, { fold: true })) + expect(Object.keys(fromDefinition.tables)).toEqual(["tpch_analytics.public_reporting.rpt_monthly_sales_by_region"]) + expect(fromDefinition.tables["tpch_analytics.public_reporting.rpt_monthly_sales_by_region"].columns.map((c: any) => c.name)).toEqual([ + "order_month", + "customer_region", + "net_revenue", + ]) + const fromFlat = JSON.parse(normalizeSchemaContext({ ORDERS: { ORDER_ID: "NUMBER", "Mixed_Col": "VARCHAR" } }, { fold: true })) + expect(fromFlat.tables.orders.columns.map((c: any) => c.name)).toEqual(["order_id", "Mixed_Col"]) + }) + + test("a fold that would collide with an entry the schema already has keeps the name as written", () => { + // Metadata with both `ORDERS` and `orders`: distinct warehouse objects, and folding one + // onto the other would drop its columns. Both survive, the uppercase one unfolded. + const tables = JSON.parse( + normalizeSchemaContext({ tables: { ORDERS: { columns: [{ name: "A", type: "INT" }] }, orders: { columns: [{ name: "b", type: "INT" }] } } }, { fold: true }), + ).tables + expect(Object.keys(tables).sort()).toEqual(["ORDERS", "orders"]) + expect(tables.ORDERS.columns[0].name).toBe("a") + // Same rule inside a table: `FOO` and `foo` columns both stay. + const cols = JSON.parse( + normalizeSchemaContext({ t: { columns: [{ name: "FOO", type: "INT" }, { name: "foo", type: "INT" }, { name: "BAR", type: "INT" }] } }, { fold: true }), + ).tables.t.columns.map((c: any) => c.name) + expect(cols).toEqual(["FOO", "foo", "bar"]) + }) + + test("a table named __PROTO__ is an entry in the folded schema, not a prototype write", () => { + const def = JSON.parse(normalizeSchemaContext({ __PROTO__: { ID: "INT" } }, { fold: true })) + expect(Object.keys(def.tables)).toEqual(["__proto__"]) + expect(def.tables.__proto__.columns).toEqual([{ name: "id", type: "INT" }]) + }) + + test("foldQuotedIdentifierCase lowercases quoted all-uppercase identifiers and nothing else", () => { + const sql = `select "ORDER_MONTH", "Mixed_Col", "ORDER", 'lit "KEEP"', x -- "NOPE"\n, /* "NOT" */ "A1$" from "ORDERS" where n = 'it''s "OK"'` + expect(foldQuotedIdentifierCase(sql)).toBe( + `select "order_month", "Mixed_Col", "order", 'lit "KEEP"', x -- "NOPE"\n, /* "NOT" */ "a1$" from "orders" where n = 'it''s "OK"'`, + ) + expect(foldQuotedIdentifierCase(sql)).toHaveLength(sql.length) + }) + + test("the lexer's other spans: backticks, doubled quotes, dotted names, dollar quoting, E-strings", () => { + // codex on #1343. BigQuery/Databricks backticks fold like double quotes. + expect(foldQuotedIdentifierCase("select `ORDER_MONTH` from `DS.ORDERS`")).toBe("select `order_month` from `ds.orders`") + // A doubled quote inside the name is not a plain identifier: kept as written. + expect(foldQuotedIdentifierCase('select "A""B" from t')).toBe('select "A""B" from t') + // A dotted quoted name folds as one identifier (metadata `A.B` folds to `a.b`). + expect(foldQuotedIdentifierCase('select x from "TPCH.PUBLIC.ORDERS"')).toBe('select x from "tpch.public.orders"') + // Dollar-quoted bodies are opaque, including quotes and comment markers inside them. + const dollar = `select $$ "KEEP" -- "KEEP" $$, $t$ 'x' "KEEP" $t$, "FOLD" from t` + expect(foldQuotedIdentifierCase(dollar)).toBe(`select $$ "KEEP" -- "KEEP" $$, $t$ 'x' "KEEP" $t$, "fold" from t`) + // An identifier containing `$` is not the start of a dollar-quoted string. + expect(foldQuotedIdentifierCase('select foo$t$ + "ORDERS" + bar$t$ from t')).toBe('select foo$t$ + "orders" + bar$t$ from t') + // E'…' strings honour backslash escapes: the escaped quote does not end the string. + expect(foldQuotedIdentifierCase(`select E'it\\'s "KEEP"', "FOLD" from t`)).toBe(`select E'it\\'s "KEEP"', "fold" from t`) + }) + + test("prepareSql folds only quoted names the schema holds, and its foldSql matches", () => { + // On MySQL/BigQuery/SQLite `"SHIPPED"` is a string literal; a query's values must + // not change under validation. (bot review) + const sql = `select "ORDER_MONTH", "NET_REVENUE" from "RPT_MONTHLY_SALES_BY_REGION" where "CUSTOMER_REGION" = "SHIPPED"` + const prepared = prepareSql(sql, undefined, UPPER) + expect(prepared.sql).toBe( + `select "order_month", "net_revenue" from "rpt_monthly_sales_by_region" where "customer_region" = "SHIPPED"`, + ) + expect(prepared.foldSql(`select "ORDER_MONTH" where x = "SHIPPED"`)).toBe(`select "order_month" where x = "SHIPPED"`) + // No schema: nothing is folded, and the sibling fold is the identity. + const bare = prepareSql(sql, undefined, undefined) + expect(bare.sql).toBe(sql) + expect(bare.hasSchema).toBe(false) + expect(bare.foldSql(sql)).toBe(sql) + }) + + test("schemaProvided counts a schema_path, and a schema_context only when it normalises to a table", () => { + expect(schemaProvided("/some/schema.json")).toBe(true) + expect(schemaProvided(undefined, UPPER)).toBe(true) + expect(schemaProvided(undefined, {})).toBe(false) + expect(schemaProvided(undefined, { tables: {} })).toBe(false) + expect(schemaProvided(undefined, { users: {} })).toBe(false) + expect(schemaProvided("", undefined)).toBe(false) + }) +}) + +describeIf("isExistenceError", () => { + let isExistenceError: typeof import("../../src/altimate/native/altimate-core").isExistenceError + beforeAll(async () => { + ;({ isExistenceError } = await import("../../src/altimate/native/altimate-core")) + }) + + test("recognises the engine's table/column-not-found by code or kind, and nothing else", () => { + expect(isExistenceError({ code: "E001", kind: { type: "TableNotFound" } })).toBe(true) + expect(isExistenceError({ code: "E002", kind: { type: "ColumnNotFound" } })).toBe(true) + expect(isExistenceError({ kind: { type: "ColumnNotFound" } })).toBe(true) + expect(isExistenceError({ code: "E001" })).toBe(true) // code alone, no kind + expect(isExistenceError({ code: "E003", kind: { type: "SyntaxError" } })).toBe(false) + expect(isExistenceError(null)).toBe(false) + expect(isExistenceError("E001")).toBe(false) + }) +}) + +describeIf("altimate_core.validate through the dispatcher (#1333)", () => { + let D: any + const telemetry = process.env.ALTIMATE_TELEMETRY_DISABLED + beforeAll(async () => { + process.env.ALTIMATE_TELEMETRY_DISABLED = "true" + D = await import("../../src/altimate/native/dispatcher") + const core = await import("../../src/altimate/native/altimate-core") + core.registerAll() + }) + afterAll(() => { + if (telemetry === undefined) delete process.env.ALTIMATE_TELEMETRY_DISABLED + else process.env.ALTIMATE_TELEMETRY_DISABLED = telemetry + // The handlers went into the process-global dispatcher; leave it as found. + D.reset() + }) + + test("a lowercase Snowflake query validates against UPPERCASE metadata", async () => { + const r = await D.call("altimate_core.validate", { sql: SQL, schema_context: UPPER }) + expect(r.data.valid).toBe(true) + expect(r.data.errors).toEqual([]) + }) + + test("the same query in uppercase validates too", async () => { + const r = await D.call("altimate_core.validate", { sql: SQL.toUpperCase(), schema_context: UPPER }) + expect(r.data.valid).toBe(true) + }) + + test("a genuinely missing column is still reported against uppercase metadata", async () => { + const r = await D.call("altimate_core.validate", { + sql: SQL.replace("customer_region", "customer_reggion"), + schema_context: UPPER, + }) + expect(r.data.valid).toBe(false) + expect(r.data.errors.some((e: any) => e.kind?.type === "ColumnNotFound")).toBe(true) + }) + + test("a quoted all-uppercase reference — dbt quote_columns style — validates against uppercase metadata", async () => { + const quoted = `select "CUSTOMER_REGION", sum("NET_REVENUE") as net_revenue +from "TPCH_ANALYTICS"."PUBLIC_REPORTING"."RPT_MONTHLY_SALES_BY_REGION" where "ORDER_MONTH" >= '1997-01-01' group by 1` + const r = await D.call("altimate_core.validate", { sql: quoted, schema_context: UPPER }) + expect(r.data.errors).toEqual([]) + expect(r.data.valid).toBe(true) + // A quoted mixed-case name is the exact-match case and still has to be exact. + const mixed = await D.call("altimate_core.validate", { + sql: `select "Customer_Region" from TPCH_ANALYTICS.PUBLIC_REPORTING.RPT_MONTHLY_SALES_BY_REGION`, + schema_context: UPPER, + }) + expect(mixed.data.valid).toBe(false) + }) + + test("a schema_context with no tables is treated as no schema", async () => { + for (const schema_context of [{ tables: {} }, { users: {} }]) { + const r = await D.call("altimate_core.validate", { sql: SQL, schema_context }) + expect(r.data.errors).toEqual([]) + expect(r.data.valid).toBe(true) + } + }) + + test("the other SQL-plus-schema handlers get the folded pair too, not only validate", async () => { + // codex on #1343: a folded schema against unfolded SQL is the mismatch the fold + // exists to remove. `lint` and `column_lineage` resolve the same quoted references. + const quoted = `select "CUSTOMER_REGION" from "TPCH_ANALYTICS"."PUBLIC_REPORTING"."RPT_MONTHLY_SALES_BY_REGION"` + const lint = await D.call("altimate_core.lint", { sql: quoted, schema_context: UPPER }) + expect(lint.success).toBe(true) + const lineage = await D.call("altimate_core.column_lineage", { sql: quoted, schema_context: UPPER }) + expect(lineage.success).toBe(true) + const out = JSON.stringify(lineage.data).toLowerCase() + expect(out).toContain("customer_region") + }) + + test("a JSON schema_path is folded like an inline context; a DDL file is left alone, base SQL included", async () => { + const fs = await import("node:fs/promises") + const dir = await fs.mkdtemp((await import("node:os")).tmpdir() + "/schema-case-") + try { + await fs.writeFile(dir + "/schema.json", JSON.stringify(UPPER)) + await fs.writeFile(dir + "/schema.sql", 'CREATE TABLE orders (order_month DATE, "STATUS" VARCHAR);') + const fromJson = await D.call("altimate_core.validate", { sql: SQL, schema_path: dir + "/schema.json" }) + expect(fromJson.data.errors).toEqual([]) + expect(fromJson.data.valid).toBe(true) + const fromDdl = await D.call("altimate_core.validate", { sql: "select order_month from orders", schema_path: dir + "/schema.sql" }) + expect(fromDdl.data.valid).toBe(true) + // The sibling fold follows the schema's preparation: identity for a DDL file, so a + // base query's quoted name is compared as written. (bot review) + const { prepareSql } = await import("../../src/altimate/native/schema-resolver") + expect(prepareSql('select "STATUS" from orders', dir + "/schema.sql").foldSql('select "STATUS" from orders')).toBe( + 'select "STATUS" from orders', + ) + expect(prepareSql("select 1", dir + "/schema.json").foldSql('select "ORDER_MONTH" from t')).toBe('select "order_month" from t') + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + test("the sql.* and lineage.check handlers prepare the pair too", async () => { + // Unquoted lowercase references: against the UNFOLDED uppercase metadata the + // engine resolves the table by its lowercased key but reports it in the + // metadata's spelling; against the folded pair everything is lowercase. The + // assertion is on the raw output, so an unfolded resolver fails it. (bot review) + const low = `select customer_region from TPCH_ANALYTICS.PUBLIC_REPORTING.RPT_MONTHLY_SALES_BY_REGION` + const { registerAllSql } = await import("../../src/altimate/native/sql/register") + registerAllSql() + const lineage = await D.call("lineage.check", { sql: low, schema_context: UPPER }) + expect(lineage.success).toBe(true) + const source = lineage.data.column_lineage[0].source + expect(source).toContain('"tpch_analytics"."public_reporting"."rpt_monthly_sales_by_region"."customer_region"') + expect(source).not.toContain("TPCH_ANALYTICS") + const analyze = await D.call("sql.analyze", { sql: low, schema_context: UPPER }) + expect(analyze.success).toBe(true) + expect(JSON.stringify(analyze.issues)).not.toMatch(/not found/i) + }) + + test("generated SQL comes back in the caller's spelling, and the diff is rendered from the raw inputs", async () => { + // The fold is a comparison form against folded metadata, not a spelling a + // case-sensitive warehouse accepts: `"ORDER_MONTH"` must not come back as + // `"order_month"` in a rewrite, a fix, or an optimisation. (bot review) + const quoted = `select "CUSTOMER_REGION", sum("NET_REVENUE") as net_revenue from "TPCH_ANALYTICS"."PUBLIC_REPORTING"."RPT_MONTHLY_SALES_BY_REGION" where "ORDER_MONTH" >= '1997-01-01' group by 1` + const { registerAllSql } = await import("../../src/altimate/native/sql/register") + registerAllSql() + for (const [method, params] of [ + ["sql.optimize", { sql: quoted, schema_context: UPPER }], + ["sql.rewrite", { sql: quoted, schema_context: UPPER }], + ["sql.fix", { sql: quoted, schema_context: UPPER }], + ["altimate_core.rewrite", { sql: quoted, schema_context: UPPER }], + ["altimate_core.fix", { sql: quoted, schema_context: UPPER }], + ["altimate_core.correct", { sql: quoted, schema_context: UPPER }], + ] as const) { + const r = await D.call(method as never, params as never) + // A handler that errored would pass the spelling checks vacuously. `sql.fix` reports + // `success: false` with a `fixed_sql` when there was nothing to fix, so the guard + // is "the handler ran and returned SQL", not `success` alone. + expect(r.success === true || typeof r.fixed_sql === "string", method).toBe(true) + expect(JSON.stringify(r), method).not.toContain("native handler") + const text = JSON.stringify(r) + expect(text, method).not.toMatch(/"\\"(order_month|customer_region|net_revenue|rpt_monthly_sales_by_region)\\""/) + expect(text, method).not.toContain('\\"order_month\\"') + } + // sql.diff: a case-only edit is a visible diff line. (What the equivalence check + // says about a quoted-lowercase spelling is the tradeoff documented on + // `foldIdentifierCase` — not asserted here either way.) + const diff = await D.call("sql.diff", { + original: `select "ORDER_MONTH" from "TPCH_ANALYTICS"."PUBLIC_REPORTING"."RPT_MONTHLY_SALES_BY_REGION"`, + modified: `select "order_month" from "TPCH_ANALYTICS"."PUBLIC_REPORTING"."RPT_MONTHLY_SALES_BY_REGION"`, + schema_context: UPPER, + } as never) + expect(diff.success).toBe(true) + expect(diff.diff).toContain('- select "ORDER_MONTH"') + expect(diff.diff).toContain('+ select "order_month"') + }) + + test("a JSON schema file with zero tables is no schema, not an engine failure", async () => { + const fs = await import("node:fs/promises") + const dir = await fs.mkdtemp((await import("node:os")).tmpdir() + "/schema-empty-") + try { + await fs.writeFile(dir + "/empty.json", JSON.stringify({ tables: {} })) + const r = await D.call("altimate_core.validate", { sql: SQL, schema_path: dir + "/empty.json" }) + expect(r.success).toBe(true) + expect(r.data.has_schema).toBe(false) + expect(r.data.valid).toBe(true) + } finally { + await fs.rm(dir, { recursive: true, force: true }) + } + }) + + test("schema-only operations see the names as written, not folded", async () => { + const r = await D.call("altimate_core.schema_diff", { + schema1_context: UPPER, + schema2_context: { ...UPPER, EXTRA_TABLE: { columns: [{ name: "ID", type: "NUMBER" }] } }, + }) + expect(r.success).toBe(true) + const text = JSON.stringify(r.data) + expect(text).toContain("EXTRA_TABLE") + expect(text).not.toContain("extra_table") + }) + + test("with no schema, existence findings are dropped and a correct query is valid", async () => { + const r = await D.call("altimate_core.validate", { sql: SQL, schema_path: "", schema_context: {} }) + expect(r.data.valid).toBe(true) + expect(r.data.errors).toEqual([]) + }) + + test("with no schema, a syntax error is still reported", async () => { + const r = await D.call("altimate_core.validate", { sql: "selec x fro t", schema_context: {} }) + expect(r.data.valid).toBe(false) + expect(r.data.errors.length).toBeGreaterThan(0) + expect(r.data.errors.every((e: any) => !["TableNotFound", "ColumnNotFound"].includes(e.kind?.type))).toBe(true) + }) +}) + +describeIf("the altimate_core_validate tool follows the handler", () => { + let tool: any + const telemetry = process.env.ALTIMATE_TELEMETRY_DISABLED + beforeAll(async () => { + process.env.ALTIMATE_TELEMETRY_DISABLED = "true" + const core = await import("../../src/altimate/native/altimate-core") + core.registerAll() + const { initTool } = await import("./tool-fixture") + const { AltimateCoreValidateTool } = await import("../../src/altimate/tools/altimate-core-validate") + tool = await initTool(AltimateCoreValidateTool) + }) + afterAll(async () => { + if (telemetry === undefined) delete process.env.ALTIMATE_TELEMETRY_DISABLED + else process.env.ALTIMATE_TELEMETRY_DISABLED = telemetry + const D = await import("../../src/altimate/native/dispatcher") + D.reset() + }) + const ctx = () => ({ sessionID: "s", messageID: "m", agent: "build", abort: new AbortController().signal, messages: [], metadata: () => {} }) + + test("a table-less schema_context is reported as no schema, not as a full validation", async () => { + // codex on #1343: the tool decided `has_schema` on its own and said "VALID" as if + // existence had been checked. + const r = await tool.execute({ sql: SQL, schema_context: { tables: {} } }, ctx()) + expect(r.metadata.has_schema).toBe(false) + expect(r.title).toContain("(no schema)") + }) + + test("a schema file that cannot be loaded is an engine failure, not a valid result", async () => { + const r = await tool.execute({ sql: SQL, schema_path: "/nonexistent/schema.json" }, ctx()) + expect(r.metadata.success).toBe(false) + expect(r.title).toBe("Validate: ERROR") + }) +})