diff --git a/docs/docs/usage/tool.md b/docs/docs/usage/tool.md index 40985774f5..af5c667c68 100644 --- a/docs/docs/usage/tool.md +++ b/docs/docs/usage/tool.md @@ -51,7 +51,7 @@ Only the data tools are exposed. Coding tools (`bash`, `read`, `edit`, ...), `db Runs are read-only by default. Reads such as `sql_execute` on a SELECT and `schema_inspect` are allowed; anything else is refused with `Permission denied: . Re-run with --allow-write to enable writes.` Tools that change local configuration (`warehouse_add`, `warehouse_remove`, `warehouse_install_driver`) also need `--allow-write`. -`DROP DATABASE`, `DROP SCHEMA` and `TRUNCATE` are refused even with `--allow-write`. The block reads the SQL text, so it is best effort: statements built at run time, for example by a stored procedure, are not seen. For a hard guarantee, connect with a database user that lacks `DROP` and `TRUNCATE`. +`DROP DATABASE`, `DROP SCHEMA` and `TRUNCATE` are refused even with `--allow-write` when the SQL check recognises them. The check reads the SQL text, so it is best effort: a statement the warehouse builds or decodes at run time (dynamic SQL assembled by concatenation, a stored routine, an unusual literal encoding) can get past it. For a hard guarantee, connect with a database user that has no `DROP` or `TRUNCATE` privilege. Tool results go to the model of the agent that called the tool. `sql_execute` returns result rows, `warehouse_list` shows one sample row per table of a local database, and `schema_inspect` and `sql_execute` with `tables` return value profiles per column; set `ALTIMATE_DISABLE_SCHEMA_VALUES=1` to leave the value profiles out. Keep passwords and keys out of `warehouse_add` arguments, which end up in the calling agent's transcript: reference credentials from a connections file or `ALTIMATE_CODE_CONN_*` variables instead. diff --git a/packages/opencode/src/altimate/native/connections/dbt-adapter-type.ts b/packages/opencode/src/altimate/native/connections/dbt-adapter-type.ts index 7c25d82b49..b2a25328a4 100644 --- a/packages/opencode/src/altimate/native/connections/dbt-adapter-type.ts +++ b/packages/opencode/src/altimate/native/connections/dbt-adapter-type.ts @@ -25,13 +25,15 @@ export async function dbtProjectAdapterType(): Promise<{ project: boolean; type? try { if (existsSync(config.path())) saved = JSON.parse(readFileSync(config.path(), "utf8")) } catch {} - projectRoot = saved === undefined ? config.findProjectRoot() : (saved?.projectRoot ?? null) + projectRoot = saved === undefined ? config.findProjectRoot() : typeof saved?.projectRoot === "string" ? saved.projectRoot : null } catch { projectRoot = null } } - // A root without dbt_project.yml is rejected by the routing, which then uses the registry default. - if (!projectRoot || !existsSync(path.join(projectRoot, "dbt_project.yml"))) return { project: false, targetTypes: [] } + // A root without dbt_project.yml is rejected by the routing, which then uses the registry default; + // an adapter already created keeps running even if the file is removed since. + if (!projectRoot || (dbtRoute.root === undefined && !existsSync(path.join(projectRoot, "dbt_project.yml")))) + return { project: false, targetTypes: [] } const targetTypes = await profileTargetTypes(projectRoot) try { const { resolveDbtTargetPath } = await import("../../validators/validator-utils") diff --git a/packages/opencode/src/altimate/tools/data-diff.ts b/packages/opencode/src/altimate/tools/data-diff.ts index c317286376..2a883ff4ad 100644 --- a/packages/opencode/src/altimate/tools/data-diff.ts +++ b/packages/opencode/src/altimate/tools/data-diff.ts @@ -119,7 +119,7 @@ export const DataDiffTool = Tool.define("data_diff", { target: args.target, key_columns: args.key_columns, source_warehouse: args.source_warehouse, - target_warehouse: args.target_warehouse, + target_warehouse: args.target_warehouse || undefined, // "" is the source warehouse, as the ask resolves it extra_columns: args.extra_columns, algorithm: args.algorithm, where_clause: args.where_clause, diff --git a/packages/opencode/src/altimate/tools/sql-classify.ts b/packages/opencode/src/altimate/tools/sql-classify.ts index 45eee7d1ef..0b8ec02c69 100644 --- a/packages/opencode/src/altimate/tools/sql-classify.ts +++ b/packages/opencode/src/altimate/tools/sql-classify.ts @@ -171,7 +171,7 @@ function otherIsReadOnly(sql: string, parsed: Array<{ category: string }>, diale // Hard-deny patterns — blocked regardless of permissions const HARD_DENY_TYPES = new Set(["DROP DATABASE", "DROP SCHEMA", "TRUNCATE", "TRUNCATE TABLE"]) -const HARD_DENY_PATTERN = /^\s*(DROP\s+(DATABASE|SCHEMA)\b|TRUNCATE(\s+TABLE)?\b)/i +const HARD_DENY_PATTERN = /^\s*(DROP\s+((EXTERNAL|PLUGGABLE)\s+)?(DATABASE|SCHEMA|CATALOG|NAMESPACE)\b|TRUNCATE(\s+TABLE)?\b)/i /** * Fail closed when the parser is unavailable or rejects the input. Lexical @@ -192,16 +192,19 @@ function classifyFallback(sql: string, dialect?: string): { queryType: "read" | // comment), so the words alone decide; the TRUNCATE() function does not count. const lexical = tokenizeSql(sql, dialect) if (!lexical.complete) { - if (mentionsHardDeny(sql)) blocked = true + // Escapes decoded too: dynamic SQL may run a literal such as '\x44ROP SCHEMA s'. + if (mentionsHardDenyAnywhere(sql)) blocked = true } else { // Read with certainty: the statement words in code (literals, identifiers and comments masked, // so DROP/**/SCHEMA reads as DROP SCHEMA) decide too, since T-SQL runs `SELECT 1 DROP DATABASE x` // as two statements with no ; and a parser may read a literal differently from the engine. - const masked = maskSql(lexical.tokens) - if (CODE_HARD_DENY.test(masked)) blocked = true - // Dynamic SQL (EXEC('...'), EXECUTE IMMEDIATE, PREPARE ... FROM, DO $$...$$) runs a string, so - // the words in its literals count too. A statement built by concatenation is not seen. - else if (DYNAMIC_SQL.test(masked) && mentionsHardDeny(sql)) blocked = true + const names = namesKept(lexical.tokens) + // Snowflake's ->> chains statements (elsewhere a JSON operator), and Snowflake Scripting's ASYNC (, + // := ( and DEFAULT ( hold one (elsewhere DEFAULT(col) is a function). + const chained = + (/^snowflake$/i.test(dialect ?? "") || !isModeledDialect(dialect)) && + /(->>|\bASYNC\s*\(|:=\s*\(|\bDEFAULT\s*\()\s*TRUNCATE\b(?!\s*\()/i.test(names) + if (CODE_HARD_DENY.test(names) || handlerTruncate(names) || chained) blocked = true } return { queryType: "write", blocked } } @@ -217,10 +220,10 @@ function batchedStatement(masked: string, dialect?: string): boolean { // or clauses (USE, FETCH, OPEN, CLOSE, ENABLE, DISABLE, RENAME, THROW, ADD, MOVE, SEND, RECEIVE, END) // count only in their statement forms. const BATCH_STATEMENT = new RegExp( - "\\b(INSERT|UPDATE|DELETE|MERGE|DROP|ALTER|CREATE|TRUNCATE|EXEC|EXECUTE|GRANT|REVOKE|DENY|BACKUP|RESTORE|DBCC|KILL|SHUTDOWN|BULK|WAITFOR|DECLARE|SET|INTO|" + + "\\bTRUNCATE\\b(?!\\s*\\()|\\b(INSERT|UPDATE|DELETE|MERGE|DROP|ALTER|CREATE|EXEC|EXECUTE|GRANT|REVOKE|DENY|BACKUP|RESTORE|DBCC|KILL|SHUTDOWN|BULK|WAITFOR|DECLARE|SET|INTO|" + "OPENROWSET|OPENQUERY|OPENDATASOURCE|OPENXML|CHECKPOINT|RECONFIGURE|RAISERROR|GOTO|BEGIN|COMMIT|ROLLBACK|SAVE|DEALLOCATE|" + "READTEXT|UPDATETEXT|WRITETEXT|SETUSER|REVERT|PRINT)\\b" + - "|\\b(ENABLE|DISABLE)\\s+TRIGGER\\b|\\bRENAME\\s+(OBJECT|DATABASE)\\b|\\bTHROW\\s*($|;|[\\d@])" + + "|\\b(ENABLE|DISABLE)\\s+TRIGGER\\b|\\bRENAME\\s+(OBJECT|DATABASE)\\b|\\bTHROW\\s+[\\d@]" + // USE except OPTION (USE HINT/PLAN) and USE INDEX; FETCH except OFFSET paging (FETCH NEXT 10 ROWS ONLY). "|\\bUSE\\b(?!\\s+(HINT|PLAN|INDEX|KEY)\\b)|\\bFETCH\\b(?!\\s+(FIRST|NEXT)\\s+([\\w@().+-]+\\s+)?(PERCENT\\s+)?ROWS?\\b)" + "|\\b(OPEN|CLOSE)\\s+(GLOBAL\\s+)?[\\w@#]+\\s*(;|\\b(SELECT|WITH)\\b)|\\bADD\\s+(SENSITIVITY|SIGNATURE|COUNTER)\\b" + @@ -228,15 +231,102 @@ const BATCH_STATEMENT = new RegExp( "i", ) -const CODE_HARD_DENY = /\bDROP\s+(DATABASE|SCHEMA)\b|\bTRUNCATE\b(?!\s*\()/i -const DYNAMIC_SQL = /\bEXEC(UTE)?\b|(^|;)\s*(PREPARE|DO)\b/i -// Sequence advances outside PostgreSQL's nextval(): NEXT VALUE FOR s (SQL Server, DB2), s.NEXTVAL -// (Oracle, Snowflake). -const SEQUENCE_ADVANCE = /\bNEXT\s+VALUE\s+FOR\b|\.\s*NEXTVAL\b/i +/** A literal's text as an engine may run it, one string per reading: numeric escapes (\xHH, \uHHHH, + * \UHHHHHHHH, octal \ooo, U&'...' \HHHH and \+HHHHHH) decoded, and a single-character escape read as a + * space when it stands for whitespace (\n \r \t \f, and \v in BigQuery) and as itself otherwise; + * and SQL Server's reading, where a backslash before a line break joins the two lines. */ +function escapeReadings(sql: string): string[] { + const char = (hex: string, base = 16) => String.fromCodePoint(Math.min(parseInt(hex, base), 0x10ffff)) + let numeric = sql + if (/U&/i.test(numeric)) numeric = numeric.replace(/\\\+([0-9a-fA-F]{6})|\\([0-9a-fA-F]{4})/g, (_, six, four) => char(six ?? four)) + const decode = (whitespace: RegExp) => + numeric.replace( + /\\(?:x([0-9a-fA-F]{1,2})|u([0-9a-fA-F]{4})|U([0-9a-fA-F]{8})|([0-7]{1,3})|([\s\S]))/g, + (_, x, u, U, octal, single) => + single !== undefined ? (whitespace.test(single) ? " " : single) : octal ? char(octal, 8) : char(x ?? u ?? U), + ) + return [sql, sql.replace(/\\\r?\n/g, ""), decode(/^[nrtf]$/), decode(/^[nrtfv]$/)] +} + +// TRUNCATE as a statement: TRUNCATE TABLE, or TRUNCATE starting a statement or a block (BEGIN [NOT +// ATOMIC], DO, LOOP, REPEAT, a handler's condition). After THEN or ELSE it must be a name and the end +// of the statement, since CASE ... THEN truncate ... is a column. A column or alias named truncate is not. +const CODE_HARD_DENY = new RegExp( + "\\bDROP\\s+((EXTERNAL|PLUGGABLE)\\s+)?(DATABASE|SCHEMA|CATALOG|NAMESPACE)\\b|\\bTRUNCATE\\s+TABLE\\b" + + // DuckDB accepts TRUNCATE after EXPLAIN (options) and PREPARE p AS ([^()] keeps the scan linear). + "|(^|;|\\bPREPARE\\s+\\w+\\s+AS\\b|\\bEXPLAIN\\s*\\([^()]*\\))\\s*TRUNCATE\\b(?!\\s*\\()" + + // After a block or handler word (or EXPLAIN ANALYZE), a target must follow: these words are also + // column names, so `SELECT atomic truncate FROM t` is an alias. + "|\\b(BEGIN|NOT\\s+ATOMIC|DO|LOOP|REPEAT|SQLEXCEPTION|SQLWARNING|FOUND|ANALYZE|ANALYSE)\\s+TRUNCATE\\s+" + + "(?!(FROM|AS|WHERE|ORDER|GROUP|HAVING|UNION|LIMIT|WINDOW|INTO|JOIN|ON|AND|OR|END)\\b)[\\w\"`$\\u0080-\\uffff]" + + // After THEN/ELSE: [IF EXISTS] a (dotted, quoted or IDENTIFIER(...)) name, then the statement's end. + "|\\b(THEN|ELSE)\\s+TRUNCATE\\s+(IF\\s+EXISTS\\s+)?(?!END\\b)" + + "(IDENTIFIER\\s*\\([^()]*\\)|[^\\s;,().]+(\\s*\\.{1,2}\\s*[^\\s;,().]+)*)(\\s+(NOWAIT|WAIT\\s+\\d+))?\\s*(;|$)" + + // Anywhere else, TRUNCATE followed by a target (a routine or task body, BEGIN ATOMIC, ALTER TABLE + // ... TRUNCATE PARTITION). A column named truncate is followed by a clause word, an operator, `,` + // or `)`, or qualified (t.truncate). A name truncate followed by any other word is refused, e.g. an + // alias without AS (`SELECT truncate x`, `FROM truncate t`). + // The t.truncate check runs only after TRUNCATE matched, over the whitespace just before it (comments + // are masked to spaces), so each run of whitespace is scanned once. + "|\\bTRUNCATE(? { + const handler = part.search(/\bHANDLER\s+FOR\b/i) + return handler >= 0 && /\bTRUNCATE\b(?!\s*\()/i.test(part.slice(handler)) + }) +} + +/** Code with literals and comments masked, and each quoted identifier as a placeholder name, so a + * statement keeps its target (TRUNCATE `t`) without the identifier's text reading as code. */ +function namesKept(tokens: ReturnType["tokens"]): string { + return tokens + .map(({ kind, text }) => + kind === "code" ? text : kind === "identifier" ? ` ${"_".repeat(Math.max(text.length - 2, 1))} ` : text.replace(/[^\r\n]/g, " "), + ) + .join("") +} + +/** Whether a write's quoted text holds a hard-denied statement. A write can run its literals as SQL: + * dynamic SQL (EXEC('...'), sp_executesql N'...', EXECUTE IMMEDIATE, PREPARE ... FROM, dblink_exec), + * or a function, procedure or trigger body created and called in the same call. Each literal is read + * word by word with its escapes decoded, as uncertain SQL is, so nothing inside it (nested quotes, + * comments) can separate the words. A statement built by concatenation is not seen, and a custom + * UESCAPE character is not decoded, so it blocks. */ +function quotedHardDeny(tokens: ReturnType["tokens"], dialect?: string): boolean { + // UESCAPE counts with a U&'...' literal: U& in code right before a literal token. + const unicode = tokens.some(({ kind, text }, i) => kind === "code" && /\bU&$/i.test(text) && tokens[i + 1]?.kind === "literal") + if (unicode && /\bUESCAPE\b/i.test(maskSql(tokens))) return true + // "..." is a string in Spark, Databricks and Hive, under SQL Server's QUOTED_IDENTIFIER OFF and in + // MySQL without ANSI_QUOTES; other identifiers never are. + const quotedStrings = + /^(sqlserver|tsql|mssql|fabric|mysql|mariadb|spark|databricks|hive)$/i.test(dialect ?? "") || !isModeledDialect(dialect) + return tokens.some( + ({ kind, text }) => + (kind === "literal" || (kind === "identifier" && quotedStrings && text[0] === '"')) && + ((/\bUESCAPE\b/i.test(text) && /\bU&'/i.test(text)) || mentionsHardDenyAnywhere(text)), + ) +} + +// Sequence advances outside PostgreSQL's nextval(): NEXT VALUE FOR s (SQL Server, DB2), and in Oracle, +// Snowflake and MariaDB (Oracle mode) s.NEXTVAL, quoted or not (elsewhere s.nextval is a column). +function advancesSequence(tokens: ReturnType["tokens"], masked: string, dialect?: string): boolean { + if (/\bNEXT\s+VALUE\s+FOR\b/i.test(masked)) return true + if (!/^(oracle|snowflake|mysql|mariadb)$/i.test(dialect ?? "") && isModeledDialect(dialect)) return false + // Comments masked to spaces and quoted names unwrapped, so s./* c */"NEXTVAL" reads as s.NEXTVAL. + return /\.\s*NEXTVAL\b/i.test(maskSql(tokens, true)) +} + +/** The word DROP with DATABASE, SCHEMA, CATALOG or NAMESPACE anywhere after it, or TRUNCATE not + * directly followed by `(` (the TRUNCATE() function; a comment before the `(` does not count, since + * where a comment ends is part of what is uncertain). Only words count: quotes and comments are ignored, since how they read * is what is uncertain, so no comment or quote placement can separate the words. One forward pass. */ function mentionsHardDeny(sql: string): boolean { let drop = false @@ -250,7 +340,7 @@ function mentionsHardDeny(sql: string): boolean { while (j < sql.length && /\w/.test(sql[j])) j++ const word = sql.slice(i, j).toUpperCase() if (word === "DROP") drop = true - else if (drop && (word === "DATABASE" || word === "SCHEMA")) return true + else if (drop && (word === "DATABASE" || word === "SCHEMA" || word === "CATALOG" || word === "NAMESPACE")) return true else if (word === "TRUNCATE" && !/^[ \t\r\n]*\(/.test(sql.slice(j, j + 64))) return true i = j } @@ -349,7 +439,21 @@ export function classifyAndCheck(sql: string, dialect?: string): { blocked: boolean parseError?: string } { - return classifySql(sql, dialect, 0) + const result = classifySql(sql, dialect, 0) + if (result.queryType === "write" && !result.blocked) { + // SQL read with certainty: a write may still run the SQL in its literals. Otherwise the word scan + // of the whole text already covered them. + const lexical = tokenizeSql(sql, dialect) + if (lexical.complete && quotedHardDeny(lexical.tokens, dialect)) return { ...result, blocked: true } + } + return result +} + +/** Whether DROP DATABASE/SCHEMA/CATALOG/NAMESPACE or TRUNCATE appears anywhere in the text, read word by word + * with quotes and comments ignored and escapes decoded (for text another layer renders, such as dbt's + * Jinja). */ +export function mentionsHardDenyAnywhere(sql: string): boolean { + return escapeReadings(sql).some(mentionsHardDeny) } function classifySql(sql: string, dialect: string | undefined, depth: number): { @@ -385,7 +489,14 @@ function classifySql(sql: string, dialect: string | undefined, depth: number): { !functions.write && !blocked && !batchedStatement(masked, dialect) && - !SEQUENCE_ADVANCE.test(masked) && + // Snowflake's ->> runs the statements it chains, DML included. + !(/^snowflake$/i.test(dialect ?? "") && masked.includes("->>")) && + // Spark/Hive SELECT TRANSFORM(...) USING 'cmd' runs a command on the cluster; any USING not + // followed by "(" counts (comments and the script masked), while JOIN ... USING (col) is a join. + !((/^(spark|databricks|hive)$/i.test(dialect ?? "") || !isModeledDialect(dialect)) && + /\bTRANSFORM\s*\(/i.test(masked) && + /\bUSING\b(?!\s*\()/i.test(masked)) && + !advancesSequence(lexical.tokens, masked, dialect) && otherIsReadOnly(sql, result.statements, dialect) ? "read" : "write", diff --git a/packages/opencode/src/altimate/tools/sql-execute.ts b/packages/opencode/src/altimate/tools/sql-execute.ts index aa0a3b3bda..78a5a11d41 100644 --- a/packages/opencode/src/altimate/tools/sql-execute.ts +++ b/packages/opencode/src/altimate/tools/sql-execute.ts @@ -9,7 +9,13 @@ import { Tool } from "../../tool/tool" import { Dispatcher } from "../native" import type { SqlExecuteResult } from "../native/types" // altimate_change start - SQL write access control + fingerprinting -import { classifyAndCheck, classifyMongoCommand, computeSqlFingerprint, isScratchSafeWrite } from "./sql-classify" +import { + classifyAndCheck, + classifyMongoCommand, + computeSqlFingerprint, + isScratchSafeWrite, + mentionsHardDenyAnywhere, +} from "./sql-classify" import { statementsOf } from "./sql-tokenize" const MONGO_WAREHOUSE_TYPES = new Set(["mongodb", "mongo"]) @@ -231,7 +237,12 @@ export const SqlExecuteTool = Tool.define( ) } const classification = mongo ?? classifyEvery(sql, dialects) - const { queryType, blocked } = classification + // dbt renders Jinja before it runs SQL, wherever the Jinja sits (a comment or literal too), + // so {% do run_query('...') %} runs while the query compiles and {# ... #} removes text. + // A Mongo command routed to dbt is rendered too. + const jinja = target?.dbt === true && /\{[{%#]/.test(sql) + const queryType = jinja ? "write" : classification.queryType + const blocked = classification.blocked || (jinja && mentionsHardDenyAnywhere(sql)) const query = mongo?.normalized ?? sql if (blocked) { throw new Error( @@ -671,14 +682,18 @@ function resolveStepRefs( * type and every target of the project's profile; when the profile's targets cannot be read, or one * is an adapter with no supported dialect, every supported dialect and the generic lexing count. * Read without starting dbt or loading the handlers. The registry warehouse is named for the ask. */ -async function defaultTarget(forceReadOnly: boolean): Promise<{ dialects: (string | undefined)[]; warehouse?: string }> { +async function defaultTarget( + forceReadOnly: boolean, +): Promise<{ dialects: (string | undefined)[]; warehouse?: string; dbt?: boolean }> { const fallback = Registry.list().warehouses[0] // before any await, as the call resolves it now const dialectOf = (type?: string) => (type ? (Registry.canonicalType(type) ?? undefined) : undefined) const dialects: (string | undefined)[] = fallback ? [dialectOf(fallback.type)] : [] + let viaDbt = false if (!forceReadOnly) { const { dbtProjectAdapterType } = await import("../native/connections/dbt-adapter-type") const dbt = await dbtProjectAdapterType() if (dbt.project) { + viaDbt = true const types = [dbt.type, ...dbt.targetTypes].filter((type): type is string => Boolean(type)) const known = types.map(dialectOf) dialects.push(...known.filter(Boolean)) @@ -686,7 +701,11 @@ async function defaultTarget(forceReadOnly: boolean): Promise<{ dialects: (strin if (!dbt.targetTypes.length || known.includes(undefined)) dialects.push(...EVERY_DIALECT, undefined) } } - return { dialects: dialects.length ? [...new Set(dialects)] : [undefined], ...(fallback && { warehouse: fallback.name }) } + return { + dialects: dialects.length ? [...new Set(dialects)] : [undefined], + ...(fallback && { warehouse: fallback.name }), + ...(viaDbt && { dbt: true }), + } } const EVERY_DIALECT = ["postgres", "redshift", "duckdb", "snowflake", "bigquery", "mysql", "sqlserver", "databricks", "clickhouse", "oracle", "trino", "sqlite"] diff --git a/packages/opencode/src/altimate/tools/sql-explain.ts b/packages/opencode/src/altimate/tools/sql-explain.ts index 77fdbbf221..1ee673d029 100644 --- a/packages/opencode/src/altimate/tools/sql-explain.ts +++ b/packages/opencode/src/altimate/tools/sql-explain.ts @@ -8,6 +8,7 @@ import * as Precedence from "../workspace/precedence" import { validateWarehouseName } from "./input-validation" import * as Registry from "../native/connections/registry" import { classifyAndCheck, isSingleQuery } from "./sql-classify" +import { statementsOf, tokenizeSql } from "./sql-tokenize" /** * Detect SQL input that cannot be meaningfully EXPLAIN'd. @@ -90,14 +91,26 @@ export const SqlExplainTool = Tool.define("sql_explain", { output: `Invalid input: ${sqlError}`, } } - // EXPLAIN ANALYZE executes the statement. Only a single read-only query may be analyzed, - // so a plan request can never perform a write on engines without a read-only transaction. - if (args.analyze) { - const dialect = Registry.canonicalType(args.warehouse ? Registry.getConfig(args.warehouse)?.type : undefined) ?? undefined + // EXPLAIN is prefixed to the text, and some drivers run every statement in it (Redshift, DuckDB), + // so only one statement is explained: a second one would run, not be planned. EXPLAIN ANALYZE + // executes the statement, so only a single read-only query may be analyzed. + { + const warehouse = args.warehouse || Registry.list().warehouses[0]?.name + const dialect = Registry.canonicalType(warehouse ? Registry.getConfig(warehouse)?.type : undefined) ?? undefined const { queryType, blocked } = classifyAndCheck(args.sql, dialect) - if (blocked || queryType !== "read" || !isSingleQuery(args.sql, dialect)) { - const error = - "EXPLAIN ANALYZE executes the statement, so only a single read-only query can be analyzed. Use analyze: false for a plan of this statement." + const statements = tokenizeSql(args.sql, dialect).complete ? statementsOf(args.sql, dialect) : [] + // The tool adds EXPLAIN itself: SQL starting with an EXPLAIN option (ANALYZE, VERBOSE, an option + // list such as ("analyze")) would turn the plan into EXPLAIN ANALYZE, which runs the statement. A + // leading "(" is a query only when a query keyword follows it. + const option = /^(EXPLAIN|ANALY[SZ]E|VERBOSE)\b|^\((?![(\s]*(SELECT|WITH|VALUES|TABLE|FROM)\b)/i.test(statements[0] ?? "") + const error = blocked + ? "DROP DATABASE, DROP SCHEMA and TRUNCATE are blocked for safety." + : statements.length !== 1 || option + ? "Only a single statement can be explained, without EXPLAIN or its options. Run each statement separately." + : args.analyze && (queryType !== "read" || !isSingleQuery(args.sql, dialect)) + ? "EXPLAIN ANALYZE executes the statement, so only a single read-only query can be analyzed. Use analyze: false for a plan of this statement." + : undefined + if (error) { return { title: "Explain: INVALID INPUT", metadata: { success: false, analyzed: false, warehouse_type: "unknown", error, error_class: "input_validation" }, diff --git a/packages/opencode/src/altimate/tools/sql-function-policy.ts b/packages/opencode/src/altimate/tools/sql-function-policy.ts index 6341156e29..675bfe026c 100644 --- a/packages/opencode/src/altimate/tools/sql-function-policy.ts +++ b/packages/opencode/src/altimate/tools/sql-function-policy.ts @@ -20,7 +20,7 @@ const PURE_PG_FUNCTION = new RegExp(`^pg_(?:${PURE_PG_FUNCTIONS})$`, "i") const SIDE_EFFECT_PG_FUNCTION = /^pg_(?:write_file|append_file|file_write|file_rename|file_unlink|try_advisory\w*|advisory_unlock\w*|advisory_lock\w*|advisory_xact\w*|terminate_backend|cancel_backend|reload_conf|rotate_logfile|promote|switch_wal|logical_emit_message|create_\w*_replication_slot|copy_\w*_replication_slot|sync_replication_slots|drop_replication_slot|replication_slot_advance|replication_origin_\w+|notify|stat\w*reset\w*|\w*_reset\w*|stat_clear\w*|stat_force\w*|wal_replay_\w+|logical_slot_\w+|replication_slot_\w+|xlog_replay_\w+|log_backend_memory_contexts|create_restore_point|backup_\w+|start_backup|stop_backup)$/i const SIDE_EFFECT_FUNCTION = - /^(?:nextval|setval|set_config|load_extension|writefile|dblink\w*|lo_\w*|lowrite|loread|checkpoint|force_checkpoint|get_lock|release_lock|release_all_locks|is_free_lock|is_used_lock|master_pos_wait|source_pos_wait|sleep|benchmark|postgres_execute|mysql_execute|sqlite_execute|SYSTEM\$\w+)$/i + /^(?:nextval|setval|set_config|load_extension|writefile|dblink\w*|lo_\w*|lowrite|loread|checkpoint|force_checkpoint|get_lock|release_lock|release_all_locks|is_free_lock|is_used_lock|master_pos_wait|source_pos_wait|sleep|benchmark|postgres_execute|mysql_execute|sqlite_execute|bigquery_execute|ducklake_\w+|jdbc|reflect|java_method|getnextval|http_request|remote_query|postgres_attach|sqlite_attach|SYSTEM\$\w+)$/i /** Accepts one decoded function identifier, without schema names or SQL text. */ export function hasSideEffectFunction(name: string, rejectUnknownPg = true): boolean { diff --git a/packages/opencode/src/altimate/tools/sql-tokenize.ts b/packages/opencode/src/altimate/tools/sql-tokenize.ts index ad4598b6ad..c980b3734e 100644 --- a/packages/opencode/src/altimate/tools/sql-tokenize.ts +++ b/packages/opencode/src/altimate/tools/sql-tokenize.ts @@ -18,7 +18,7 @@ export function tokenizeSql(sql: string, dialect?: string): { tokens: SqlToken[] const tokens: SqlToken[] = [] const mysqlComments = !dialect || /^(mysql|mariadb)$/i.test(dialect) const hashComments = /^(mysql|mariadb|bigquery)$/i.test(dialect ?? "") - const clickhouseComments = /^clickhouse$/i.test(dialect ?? "") + const clickhouse = /^clickhouse$/i.test(dialect ?? "") // Snowflake also starts a line comment with //. const slashComments = /^snowflake$/i.test(dialect ?? "") // E'...' escape strings exist only in PostgreSQL and the engines that follow it; elsewhere E is @@ -35,7 +35,8 @@ export function tokenizeSql(sql: string, dialect?: string): { tokens: SqlToken[] const nestedComments = /^(postgres|duckdb|sqlserver|tsql|mssql|fabric|clickhouse|spark|databricks)$/i.test(dialect ?? "") // Where nesting is not known either way, a /* inside a comment makes its end uncertain. const nestingKnown = nestedComments || /^(mysql|mariadb|sqlite|oracle|trino)$/i.test(dialect ?? "") - const dollarQuotes = /^(postgres|redshift|duckdb|snowflake)$/i.test(dialect ?? "") + // ClickHouse heredocs ($tag$...$tag$) read like PostgreSQL dollar quotes. + const dollarQuotes = /^(postgres|redshift|duckdb|snowflake|clickhouse)$/i.test(dialect ?? "") // Quoting is dialect-specific: brackets can contain executable array expressions, // and PostgreSQL backticks can be operators. Unknown dialects keep both visible. const backtickIdentifiers = /^(mysql|mariadb|sqlite|bigquery|clickhouse|spark|databricks|hive)$/i.test(dialect ?? "") @@ -44,6 +45,9 @@ export function tokenizeSql(sql: string, dialect?: string): { tokens: SqlToken[] // may still escape a quote with a backslash or start a comment with #, so both make the SQL // uncertain. Oracle, Trino and Fabric do neither. const unmodeled = !isModeledDialect(dialect) + const sparkHints = unmodeled || /^(spark|databricks|hive|mysql|mariadb)$/i.test(dialect ?? "") + // Oracle q'...' quotes end at a delimiter this lexer does not track. + const qQuotes = unmodeled || /^oracle$/i.test(dialect ?? "") const ambiguousBackslash = unmodeled || /^(postgres|postgresql|spark|databricks|hive)$/i.test(dialect ?? "") let complete = true let i = 0 @@ -56,7 +60,7 @@ export function tokenizeSql(sql: string, dialect?: string): { tokens: SqlToken[] if ( (sql.startsWith("--", i) && (!mysqlComments || i + 2 === sql.length || /[\x00-\x20\x7f]/.test(sql[i + 2]))) || (hashComments && sql[i] === "#") || - (clickhouseComments && (sql.startsWith("//", i) || sql.startsWith("# ", i) || sql.startsWith("#!", i))) || + (clickhouse && (sql.startsWith("//", i) || sql.startsWith("# ", i) || sql.startsWith("#!", i))) || (slashComments && sql.startsWith("//", i)) ) { kind = "comment" @@ -66,7 +70,8 @@ export function tokenizeSql(sql: string, dialect?: string): { tokens: SqlToken[] } else if (sql.startsWith("/*", i)) { // MySQL/MariaDB execute these comments. Generic SQL parsers discard them, // so they cannot establish that a query is read-only or scratch-confined. - if (/^\/\*(?:!|[mM]!)/.test(sql.slice(i))) complete = false + if (/^\/\*(?:!|[mM]!)/.test(sql.slice(i, i + 4))) complete = false + const hint = sparkHints && sql.startsWith("/*+", i) kind = "comment" let depth = 1 i += 2 @@ -81,6 +86,8 @@ export function tokenizeSql(sql: string, dialect?: string): { tokens: SqlToken[] } else i++ } if (depth) complete = false + // Spark and MySQL 8 lex a /*+ hint's body as code, so a quote there can hold the */ that seems to end it. + if (hint && /['"`]/.test(sql.slice(start, i))) complete = false } else if (tripleQuotes && (sql.startsWith("'''", i) || sql.startsWith('"""', i))) { // Ends only at the same three quotes; a backslash keeps the next character even in a raw // r'''...''' string (BigQuery reads the escape but keeps it as text). @@ -130,7 +137,14 @@ export function tokenizeSql(sql: string, dialect?: string): { tokens: SqlToken[] // After an odd run of backslashes a quote is escaped in some modes and closes the literal in // others, so where it ends cannot be known: in an unmodeled dialect, in Spark, Databricks // and Hive (escapedStringLiterals) and in PostgreSQL (standard_conforming_strings off). - if (ambiguousBackslash && !escaped && quote !== "[" && sql[i] === end) { + // Not PostgreSQL "..." or `...` identifiers, which take no backslash escapes. + if ( + ambiguousBackslash && + !escaped && + quote !== "`" && + !(quote === '"' && /^(postgres|postgresql)$/i.test(dialect ?? "")) && + sql[i] === end + ) { let run = 0 for (let j = i - 1; j > start && sql[j] === "\\"; j--) run++ if (run % 2 === 1) complete = false @@ -149,9 +163,19 @@ export function tokenizeSql(sql: string, dialect?: string): { tokens: SqlToken[] } if (!closed) complete = false i = Math.min(i, sql.length) - } else if (dollarQuotes && sql[i] === "$" && !IDENTIFIER_CHAR.test(sql[i - 1] ?? "")) { + } else if (dollarQuotes && sql[i] === "$" && !(clickhouse ? /[\w$]/ : IDENTIFIER_CHAR).test(sql[i - 1] ?? "")) { DOLLAR_TAG.lastIndex = i - const tag = /^snowflake$/i.test(dialect ?? "") ? (sql.startsWith("$$", i) ? "$$" : undefined) : DOLLAR_TAG.exec(sql)?.[0] + // ClickHouse's heredoc tag is whatever lies between the two $ (ASCII words only before it). + const close = clickhouse ? sql.indexOf("$", i + 1) : -1 + const tag = /^snowflake$/i.test(dialect ?? "") + ? sql.startsWith("$$", i) + ? "$$" + : undefined + : clickhouse + ? close < 0 + ? undefined + : sql.slice(i, close + 1) + : DOLLAR_TAG.exec(sql)?.[0] if (tag) { kind = "literal" const end = sql.indexOf(tag, i + tag.length) @@ -165,6 +189,8 @@ export function tokenizeSql(sql: string, dialect?: string): { tokens: SqlToken[] if (!kind) { // Unmodeled dialect: # starts a comment in MySQL and BigQuery, so what follows cannot be read. if (unmodeled && sql[start] === "#") complete = false + if (qQuotes && /[nNqQ]/.test(sql[start]) && /^[nN]?[qQ]'/.test(sql.slice(start, start + 3)) && !/[\w$]/.test(sql[start - 1] ?? "")) + complete = false i++ continue } diff --git a/packages/opencode/test/altimate/native/dbt-adapter-type.test.ts b/packages/opencode/test/altimate/native/dbt-adapter-type.test.ts index 0dade10d7c..09340eff1c 100644 --- a/packages/opencode/test/altimate/native/dbt-adapter-type.test.ts +++ b/packages/opencode/test/altimate/native/dbt-adapter-type.test.ts @@ -118,4 +118,10 @@ test("once dbt's adapter is resolved, its project is the one read; a saved confi mkdirSync(path.dirname(configPath()), { recursive: true }) writeFileSync(configPath(), "null") expect((await dbtProjectAdapterType()).project).toBe(false) + writeFileSync(configPath(), JSON.stringify({ projectRoot: 42 })) + expect((await dbtProjectAdapterType()).project).toBe(false) + // An adapter already created keeps running after its dbt_project.yml is removed. + rmSync(path.join(other, "dbt_project.yml")) + dbtRoute.root = other + expect((await dbtProjectAdapterType()).project).toBe(true) }) diff --git a/packages/opencode/test/altimate/sql-validation-adversarial.test.ts b/packages/opencode/test/altimate/sql-validation-adversarial.test.ts index ef373e8469..e536bc4f38 100644 --- a/packages/opencode/test/altimate/sql-validation-adversarial.test.ts +++ b/packages/opencode/test/altimate/sql-validation-adversarial.test.ts @@ -113,10 +113,12 @@ describe("Adversarial: sql-classify bypass attempts", () => { expect(r.blocked).toBe(false) }) - test("TRUNCATE inside string literal is not blocked", () => { + test("TRUNCATE inside a write's string literal is blocked", () => { + // A write can run its literals as SQL (dynamic SQL, routine bodies), so a write's literal that + // names TRUNCATE is refused even with --allow-write: an accepted limit of the best-effort block. const r = classifyAndCheck("INSERT INTO logs VALUES ('TRUNCATE TABLE users happened')") expect(r.queryType).toBe("write") - expect(r.blocked).toBe(false) // string literal, not real TRUNCATE + expect(r.blocked).toBe(true) }) test("DROP SCHEMA inside double-quoted identifier is not blocked", () => { diff --git a/packages/opencode/test/altimate/tool-cli/runner.test.ts b/packages/opencode/test/altimate/tool-cli/runner.test.ts index 7e43e7e665..7fcf0bd60a 100644 --- a/packages/opencode/test/altimate/tool-cli/runner.test.ts +++ b/packages/opencode/test/altimate/tool-cli/runner.test.ts @@ -10,6 +10,7 @@ import { Effect, Exit } from "effect" import { Global } from "../../../src/global" import * as Registry from "../../../src/altimate/native/connections/registry" import { registerDefaultNativeHandlers } from "../../lib/native-handlers" +import * as Dispatcher from "../../../src/altimate/native/dispatcher" let dir: string let runner: Runner @@ -401,6 +402,52 @@ describe("data_diff write-carrying inputs", () => { }) }) +describe("sql_explain", () => { + test("explains one statement only, since some drivers run every statement in the text", async () => { + Registry.setConfigs({ rs: { type: "redshift" } as any }) + const sent: string[] = [] + const get = spyOn(Registry, "get").mockResolvedValue({ + execute: async (sql: string) => { + sent.push(sql) + return { columns: ["QUERY PLAN"], rows: [["Seq Scan"]], row_count: 1, truncated: false } + }, + } as any) + try { + for (const sql of ["SELECT 1; DELETE FROM t", "SELECT 1; TRUNCATE t", "ANALYZE DELETE FROM t", "/* x */ ANALYZE DELETE FROM t", "(ANALYZE) DELETE FROM t", '("analyze") DELETE FROM t']) { + const result = await runner.run("sql_explain", { sql, warehouse: "rs" }) + expect(JSON.stringify(result)).toMatch(/single statement|blocked for safety/) + } + expect(sent).toEqual([]) + // A long run of whitespace after "(" is refused quickly. + const started = performance.now() + await runner.run("sql_explain", { sql: "(" + " ".repeat(200_000) + "x)", warehouse: "rs" }) + expect(performance.now() - started).toBeLessThan(5_000) + // A parenthesized query is a query, not an EXPLAIN option list. + const plan = await runner.run("sql_explain", { sql: "(SELECT 1) UNION (SELECT 2)", warehouse: "rs" }) + expect(JSON.stringify(plan)).not.toMatch(/single statement/) + } finally { + get.mockRestore() + Registry.reset() + } + }) +}) + +describe("data_diff empty target_warehouse", () => { + test("dispatches the source warehouse", async () => { + Registry.setConfigs({ src: { type: "postgres" } as any, other: { type: "mysql" } as any }) + const call = spyOn(Dispatcher, "call").mockResolvedValue({ success: true, outcome: { mode: "diff" } } as any) + try { + await runner.run("data_diff", { source: "orders", target: "orders_v2", key_columns: ["id"], source_warehouse: "src", target_warehouse: "" }) + const params = call.mock.calls.find(([method]) => method === "data.diff")?.[1] as Record + expect(params.source_warehouse).toBe("src") + expect(params.target_warehouse).toBeUndefined() + } finally { + call.mockRestore() + Registry.reset() + } + }) +}) + describe("data_diff hard block on long SQL", () => { test("is checked on the whole SQL, not the 120-character preview", async () => { Registry.setConfigs({ pg: { type: "postgres" } as any }) diff --git a/packages/opencode/test/altimate/tools/sql-classify.test.ts b/packages/opencode/test/altimate/tools/sql-classify.test.ts index b39374c221..6d726db42c 100644 --- a/packages/opencode/test/altimate/tools/sql-classify.test.ts +++ b/packages/opencode/test/altimate/tools/sql-classify.test.ts @@ -120,7 +120,7 @@ describe("classify", () => { }) describe("classifyMulti", () => { - for (const dialect of ["mysql", "sqlite", "bigquery", "clickhouse", "unknown", undefined]) { + for (const dialect of ["mysql", "sqlite", "bigquery", "unknown", undefined]) { test(`dollar-delimited writes fail closed for ${dialect ?? "omitted"} dialect`, () => { const sql = "SELECT $x$; DELETE FROM t; $x$" expect(classifyMulti(sql, dialect)).toBe("write") @@ -440,6 +440,8 @@ describe("each engine's own literal and statement rules", () => { ["SELECT 1 USE master", "sqlserver", "write", false], ["SELECT enable, disable, rename, throw FROM t", "sqlserver", "read", false], ["SELECT 1 THROW 50000, 'x', 1", "sqlserver", "write", false], + ["SELECT throw FROM t ORDER BY throw", "sqlserver", "read", false], + ["SELECT 1 AS throw", "fabric", "read", false], // Spark continues a -- comment past a backslash before the line end. ["-- x\\\nSELECT '\nDELETE FROM t -- '", "databricks", "write", false], ["-- x\\\nSELECT '\nDROP SCHEMA s CASCADE -- '", "spark", "write", true], @@ -453,7 +455,174 @@ describe("each engine's own literal and statement rules", () => { ["EXECUTE IMMEDIATE \"DROP SCHEMA ds CASCADE\"", "bigquery", "write", true], ["DO $$BEGIN EXECUTE 'DROP SCHEMA s CASCADE'; END$$", "postgres", "write", true], ["PREPARE s FROM 'DROP DATABASE prod'", "mysql", "write", true], + // ...with the literal's escapes decoded. + ["EXECUTE IMMEDIATE '\\x44ROP SCHEMA ds CASCADE'", "bigquery", "write", true], + ["EXECUTE IMMEDIATE '\\x44ROP DATABASE victim'", "snowflake", "write", true], + ["EXECUTE IMMEDIATE '\\104ROP SCHEMA ds'", "bigquery", "write", true], + ["DO $$BEGIN EXECUTE E'\\x44ROP SCHEMA victim CASCADE'; END$$", "postgres", "write", true], + ["DO $$BEGIN EXECUTE U&'\\0044ROP SCHEMA victim CASCADE'; END$$", "postgres", "write", true], + ["DO $$BEGIN EXECUTE U&'!0044ROP SCHEMA v' UESCAPE '!'; END$$", "postgres", "write", true], ["EXEC dbo.refresh @note = 'nightly'", "sqlserver", "write", false], + // A write runs its literals as SQL (dynamic SQL, routine bodies): each is read with escapes decoded. + ["sp_executesql N'DROP DATABASE prod'", "sqlserver", "write", true], + ["CREATE FUNCTION f() RETURNS void LANGUAGE sql AS $$ DROP SCHEMA s CASCADE $$; SELECT f()", "postgres", "write", true], + ["CREATE PROCEDURE p() AS $$ BEGIN TRUNCATE t; END; $$ LANGUAGE plpgsql; CALL p()", "redshift", "write", true], + ["SELECT dblink_exec('dbname=postgres', 'DROP DATABASE prod')", "postgres", "write", true], + ["WITH p AS PROCEDURE() RETURNS VARCHAR LANGUAGE SQL AS $$ BEGIN DROP DATABASE prod; RETURN 'x'; END $$ CALL p()", "snowflake", "write", true], + // Read word by word: escapes, nested quotes and comments in the literal cannot separate the words, + // so a write's text that merely mentions them is blocked too. + ["EXECUTE IMMEDIATE 'DROP\\nSCHEMA ds CASCADE'", "bigquery", "write", true], + ["DO $$BEGIN EXECUTE E'DROP\\tSCHEMA s CASCADE'; END$$", "postgres", "write", true], + ["EXEC(N'SELECT ''--''; DROP DATABASE victim')", "sqlserver", "write", true], + ["DO $$BEGIN EXECUTE $q$SELECT '--'; DROP SCHEMA s CASCADE$q$; END$$", "postgres", "write", true], + ["INSERT INTO log VALUES ('nightly truncate of staging')", "postgres", "write", true], + ["INSERT INTO log VALUES ('dropped rows', 'schema v2')", "postgres", "write", false], + ["SELECT * FROM audit WHERE action = 'DROP SCHEMA'", "postgres", "read", false], + // An escaped uppercase letter stands for itself; MySQL reads an unknown lowercase escape the same way. + ["DO E'BEGIN DROP SCH\\EMA s CASCADE; END'", "postgres", "write", true], + ["CREATE PROCEDURE p() AS 'BEGIN TRUNC\\ATE t; END' LANGUAGE plpgsql; CALL p()", "redshift", "write", true], + ["EXECUTE IMMEDIATE 'DROP SCH\\EMA s'", "snowflake", "write", true], + ["PREPARE s FROM 'D\\ROP DATABASE x'", "mysql", "write", true], + ["PREPARE s FROM 'DROP sch\\ema x'", "mysql", "write", true], + // TRUNCATE starting a block or a handler is the statement. + ["BEGIN REPEAT TRUNCATE t; UNTIL (TRUE) END REPEAT; END", "snowflake", "write", true], + ["CREATE PROCEDURE p() BEGIN DECLARE CONTINUE HANDLER FOR SQLEXCEPTION TRUNCATE t; END", "mysql", "write", true], + ["SELECT truncate, x FROM t ORDER BY truncate DESC", "postgres", "read", false], + // An alias written without AS after a column named truncate is refused (best effort, fails closed). + ["SELECT truncate result FROM (VALUES (1)) AS t(truncate)", "postgres", "write", true], + ["SELECT truncate AS result FROM (VALUES (1)) AS t(truncate)", "postgres", "read", false], + // TRUNCATE followed by a target, wherever it sits. + ["CREATE TASK k AS TRUNCATE t", "snowflake", "write", true], + ["CREATE OR REPLACE TASK k WAREHOUSE = w AS TRUNCATE t", "snowflake", "write", true], + ["CREATE PROCEDURE p() TRUNCATE t", "mysql", "write", true], + ["CREATE PROCEDURE p() SQL SECURITY INVOKER TRUNCATE t", "mysql", "write", true], + ["CREATE PROCEDURE p() LANGUAGE sql BEGIN ATOMIC TRUNCATE t; END; CALL p()", "postgres", "write", true], + ["ALTER TABLE t TRUNCATE PARTITION p", "oracle", "write", true], + ["ALTER TABLE t TRUNCATE SUBPARTITION p", "oracle", "write", true], + ["ALTER TABLE t TRUNCATE PARTITION p", "mysql", "write", true], + ["ALTER TABLE t TRUNCATE PARTITION (d = 1)", "databricks", "write", true], + ["SELECT t.truncate x FROM t", "postgres", "read", false], + ["SELECT t./* column comment */truncate x FROM t", "postgres", "read", false], + ["SELECT truncate ISNULL, truncate NOTNULL FROM t", "postgres", "read", false], + ["SELECT truncate GLOB 'a*' FROM t", "sqlite", "read", false], + ["SELECT sum(x) OVER (ORDER BY truncate ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) FROM t", "postgres", "read", false], + ["SELECT truncate, COUNT(*) FROM t GROUP BY truncate WITH ROLLUP", "mysql", "read", false], + ["SELECT id FROM t CONNECT BY PRIOR id = truncate START WITH id = 1", "oracle", "read", false], + ["SELECT truncate OPERATOR(pg_catalog.+) 1 FROM t", "postgres", "read", false], + ["SELECT * FROM t ORDER BY truncate ASCENDING", "clickhouse", "read", false], + ["SELECT sum(x) OVER (ORDER BY truncate RANGE BETWEEN 1 PRECEDING AND CURRENT ROW) FROM t", "postgres", "read", false], + ["SELECT * FROM t ORDER BY truncate SETTINGS max_threads = 1", "clickhouse", "read", false], + ["BEGIN NOT ATOMIC TRUNCATE $t; END", "mysql", "write", true], + ["BEGIN NOT ATOMIC TRUNCATE \u00fcber; END", "mysql", "write", true], + ["BEGIN NOT ATOMIC DECLARE CONTINUE HANDLER FOR 1062 TRUNCATE nulls; SELECT 1; END", "mysql", "write", true], + ["BEGIN NOT ATOMIC DECLARE EXIT HANDLER FOR SQLSTATE '23000' TRUNCATE t; SELECT 1; END", "mysql", "write", true], + ["BEGIN NOT ATOMIC DECLARE CONTINUE HANDLER FOR 1146" + " ".repeat(300) + "TRUNCATE t; SELECT * FROM missing; END", "mysql", "write", true], + ["IF 1 THEN TRUNCATE t; END IF", "mysql", "write", true], + ['BEGIN IF (x) THEN TRUNCATE "db"."t"; END IF; END', "snowflake", "write", true], + ['BEGIN IF (x) THEN TRUNCATE db."t"; END IF; END', "snowflake", "write", true], + ["BEGIN IF (x) THEN TRUNCATE IF EXISTS t; END IF; END", "snowflake", "write", true], + ["BEGIN IF (x) THEN TRUNCATE IDENTIFIER('t'); END IF; END", "snowflake", "write", true], + ["BEGIN NOT ATOMIC IF 1 THEN TRUNCATE `db`.`t`; END IF; END", "mysql", "write", true], + ["SELECT CASE WHEN a THEN truncate.x END FROM t", "postgres", "read", false], + ["EXPLAIN (ANALYZE) TRUNCATE t", "duckdb", "write", true], + ["EXPLAIN (ANALYZE, FORMAT JSON) TRUNCATE t", "duckdb", "write", true], + ["SELECT DEFAULT(truncate) FROM t", "mysql", "read", false], + ["SELECT atomic truncate FROM (VALUES (1)) AS t(atomic)", "postgres", "read", false], + ["SELECT found truncate, x FROM t", "postgres", "read", false], + ["SELECT seq.NEXTVAL", "mysql", "write", false], + ["SELECT s./*comment*/NEXTVAL FROM dual WHERE ROWNUM = 1", "oracle", "write", false], + ['SELECT s . /* c */ "NEXTVAL" FROM dual', "snowflake", "write", false], + ["BEGIN NOT ATOMIC IF 1 THEN TRUNCATE t NOWAIT; END IF; END", "mysql", "write", true], + ["BEGIN NOT ATOMIC IF 1 THEN TRUNCATE t WAIT 5; END IF; END", "mysql", "write", true], + ["IF 1 THEN TRUNCATE t NOWAIT; END IF", "mysql", "write", true], + ["CASE WHEN 1 THEN TRUNCATE t NOWAIT; END CASE", "mysql", "write", true], + ["IF 1 THEN TRUNCATE `d`.`t` WAIT 5; END IF", "mysql", "write", true], + // ClickHouse heredocs are strings. + ["SELECT $x$'$x$ AS a FROM jdbc($x$ds$x$, $x$INSERT INTO t VALUES (1)$x$) WHERE $x$'$x$ = 1", "clickhouse", "write", false], + ["SELECT $heredoc$it's$heredoc$ AS v", "clickhouse", "read", false], + ["SELECT $1$'$1$ AS a FROM jdbc($x$ds$x$, $x$INSERT INTO t VALUES (1)$x$) WHERE $1$'$1$ = 1", "clickhouse", "write", false], + ["SELECT $-$'$-$ AS a FROM jdbc($x$ds$x$, $x$INSERT INTO t VALUES (1)$x$) WHERE $-$'$-$ = 1", "clickhouse", "write", false], + ["SELECT\u00a0$x$'$x$ AS a FROM jdbc($x$ds$x$, $x$INSERT INTO t VALUES (1)$x$) WHERE\u00a0$x$'$x$ = $x$a$x$", "clickhouse", "write", false], + ["BEGIN IF (1=1) THEN TRUNCATE db..t; END IF; END", "snowflake", "write", true], + ["SELECT * FROM postgres_attach('dbname=x')", "duckdb", "write", false], + // Databricks functions that call outside systems. + ["SELECT http_request(conn => 'c', method => 'POST', path => '/x')", "databricks", "write", false], + ["SELECT * FROM remote_query('pg', database => 'db', query => 'DELETE FROM t RETURNING *')", "databricks", "write", false], + ["SELECT * FROM TABLE(GETNEXTVAL(s))", "snowflake", "write", false], + ["SELECT CASE WHEN true THEN truncate ELSE 0 END FROM metrics", "postgres", "read", false], + ["SELECT CASE WHEN a THEN 1 ELSE truncate END FROM metrics", "duckdb", "read", false], + // MySQL 8 reads quotes in a /*+ hint, so one there makes the SQL uncertain. + ["SELECT /*+ QB_NAME(`*/`) */ 1 INTO OUTFILE '/tmp/x' -- `", "mysql", "write", false], + ["SELECT /*+ SET_VAR(sql_mode='*/') */ 1 INTO OUTFILE '/tmp/x' -- '", "mysql", "write", false], + ["SELECT /*+ MAX_EXECUTION_TIME(1000) */ * FROM t", "mysql", "read", false], + // Snowflake: ->> chains statements; ASYNC ( and := ( hold one. + ["SELECT 1 ->> TRUNCATE t -- LIMIT", "snowflake", "write", true], + ["SELECT 1 ->> DELETE FROM t", "snowflake", "write", false], + ["BEGIN ASYNC (TRUNCATE t); AWAIT ALL; END", "snowflake", "write", true], + ["BEGIN LET r RESULTSET := (TRUNCATE t); RETURN TABLE(r); END", "snowflake", "write", true], + ["SELECT data ->> 'a' FROM t", "postgres", "read", false], + ["SELECT j ->> truncate FROM t", "postgres", "read", false], + ["BEGIN LET r RESULTSET DEFAULT (TRUNCATE t); RETURN TABLE(r); END", "snowflake", "write", true], + // DuckDB runs TRUNCATE under EXPLAIN ANALYZE and PREPARE ... AS. + ["EXPLAIN ANALYZE TRUNCATE t", "duckdb", "write", true], + ["PREPARE p AS TRUNCATE t; EXECUTE p", "duckdb", "write", true], + // Spark/Hive TRANSFORM ... USING runs a command; reflect() calls any Java method. + ["SELECT TRANSFORM(a) USING 'rm -rf /tmp/x' FROM t", "databricks", "write", false], + ["SELECT reflect('java.lang.System', 'exit', 0)", "spark", "write", false], + ["SELECT transform(arr, x -> x + 1) FROM t", "databricks", "read", false], + ["SELECT transform(arr, x -> x + 1) FROM t JOIN u USING (id)", "spark", "read", false], + ["SELECT TRANSFORM(a) USING /*c*/ 'touch /tmp/proof' FROM t", "spark", "write", false], + ['SELECT TRANSFORM(a) USING "touch /tmp/proof" FROM t', "databricks", "write", false], + // UESCAPE without a U& literal is a word, not an escape character. + ["UPDATE t SET uescape = 1", "postgres", "write", false], + ["UPDATE t SET uescape = menu&1", "postgres", "write", false], + // Sequence advances through a quoted name, and DuckDB extension functions that write. + ['SELECT s."NEXTVAL"', "snowflake", "write", false], + ['SELECT "NEXT VALUE FOR" FROM metrics', "postgres", "read", false], + ["SELECT s.nextval FROM metrics s", "postgres", "read", false], + ["SELECT CASE WHEN true THEN truncate SIMILAR TO 'a%' ELSE false END FROM metrics", "postgres", "read", false], + ["SELECT CASE WHEN a THEN truncate AT TIME ZONE 'UTC' END FROM metrics", "postgres", "read", false], + ["SELECT CASE WHEN a THEN 1 ELSE truncate END;", "postgres", "read", false], + ["SELECT * FROM bigquery_execute('p', 'DELETE FROM d.t WHERE true')", "duckdb", "write", false], + ["SELECT * FROM ducklake_expire_snapshots('lake', older_than => now())", "duckdb", "write", false], + // ClickHouse jdbc() can run a mutation through the bridge. + ["SELECT * FROM jdbc('ch-server?mutation', 'INSERT INTO audit VALUES (1)')", "clickhouse", "write", false], + // Identifiers that are never strings are not read as SQL. + ['CREATE TABLE t ("truncate" int)', "postgres", "write", false], + ["INSERT INTO t ([truncate]) VALUES (1)", "sqlserver", "write", false], + ['EXEC("DROP DATABASE prod")', "sqlserver", "write", true], + // Each engine's escapes: whitespace escapes are spaces, letter escapes the letter, and SQL Server + // joins a line ending in a backslash. + ["DO E'BEGIN DROP\\nSCH\\eMA s CASCADE; END'", "postgres", "write", true], + ["EXECUTE IMMEDIATE 'DROP\\nSCH\\eMA s'", "snowflake", "write", true], + ["PREPARE s FROM 'DROP\\nDAT\\aBASE x'", "mysql", "write", true], + ["EXEC('DROP DATA\\\nBASE x')", "sqlserver", "write", true], + ["EXEC sp_executesql N'TRUN\\\r\nCATE TABLE t'", "fabric", "write", true], + // A Spark hint's body is code: a quote in it makes the SQL uncertain. + ["WITH c AS (SELECT /*+ X('*/') */ ' AS a) SELECT 1 AS z /*' AS a) INSERT INTO t SELECT * FROM c -- */", "databricks", "write", false], + ["SELECT /*+ BROADCAST(t) */ * FROM t", "databricks", "read", false], + // Oracle q-quotes are uncertain. + ["DECLARE v VARCHAR2(9); w VARCHAR2(9); BEGIN v := q'{ ' }'; w := ' -- '; EXECUTE IMMEDIATE 'TRUNCATE TABLE t'; END;", "oracle", "write", true], + // A quoted target is still a target; Spark "..." is a string dynamic SQL can run. + ["BEGIN NOT ATOMIC TRUNCATE `t`; END", "mysql", "write", true], + ["SELECT \"truncate\" FROM t", "postgres", "read", false], + ['EXECUTE IMMEDIATE "TRUNCATE TABLE t"', "databricks", "write", true], + ['EXECUTE IMMEDIATE "DROP SCHEMA s CASCADE"', "spark", "write", true], + // DROP NAMESPACE is Spark's DROP SCHEMA, nested too. + ["EXECUTE IMMEDIATE 'DROP NAMESPACE s CASCADE'", "databricks", "write", true], + ["BEGIN DROP NAMESPACE s CASCADE; END", "databricks", "write", true], + ["SELECT dbo.truncate(x, 2) FROM t", "sqlserver", "read", false], + // Other names for dropping a database or schema. + ["DROP CATALOG prod CASCADE", "databricks", "write", true], + ["DROP EXTERNAL SCHEMA ds", "bigquery", "write", true], + ["DROP PLUGGABLE DATABASE pdb1 INCLUDING DATAFILES", "oracle", "write", true], + // A column or alias named truncate is not the statement. + ["SELECT truncate FROM t", "postgres", "read", false], + ["SELECT 1 AS truncate", "postgres", "read", false], + ["BEGIN TRUNCATE t; END", "snowflake", "write", true], + // Identifiers take no backslash escapes. + ['SELECT "a\\" FROM t', "postgres", "read", false], + ["SELECT `a\\` FROM t", "databricks", "read", false], // Sequence advances are writes in every dialect. ["SELECT NEXT VALUE FOR dbo.s", "sqlserver", "write", false], ["SELECT s.NEXTVAL FROM dual", "oracle", "write", false], @@ -487,11 +656,28 @@ describe("each engine's own literal and statement rules", () => { expect(result.blocked).toBe(blocked) }) - test("stays linear on repeated TRUNCATE inside a long comment", () => { - const started = performance.now() - classifyAndCheck("TRUNCATE /* ".repeat(40_000) + "*/ ( '", "mysql") - // Every TRUNCATE reaching the same comment close and whitespace after it. - classifyAndCheck("SELECT " + "TRUNCATE /*".repeat(40_000) + "*/" + " ".repeat(400_000) + "(", "postgres") - expect(performance.now() - started).toBeLessThan(2_000) + test("stays linear on large crafted inputs", () => { + // Each input on its own: a quadratic scan takes seconds to minutes here, a linear one well under a second. + const inputs: [string, string][] = [ + ["TRUNCATE /* ".repeat(40_000) + "*/ ( '", "mysql"], + // An unclosed EXPLAIN option list, and a long dotted name after THEN truncate. + ["EXPLAIN (" + "ANALYZE ".repeat(40_000), "duckdb"], + ["EXPLAIN (".repeat(40_000), "duckdb"], + ["IF 1 THEN TRUNCATE IDENTIFIER(".repeat(20_000), "snowflake"], + // Long whitespace in SQL read with certainty (the TRUNCATE rules look behind only after a match). + ["SELECT 1" + " ".repeat(400_000) + "FROM t", "postgres"], + ["SELECT t." + " ".repeat(400_000) + "truncate x FROM t", "postgres"], + ["SELECT " + "t. truncate x, ".repeat(40_000) + "1 FROM t", "postgres"], + ["SELECT CASE WHEN true THEN truncate -" + "a.".repeat(2_000) + "a + 0 ELSE 0 END FROM t", "duckdb"], + // A write's literal full of comment openers. + ["INSERT INTO t VALUES ($$" + "/*x*/".repeat(100_000) + "x$$)", "postgres"], + // Every TRUNCATE reaching the same comment close and whitespace after it. + ["SELECT " + "TRUNCATE /*".repeat(40_000) + "*/" + " ".repeat(400_000) + "(", "postgres"], + ] + for (const [sql, dialect] of inputs) { + const started = performance.now() + classifyAndCheck(sql, dialect) + expect(performance.now() - started).toBeLessThan(2_000) + } }) }) diff --git a/packages/opencode/test/altimate/tools/sql-comment-safety.test.ts b/packages/opencode/test/altimate/tools/sql-comment-safety.test.ts index 2ba8be9012..96eca55685 100644 --- a/packages/opencode/test/altimate/tools/sql-comment-safety.test.ts +++ b/packages/opencode/test/altimate/tools/sql-comment-safety.test.ts @@ -233,3 +233,50 @@ test("omitted SQL warehouse cannot assume the registry dialect before dbt execut dbt.mockRestore() } }) + +test("on the dbt route, Jinja in a comment or literal is a write, and a hard-denied run_query is blocked", async () => { + Registry.setConfigs({ fallback: { type: "postgres" } }) + const dbt = spyOn(DbtAdapterType, "dbtProjectAdapterType").mockResolvedValue({ project: true, type: "postgres", targetTypes: ["postgres"] }) + const dispatch = spyOn(Dispatcher, "call").mockResolvedValue({ columns: ["x"], rows: [[1]], row_count: 1, truncated: false } as never) + const run = async (query: string) => { + const asked: string[] = [] + const tool = await initTool(SqlExecuteTool) + const result = tool.execute( + { query, limit: 10 }, + { + sessionID: SessionID.make("ses_dbt_jinja"), + messageID: MessageID.make("msg_dbt_jinja"), + callID: "", + agent: "analyst", + abort: AbortSignal.any([]), + messages: [], + metadata: () => {}, + ask: async (request: { permission: string }) => { + asked.push(request.permission) + throw new Error("write permission denied") + }, + }, + ) + return { result, asked } + } + try { + const write = await run("SELECT 1 AS x /* {% do run_query('DELETE FROM t') %} */") + await expect(write.result).rejects.toThrow("write permission denied") + expect(write.asked).toEqual(["sql_execute_write"]) + const drop = await run("SELECT 1 AS x -- {% do run_query('DROP SCHEMA prod CASCADE') %}") + await expect(drop.result).rejects.toThrow("blocked for safety") + // A Jinja comment removes text, which can expose a statement the SQL lexing reads as a literal. + const hidden = await run("SELECT 'x' /* {# */ || ' #} */) AS q ; DROP SCHEMA s CASCADE ; SELECT 1 -- '") + await expect(hidden.result).rejects.toThrow("blocked for safety") + // A Mongo default's command goes to dbt first as well. + Registry.setConfigs({ fallback: { type: "mongodb" } }) + const mongo = await run(`{"command":"find","collection":"c","filter":{"x":"{% do run_query('DELETE FROM t') %}"}}`) + await expect(mongo.result).rejects.toThrow("write permission denied") + expect(mongo.asked).toEqual(["sql_execute_write"]) + expect(dispatch).not.toHaveBeenCalled() + } finally { + dispatch.mockRestore() + dbt.mockRestore() + Registry.reset() + } +}) diff --git a/packages/opencode/test/altimate/tools/sql-explain.test.ts b/packages/opencode/test/altimate/tools/sql-explain.test.ts index b3bdccbc8b..09c9b0bc14 100644 --- a/packages/opencode/test/altimate/tools/sql-explain.test.ts +++ b/packages/opencode/test/altimate/tools/sql-explain.test.ts @@ -501,7 +501,7 @@ describe("SqlExplainTool.execute", () => { expect(result.metadata.success).toBe(false) expect(result.metadata.error_class).toBe("input_validation") - expect(String(result.output)).toContain("only a single read-only query can be analyzed") + expect(String(result.output)).toMatch(/only a single read-only query can be analyzed|single statement can be explained/) expect(spy).not.toHaveBeenCalled() spy.mockRestore() }) diff --git a/packages/opencode/test/altimate/tools/sql-read-safety.test.ts b/packages/opencode/test/altimate/tools/sql-read-safety.test.ts index 024d8f1be5..f0cfe21491 100644 --- a/packages/opencode/test/altimate/tools/sql-read-safety.test.ts +++ b/packages/opencode/test/altimate/tools/sql-read-safety.test.ts @@ -106,7 +106,7 @@ describe("read-only SQL parser boundary", () => { expect(maskSql(tokenizeSql("SELECT $$-- ; DELETE$$ AS a -- actual comment", "postgres").tokens)).toMatch(/^SELECT\s+AS a\s+$/) }) - for (const dialect of ["mysql", "sqlite", "bigquery", "clickhouse", "unknown", undefined]) { + for (const dialect of ["mysql", "sqlite", "bigquery", "unknown", undefined]) { test(`dollar delimiters leave statements visible for ${dialect ?? "omitted"} dialect`, () => { const sql = "SELECT $x$; DELETE FROM t; $x$" const { tokens, complete } = tokenizeSql(sql, dialect)