Make the engine's own checks fire, and version core 0.7.0

packages/browser-extension/src/storage.ts

@@ -41,13 +41,6 @@ export interface SidecarConnection {
   readonly database?: string;
 }
 
-export interface AskSqlSettings {
-  readonly provider: ProviderSettings;
-  readonly engine: EngineSettings;
-  readonly connections: readonly SidecarConnection[];
-  readonly warningAcknowledged: boolean;
-}
-
 export const DEFAULT_ENGINE_SETTINGS: EngineSettings = {
   maxRows: 200,
   requireApproval: false,

packages/core/CHANGELOG.md

@@ -1,5 +1,58 @@
 # @asksql/core
 
+## 0.7.0
+
+### Minor Changes
+
+- Answer the questions a model kept guessing at, and hold the line on the ones it should not answer.
+
+  Structure questions now run SQL written here rather than guessed. "How many rows are in each table?",
+  "which tables have no primary key?" and "what tables are in this database?" are answered from each
+  engine's own catalog, with every name quoted, because a model that has never seen
+  `information_schema` invents columns on it. A name holding a quote character survives, and the
+  generated statement is validated by the guard like any other.
+
+  Question routing decides what a question actually is before a query is written. A relationship
+  question ("how do customers and orders relate?") is answered from the foreign keys rather than by
+  returning rows of a join. A question about nothing in the database is declined in one sentence
+  instead of producing a confident answer about the wrong thing, and a question that names a table the
+  catalog has never seen triggers one bounded re-read in case the schema changed, at most once every 30
+  seconds per connection. A write request still reaches the proposal path and is never executed.
+
+  Oracle no longer refuses a query for the one thing a small model cannot stop doing. Oracle has no
+  LIMIT clause, so `SELECT ... LIMIT 10` was refused and the repair loop spent every attempt failing to
+  talk the model out of it, identically each time. A plain trailing `LIMIT n` is now translated to
+  `FETCH FIRST n ROWS ONLY`, which is the same query, and lowered to the row cap as usual. Forms with
+  no single-clause equivalent, `LIMIT n OFFSET m` and a placeholder count, are still refused. Consumers
+  that relied on `limit_unsupported` for the plain form will now see the translated statement.
+
+  MongoDB gained the same care. A distinct count written as `$addToSet` plus `$size` is rewritten to a
+  grouped count that spills to disk rather than being refused for the 16MB document limit, and the
+  rewrite drops documents missing the field so the count matches what `$addToSet` would have produced.
+  A pipeline naming a field no stage can resolve is caught before it runs, and refusals now say what to
+  write instead.
+
+  Database error text is redacted before it reaches a model. A driver quotes the offending row, and
+  Postgres appends the whole row as a DETAIL, so a repair prompt carried cell values the user never
+  agreed to send.
+
+  New exports: `isRelationshipQuestion`, `isCapabilityQuestion`, `isPromptInjection`,
+  `danglingReference`, and from the mongo entry point `rewriteDistinctCount`, `firstMisquotedField` and
+  `firstUnknownStageField`.
+
+  Five checks were silently switched off on Oracle. A top-N question makes the model write
+  `FETCH FIRST n ROWS ONLY`, the parser these checks use cannot read that clause, and each of them
+  fails open by design, so the column floor, the table floor, the fan-out floor, the ungrouped-aggregate
+  lint and the ambiguous-column floor all went quiet. A query selecting a column no table has reached
+  the database instead of being caught and corrected. Every check now reads the statement with that
+  tail removed, and a test gives each one a query it must flag on every dialect, so one going quiet
+  fails the suite rather than shipping.
+
+  A correction now names where the column actually lives. Telling a model only that a column does not
+  exist leaves it renaming the alias and failing the same way; naming the table that has the column and
+  the join that reaches it recovers the query. Measured on a 7B model against Oracle: 0 of 3 before,
+  3 of 3 after.
+
 ## 0.6.3
 
 ### Patch Changes

packages/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@asksql/core",
-  "version": "0.6.3",
+  "version": "0.7.0",
   "description": "AskSQL engine: schema catalog, AST SQL guard, prompt pipeline, LLM orchestration. Zero database drivers.",
   "type": "module",
   "main": "./dist/index.js",

packages/core/src/catalog-answers.ts

@@ -0,0 +1,144 @@
+/**
+ * Structure questions answered with SQL written here rather than guessed by a model, which has never
+ * seen the system catalogs and invents columns on `information_schema` and `pg_stat_*`.
+ *
+ * Always a statement, never a cached answer: the catalog supplies only names, and it can be minutes
+ * stale where a query cannot. Matching is narrow, since hijacking a data question is worse than
+ * missing one of these.
+ */
+
+import type { DialectInfo, EngineKind, SchemaCatalog, TableInfo } from './types.js';
+
+export interface CatalogQuery {
+  readonly sql: string;
+  /** Shown in place of the model's explanation, since no model wrote this. */
+  readonly explanation: string;
+}
+
+const EVERY_TABLE = /\b(each|every|per|all)\s+(?:the\s+)?tables?\b/i;
+const ROWS = /\b(rows?|records?)\b/i;
+const MOST_ROWS =
+  /\b(most|largest|biggest|highest)\b[^.?!]{0,24}\b(rows?|records?)\b|\b(rows?|records?)\b[^.?!]{0,24}\b(most|largest|biggest)\b/i;
+const NEGATED = /\b(without|no|missing|lack(?:ing|s)?|do(?:es)?\s*n[o']?t have|have no)\b/i;
+const TABLES = /\btables?\b/i;
+const PRIMARY_KEY = /\bprimary\s+keys?\b|\bpk\b/i;
+/** "the orders table" names one table, so the question is about its rows, not about every table. */
+/** The subject has to be tables. "which rows ... have no pk" asks about rows in one table. */
+const TABLE_SUBJECT = /\b(?:which|what|list|show|find|any)\b[^.?!]{0,24}\btables?\b/i;
+const ROW_SUBJECT = /\b(?:rows?|records?)\b/i;
+
+const NAMED_TABLE = /\b(?:the|this|that|a|an|our|my)\s+[\w"`\]]+\s+tables?\b/i;
+
+const escapeRe = (s: string): string => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+
+const qualified = (t: TableInfo): string => (t.schema ? `${t.schema}.${t.name}` : t.name);
+
+/** Views have no rows of their own, and a partition is counted through its parent. */
+const countableTables = (catalog: SchemaCatalog): TableInfo[] =>
+  catalog.tables.filter((t) => t.kind === 'table' && !t.partitionOf);
+
+function quoteFor(name: string, dialect: DialectInfo): string {
+  const q = dialect.quoteChar;
+  return `${q}${name.split(q).join(q + q)}${q}`;
+}
+
+/**
+ * Tables with no primary key, in each engine's own catalog. Written per engine because this is
+ * exactly where a model guesses: the shapes differ, and only Oracle and MySQL expose it simply.
+ */
+function tablesWithoutPrimaryKey(engine: EngineKind, schemas: readonly string[]): string | null {
+  // The catalog spans every schema introspected, so answering for current_schema() alone reports a
+  // narrower truth than the schema tree the reader is looking at.
+  // With no schema on the catalog's tables there is nothing better than the session's own.
+  const inList = schemas.length > 0 ? schemas.map((s) => `'${s.replace(/'/g, "''")}'`).join(', ') : 'current_schema()';
+  switch (engine) {
+    case 'postgres':
+      return `SELECT t.table_name
+FROM information_schema.tables t
+WHERE t.table_schema IN (${inList})
+  AND t.table_type = 'BASE TABLE'
+  AND NOT EXISTS (
+    SELECT 1 FROM information_schema.table_constraints c
+    WHERE c.table_schema = t.table_schema
+      AND c.table_name = t.table_name
+      AND c.constraint_type = 'PRIMARY KEY'
+  )
+ORDER BY t.table_name`;
+    case 'mysql':
+      return `SELECT t.TABLE_NAME
+FROM information_schema.TABLES t
+WHERE t.TABLE_SCHEMA = DATABASE()
+  AND t.TABLE_TYPE = 'BASE TABLE'
+  AND NOT EXISTS (
+    SELECT 1 FROM information_schema.TABLE_CONSTRAINTS c
+    WHERE c.TABLE_SCHEMA = t.TABLE_SCHEMA
+      AND c.TABLE_NAME = t.TABLE_NAME
+      AND c.CONSTRAINT_TYPE = 'PRIMARY KEY'
+  )
+ORDER BY t.TABLE_NAME`;
+    case 'oracle':
+      return `SELECT t.table_name
+FROM user_tables t
+WHERE NOT EXISTS (
+  SELECT 1 FROM user_constraints c
+  WHERE c.table_name = t.table_name AND c.constraint_type = 'P'
+)
+ORDER BY t.table_name`;
+    case 'sqlite':
+      // sqlite_master has no constraint view; pragma_table_info exposes the key flag per column.
+      return `SELECT m.name
+FROM sqlite_master m
+WHERE m.type = 'table'
+  AND m.name NOT LIKE 'sqlite_%'
+  AND NOT EXISTS (SELECT 1 FROM pragma_table_info(m.name) p WHERE p.pk > 0)
+ORDER BY m.name`;
+    default:
+      return null; // DuckDB and anything else: let the model try rather than guess a shape here
+  }
+}
+
+/**
+ * Returns a statement for the structure questions worth writing exactly, or null for everything
+ * else, which is the common case.
+ */
+export function catalogQueryFor(question: string, catalog: SchemaCatalog, dialect: DialectInfo): CatalogQuery | null {
+  const q = question.trim();
+  // Only Postgres needs this: MySQL's DATABASE() and SQLite's file are already the whole catalog,
+  // and Oracle is introspected for one owner.
+  const schemas = [...new Set(catalog.tables.map((t) => t.schema).filter((x): x is string => !!x))];
+  if (!TABLES.test(q)) return null;
+
+  if (NEGATED.test(q) && PRIMARY_KEY.test(q) && TABLE_SUBJECT.test(q) && !ROW_SUBJECT.test(q)) {
+    const sql = tablesWithoutPrimaryKey(dialect.engine, schemas);
+    if (sql) {
+      return { sql, explanation: 'Lists tables with no primary key, read from the database catalog.' };
+    }
+  }
+
+  // Row counts, one branch per table: a model writes this as an information_schema join and gets an
+  // ambiguous column. Naming a table makes it a data question about that table's rows instead.
+  if (NAMED_TABLE.test(q) || catalog.tables.some((t) => new RegExp(`\\b${escapeRe(t.name)}\\b`, 'i').test(q))) {
+    return null;
+  }
+  // A condition on the rows ("...that have no pk", "...where status is null") makes it a data
+  // question about rows, not a count of every table.
+  if (NEGATED.test(q) || /\b(?:where|that (?:are|have)|with a|having)\b/i.test(q)) return null;
+  if ((EVERY_TABLE.test(q) && ROWS.test(q)) || MOST_ROWS.test(q)) {
+    const tables = countableTables(catalog);
+    if (tables.length === 0) return null;
+    const branches = tables.map((t) => {
+      const label = qualified(t).replace(/'/g, "''");
+      const from = t.schema ? `${quoteFor(t.schema, dialect)}.${quoteFor(t.name, dialect)}` : quoteFor(t.name, dialect);
+      return `SELECT '${label}' AS table_name, COUNT(*) AS row_count FROM ${from}`;
+    });
+    const body = branches.join('\nUNION ALL\n');
+    // Always ordered: the guard appends its row cap, and an unordered UNION ALL truncated to the cap
+    // drops tables at random while the explanation claims to have counted them all.
+    return {
+      sql: `SELECT * FROM (\n${body}\n) counts ORDER BY row_count DESC`,
+      explanation: `Counts the rows in each of the ${tables.length} tables, largest first.`,
+    };
+  }
+
+  return null;
+}

packages/core/src/dialects.ts

@@ -14,6 +14,7 @@ export const POSTGRES_DIALECT: DialectInfo = Object.freeze({
   promptNotes: Object.freeze([
     'Quote mixed-case or reserved identifiers with double quotes.',
     'Use ILIKE for case-insensitive text matching.',
+    "Combine values into one string with string_agg(col, ', ').",
     "Use date_trunc / interval arithmetic for date math (e.g. now - interval '30 days').",
   ]),
 });
@@ -27,6 +28,7 @@ export const MYSQL_DIALECT: DialectInfo = Object.freeze({
   promptNotes: Object.freeze([
     'Quote identifiers with backticks when needed.',
     'Use DATE_SUB / DATE_ADD / DATE_FORMAT for date math.',
+    "Combine values into one string with GROUP_CONCAT(col SEPARATOR ', ').",
   ]),
 });
 
@@ -40,6 +42,7 @@ export const SQLITE_DIALECT: DialectInfo = Object.freeze({
   promptNotes: Object.freeze([
     "Use date/datetime/strftime for date math (e.g. date('now','-30 days')).",
     'There are no schemas; refer to tables by bare name.',
+    "Combine values into one string with group_concat(col, ', ').",
   ]),
 });
 
@@ -57,6 +60,7 @@ export const ORACLE_DIALECT: DialectInfo = Object.freeze({
     'Unquoted identifiers are case-insensitive and stored upper case; double-quote to preserve case.',
     'Select a literal from the DUAL table (e.g. SELECT 1 FROM DUAL), not a bare SELECT 1.',
     'There is no boolean type; a comparison is not a directly selectable value.',
+    'The safety validator cannot read LISTAGG ... WITHIN GROUP, so return the rows themselves rather than combining them into one string.',
   ]),
 });
 
@@ -68,6 +72,7 @@ export const DUCKDB_DIALECT: DialectInfo = Object.freeze({
   limitStyle: 'limit',
   promptNotes: Object.freeze([
     'DuckDB follows PostgreSQL syntax for queries.',
+    "Combine values into one string with string_agg(col, ', '); SEPARATOR is MySQL syntax and is rejected here.",
     'Uploaded files are already registered as tables - query them by table name, never by file path.',
   ]),
 });