Read a wide schema the way real databases actually demand

docs/deployment.md

@@ -65,7 +65,7 @@ const app = express();
 app.use(express.json());
 app.use('/asksql', asksqlMiddleware({
   connectors: [new PostgresConnector({ id: 'shop', name: 'Shop', connectionString: process.env.DATABASE_URL })],
-  engine: { model: await resolveModel({ provider: 'groq', model: 'llama-3.3-70b-versatile', apiKey: process.env.GROQ_API_KEY }) },
+  engine: { model: await resolveModel({ provider: 'groq', model: 'openai/gpt-oss-20b', apiKey: process.env.GROQ_API_KEY }) },
   auth: (req) => ({ userId: lookUpSession(req), allowedConnectionIds: ['shop'] }), // your auth
 }));
 ```

docs/providers.md

@@ -23,7 +23,7 @@ What each field means:
 | Field | Meaning | Required when |
 |-------|---------|---------------|
 | `provider` | Which SDK adapter to load. Picks the wire protocol and default endpoint. | always |
-| `model` | The model id to call (`gpt-4o-mini`, `llama-3.3-70b-versatile`, ...). For **classic Azure**, this is your **deployment name**, not the base model name. | always |
+| `model` | The model id to call (`gpt-4o-mini`, `openai/gpt-oss-20b`, ...). For **classic Azure**, this is your **deployment name**, not the base model name. | always |
 | `apiKey` | Your provider secret, sent as the bearer token. Keep it on the server, never in the browser. | `openai`, `anthropic`, `google`, `azure`, `groq`, `nvidia`. Not required for `ollama` or `openai-compatible` (pass one if your endpoint wants it) |
 | `baseURL` | Full endpoint URL to override the provider default. Point it at a local runtime (Ollama), any OpenAI-compatible host, or an Azure AI Foundry endpoint. | `openai-compatible`; optional for `ollama` (defaults to `http://localhost:11434/v1`) |
 | `resourceName` | Classic Azure OpenAI resource subdomain, from `https://<resourceName>.openai.azure.com`. Used only to build the classic Azure endpoint. | classic `azure` when `baseURL` is not set |
@@ -62,7 +62,7 @@ resolveModel({ provider: 'anthropic', model: 'claude-3-5-haiku-latest', apiKey }
 resolveModel({ provider: 'google', model: 'gemini-2.0-flash', apiKey });
 
 // Groq
-resolveModel({ provider: 'groq', model: 'llama-3.3-70b-versatile', apiKey });
+resolveModel({ provider: 'groq', model: 'openai/gpt-oss-20b', apiKey });
 
 // NVIDIA (build.nvidia.com; OpenAI-compatible, endpoint pre-seeded for you)
 resolveModel({ provider: 'nvidia', model: 'meta/llama-3.3-70b-instruct', apiKey });

examples/browser-duckdb/src/main.tsx

@@ -53,7 +53,7 @@ function App() {
       });
       await connector.connect();
       const model =
-        window.__asksqlModel ?? (await resolveModel({ provider: 'groq', model: 'llama-3.3-70b-versatile', apiKey }));
+        window.__asksqlModel ?? (await resolveModel({ provider: 'groq', model: 'openai/gpt-oss-20b', apiKey }));
       const engine = createAskSql({ connectors: [connector], model, policy: { maxRows: 200 } });
       setTransport(new LocalTransport(engine));
       setStatus(`Ready - ask about "${table}". Nothing left your browser.`);

examples/express-postgres/server.mjs

@@ -16,7 +16,7 @@ const PG_URL = process.env.ASKSQL_PG_URL ?? 'postgres://postgres:root@localhost:
 const PORT = Number(process.env.PORT ?? 4000);
 
 const model = process.env.GROQ_API_KEY
-  ? await resolveModel({ provider: 'groq', model: 'llama-3.3-70b-versatile', apiKey: process.env.GROQ_API_KEY })
+  ? await resolveModel({ provider: 'groq', model: 'openai/gpt-oss-20b', apiKey: process.env.GROQ_API_KEY })
   : await resolveModel({
       provider: 'ollama',
       model: process.env.OLLAMA_MODEL ?? 'qwen2.5-coder:7b',

examples/node-duckdb/demo.mjs

@@ -18,8 +18,8 @@ const dir = dirname(fileURLToPath(import.meta.url));
 
 async function pickModel() {
   if (process.env.GROQ_API_KEY) {
-    console.log('· model: Groq llama-3.3-70b-versatile');
-    return resolveModel({ provider: 'groq', model: 'llama-3.3-70b-versatile', apiKey: process.env.GROQ_API_KEY });
+    console.log('· model: Groq openai/gpt-oss-20b');
+    return resolveModel({ provider: 'groq', model: 'openai/gpt-oss-20b', apiKey: process.env.GROQ_API_KEY });
   }
   console.log('· model: local Ollama qwen2.5-coder:7b');
   return resolveModel({

examples/node-mongodb/demo.mjs

@@ -16,7 +16,7 @@ import { createMongoAskSql } from '@asksql/core/mongo';
 import { MongodbConnector } from '@asksql/mongodb';
 
 const model = process.env.GROQ_API_KEY
-  ? await resolveModel({ provider: 'groq', model: 'llama-3.3-70b-versatile', apiKey: process.env.GROQ_API_KEY })
+  ? await resolveModel({ provider: 'groq', model: 'openai/gpt-oss-20b', apiKey: process.env.GROQ_API_KEY })
   : await resolveModel({
       provider: 'ollama',
       model: process.env.OLLAMA_MODEL ?? 'qwen2.5-coder:7b',

examples/node-oracle/demo.mjs

@@ -16,7 +16,7 @@ import { createAskSql, resolveModel } from '@asksql/core';
 import { OracleConnector } from '@asksql/oracle';
 
 const model = process.env.GROQ_API_KEY
-  ? await resolveModel({ provider: 'groq', model: 'llama-3.3-70b-versatile', apiKey: process.env.GROQ_API_KEY })
+  ? await resolveModel({ provider: 'groq', model: 'openai/gpt-oss-20b', apiKey: process.env.GROQ_API_KEY })
   : await resolveModel({
       provider: 'ollama',
       model: process.env.OLLAMA_MODEL ?? 'qwen2.5-coder:7b',

packages/browser-extension/test/storage.test.ts

@@ -59,8 +59,8 @@ describe('provider settings', () => {
   });
 
   it('round-trips a saved value', async () => {
-    await setProviderSettings({ provider: 'groq', model: 'llama-3.3-70b-versatile', apiKey: 'sk-1' });
-    expect(await getProviderSettings()).toEqual({ provider: 'groq', model: 'llama-3.3-70b-versatile', apiKey: 'sk-1' });
+    await setProviderSettings({ provider: 'groq', model: 'openai/gpt-oss-20b', apiKey: 'sk-1' });
+    expect(await getProviderSettings()).toEqual({ provider: 'groq', model: 'openai/gpt-oss-20b', apiKey: 'sk-1' });
   });
 
   it('merges partial stored data over the defaults', async () => {

packages/core/CHANGELOG.md

@@ -1,5 +1,29 @@
 # @asksql/core
 
+## 0.10.0
+
+### Minor Changes
+
+- ea596d5: A schema wider than about 50 tables used to leave every table past that point with no column hints
+  at all, since the probe budget was spent first-come rather than shared. It is now split fairly across
+  every table.
+
+  The schema pruner's table cap was a fixed 40 regardless of the token budget, so a wide schema could
+  drop a table the budget genuinely had room for; the cap now only guards against a pathological
+  schema, and the token budget decides what is actually sent. A single unusually wide table no longer
+  evicts every smaller table behind it in the same pass.
+
+  A result whose row cap was reached by the guard _lowering_ an oversized LIMIT - rather than adding
+  one where there was none - is now correctly reported as truncated, on both the flag and the warning
+  text; previously only the "added a limit" case was caught.
+
+  DuckDB: introspecting a schema past 100,000 total columns no longer truncates the catalog silently.
+  Uploading a `.sql` dump over 20 MB is now rejected before it is read into memory, rather than risking
+  an out-of-memory crash partway through.
+
+  Oracle: the JSON-array column hint left one identifier unquoted, so a lower/mixed-case column name
+  produced a hint the database would reject with ORA-00904. Both branches now quote consistently.
+
 ## 0.9.0
 
 ### Minor Changes

packages/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@asksql/core",
-  "version": "0.9.0",
+  "version": "0.10.0",
   "description": "AskSQL engine: schema catalog, AST SQL guard, prompt pipeline, LLM orchestration. Zero database drivers.",
   "type": "module",
   "main": "./dist/index.js",

packages/core/src/catalog.ts

@@ -66,6 +66,18 @@ function qualifiedName(t: TableInfo, multiSchema: boolean, quote: string, engine
 /** Bounds so a large catalog cannot crowd out the tables themselves. */
 const MAX_INDEXES_PER_TABLE = 8;
 const MAX_OBJECTS = 30;
+/** Max join paths rendered; a wide schema has far more edges than the model can use. */
+const MAX_EDGES = 200;
+/** Max callable functions rendered. */
+const MAX_FUNCTIONS = 40;
+
+/**
+ * Marks a list the renderer cut short. A silent cut reads as the complete set, so the model treats a
+ * name it was never shown as one that does not exist.
+ */
+function andMore(total: number, shown: number): string {
+  return total > shown ? ` (and ${total - shown} more not shown)` : '';
+}
 
 export function formatCatalogForPrompt(catalog: SchemaCatalog): string {
   const multiSchema = catalog.schemas.length > 1;
@@ -121,7 +133,7 @@ export function formatCatalogForPrompt(catalog: SchemaCatalog): string {
   }
 
   if (catalog.triggers.length > 0) {
-    lines.push('TRIGGERS:');
+    lines.push(`TRIGGERS:${andMore(catalog.triggers.length, MAX_OBJECTS)}`);
     for (const tr of catalog.triggers.slice(0, MAX_OBJECTS)) {
       const on = tr.schema ? `${tr.schema}.${tr.table}` : tr.table;
       lines.push(` ${tr.name} ${tr.timing} ${tr.events.join('/')} ON ${on}${tr.enabled ? '' : ' [disabled]'}`);
@@ -131,7 +143,9 @@ export function formatCatalogForPrompt(catalog: SchemaCatalog): string {
   const procedures = catalog.routines.filter((r) => r.kind === 'procedure');
   if (procedures.length > 0) {
     // Listed so "what procedures exist" can be answered; never offered as something to call.
-    lines.push('STORED PROCEDURES (reference only - NEVER call these; a read-only query cannot invoke them):');
+    lines.push(
+      `STORED PROCEDURES (reference only - NEVER call these; a read-only query cannot invoke them):${andMore(procedures.length, MAX_OBJECTS)}`,
+    );
     for (const r of procedures.slice(0, MAX_OBJECTS)) {
       lines.push(` ${multiSchema && r.schema ? `${r.schema}.${r.name}` : r.name}(${r.args})`);
     }
@@ -141,12 +155,12 @@ export function formatCatalogForPrompt(catalog: SchemaCatalog): string {
     const names = catalog.sequences
       .slice(0, MAX_OBJECTS)
       .map((q) => (multiSchema && q.schema ? `${q.schema}.${q.name}` : q.name));
-    lines.push(`SEQUENCES: ${names.join(', ')}`);
+    lines.push(`SEQUENCES: ${names.join(', ')}${andMore(catalog.sequences.length, MAX_OBJECTS)}`);
   }
 
   if (catalog.enums.length > 0) {
-    lines.push('ENUM TYPES:');
-    for (const e of catalog.enums) {
+    lines.push(`ENUM TYPES:${andMore(catalog.enums.length, MAX_OBJECTS)}`);
+    for (const e of catalog.enums.slice(0, MAX_OBJECTS)) {
       lines.push(` ${e.name}: ${e.values.slice(0, 32).map(sanitizeValue).join('|')}`);
     }
   }
@@ -155,17 +169,19 @@ export function formatCatalogForPrompt(catalog: SchemaCatalog): string {
     (r) => r.kind === 'function' && (r.volatility === 'immutable' || r.volatility === 'stable'),
   );
   if (callable.length > 0) {
-    lines.push('CALLABLE READ-ONLY FUNCTIONS (safe to use in SELECT; call by the exact name shown):');
-    for (const r of callable.slice(0, 40)) {
+    lines.push(
+      `CALLABLE READ-ONLY FUNCTIONS (safe to use in SELECT; call by the exact name shown):${andMore(callable.length, MAX_FUNCTIONS)}`,
+    );
+    for (const r of callable.slice(0, MAX_FUNCTIONS)) {
       const fnName = multiSchema && r.schema ? `${r.schema}.${r.name}` : r.name;
       lines.push(` ${fnName}(${r.args})${r.returns ? ` -> ${r.returns}` : ''}`);
     }
   }
 
   const edges = joinGraph(catalog);
   if (edges.length > 0) {
-    lines.push('RELATIONSHIPS (join paths):');
-    for (const e of edges.slice(0, 200)) lines.push(` ${e}`);
+    lines.push(`RELATIONSHIPS (join paths):${andMore(edges.length, MAX_EDGES)}`);
+    for (const e of edges.slice(0, MAX_EDGES)) lines.push(` ${e}`);
   }
 
   return lines.join('\n');
@@ -418,7 +434,8 @@ function trimColumns(
 const fullRenderCache = new WeakMap<SchemaCatalog, string>();
 
 export function pruneCatalog(catalog: SchemaCatalog, question: string, settings?: PrunerSettings): PruneResult {
-  const maxTables = settings?.maxTables ?? 40;
+  // Guards the full render below from a pathological schema; the token budget decides what is sent.
+  const maxTables = settings?.maxTables ?? 200;
   const maxSchemaTokens = settings?.maxSchemaTokens ?? 6000;
   const all = catalog.tables.filter((t) => !t.partitionOf);
 
@@ -489,7 +506,14 @@ export function pruneCatalog(catalog: SchemaCatalog, question: string, settings?
   for (const t of candidate) {
     if (kept.length >= maxTables) break;
     const cost = estimateTableTokens(t);
-    if (kept.length >= 1 && used + cost > perTableBudget) break;
+    if (kept.length === 0) {
+      // The best-scoring table is always kept, charged at most half the budget so siblings still fit.
+      kept.push(t);
+      used += Math.min(cost, Math.floor(perTableBudget / 2));
+      continue;
+    }
+    // Skip what does not fit rather than stopping: smaller tables behind it may still have room.
+    if (used + cost > perTableBudget) continue;
     kept.push(t);
     used += cost;
   }

packages/core/src/column-hints.ts

@@ -146,6 +146,16 @@ export const MAX_HINT_PROBES = 200;
 
 /** Per table, so a wide schema degrades evenly instead of the first tables taking every probe. */
 export const MAX_HINT_PROBES_PER_TABLE = 4;
+
+/**
+ * The per-table probe share for a schema of `tableCount` tables, so the global cap spreads evenly
+ * instead of the first tables spending it all. Never returns 0, and total spend across all tables
+ * never exceeds MAX_HINT_PROBES.
+ */
+export function hintProbesPerTable(tableCount: number): number {
+  const fairShare = Math.floor(MAX_HINT_PROBES / Math.max(1, tableCount));
+  return Math.max(1, Math.min(MAX_HINT_PROBES_PER_TABLE, fairShare));
+}
 export const JSON_SAMPLE_ROWS = 20;
 
 /** A probe reads only enough of a cell to judge its shape; the rest is bandwidth and parse cost. */

packages/core/src/engine.ts

@@ -540,8 +540,9 @@ export function createAskSql(config: AskSqlConfig): AskSqlEngine {
       if (loweredLimit) {
         warnings.push(`The row limit was lowered to ${policy.maxRows}.`);
       }
-      // An auto-limited result that filled the cap counts as truncated: the injected LIMIT hides the overflow row.
-      const truncated = result.truncated || (autoLimited && result.rowCount >= cappedMax);
+      // A capped result that filled the cap counts as truncated: a LIMIT we injected OR lowered hides
+      // the overflow row, so the connector never sees one.
+      const truncated = result.truncated || ((autoLimited || loweredLimit) && result.rowCount >= cappedMax);
       return { ...result, warnings, truncated };
     } catch (err) {
       // A driver may reject a cancelled query with its own AbortError rather than
@@ -654,7 +655,17 @@ export function createAskSql(config: AskSqlConfig): AskSqlEngine {
     let pruned = pruneCatalog(fullCatalog, q, config.pruner);
     let schemaText = pruned.schemaText;
     if (pruned.dropped > 0) {
-      emit({ type: 'warning', message: `Schema narrowed to ${pruned.catalog.tables.length} relevant tables.` }, opts);
+      // The count that matters is the one the model never saw, not the one that survived.
+      emit(
+        {
+          type: 'warning',
+          message:
+            `Schema narrowed to the ${pruned.catalog.tables.length} tables most relevant to this question; ` +
+            `${pruned.dropped} of ${pruned.catalog.tables.length + pruned.dropped} were not sent to the model. ` +
+            `If the answer missed a table, name it in the question or raise the schema token budget.`,
+        },
+        opts,
+      );
     }
 
     // Few-shot retrieval, scoped to the connection and (in server mode) the requesting user.

packages/core/src/index.ts

@@ -69,6 +69,7 @@ export {
   jsonHint,
   MAX_HINT_PROBES,
   MAX_HINT_PROBES_PER_TABLE,
+  hintProbesPerTable,
   JSON_SAMPLE_ROWS,
   HINT_VALUE_CAP,
 } from './column-hints.js';

packages/core/test/catalog-objects.test.ts

@@ -114,3 +114,41 @@ describe('a DBA question can be answered from the prompt', () => {
     expect(bare).not.toContain('INDEXES:');
   });
 });
+
+describe('a list the renderer cut short says so', () => {
+  // A silent cut reads as the complete set: the model concludes a name it was never shown does not
+  // exist, and answers "there is no such table" with total confidence.
+  const many = <T,>(n: number, make: (i: number) => T): T[] => Array.from({ length: n }, (_v, i) => make(i));
+
+  it('marks triggers, procedures, sequences and enums past the cap', () => {
+    const cat: SchemaCatalog = {
+      ...CATALOG,
+      enums: many(45, (i) => ({ name: `enum_${i}`, values: ['a', 'b'] })),
+      sequences: many(45, (i) => ({ schema: 'shop', name: `seq_${i}` })),
+      triggers: many(45, (i) => ({
+        name: `trg_${i}`,
+        schema: 'shop',
+        table: 'orders',
+        timing: 'AFTER' as const,
+        events: ['INSERT'],
+        enabled: true,
+      })),
+      routines: many(45, (i) => ({ schema: 'shop', name: `proc_${i}`, kind: 'procedure' as const, args: '' })),
+    };
+    const text = formatCatalogForPrompt(cat);
+    expect(text).toMatch(/TRIGGERS:.*15 more not shown/);
+    expect(text).toMatch(/STORED PROCEDURES.*15 more not shown/);
+    expect(text).toMatch(/SEQUENCES:.*15 more not shown/);
+    expect(text).toMatch(/ENUM TYPES:.*15 more not shown/);
+  });
+
+  it('caps enums at all, which it previously did not', () => {
+    const cat: SchemaCatalog = { ...CATALOG, enums: many(200, (i) => ({ name: `enum_${i}`, values: ['a'] })) };
+    const rendered = formatCatalogForPrompt(cat).split('\n').filter((l) => /^ enum_\d+:/.test(l));
+    expect(rendered.length).toBe(30);
+  });
+
+  it('says nothing extra when everything fits', () => {
+    expect(formatCatalogForPrompt(CATALOG)).not.toMatch(/more not shown/);
+  });
+});

packages/core/test/hint-probe-budget.test.ts

@@ -0,0 +1,38 @@
+/**
+ * The probe budget is global and spent in catalog order. A flat per-table cap does not spread it: the
+ * first tables take everything and the rest get no hints at all, silently. An application database of
+ * 186 tables is ordinary, and the flat cap covered only the first 50 of them.
+ */
+import { describe, expect, it } from 'vitest';
+import { MAX_HINT_PROBES, MAX_HINT_PROBES_PER_TABLE, hintProbesPerTable } from '../src/column-hints.js';
+
+describe('the hint probe budget is shared, not first-come', () => {
+  it('gives every table a share on a schema too wide for the flat cap', () => {
+    expect(hintProbesPerTable(186)).toBeGreaterThanOrEqual(1);
+    // Total spend stays inside the global cap, so this costs no more time than the flat cap did.
+    expect(hintProbesPerTable(186) * 186).toBeLessThanOrEqual(MAX_HINT_PROBES + 186);
+  });
+
+  it('never starves a table to zero, however wide the schema', () => {
+    for (const n of [51, 200, 1000, 5000]) {
+      expect(hintProbesPerTable(n), `${n} tables`).toBeGreaterThanOrEqual(1);
+    }
+  });
+
+  it('keeps the full per-table allowance on a schema that fits', () => {
+    for (const n of [1, 5, 50]) {
+      expect(hintProbesPerTable(n), `${n} tables`).toBe(MAX_HINT_PROBES_PER_TABLE);
+    }
+  });
+
+  it('never exceeds the per-table cap', () => {
+    for (const n of [1, 10, 186, 5000]) {
+      expect(hintProbesPerTable(n)).toBeLessThanOrEqual(MAX_HINT_PROBES_PER_TABLE);
+    }
+  });
+
+  it('treats an empty schema as one table rather than dividing by zero', () => {
+    expect(Number.isFinite(hintProbesPerTable(0))).toBe(true);
+    expect(hintProbesPerTable(0)).toBe(MAX_HINT_PROBES_PER_TABLE);
+  });
+});

packages/core/test/limits.test.ts

@@ -174,6 +174,17 @@ describe('maxRows clamp (H3) and truncation signal (M1)', () => {
     expect(res.warnings.join(' ')).not.toMatch(/export/i);
   });
 
+  it('reports truncated when a LOWERED limit fills the cap', async () => {
+    // The model asked for more than the ceiling, so the guard lowers its LIMIT rather than adding one:
+    // autoLimited stays false. Crediting only autoLimited reported a full answer while the rows the
+    // question asked for were missing.
+    const conn = new Capturing(100);
+    const engine = createAskSql({ connectors: [conn], model: async () => 'x', policy: { maxRows: 100 } });
+    const res = await engine.execute('SELECT * FROM users LIMIT 5000');
+    expect(res.truncated).toBe(true);
+    expect(res.warnings.join(' ')).toMatch(/lowered/i);
+  });
+
   it('does not over-report truncation for a result under the cap', async () => {
     const conn = new Capturing(3);
     const engine = createAskSql({ connectors: [conn], model: async () => 'x', policy: { maxRows: 100 } });