feat: plan-first turn for multi-part questions

CHANGELOG.md

@@ -5,6 +5,24 @@ and public product updates.
 
 ## Unreleased
 
+### Plan-first turn for multi-part questions
+
+- Asked for revenue *and* ticket counts per customer, the agent wrote one query joining both
+  children of the same parent and silently multiplied the rows, returning 3,405 for a customer
+  whose revenue is 1,702.50. A warning in the grounding context did not prevent it - the agent
+  wrote one confident query in a single step - so the fix is a turn with **no tools offered**,
+  forcing it to commit to an approach in writing first
+- The trigger is deliberately narrow: a false negative costs nothing, a false positive spends a
+  turn on every question. Data-modification questions are excluded outright, because planning
+  them measured *worse* - it turned a wrong number into a blank refusal. Safety stays the
+  verification pass's job
+- **Measured**: 1/8 runs without planning, 8/8 with, on the one case in either suite that
+  triggers it. The hard A/B moved +30.6 to +33.3 - close to one noise unit, and necessarily so,
+  since planning fires on 1 of 12 cases and could move the aggregate by at most 8.3 points. The
+  case-level result is strong; the suite-level result is not, and both are reported
+- `--no-plan` disables it, and a test asserts a non-triggering question takes a byte-identical
+  path, so the feature is provably free when it does not apply
+
 ### Measure names are unique, and the fan-out case is gradeable (bug fix)
 
 - Two tables with the same numeric column produced the same measure name, and the metric compiler

ROADMAP.md

@@ -374,7 +374,31 @@ see the measure-grain defect in step 6.2. Build one step at a time:
       **Re-measured after the fix**: the hard A/B is unchanged at **+30.6** (grounded 31/36,
       raw-sql 20/36) across two runs at different code states, which is a useful reproducibility
       signal for the harness itself.
-7. Short planning/decomposition for multi-part questions (bounded).
+7. **Short planning/decomposition for multi-part questions** - ✅ Built (`PLANNING_PROMPT`,
+   `buildPlanningTurn`, `src/core/agent/loop.ts`; `--no-plan` to disable).
+   Before acting on a question that asks for more than one thing, the agent writes a plan in a
+   turn with **no tools offered**, so it has to commit to an approach in writing first. That
+   forcing function is the point: the grain warning from 6.2 was already in the context for the
+   failing entity and did not prevent the double count, because the agent wrote one confident
+   query in a single step.
+   **Deliberately narrow trigger.** A false negative costs nothing; a false positive spends a
+   turn on every question. A bare "and" would have fired on "give the customer name and the
+   amount" - one quantity in two columns, already answered correctly - so the trigger keys on a
+   genuine second quantity or instruction. Write-verb questions are excluded outright: measured,
+   planning them turned a wrong number into a blank refusal, the over-refusal failure the
+   verification pass exists to prevent. Their failure was never decomposition, so safety stays
+   verification's job.
+   **Measured** on `hard-fanout-revenue-and-cases`, the only case in either suite that triggers
+   it: **1/8 runs without planning, 8/8 with** (a repeat-5 run plus a full-suite repeat-3 run
+   each way). The 2x fan-out inflation it was returning is gone.
+   **Suite-level, stated honestly**: the hard A/B moved **+30.6 to +33.3** (grounded 31/36 to
+   33/36). That is close to one noise unit, and it has to be: planning fires on 1 of 12 cases, so
+   the most it could ever move the aggregate is 8.3 points. The case-level result is strong, the
+   suite-level result is weak, and those are different claims.
+   **Two implementation bugs the measurement caught**, neither visible to the scripted tests:
+   the transcript ended on an assistant turn (which the API rejects outright), and the agent
+   finalized on the plan without running anything in 2 of 5 runs until the handoff message said
+   plainly that the plan is not the answer.
 8. **Native desktop app** (decided 2026-07-25) — the flagship product surface.
    A native macOS app (Swift + AppKit) embeds a libghostty terminal pane running
    Claude Code / Codex under the user's own subscription (no BYOK).

src/adapters/cli/ask.ts

@@ -52,6 +52,8 @@ export interface RunAskOptions {
   maxSteps?: number;
   /** Run a self-critique pass before answering (default true). */
   verify?: boolean;
+  /** Plan first on multi-part questions (default true). */
+  plan?: boolean;
   /** Print each agent tool step. */
   verbose?: boolean;
   /** Injected in tests; built from env credentials otherwise. */
@@ -160,6 +162,7 @@ export async function runAsk(options: RunAskOptions): Promise<AskResult> {
         complete: ai.agentComplete,
         maxSteps: options.maxSteps,
         verify: options.verify ?? true,
+        plan: options.plan ?? true,
         onStep: options.verbose ? (step) => log(formatStep(step)) : undefined,
       });
 

src/adapters/cli/eval.ts

@@ -16,6 +16,8 @@ export interface RunEvalOptions {
   verify?: boolean;
   /** Agent suite: which configuration to measure (default "grounded"). */
   arm?: string;
+  /** Agent suite: plan first on multi-part questions (default true). */
+  plan?: boolean;
   /** Agent suite: run both arms interleaved and print the comparison. */
   ab?: boolean;
   /** Agent suite: the agent's turn budget. */
@@ -55,6 +57,7 @@ export async function runEval(options: RunEvalOptions): Promise<number> {
         repeat: options.repeat,
         provider: options.provider,
         verify: options.verify,
+        plan: options.plan,
         maxSteps: options.steps,
         onProgress: options.json ? undefined : (line: string) => log(line),
       };

src/adapters/cli/index.ts

@@ -32,8 +32,8 @@ Usage:
                                    (needs a key) against evals/cases/. --json for raw output;
                                    agent accepts --cases <id,…>, --repeat <n>, --steps <n>,
                                    --no-verify (baseline with the self-critique pass off),
-                                   --arm <grounded|raw-sql>, and --ab (run both arms
-                                   interleaved and print the grounding comparison).
+                                   --arm <grounded|raw-sql>, --no-plan, and --ab (run
+                                   both arms interleaved and print the comparison).
                                    Both suites accept --dataset <folder>, --cases-file <path>
                                    and --glossary <path> to score a different dataset.
   querypad help                    Show this help
@@ -56,6 +56,8 @@ Options for ask:
   --no-verify                      Skip the self-critique pass (on by default:
                                    the agent re-checks projection/ranking/safety
                                    before answering)
+  --no-plan                        Skip the plan-first turn (on by default, and only
+                                   fires for questions asking for more than one thing)
   --verbose                        Print each agent tool step
 
 Environment: ANTHROPIC_API_KEY or OPENAI_API_KEY for the chosen provider.
@@ -142,6 +144,7 @@ async function main(argv: string[]): Promise<number> {
         showSql: flags["show-sql"] === true,
         maxSteps: steps && Number.isFinite(steps) ? steps : undefined,
         verify: flags["no-verify"] !== true,
+        plan: flags["no-plan"] !== true,
         verbose: flags.verbose === true,
       });
       return 0;
@@ -193,6 +196,7 @@ async function main(argv: string[]): Promise<number> {
         repeat: repeat && Number.isFinite(repeat) ? repeat : undefined,
         provider: stringFlag(flags, "provider"),
         verify: flags["no-verify"] !== true,
+        plan: flags["no-plan"] !== true,
         arm: stringFlag(flags, "arm"),
         ab: flags.ab === true,
         dataset: stringFlag(flags, "dataset"),

src/core/agent/loop.ts

@@ -25,6 +25,48 @@ Guidance:
 /** Write verbs that mean the user asked to modify data — refuse those, answer the rest. */
 const WRITE_VERB = /\b(delete|drop|truncate|update|insert|alter|remove)\b/i;
 
+/**
+ * Questions that ask for a second quantity or a second instruction. Deliberately
+ * narrow: it is the trigger for spending an extra model turn, so a false negative
+ * costs nothing (the agent proceeds as before) while a false positive costs a turn
+ * on every question. Derived from observed failures, not from intuition - a bare
+ * "and" would fire on "give the customer name and the amount", which is one
+ * quantity with two columns and already answered correctly.
+ */
+const MULTI_PART =
+  /\bthen\b|\band how many\b|\band their\b|\band the (number|count)\b|\balong with\b|\bas well as\b/i;
+
+/**
+ * Plan-first turn for a question that asks for more than one thing. The failure it
+ * targets is specific and measured: asked for revenue *and* ticket counts per
+ * customer, the agent writes a single query joining both children of the same
+ * parent and silently multiplies the rows - returning 3405 for a customer whose
+ * revenue is 1702.50, because that customer has two tickets.
+ *
+ * Note the last line. Our grader compares one result set, and more importantly the
+ * question genuinely asks for one table, so a plan that splits the work into two
+ * separate answers would be wrong for the user as well as unscoreable.
+ */
+export const PLANNING_PROMPT = `Before running anything, write a short plan (2 lines at most):
+1. List each quantity the question asks for, and which table each one comes from.
+2. If two quantities come from different tables that both hang off the same parent, joining them in one query repeats rows and inflates the totals. Compute each at its own grain - a subquery or CTE per quantity - then combine.
+Then carry out the plan and return a single result set that answers the whole question.`;
+
+/**
+ * The planning turn for a question, or null when the extra turn would buy nothing.
+ * Pure — unit-tested without the network.
+ *
+ * Data-modification questions are deliberately excluded even though they read as
+ * multi-part. Measured: planning them made things worse, turning a wrong number into
+ * a blank refusal — the over-refusal failure the verification pass exists to prevent.
+ * Their failure was never decomposition (the agent already separates the parts), so
+ * safety stays verification's job and planning stays decomposition's.
+ */
+export function buildPlanningTurn(question: string): string | null {
+  if (WRITE_VERB.test(question)) return null;
+  return MULTI_PART.test(question) ? PLANNING_PROMPT : null;
+}
+
 /**
  * Self-critique checklist injected once before the agent finalizes. Targets the
  * mistakes a naive agent makes: over-projecting columns, ignoring ranking words,
@@ -102,6 +144,11 @@ export interface RunAgentQueryOptions {
   maxSteps?: number;
   /** Run one self-critique turn before accepting the final answer (default false). */
   verify?: boolean;
+  /**
+   * Write a plan first, but only for questions that ask for more than one thing
+   * (default false). Costs one model turn when it fires and nothing when it does not.
+   */
+  plan?: boolean;
   /** Restrict the agent to these tool names (default: the whole toolkit). */
   tools?: string[];
   /**
@@ -120,6 +167,17 @@ function textFrom(content: ContentBlock[]): string {
     .trim();
 }
 
+/**
+ * Trim text blocks and drop empties. The API rejects an assistant message whose final
+ * text block ends in whitespace, which a model writing a bulleted plan does routinely.
+ */
+function trimTextBlocks(content: ContentBlock[]): ContentBlock[] {
+  const isText = (b: ContentBlock): b is { type: "text"; text: string } => b.type === "text";
+  return content
+    .map((block) => (isText(block) ? { ...block, text: block.text.trim() } : block))
+    .filter((block) => !isText(block) || block.text.length > 0);
+}
+
 function toolUsesFrom(content: ContentBlock[]): ToolUseBlock[] {
   return content.filter((block): block is ToolUseBlock => block.type === "tool_use");
 }
@@ -153,6 +211,29 @@ export async function runAgentQuery(options: RunAgentQueryOptions): Promise<Agen
 
   const messages: ChatMessage[] = [{ role: "user", content: question }];
 
+  // Plan first, with no tools available, so the model has to commit to an approach in
+  // writing before it can act. A warning in the grounding context was not enough: the
+  // context already says which joins repeat rows, and the agent still wrote the
+  // double-counting query in a single confident step.
+  if (options.plan) {
+    const planPrompt = buildPlanningTurn(question);
+    if (planPrompt) {
+      messages.push({ role: "user", content: planPrompt });
+      const { content } = await complete({ system, messages, tools: [] });
+      // Trailing whitespace on a final assistant block is rejected by the API, and a
+      // transcript may not end on an assistant turn — so trim, then hand the turn back.
+      messages.push({ role: "assistant", content: trimTextBlocks(content) });
+      // Be explicit that the plan is not the answer. Left at "now carry out that plan",
+      // the model finalized on the plan alone in 2 of 5 runs and returned no rows at all.
+      messages.push({
+        role: "user",
+        content:
+          "Now carry out that plan with the tools. You have not answered yet: run the query and " +
+          "return the actual rows before giving your finding.",
+      });
+    }
+  }
+
   for (let turn = 0; turn < maxSteps; turn += 1) {
     const { content } = await complete({ system, messages, tools: toolkit.definitions });
     messages.push({ role: "assistant", content });

src/evals/run-agent.ts

@@ -62,6 +62,8 @@ export interface AgentSuiteOptions extends EvalDatasetOptions {
   verify?: boolean;
   /** Which configuration to measure (default "grounded" — what ships). */
   arm?: ArmId;
+  /** Plan first on multi-part questions (default true; mirrors shipped `ask`). */
+  plan?: boolean;
   /**
    * Apply the case file's behavioral assertions (mustUseTool / mustNotUseTool /
    * maxSteps). Default true, which is right for the product-regression suite.
@@ -144,6 +146,7 @@ interface CaseDeps {
   repeat: number;
   maxSteps?: number;
   verify: boolean;
+  plan: boolean;
   behavioral: boolean;
   onProgress?: (line: string) => void;
 }
@@ -178,6 +181,7 @@ async function runCase(
         runner: deps.runner,
         maxSteps: deps.maxSteps,
         verify: deps.verify,
+        plan: deps.plan,
         tools: arm.tools,
         systemPrompt: arm.systemPrompt,
         complete: deps.complete,
@@ -258,6 +262,7 @@ export async function runAgentSuite(options: AgentSuiteOptions = {}): Promise<Su
       repeat,
       maxSteps: options.maxSteps,
       verify: options.verify ?? true,
+      plan: options.plan ?? true,
       behavioral: options.behavioral ?? true,
       onProgress: options.onProgress,
     };
@@ -287,6 +292,7 @@ function describeConfig(armId: ArmId, deps: CaseDeps): SuiteConfig {
     arm: armId,
     repeat: deps.repeat,
     verify: deps.verify,
+    plan: deps.plan,
     behavioral: deps.behavioral,
     maxSteps: deps.maxSteps ?? DEFAULT_MAX_STEPS,
     tools: ARMS[armId].tools ?? DATA_TOOL_DEFINITIONS.map((t) => t.name),
@@ -324,6 +330,7 @@ export async function runAbSuite(
       repeat,
       maxSteps: options.maxSteps ?? AB_MAX_STEPS,
       verify: options.verify ?? true,
+      plan: options.plan ?? true,
       // Accuracy-only for both arms: identical rubric is the whole point.
       behavioral: false,
       onProgress: options.onProgress,

src/evals/types.ts

@@ -86,6 +86,7 @@ export interface SuiteConfig {
   arm: string;
   repeat: number;
   verify: boolean;
+  plan: boolean;
   behavioral: boolean;
   maxSteps: number;
   tools: string[];