feat(auto-routing): add model-switch telemetry to decision logging

apps/mobile/src/components/agents/session-message-list.tsx

@@ -158,6 +158,7 @@ export function SessionMessageList<T>({
             accessibilityRole="button"
             accessibilityLabel="Scroll to bottom"
             onPress={scrollToLatestAnimated}
+            hitSlop={2}
             className="h-10 w-10 items-center justify-center rounded-full border border-border bg-card shadow-lg shadow-black/25 active:opacity-70"
           >
             <ChevronDown size={20} color={colors.foreground} />

packages/auto-routing-contracts/src/index.ts

@@ -111,6 +111,13 @@ const BenchmarkAutoRoutingDecisionSchema = z.object({
   // pick. Defaulted so responses from a not-yet-redeployed worker still
   // parse.
   sticky: z.boolean().default(false),
+  // Why the session's incumbent model was abandoned, when it was:
+  // 'threshold' — the incumbent is denied, off the route, or below the
+  // accuracy bar; 'cost' — eligible, but the mode's switch condition made
+  // the fresh pick worth it; 'capability' — ejected by the modality/context
+  // capability filters. Null when there was no incumbent or it was kept.
+  // Optional so responses from a not-yet-redeployed worker still parse.
+  switchReason: z.enum(['threshold', 'cost', 'capability']).nullable().optional(),
 });
 
 const CodingPlanDefaultDecisionSchema = z.object({

packages/auto-routing-contracts/src/normalize.test.ts

@@ -255,6 +255,23 @@ describe('detectRequiredInputModalities', () => {
     ).toEqual(['image']);
   });
 
+  it('does not treat a bare `file` key in tool output as a file request', () => {
+    // A `read_file`-style tool result whose `content` is a structured object
+    // with a `file` property must not be misdetected as a document request.
+    expect(
+      detectRequiredInputModalities({
+        model: 'gpt-4o',
+        messages: [
+          { role: 'user', content: 'Read src/webhook.ts' },
+          {
+            role: 'tool',
+            content: [{ type: 'tool_result', content: { file: 'src/webhook.ts', lines: 42 } }],
+          },
+        ],
+      })
+    ).toEqual([]);
+  });
+
   it('does not claim support for Gemini native contents[].parts[] bodies', () => {
     // Native Gemini request bodies never reach this helper: the gateway only
     // invokes auto-routing for chat_completions / responses / messages shapes,

packages/auto-routing-contracts/src/normalize.ts

@@ -390,11 +390,15 @@ function collectModalities(value: unknown, out: Set<string>): void {
   }
 
   // Guards against callers that omit the `type` discriminator — the
-  // presence of a known media field is itself sufficient signal.
+  // presence of a known media field is itself sufficient signal. We do NOT
+  // treat a bare `file` key as a signal: it collides with ordinary tool
+  // output (e.g. a `read_file` result whose `content` is `{ file: ... }`),
+  // and the typed `file`/`input_file`/`document` cases above already cover
+  // the real Responses/Anthropic file shapes.
   if ('image_url' in value || 'input_image' in value) {
     out.add('image');
   }
-  if ('input_file' in value || 'file' in value) {
+  if ('input_file' in value) {
     out.add('file');
   }
 }

services/auto-routing/src/decide.ts

@@ -1,4 +1,4 @@
-import { MirrorPayloadSchema } from '@kilocode/auto-routing-contracts';
+import { MirrorPayloadSchema, taxonomyRouteKey } from '@kilocode/auto-routing-contracts';
 import type {
   AutoRoutingDecision,
   AutoRoutingDecisionResponse,
@@ -25,44 +25,16 @@ import {
   putCachedClassification,
   putStickyDecision,
 } from './decision-cache';
-import { computeDecision, ENFORCED_MODALITIES } from './decision-engine';
+import type { StickyDecision } from './decision-cache';
+import { computeDecision, modelSatisfiesConstraints } from './decision-engine';
 import { ClassifierRunError, classifyNormalizedInput } from './model-classifier';
 import type { ClassifierRunResult } from './model-classifier';
 import { getRoutingTable } from './routing-table';
 import { getAutoRoutingMode } from './routing-mode';
 import type { HonoEnv } from './hono-env';
 import { codingPlanDefaultDecision, getCodingPlanPreference } from './coding-plan-preference';
 import { getModelCapabilities } from './model-capabilities';
-import type { ModelCapabilities, ModelCapabilitiesMap } from './model-capabilities';
-
-// Check whether the coding-plan default model satisfies a constrained
-// request. Mirrors the same `fail-closed when required` policy used in
-// `decision-engine.ts`:
-//   * Unknown capability metadata fails when a required+enforced modality
-//     is set (the model might or might not support it, so we do not trust
-//     the short-circuit).
-//   * Unknown context length is treated as "still fits" (consistent with
-//     the unknown-keeps-rank policy in the engine).
-//   * A known context that is provably smaller than the estimate fails
-//     the check.
-function codingPlanSatisfiesConstraints(
-  caps: ModelCapabilities | undefined,
-  constraints: RoutingConstraints
-): boolean {
-  const required = constraints.requiredInputModalities ?? [];
-  const enforcedAndRequired = required.filter(m => ENFORCED_MODALITIES.includes(m));
-  if (enforcedAndRequired.length > 0) {
-    if (!caps) return false;
-    for (const modality of enforcedAndRequired) {
-      if (!caps.inputModalities.has(modality)) return false;
-    }
-  }
-  const estimate = constraints.promptTokensEstimate;
-  if (typeof estimate === 'number' && caps && typeof caps.contextLength === 'number') {
-    if (caps.contextLength < estimate) return false;
-  }
-  return true;
-}
+import type { ModelCapabilitiesMap } from './model-capabilities';
 
 // Isolate-scoped request counter, used to correlate latency with isolate
 // warm-up in logs.
@@ -232,14 +204,16 @@ function summarizeOutcome(outcome: DecisionOutcome): DecisionSummary {
 
 // Single sink for decision telemetry: one Analytics Engine data point and
 // one `auto_routing_decision` log line per decision. Successes are sampled
-// per the KV-configured rate; fallbacks and errors always log (at warn).
+// per the KV-configured rate; fallbacks, errors, and real model switches
+// always log (failures at warn).
 function recordDecision(
   env: Env,
   ctx: DecisionContext,
   durationMs: number,
   outcome: DecisionOutcome,
   autoRoutingMode: string,
-  decision: AutoRoutingDecision | null = null
+  decision: AutoRoutingDecision | null = null,
+  incumbent: StickyDecision | null = null
 ): void {
   const summary = summarizeOutcome(outcome);
 
@@ -253,10 +227,21 @@ function recordDecision(
     cacheHit: summary.cacheHit,
   });
 
+  const incumbentModel = incumbent?.model ?? null;
+  const switched =
+    decision !== null && incumbentModel !== null && decision.model !== incumbentModel;
+  const routeKey = summary.classification ? taxonomyRouteKey(summary.classification) : null;
+  // Null when there is no incumbent route to compare against (no incumbent,
+  // a pre-routeKey cache entry, or no classification this request).
+  const routeChanged =
+    routeKey !== null && incumbent?.routeKey != null ? routeKey !== incumbent.routeKey : null;
+
   // Retried decisions are rare and diagnostically valuable, so they bypass
-  // sampling along with failures.
+  // sampling along with failures. Real model switches also always log:
+  // within-session switch sequences are the signal the sampled stream
+  // decimates.
   const isFailure = summary.status !== 'classified';
-  const alwaysLog = isFailure || summary.retried;
+  const alwaysLog = isFailure || summary.retried || switched;
   if (!alwaysLog && Math.random() >= ctx.successSampleRate) {
     return;
   }
@@ -294,6 +279,14 @@ function recordDecision(
       decidedSubtaskType: decision?.subtaskType ?? null,
       decisionSource: decision?.source ?? null,
       sticky: decision?.sticky ?? null,
+      incumbentModel,
+      switched,
+      switchReason: decision?.source === 'benchmark' ? (decision.switchReason ?? null) : null,
+      routeChanged,
+      contextTokens: ctx.payload.constraints?.promptTokensEstimate ?? null,
+      // False when this line bypassed the success sample rate (switch,
+      // retry, failure); downstream rate math must only scale sampled rows.
+      sampled: !alwaysLog,
       ...summary.details,
     })
   );
@@ -345,10 +338,7 @@ export const decideHandler: Handler<HonoEnv> = async c => {
   if (codingPlanActive) {
     const canTakeShortCircuit =
       hasConstraints && constraints
-        ? codingPlanSatisfiesConstraints(
-            capabilities.get(codingPlanPreference.modelId),
-            constraints
-          )
+        ? modelSatisfiesConstraints(capabilities.get(codingPlanPreference.modelId), constraints)
         : true;
     if (canTakeShortCircuit) {
       const decision = codingPlanDefaultDecision(codingPlanPreference);
@@ -390,15 +380,15 @@ export const decideHandler: Handler<HonoEnv> = async c => {
   };
 
   // Both live in the conversation's Durable Object; fetch them together.
-  const [cached, stickyModel] = await Promise.all([
+  const [cached, sticky] = await Promise.all([
     getCachedClassification(c.env, ctx.conversationKey, hashes.exact, classifierModel),
     getStickyDecision(c.env, ctx.conversationKey),
   ]);
   if (cached) {
     const decision = computeDecision(
       cached,
       routingTable,
-      stickyModel,
+      sticky?.model ?? null,
       deniedModelIds,
       routingMode,
       {
@@ -407,15 +397,18 @@ export const decideHandler: Handler<HonoEnv> = async c => {
       }
     );
     if (decision) {
-      c.executionCtx.waitUntil(putStickyDecision(c.env, ctx.conversationKey, decision.model));
+      c.executionCtx.waitUntil(
+        putStickyDecision(c.env, ctx.conversationKey, decision.model, taxonomyRouteKey(cached))
+      );
     }
     recordDecision(
       c.env,
       ctx,
       performance.now() - startedAt,
       { kind: 'cache_hit', classifierModel, classification: cached },
       routingMode,
-      decision
+      decision,
+      sticky
     );
     return c.json(decisionResponse(0, cached, payload.input, decision));
   }
@@ -438,7 +431,7 @@ export const decideHandler: Handler<HonoEnv> = async c => {
     const decision = computeDecision(
       classifier.classification,
       routingTable,
-      stickyModel,
+      sticky?.model ?? null,
       deniedModelIds,
       routingMode,
       {
@@ -449,15 +442,23 @@ export const decideHandler: Handler<HonoEnv> = async c => {
     // Like the classification cache, sticky state only trusts real classifier
     // output: a heuristic fallback must not re-anchor the session's model.
     if (decision && !classifier.fallback) {
-      c.executionCtx.waitUntil(putStickyDecision(c.env, ctx.conversationKey, decision.model));
+      c.executionCtx.waitUntil(
+        putStickyDecision(
+          c.env,
+          ctx.conversationKey,
+          decision.model,
+          taxonomyRouteKey(classifier.classification)
+        )
+      );
     }
     recordDecision(
       c.env,
       ctx,
       performance.now() - startedAt,
       { kind: 'model', classifier },
       routingMode,
-      decision
+      decision,
+      sticky
     );
     return c.json(
       decisionResponse(classifier.cost ?? 0, classifier.classification, payload.input, decision)
@@ -468,7 +469,9 @@ export const decideHandler: Handler<HonoEnv> = async c => {
       ctx,
       performance.now() - startedAt,
       { kind: 'error', error },
-      routingMode
+      routingMode,
+      null,
+      sticky
     );
     // A failed run can still have billed the first attempt (e.g. a valid-but-
     // invalid response followed by a throwing retry), so report that cost

services/auto-routing/src/decision-cache.test.ts

@@ -126,17 +126,20 @@ describe('sticky decision storage', () => {
     return { env, cacheDO, storage };
   }
 
-  it('round-trips the sticky model for a conversation', async () => {
+  it('round-trips the sticky model and route for a conversation', async () => {
     const { env } = createStickyEnv();
     await expect(getStickyDecision(env, 'conversation-1')).resolves.toBeNull();
 
-    await putStickyDecision(env, 'conversation-1', 'mid/chat');
-    await expect(getStickyDecision(env, 'conversation-1')).resolves.toBe('mid/chat');
+    await putStickyDecision(env, 'conversation-1', 'mid/chat', 'implementation/code_generation');
+    await expect(getStickyDecision(env, 'conversation-1')).resolves.toEqual({
+      model: 'mid/chat',
+      routeKey: 'implementation/code_generation',
+    });
   });
 
   it('expires sticky entries after the TTL', async () => {
     const { env } = createStickyEnv();
-    await putStickyDecision(env, 'conversation-1', 'mid/chat');
+    await putStickyDecision(env, 'conversation-1', 'mid/chat', 'implementation/code_generation');
 
     vi.advanceTimersByTime(31 * 60 * 1000);
     await expect(getStickyDecision(env, 'conversation-1')).resolves.toBeNull();
@@ -148,4 +151,14 @@ describe('sticky decision storage', () => {
 
     await expect(getStickyDecision(env, 'conversation-1')).resolves.toBeNull();
   });
+
+  it('serves entries written before the routeKey field existed', async () => {
+    const { env, cacheDO } = createStickyEnv();
+    await cacheDO.putEntry('sticky', { model: 'mid/chat' } as unknown as ClassifierOutput);
+
+    await expect(getStickyDecision(env, 'conversation-1')).resolves.toEqual({
+      model: 'mid/chat',
+      routeKey: null,
+    });
+  });
 });

services/auto-routing/src/decision-cache.ts

@@ -86,8 +86,13 @@ function entryKey(contentHash: string, classifierModel: string): string {
 // Cannot collide with classification keys, which always contain a ':'.
 const STICKY_DECISION_KEY = 'sticky';
 
-const StickyDecisionSchema = z.object({ model: z.string().min(1) });
-type StickyDecision = z.infer<typeof StickyDecisionSchema>;
+const StickyDecisionSchema = z.object({
+  model: z.string().min(1),
+  // Taxonomy route the model was decided on, for route-change telemetry.
+  // Nullable/defaulted so entries written before the field existed parse.
+  routeKey: z.string().min(1).nullish().default(null),
+});
+export type StickyDecision = z.infer<typeof StickyDecisionSchema>;
 
 export async function getCachedClassification(
   env: DecisionCacheEnv,
@@ -112,14 +117,14 @@ export async function getCachedClassification(
 export async function getStickyDecision(
   env: DecisionCacheEnv,
   conversationKey: string
-): Promise<string | null> {
+): Promise<StickyDecision | null> {
   try {
     const value = await cacheStub(env, conversationKey).getEntry(STICKY_DECISION_KEY);
     if (!value) return null;
     // Entries may have been written by an older worker version; validate
     // before serving.
     const parsed = StickyDecisionSchema.safeParse(value);
-    return parsed.success ? parsed.data.model : null;
+    return parsed.success ? parsed.data : null;
   } catch {
     return null;
   }
@@ -128,11 +133,13 @@ export async function getStickyDecision(
 export async function putStickyDecision(
   env: DecisionCacheEnv,
   conversationKey: string,
-  model: string
+  model: string,
+  routeKey: string
 ): Promise<void> {
   try {
     await cacheStub(env, conversationKey).putEntry(STICKY_DECISION_KEY, {
       model,
+      routeKey,
     } satisfies StickyDecision);
   } catch {
     // Sticky writes are best effort and must not fail the decision.