diff --git a/apps/mobile/src/components/agents/session-message-list.tsx b/apps/mobile/src/components/agents/session-message-list.tsx index 3c166c4b5d..5dcf6ed40b 100644 --- a/apps/mobile/src/components/agents/session-message-list.tsx +++ b/apps/mobile/src/components/agents/session-message-list.tsx @@ -158,6 +158,7 @@ export function SessionMessageList({ accessibilityRole="button" accessibilityLabel="Scroll to bottom" onPress={scrollToLatestAnimated} + hitSlop={2} className="h-10 w-10 items-center justify-center rounded-full border border-border bg-card shadow-lg shadow-black/25 active:opacity-70" > diff --git a/packages/auto-routing-contracts/src/index.ts b/packages/auto-routing-contracts/src/index.ts index 5bac183e40..608f991a95 100644 --- a/packages/auto-routing-contracts/src/index.ts +++ b/packages/auto-routing-contracts/src/index.ts @@ -111,6 +111,13 @@ const BenchmarkAutoRoutingDecisionSchema = z.object({ // pick. Defaulted so responses from a not-yet-redeployed worker still // parse. sticky: z.boolean().default(false), + // Why the session's incumbent model was abandoned, when it was: + // 'threshold' — the incumbent is denied, off the route, or below the + // accuracy bar; 'cost' — eligible, but the mode's switch condition made + // the fresh pick worth it; 'capability' — ejected by the modality/context + // capability filters. Null when there was no incumbent or it was kept. + // Optional so responses from a not-yet-redeployed worker still parse. + switchReason: z.enum(['threshold', 'cost', 'capability']).nullable().optional(), }); const CodingPlanDefaultDecisionSchema = z.object({ diff --git a/packages/auto-routing-contracts/src/normalize.test.ts b/packages/auto-routing-contracts/src/normalize.test.ts index 4cfc21d2e1..8d1f0bd84d 100644 --- a/packages/auto-routing-contracts/src/normalize.test.ts +++ b/packages/auto-routing-contracts/src/normalize.test.ts @@ -255,6 +255,23 @@ describe('detectRequiredInputModalities', () => { ).toEqual(['image']); }); + it('does not treat a bare `file` key in tool output as a file request', () => { + // A `read_file`-style tool result whose `content` is a structured object + // with a `file` property must not be misdetected as a document request. + expect( + detectRequiredInputModalities({ + model: 'gpt-4o', + messages: [ + { role: 'user', content: 'Read src/webhook.ts' }, + { + role: 'tool', + content: [{ type: 'tool_result', content: { file: 'src/webhook.ts', lines: 42 } }], + }, + ], + }) + ).toEqual([]); + }); + it('does not claim support for Gemini native contents[].parts[] bodies', () => { // Native Gemini request bodies never reach this helper: the gateway only // invokes auto-routing for chat_completions / responses / messages shapes, diff --git a/packages/auto-routing-contracts/src/normalize.ts b/packages/auto-routing-contracts/src/normalize.ts index e668c85d87..f140c99b99 100644 --- a/packages/auto-routing-contracts/src/normalize.ts +++ b/packages/auto-routing-contracts/src/normalize.ts @@ -390,11 +390,15 @@ function collectModalities(value: unknown, out: Set): void { } // Guards against callers that omit the `type` discriminator — the - // presence of a known media field is itself sufficient signal. + // presence of a known media field is itself sufficient signal. We do NOT + // treat a bare `file` key as a signal: it collides with ordinary tool + // output (e.g. a `read_file` result whose `content` is `{ file: ... }`), + // and the typed `file`/`input_file`/`document` cases above already cover + // the real Responses/Anthropic file shapes. if ('image_url' in value || 'input_image' in value) { out.add('image'); } - if ('input_file' in value || 'file' in value) { + if ('input_file' in value) { out.add('file'); } } diff --git a/services/auto-routing/src/decide.ts b/services/auto-routing/src/decide.ts index 12fc4aeb9f..a01f9e041c 100644 --- a/services/auto-routing/src/decide.ts +++ b/services/auto-routing/src/decide.ts @@ -1,4 +1,4 @@ -import { MirrorPayloadSchema } from '@kilocode/auto-routing-contracts'; +import { MirrorPayloadSchema, taxonomyRouteKey } from '@kilocode/auto-routing-contracts'; import type { AutoRoutingDecision, AutoRoutingDecisionResponse, @@ -25,7 +25,8 @@ import { putCachedClassification, putStickyDecision, } from './decision-cache'; -import { computeDecision, ENFORCED_MODALITIES } from './decision-engine'; +import type { StickyDecision } from './decision-cache'; +import { computeDecision, modelSatisfiesConstraints } from './decision-engine'; import { ClassifierRunError, classifyNormalizedInput } from './model-classifier'; import type { ClassifierRunResult } from './model-classifier'; import { getRoutingTable } from './routing-table'; @@ -33,36 +34,7 @@ import { getAutoRoutingMode } from './routing-mode'; import type { HonoEnv } from './hono-env'; import { codingPlanDefaultDecision, getCodingPlanPreference } from './coding-plan-preference'; import { getModelCapabilities } from './model-capabilities'; -import type { ModelCapabilities, ModelCapabilitiesMap } from './model-capabilities'; - -// Check whether the coding-plan default model satisfies a constrained -// request. Mirrors the same `fail-closed when required` policy used in -// `decision-engine.ts`: -// * Unknown capability metadata fails when a required+enforced modality -// is set (the model might or might not support it, so we do not trust -// the short-circuit). -// * Unknown context length is treated as "still fits" (consistent with -// the unknown-keeps-rank policy in the engine). -// * A known context that is provably smaller than the estimate fails -// the check. -function codingPlanSatisfiesConstraints( - caps: ModelCapabilities | undefined, - constraints: RoutingConstraints -): boolean { - const required = constraints.requiredInputModalities ?? []; - const enforcedAndRequired = required.filter(m => ENFORCED_MODALITIES.includes(m)); - if (enforcedAndRequired.length > 0) { - if (!caps) return false; - for (const modality of enforcedAndRequired) { - if (!caps.inputModalities.has(modality)) return false; - } - } - const estimate = constraints.promptTokensEstimate; - if (typeof estimate === 'number' && caps && typeof caps.contextLength === 'number') { - if (caps.contextLength < estimate) return false; - } - return true; -} +import type { ModelCapabilitiesMap } from './model-capabilities'; // Isolate-scoped request counter, used to correlate latency with isolate // warm-up in logs. @@ -232,14 +204,16 @@ function summarizeOutcome(outcome: DecisionOutcome): DecisionSummary { // Single sink for decision telemetry: one Analytics Engine data point and // one `auto_routing_decision` log line per decision. Successes are sampled -// per the KV-configured rate; fallbacks and errors always log (at warn). +// per the KV-configured rate; fallbacks, errors, and real model switches +// always log (failures at warn). function recordDecision( env: Env, ctx: DecisionContext, durationMs: number, outcome: DecisionOutcome, autoRoutingMode: string, - decision: AutoRoutingDecision | null = null + decision: AutoRoutingDecision | null = null, + incumbent: StickyDecision | null = null ): void { const summary = summarizeOutcome(outcome); @@ -253,10 +227,21 @@ function recordDecision( cacheHit: summary.cacheHit, }); + const incumbentModel = incumbent?.model ?? null; + const switched = + decision !== null && incumbentModel !== null && decision.model !== incumbentModel; + const routeKey = summary.classification ? taxonomyRouteKey(summary.classification) : null; + // Null when there is no incumbent route to compare against (no incumbent, + // a pre-routeKey cache entry, or no classification this request). + const routeChanged = + routeKey !== null && incumbent?.routeKey != null ? routeKey !== incumbent.routeKey : null; + // Retried decisions are rare and diagnostically valuable, so they bypass - // sampling along with failures. + // sampling along with failures. Real model switches also always log: + // within-session switch sequences are the signal the sampled stream + // decimates. const isFailure = summary.status !== 'classified'; - const alwaysLog = isFailure || summary.retried; + const alwaysLog = isFailure || summary.retried || switched; if (!alwaysLog && Math.random() >= ctx.successSampleRate) { return; } @@ -294,6 +279,14 @@ function recordDecision( decidedSubtaskType: decision?.subtaskType ?? null, decisionSource: decision?.source ?? null, sticky: decision?.sticky ?? null, + incumbentModel, + switched, + switchReason: decision?.source === 'benchmark' ? (decision.switchReason ?? null) : null, + routeChanged, + contextTokens: ctx.payload.constraints?.promptTokensEstimate ?? null, + // False when this line bypassed the success sample rate (switch, + // retry, failure); downstream rate math must only scale sampled rows. + sampled: !alwaysLog, ...summary.details, }) ); @@ -345,10 +338,7 @@ export const decideHandler: Handler = async c => { if (codingPlanActive) { const canTakeShortCircuit = hasConstraints && constraints - ? codingPlanSatisfiesConstraints( - capabilities.get(codingPlanPreference.modelId), - constraints - ) + ? modelSatisfiesConstraints(capabilities.get(codingPlanPreference.modelId), constraints) : true; if (canTakeShortCircuit) { const decision = codingPlanDefaultDecision(codingPlanPreference); @@ -390,7 +380,7 @@ export const decideHandler: Handler = async c => { }; // Both live in the conversation's Durable Object; fetch them together. - const [cached, stickyModel] = await Promise.all([ + const [cached, sticky] = await Promise.all([ getCachedClassification(c.env, ctx.conversationKey, hashes.exact, classifierModel), getStickyDecision(c.env, ctx.conversationKey), ]); @@ -398,7 +388,7 @@ export const decideHandler: Handler = async c => { const decision = computeDecision( cached, routingTable, - stickyModel, + sticky?.model ?? null, deniedModelIds, routingMode, { @@ -407,7 +397,9 @@ export const decideHandler: Handler = async c => { } ); if (decision) { - c.executionCtx.waitUntil(putStickyDecision(c.env, ctx.conversationKey, decision.model)); + c.executionCtx.waitUntil( + putStickyDecision(c.env, ctx.conversationKey, decision.model, taxonomyRouteKey(cached)) + ); } recordDecision( c.env, @@ -415,7 +407,8 @@ export const decideHandler: Handler = async c => { performance.now() - startedAt, { kind: 'cache_hit', classifierModel, classification: cached }, routingMode, - decision + decision, + sticky ); return c.json(decisionResponse(0, cached, payload.input, decision)); } @@ -438,7 +431,7 @@ export const decideHandler: Handler = async c => { const decision = computeDecision( classifier.classification, routingTable, - stickyModel, + sticky?.model ?? null, deniedModelIds, routingMode, { @@ -449,7 +442,14 @@ export const decideHandler: Handler = async c => { // Like the classification cache, sticky state only trusts real classifier // output: a heuristic fallback must not re-anchor the session's model. if (decision && !classifier.fallback) { - c.executionCtx.waitUntil(putStickyDecision(c.env, ctx.conversationKey, decision.model)); + c.executionCtx.waitUntil( + putStickyDecision( + c.env, + ctx.conversationKey, + decision.model, + taxonomyRouteKey(classifier.classification) + ) + ); } recordDecision( c.env, @@ -457,7 +457,8 @@ export const decideHandler: Handler = async c => { performance.now() - startedAt, { kind: 'model', classifier }, routingMode, - decision + decision, + sticky ); return c.json( decisionResponse(classifier.cost ?? 0, classifier.classification, payload.input, decision) @@ -468,7 +469,9 @@ export const decideHandler: Handler = async c => { ctx, performance.now() - startedAt, { kind: 'error', error }, - routingMode + routingMode, + null, + sticky ); // A failed run can still have billed the first attempt (e.g. a valid-but- // invalid response followed by a throwing retry), so report that cost diff --git a/services/auto-routing/src/decision-cache.test.ts b/services/auto-routing/src/decision-cache.test.ts index 8f97f5c3ca..c4e529cf7d 100644 --- a/services/auto-routing/src/decision-cache.test.ts +++ b/services/auto-routing/src/decision-cache.test.ts @@ -126,17 +126,20 @@ describe('sticky decision storage', () => { return { env, cacheDO, storage }; } - it('round-trips the sticky model for a conversation', async () => { + it('round-trips the sticky model and route for a conversation', async () => { const { env } = createStickyEnv(); await expect(getStickyDecision(env, 'conversation-1')).resolves.toBeNull(); - await putStickyDecision(env, 'conversation-1', 'mid/chat'); - await expect(getStickyDecision(env, 'conversation-1')).resolves.toBe('mid/chat'); + await putStickyDecision(env, 'conversation-1', 'mid/chat', 'implementation/code_generation'); + await expect(getStickyDecision(env, 'conversation-1')).resolves.toEqual({ + model: 'mid/chat', + routeKey: 'implementation/code_generation', + }); }); it('expires sticky entries after the TTL', async () => { const { env } = createStickyEnv(); - await putStickyDecision(env, 'conversation-1', 'mid/chat'); + await putStickyDecision(env, 'conversation-1', 'mid/chat', 'implementation/code_generation'); vi.advanceTimersByTime(31 * 60 * 1000); await expect(getStickyDecision(env, 'conversation-1')).resolves.toBeNull(); @@ -148,4 +151,14 @@ describe('sticky decision storage', () => { await expect(getStickyDecision(env, 'conversation-1')).resolves.toBeNull(); }); + + it('serves entries written before the routeKey field existed', async () => { + const { env, cacheDO } = createStickyEnv(); + await cacheDO.putEntry('sticky', { model: 'mid/chat' } as unknown as ClassifierOutput); + + await expect(getStickyDecision(env, 'conversation-1')).resolves.toEqual({ + model: 'mid/chat', + routeKey: null, + }); + }); }); diff --git a/services/auto-routing/src/decision-cache.ts b/services/auto-routing/src/decision-cache.ts index 0aed20c63e..4e0fbf5770 100644 --- a/services/auto-routing/src/decision-cache.ts +++ b/services/auto-routing/src/decision-cache.ts @@ -86,8 +86,13 @@ function entryKey(contentHash: string, classifierModel: string): string { // Cannot collide with classification keys, which always contain a ':'. const STICKY_DECISION_KEY = 'sticky'; -const StickyDecisionSchema = z.object({ model: z.string().min(1) }); -type StickyDecision = z.infer; +const StickyDecisionSchema = z.object({ + model: z.string().min(1), + // Taxonomy route the model was decided on, for route-change telemetry. + // Nullable/defaulted so entries written before the field existed parse. + routeKey: z.string().min(1).nullish().default(null), +}); +export type StickyDecision = z.infer; export async function getCachedClassification( env: DecisionCacheEnv, @@ -112,14 +117,14 @@ export async function getCachedClassification( export async function getStickyDecision( env: DecisionCacheEnv, conversationKey: string -): Promise { +): Promise { try { const value = await cacheStub(env, conversationKey).getEntry(STICKY_DECISION_KEY); if (!value) return null; // Entries may have been written by an older worker version; validate // before serving. const parsed = StickyDecisionSchema.safeParse(value); - return parsed.success ? parsed.data.model : null; + return parsed.success ? parsed.data : null; } catch { return null; } @@ -128,11 +133,13 @@ export async function getStickyDecision( export async function putStickyDecision( env: DecisionCacheEnv, conversationKey: string, - model: string + model: string, + routeKey: string ): Promise { try { await cacheStub(env, conversationKey).putEntry(STICKY_DECISION_KEY, { model, + routeKey, } satisfies StickyDecision); } catch { // Sticky writes are best effort and must not fail the decision. diff --git a/services/auto-routing/src/decision-engine.test.ts b/services/auto-routing/src/decision-engine.test.ts index b81cb29522..ae51d84c89 100644 --- a/services/auto-routing/src/decision-engine.test.ts +++ b/services/auto-routing/src/decision-engine.test.ts @@ -92,6 +92,7 @@ describe('computeDecision', () => { tableVersion: 'run-1', reasoningEffort: null, sticky: false, + switchReason: null, }); }); it('defaults to the best accuracy per dollar candidate', () => { @@ -108,11 +109,12 @@ describe('computeDecision', () => { tableVersion: 'run-1', reasoningEffort: null, sticky: false, + switchReason: null, }); }); it('does not keep a lower-accuracy incumbent in best accuracy mode', () => { const decision = computeDecision(classification, table, 'mid/chat', new Set(), 'best_accuracy'); - expect(decision).toMatchObject({ model: 'pricey/chat', sticky: false }); + expect(decision).toMatchObject({ model: 'pricey/chat', sticky: false, switchReason: 'cost' }); }); it('keeps a best-accuracy incumbent when the fresh pick is less than five points better', () => { const nearTieTable: RoutingTable = { @@ -199,6 +201,7 @@ describe('computeDecision', () => { tableVersion: 'run-1', reasoningEffort: 'medium', sticky: false, + switchReason: null, }); }); it('skips virtual auto-model candidates', () => { @@ -251,6 +254,7 @@ describe('computeDecision', () => { // The incumbent's benchmarked effort, not the fresh pick's. reasoningEffort: 'medium', sticky: true, + switchReason: null, }); }); it('keeps the incumbent at the exact switch-cost boundary', () => { @@ -280,23 +284,35 @@ describe('computeDecision', () => { it('switches when the fresh pick is cheaper by more than the factor', () => { // pricey/chat at 0.02 vs fresh 0.002 * 3 = 0.006: switch pays off. const decision = computeDecision(classification, table, 'pricey/chat'); - expect(decision).toMatchObject({ model: 'cheap/chat', sticky: false }); + expect(decision).toMatchObject({ model: 'cheap/chat', sticky: false, switchReason: 'cost' }); }); it('does not keep a denied incumbent', () => { const decision = computeDecision(classification, table, 'mid/chat', new Set(['mid/chat'])); - expect(decision).toMatchObject({ model: 'cheap/chat', sticky: false }); + expect(decision).toMatchObject({ + model: 'cheap/chat', + sticky: false, + switchReason: 'threshold', + }); }); it('switches when the incumbent no longer meets the route threshold', () => { const decision = computeDecision(classification, table, 'weak/chat'); - expect(decision).toMatchObject({ model: 'cheap/chat', sticky: false }); + expect(decision).toMatchObject({ + model: 'cheap/chat', + sticky: false, + switchReason: 'threshold', + }); }); it('serves the fresh pick when the incumbent is not in the route', () => { const decision = computeDecision(classification, table, 'gone/model'); - expect(decision).toMatchObject({ model: 'cheap/chat', sticky: false }); + expect(decision).toMatchObject({ + model: 'cheap/chat', + sticky: false, + switchReason: 'threshold', + }); }); it('is not sticky when the incumbent is the fresh pick', () => { const decision = computeDecision(classification, table, 'cheap/chat'); - expect(decision).toMatchObject({ model: 'cheap/chat', sticky: false }); + expect(decision).toMatchObject({ model: 'cheap/chat', sticky: false, switchReason: null }); }); }); @@ -444,7 +460,11 @@ describe('computeDecision', () => { capabilityMap: caps, } ); - expect(decision).toMatchObject({ model: 'vision/chat', sticky: false }); + expect(decision).toMatchObject({ + model: 'vision/chat', + sticky: false, + switchReason: 'capability', + }); }); it('a fitting lower-ranked candidate wins over a provably-too-small top candidate', () => { @@ -531,7 +551,11 @@ describe('computeDecision', () => { capabilityMap: caps, } ); - expect(decision).toMatchObject({ model: 'huge/chat', sticky: false }); + expect(decision).toMatchObject({ + model: 'huge/chat', + sticky: false, + switchReason: 'capability', + }); }); it('falls back to the max-known-context candidate when every known context is too small', () => { diff --git a/services/auto-routing/src/decision-engine.ts b/services/auto-routing/src/decision-engine.ts index 28b567030c..847c6d3d01 100644 --- a/services/auto-routing/src/decision-engine.ts +++ b/services/auto-routing/src/decision-engine.ts @@ -9,7 +9,7 @@ import { type RoutingConstraints, type RoutingTable, } from '@kilocode/auto-routing-contracts'; -import type { ModelCapabilitiesMap } from './model-capabilities'; +import type { ModelCapabilities, ModelCapabilitiesMap } from './model-capabilities'; // Modalities the worker actively enforces against `model_stats.input_modalities`. // Required modalities outside this set are intentionally ignored: they pass @@ -21,6 +21,52 @@ import type { ModelCapabilitiesMap } from './model-capabilities'; // `model_stats.inputModalities` (`apps/web/src/lib/model-stats/sync-openrouter.ts:77,95,124`). export const ENFORCED_MODALITIES: ReadonlyArray = ['image', 'file']; +// Single source of the constraint policy shared by benchmark routing +// (`applyCapabilityFilters`) and the coding-plan short-circuit in `decide.ts`. +// Keeping these in one place stops the two paths from silently diverging when +// the policy changes. + +// A required+enforced modality the model does not (or might not) support fails +// closed: unknown capability data is treated as unsupported. +export function satisfiesRequiredModalities( + caps: ModelCapabilities | undefined, + constraints: RoutingConstraints +): boolean { + const enforcedAndRequired = (constraints.requiredInputModalities ?? []).filter(m => + ENFORCED_MODALITIES.includes(m) + ); + if (enforcedAndRequired.length === 0) return true; + if (!caps) return false; + return enforcedAndRequired.every(modality => caps.inputModalities.has(modality)); +} + +// True only when the model has a known context length provably smaller than the +// estimate. Unknown context length is NOT proof of unfitness (keeps its rank). +export function contextProvablyTooSmall( + caps: ModelCapabilities | undefined, + constraints: RoutingConstraints +): boolean { + const estimate = constraints.promptTokensEstimate; + return ( + typeof estimate === 'number' && + !!caps && + typeof caps.contextLength === 'number' && + caps.contextLength < estimate + ); +} + +// Combined fitness used by the coding-plan short-circuit, which needs a single +// yes/no. Benchmark routing calls the two predicates separately because a +// modality miss drops the candidate while a too-small context only demotes it. +export function modelSatisfiesConstraints( + caps: ModelCapabilities | undefined, + constraints: RoutingConstraints +): boolean { + return ( + satisfiesRequiredModalities(caps, constraints) && !contextProvablyTooSmall(caps, constraints) + ); +} + function pickFreshCandidate( candidates: ReadonlyArray, mode: AutoRoutingMode @@ -67,34 +113,17 @@ function applyCapabilityFilters( return { filtered: candidates, reason: 'no_constraints' }; } - const required = constraints.requiredInputModalities ?? []; - const enforcedAndRequired = required.filter(m => ENFORCED_MODALITIES.includes(m)); - - const modalityOk = (model: string): boolean => { - if (enforcedAndRequired.length === 0) return true; - const caps = capabilityMap?.get(model); - if (!caps) return false; - for (const modality of enforcedAndRequired) { - if (!caps.inputModalities.has(modality)) return false; - } - return true; - }; - - const afterModality = candidates.filter(c => modalityOk(c.model)); + const afterModality = candidates.filter(c => + satisfiesRequiredModalities(capabilityMap?.get(c.model), constraints) + ); if (afterModality.length === 0) { return { filtered: [], reason: 'empty' }; } - const estimate = constraints.promptTokensEstimate; - if (typeof estimate !== 'number') { - return { filtered: afterModality, reason: 'ok' }; - } - const eligible: RankedCandidate[] = []; const provablyTooSmall: RankedCandidate[] = []; for (const candidate of afterModality) { - const caps = capabilityMap?.get(candidate.model); - if (caps && typeof caps.contextLength === 'number' && caps.contextLength < estimate) { + if (contextProvablyTooSmall(capabilityMap?.get(candidate.model), constraints)) { provablyTooSmall.push(candidate); } else { eligible.push(candidate); @@ -179,9 +208,11 @@ export function computeDecision( tableVersion: table.version, reasoningEffort: incumbent.reasoningEffort ?? null, sticky: true, + switchReason: null, }; } + const switched = incumbentModel !== null && incumbentModel !== freshPick.model; return { model: freshPick.model, taskType: classification.taskType, @@ -190,5 +221,18 @@ export function computeDecision( tableVersion: table.version, reasoningEffort: freshPick.reasoningEffort ?? null, sticky: false, + // 'cost': the incumbent was eligible but the mode's switch condition + // (cost factor / accuracy gap) made the fresh pick worth it; + // 'capability': the modality/context filters ejected it from the route; + // 'threshold': it is denied, off the route, or below the accuracy bar. + switchReason: !switched + ? null + : incumbent + ? incumbent.meetsThreshold + ? 'cost' + : 'threshold' + : routeCandidates.some(c => c.model === incumbentModel) + ? 'capability' + : 'threshold', }; } diff --git a/services/auto-routing/src/index.test.ts b/services/auto-routing/src/index.test.ts index 01a682309a..2ef838dc67 100644 --- a/services/auto-routing/src/index.test.ts +++ b/services/auto-routing/src/index.test.ts @@ -582,6 +582,7 @@ describe('auto routing worker', () => { tableVersion: 'bench-run-1', reasoningEffort: null, sticky: false, + switchReason: null, }, classifierResult: { classification: mockClassification, @@ -626,6 +627,12 @@ describe('auto routing worker', () => { uaPrefix: 'Kilo-Code/4.106.0', bodyBytes: 2048, sticky: false, + incumbentModel: null, + switched: false, + switchReason: null, + routeChanged: null, + contextTokens: null, + sampled: true, }); // The raw user id (which embeds the client IP for anonymous users) must // never reach persisted logs. @@ -947,7 +954,10 @@ describe('auto routing worker', () => { // The classification is not re-cached; only the served model is // remembered for session stickiness. expect(cachePutEntry).toHaveBeenCalledTimes(1); - expect(cachePutEntry).toHaveBeenCalledWith('sticky', { model: expect.any(String) }); + expect(cachePutEntry).toHaveBeenCalledWith('sticky', { + model: expect.any(String), + routeKey: 'implementation/feature_development', + }); expect(writeDataPoint).toHaveBeenCalledWith( expect.objectContaining({ doubles: [0, 0, mockClassification.confidence, 1], @@ -1013,6 +1023,59 @@ describe('auto routing worker', () => { }); }); + it('always logs decisions that switch away from the incumbent, marking them unsampled', async () => { + const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + // 0.99 >= the 0.01 default sample rate: a routine decision would be + // dropped, but a real switch must always be observed. + vi.spyOn(Math, 'random').mockReturnValue(0.99); + // Incumbent off the current route: the decision switches away from it. + cacheGetEntry.mockImplementation(async (key: string) => + key === 'sticky' ? { model: 'gone/model', routeKey: 'planning_design/system_design' } : null + ); + + const response = await decideRequest(mirrorPayload()); + + expect(response.status).toBe(200); + expect(logSpy).toHaveBeenCalledTimes(1); + const [logMessage] = logSpy.mock.calls[0] ?? []; + expect(JSON.parse(String(logMessage))).toMatchObject({ + event: 'auto_routing_decision', + incumbentModel: 'gone/model', + switched: true, + switchReason: 'threshold', + routeChanged: true, + sampled: false, + }); + }); + + it('drops routine non-switch decisions outside the sample rate', async () => { + const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + vi.spyOn(Math, 'random').mockReturnValue(0.99); + + const response = await decideRequest(mirrorPayload()); + + expect(response.status).toBe(200); + expect(logSpy).not.toHaveBeenCalled(); + }); + + it('leaves routeChanged unknown for sticky entries written before routeKey existed', async () => { + const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + vi.spyOn(Math, 'random').mockReturnValue(0.99); + cacheGetEntry.mockImplementation(async (key: string) => + key === 'sticky' ? { model: 'gone/model' } : null + ); + + const response = await decideRequest(mirrorPayload()); + + expect(response.status).toBe(200); + expect(logSpy).toHaveBeenCalledTimes(1); + const [logMessage] = logSpy.mock.calls[0] ?? []; + expect(JSON.parse(String(logMessage))).toMatchObject({ + switched: true, + routeChanged: null, + }); + }); + it('falls back to a machine-scoped conversation key without a session id', async () => { const response = await decideRequest(mirrorPayload({ sessionId: null }));