From 9112728e3a973dab92d023a97fcc28aacdbf0177 Mon Sep 17 00:00:00 2001
From: chrarnoldus <12196001+chrarnoldus@users.noreply.github.com>
Date: Fri, 17 Jul 2026 08:11:44 +0000
Subject: [PATCH] fix(ai-gateway): nest messages cache breakpoints

Co-authored-by: kiloconnect[bot] <240665456+kiloconnect[bot]@users.noreply.github.com>
---
 .../providers/openrouter/request-helpers.ts   | 49 +++++++++++--
 .../tests/openrouter-request-helpers.test.ts  | 68 ++++++++++++++++++-
 2 files changed, 111 insertions(+), 6 deletions(-)

diff --git a/apps/web/src/lib/ai-gateway/providers/openrouter/request-helpers.ts b/apps/web/src/lib/ai-gateway/providers/openrouter/request-helpers.ts
index 1c0400ed6d..399537fb7d 100644
--- a/apps/web/src/lib/ai-gateway/providers/openrouter/request-helpers.ts
+++ b/apps/web/src/lib/ai-gateway/providers/openrouter/request-helpers.ts
@@ -1,10 +1,11 @@
+import type Anthropic from '@anthropic-ai/sdk';
+import type OpenAI from 'openai';
 import type {
   GatewayRequest,
   GatewayResponsesRequest,
   OpenCodeSpecificProperties,
   OpenRouterChatCompletionRequest,
 } from '@/lib/ai-gateway/providers/openrouter/types';
-import type OpenAI from 'openai';
 
 export function getMaxTokens(request: GatewayRequest) {
   if (request.kind === 'responses') {
@@ -80,6 +81,41 @@ function setCacheControlOnResponsesMessage(message: OpenAI.Responses.ResponseInp
   }
 }
 
+function setCacheControlOnMessagesMessage(
+  message: Anthropic.MessageParam,
+  cacheControl: Anthropic.CacheControlEphemeral
+) {
+  if (typeof message.content === 'string') {
+    message.content = [
+      {
+        type: 'text',
+        text: message.content,
+        cache_control: cacheControl,
+      },
+    ];
+  } else {
+    const lastItem = message.content.findLast(isCacheableMessagesContentBlock);
+    if (lastItem) {
+      lastItem.cache_control = cacheControl;
+    }
+  }
+}
+
+function isCacheableMessagesContentBlock(
+  item: Anthropic.ContentBlockParam
+): item is Exclude<
+  Anthropic.ContentBlockParam,
+  Anthropic.ThinkingBlockParam | Anthropic.RedactedThinkingBlockParam
+> {
+  return item.type !== 'thinking' && item.type !== 'redacted_thinking';
+}
+
+function hasCacheableMessagesContent(message: Anthropic.MessageParam) {
+  return typeof message.content === 'string'
+    ? message.content.length > 0
+    : message.content.some(isCacheableMessagesContentBlock);
+}
+
 function isObjectRecord(value: unknown): value is Record<string, unknown> {
   return typeof value === 'object' && value !== null;
 }
@@ -163,11 +199,16 @@ export function addCacheBreakpoints(request: GatewayRequest) {
   } else if (
     request.kind === 'messages' &&
     request.body.messages.length > 1 &&
-    !request.body.cache_control &&
     !containsCacheControl(request.body.messages)
   ) {
-    console.debug('[addCacheBreakpoints] setting cache breakpoint on messages request');
-    request.body.cache_control = { type: 'ephemeral' };
+    const lastMessage = request.body.messages.findLast(hasCacheableMessagesContent);
+    if (lastMessage) {
+      console.debug('[addCacheBreakpoints] setting cache breakpoint on last messages message');
+      // Vercel AI Gateway does not honor top-level cache_control on Messages API requests.
+      const cacheControl = request.body.cache_control ?? { type: 'ephemeral' };
+      delete request.body.cache_control;
+      setCacheControlOnMessagesMessage(lastMessage, cacheControl);
+    }
   }
 }
 
diff --git a/apps/web/src/tests/openrouter-request-helpers.test.ts b/apps/web/src/tests/openrouter-request-helpers.test.ts
index bb86d93b59..90a9fa452a 100644
--- a/apps/web/src/tests/openrouter-request-helpers.test.ts
+++ b/apps/web/src/tests/openrouter-request-helpers.test.ts
@@ -207,7 +207,7 @@ describe('addCacheBreakpoints', () => {
     });
   });
 
-  test('adds top-level cache_control on messages request when none is present', () => {
+  test('adds cache_control to the last content block of a messages request', () => {
     const request: GatewayRequest = {
       kind: 'messages',
       body: {
@@ -223,7 +223,71 @@ describe('addCacheBreakpoints', () => {
 
     addCacheBreakpoints(request);
 
-    expect(request.body.cache_control).toEqual({ type: 'ephemeral' });
+    expect(request.body.cache_control).toBeUndefined();
+    expect(request.body.messages.at(-1)?.content).toEqual([
+      { type: 'text', text: 'Latest prompt', cache_control: { type: 'ephemeral' } },
+    ]);
+  });
+
+  test('adds nested cache_control when an unhonored top-level value is present', () => {
+    const request: GatewayRequest = {
+      kind: 'messages',
+      body: {
+        model: 'anthropic/claude-sonnet-4-5',
+        max_tokens: 1024,
+        cache_control: { type: 'ephemeral', ttl: '1h' },
+        messages: [
+          { role: 'user', content: 'First prompt' },
+          { role: 'assistant', content: 'First response' },
+          {
+            role: 'user',
+            content: [
+              { type: 'text', text: 'Latest prompt' },
+              { type: 'text', text: 'Latest detail' },
+            ],
+          },
+        ],
+      },
+    };
+
+    addCacheBreakpoints(request);
+
+    expect(request.body.cache_control).toBeUndefined();
+    expect(request.body.messages.at(-1)?.content).toEqual([
+      { type: 'text', text: 'Latest prompt' },
+      {
+        type: 'text',
+        text: 'Latest detail',
+        cache_control: { type: 'ephemeral', ttl: '1h' },
+      },
+    ]);
+  });
+
+  test('adds cache_control to the last cacheable content block', () => {
+    const request: GatewayRequest = {
+      kind: 'messages',
+      body: {
+        model: 'anthropic/claude-sonnet-4-5',
+        max_tokens: 1024,
+        messages: [
+          { role: 'user', content: 'Prompt' },
+          {
+            role: 'assistant',
+            content: [
+              { type: 'text', text: 'Response' },
+              { type: 'thinking', thinking: 'Reasoning', signature: 'signature' },
+            ],
+          },
+        ],
+      },
+    };
+
+    addCacheBreakpoints(request);
+
+    expect(request.body.messages.at(-1)?.content).toEqual([
+      { type: 'text', text: 'Response', cache_control: { type: 'ephemeral' } },
+      { type: 'thinking', thinking: 'Reasoning', signature: 'signature' },
+    ]);
   });
 
   test('does nothing for messages request when any cache_control is already present', () => {