fix: preserve and verify image input through Responses

.rust-file-sizes.json

@@ -3,7 +3,6 @@
   "baseline": {
     "crates/agentic-server-core/src/executor/accumulator/mod.rs": 706,
     "crates/agentic-server-core/src/executor/accumulator/slot.rs": 679,
-    "crates/agentic-server-core/src/executor/compaction.rs": 530,
     "crates/agentic-server-core/src/executor/engine.rs": 823,
     "crates/agentic-server-core/src/executor/gateway.rs": 653,
     "crates/agentic-server-core/src/executor/messages_stream.rs": 520,
@@ -12,7 +11,7 @@
     "crates/agentic-server-core/src/tool/registry.rs": 515,
     "crates/agentic-server-core/src/tool/tool_search.rs": 1764,
     "crates/agentic-server-core/src/tool/web_search/mod.rs": 535,
-    "crates/agentic-server-core/src/types/io/input.rs": 678,
+    "crates/agentic-server-core/src/types/io/input.rs": 629,
     "crates/agentic-server-core/src/types/io/output.rs": 1110,
     "crates/agentic-server-core/src/types/request_response.rs": 582,
     "crates/agentic-server-core/src/types/tools/params.rs": 537,

CHANGELOG.md

@@ -84,6 +84,17 @@ All notable changes to Agentic API are documented here.
   stream, and bounded concurrency across streams (#240).
 - Added compile-time OpenAPI 3.1 schema generation and checked-in schema validation for the HTTP API (#229).
 - Added pinned SGLang conformance recordings, replay coverage, and launch and recording guidance (#267).
+- Verified image preservation through the Responses gateway end to end (#253): integration coverage for mixed
+  text/image ordering, multiple images per turn, client-executed `view_image` tool output, `previous_response_id`
+  continuation, `conversation_id` rehydration, stateless `store: false` proxying, and compaction of retained
+  image-bearing user messages, over both the HTTP and WebSocket transports.
+- Recorded paired image cassettes — client → OpenAI as the reference and client → gateway → vLLM serving
+  `Qwen/Qwen2.5-VL-3B-Instruct` — for a text-and-image message, two interleaved images, a `previous_response_id`
+  follow-up, and a client-executed tool returning an image through a structured `function_call_output`, each
+  streaming and non-streaming. Replay coverage compares request shape, completed-response structure, the streaming
+  event lifecycle, and the history the gateway forwards on continuation; model wording is never compared (#253).
+  The cassette recorder accepts `--input-file` for the first of several turns and sends a tool handler's list of
+  content parts as a structured output array.
 
 ### Changed
 
@@ -99,6 +110,8 @@ All notable changes to Agentic API are documented here.
   architecture (#246).
 - Updated the execution architecture documentation to match the current scheduler and llm-d backend (#270).
 - Preserved the typed `ignore_eos` extension when forwarding Responses requests to vLLM (#268).
+- Modeled `refusal` as an assistant-history content part so OpenAI-style history replays through the typed
+  Responses executor instead of being rejected as unmodeled (#253).
 
 ### Fixed
 
@@ -110,6 +123,12 @@ All notable changes to Agentic API are documented here.
 - Required a healthy packaged gateway before `agentic-api doctor --mode local` reports success (#223).
 - Rebuilt workspace crates after `cargo-chef` dependency cooking so container binaries carry current source and package
   metadata (#208, #209).
+- Rejected message content the typed Responses executor cannot convey — unmodeled part types and empty part arrays,
+  alongside the existing `input_file` rejection — with a `400` naming the offending part, instead of forwarding a
+  synthetic `{"type": "unknown"}` part or silently dropping it. Modeled parts keep their unmodeled extension fields
+  through the typed path, so a message is never mutated in transit, never means something different on the typed
+  path than on the raw `store: false` path, and is never persisted with content the client did not send (#253).
+- Counted an image referenced by `file_id` as retained context during compaction, matching inline images (#253).
 - Hardened split execution with atomic duplicate persistence, strict relayed-response validation, independent secret
   validation, bounded hydrate and persist payloads, stable error envelopes, and graceful shutdown error propagation
   (#235).

crates/agentic-server-core/src/executor/compaction.rs

@@ -1,3 +1,7 @@
+mod context;
+
+use context::item_has_meaningful_context;
+
 use crate::executor::error::{ExecutorError, ExecutorResult};
 use crate::executor::persist::persist_prepared_turn;
 use crate::executor::prepare::prepare_request_tools;
@@ -128,47 +132,6 @@ fn response_output_text(output: &[OutputItem]) -> Option<String> {
     (!text.is_empty()).then_some(text)
 }
 
-fn value_has_content(value: &serde_json::Value) -> bool {
-    match value {
-        serde_json::Value::Null => false,
-        serde_json::Value::String(text) => !text.trim().is_empty(),
-        serde_json::Value::Array(values) => values.iter().any(value_has_content),
-        serde_json::Value::Object(values) => values.values().any(value_has_content),
-        serde_json::Value::Bool(_) | serde_json::Value::Number(_) => true,
-    }
-}
-
-fn item_has_meaningful_context(item: &InputItem) -> bool {
-    match item {
-        InputItem::Message(message) => match &message.content {
-            InputMessageContent::Text(text) => !text.trim().is_empty(),
-            InputMessageContent::Parts(parts) => parts.iter().any(|part| match part {
-                InputContent::InputText(text) | InputContent::OutputText(text) | InputContent::ReasoningText(text) => {
-                    !text.text.trim().is_empty()
-                }
-                InputContent::InputImage(image) => image.image_url.as_deref().is_some_and(|url| !url.trim().is_empty()),
-                // Message files are rejected during typed input validation.
-                InputContent::InputFile(_) | InputContent::Unknown => false,
-            }),
-        },
-        InputItem::FunctionCall(call) => !call.name.trim().is_empty() || !call.arguments.trim().is_empty(),
-        InputItem::FunctionCallOutput(output) => output.output.has_content(),
-        InputItem::ToolSearchCall(call) => !call.call_id.trim().is_empty() || value_has_content(&call.arguments),
-        InputItem::ToolSearchOutput(output) => !output.call_id.trim().is_empty() || !output.tools.is_empty(),
-        InputItem::CustomToolCall(call) => !call.name.trim().is_empty() || !call.input.trim().is_empty(),
-        InputItem::CustomToolCallOutput(output) => output.output.has_content(),
-        InputItem::ShellCall(call) => !call.action.commands.is_empty(),
-        InputItem::ShellCallOutput(output) => !output.output.is_empty(),
-        InputItem::Reasoning(reasoning) => {
-            reasoning.content.iter().any(|content| !content.text.trim().is_empty())
-                || reasoning.summary.iter().any(value_has_content)
-                || reasoning.encrypted_content.as_ref().is_some_and(value_has_content)
-        }
-        InputItem::Compaction(compaction) => !compaction.encrypted_content.trim().is_empty(),
-        InputItem::McpListTools(_) | InputItem::CompactionTrigger | InputItem::Unknown => false,
-    }
-}
-
 fn completed_summary_text(response: &ResponsePayload) -> ExecutorResult<String> {
     if response.status != "completed" || response.error.is_some() {
         let details = response
@@ -205,9 +168,13 @@ fn add_message_content(estimate: &mut InputTokenEstimate, content: &InputMessage
                         estimate.add_tokens(ESTIMATED_CONTENT_PART_OVERHEAD_TOKENS);
                         estimate.add_text(&text.text);
                     }
+                    InputContent::Refusal(refusal) => {
+                        estimate.add_tokens(ESTIMATED_CONTENT_PART_OVERHEAD_TOKENS);
+                        estimate.add_text(&refusal.refusal);
+                    }
                     InputContent::InputImage(_) => estimate.add_tokens(ESTIMATED_IMAGE_TOKENS),
                     InputContent::InputFile(file) => add_file_content(estimate, file),
-                    InputContent::Unknown => estimate.add_tokens(ESTIMATED_CONTENT_PART_OVERHEAD_TOKENS),
+                    InputContent::Unknown(_) => estimate.add_tokens(ESTIMATED_CONTENT_PART_OVERHEAD_TOKENS),
                 }
             }
         }
@@ -565,9 +532,9 @@ mod tests {
 
     fn inline_image(encoded_bytes: usize) -> InputImageContent {
         InputImageContent {
-            file_id: None,
             image_url: Some(format!("data:image/png;base64,{}", "A".repeat(encoded_bytes))),
             detail: Some("auto".to_owned()),
+            ..InputImageContent::default()
         }
     }
 
@@ -842,6 +809,29 @@ mod tests {
         }
     }
 
+    #[test]
+    fn an_image_referenced_by_file_id_is_meaningful_context() {
+        let image_by = |content: InputImageContent| {
+            InputItem::Message(InputMessage {
+                id: None,
+                role: "user".to_owned(),
+                status: None,
+                content: InputMessageContent::Parts(vec![InputContent::InputImage(content)]),
+            })
+        };
+
+        assert!(super::item_has_meaningful_context(&image_message(1)));
+        assert!(super::item_has_meaningful_context(&image_by(InputImageContent {
+            file_id: Some("file_diagram".to_owned()),
+            ..InputImageContent::default()
+        })));
+        assert!(!super::item_has_meaningful_context(&image_by(InputImageContent {
+            file_id: Some("  ".to_owned()),
+            image_url: Some(String::new()),
+            ..InputImageContent::default()
+        })));
+    }
+
     #[test]
     fn each_image_adds_the_fixed_image_budget() {
         let estimate_with_images = |count| {

crates/agentic-server-core/src/executor/compaction/context.rs

@@ -0,0 +1,49 @@
+//! Identify whether resolved input contains context worth compacting.
+
+use crate::types::io::{InputContent, InputItem, InputMessageContent};
+
+fn value_has_content(value: &serde_json::Value) -> bool {
+    match value {
+        serde_json::Value::Null => false,
+        serde_json::Value::String(text) => !text.trim().is_empty(),
+        serde_json::Value::Array(values) => values.iter().any(value_has_content),
+        serde_json::Value::Object(values) => values.values().any(value_has_content),
+        serde_json::Value::Bool(_) | serde_json::Value::Number(_) => true,
+    }
+}
+
+pub(super) fn item_has_meaningful_context(item: &InputItem) -> bool {
+    match item {
+        InputItem::Message(message) => match &message.content {
+            InputMessageContent::Text(text) => !text.trim().is_empty(),
+            InputMessageContent::Parts(parts) => parts.iter().any(|part| match part {
+                InputContent::InputText(text) | InputContent::OutputText(text) | InputContent::ReasoningText(text) => {
+                    !text.text.trim().is_empty()
+                }
+                // An image is context whether it is inline or a file reference.
+                InputContent::InputImage(image) => [image.image_url.as_deref(), image.file_id.as_deref()]
+                    .into_iter()
+                    .flatten()
+                    .any(|reference| !reference.trim().is_empty()),
+                InputContent::Refusal(refusal) => !refusal.refusal.trim().is_empty(),
+                // Message files and unmodeled parts are rejected during typed input validation.
+                InputContent::InputFile(_) | InputContent::Unknown(_) => false,
+            }),
+        },
+        InputItem::FunctionCall(call) => !call.name.trim().is_empty() || !call.arguments.trim().is_empty(),
+        InputItem::FunctionCallOutput(output) => output.output.has_content(),
+        InputItem::ToolSearchCall(call) => !call.call_id.trim().is_empty() || value_has_content(&call.arguments),
+        InputItem::ToolSearchOutput(output) => !output.call_id.trim().is_empty() || !output.tools.is_empty(),
+        InputItem::CustomToolCall(call) => !call.name.trim().is_empty() || !call.input.trim().is_empty(),
+        InputItem::CustomToolCallOutput(output) => output.output.has_content(),
+        InputItem::ShellCall(call) => !call.action.commands.is_empty(),
+        InputItem::ShellCallOutput(output) => !output.output.is_empty(),
+        InputItem::Reasoning(reasoning) => {
+            reasoning.content.iter().any(|content| !content.text.trim().is_empty())
+                || reasoning.summary.iter().any(value_has_content)
+                || reasoning.encrypted_content.as_ref().is_some_and(value_has_content)
+        }
+        InputItem::Compaction(compaction) => !compaction.encrypted_content.trim().is_empty(),
+        InputItem::McpListTools(_) | InputItem::CompactionTrigger | InputItem::Unknown => false,
+    }
+}

crates/agentic-server-core/src/executor/rehydrate.rs

@@ -16,25 +16,47 @@ use crate::types::io::{
 use crate::types::request_response::RequestPayload;
 use crate::utils::uuid7_str;
 
-/// Reject unsupported message files on typed paths, including restored history.
+/// Reject message content the typed executor cannot convey, including restored history.
 ///
 /// Keep this out of deserialization: eligible raw proxy requests must retain their
 /// original bytes and leave support decisions to the upstream. Structured tool
 /// call outputs have a separate content type and are deliberately not rejected.
-pub(super) fn validate_message_files(input: &ResponsesInput) -> ExecutorResult<()> {
+///
+/// A part is rejected rather than dropped so a message is never mutated in
+/// transit: `input_file` because support is decided after routing, an unmodeled
+/// type because it cannot be forwarded or persisted without inventing a
+/// synthetic part, and an empty part array because the turn would carry nothing.
+pub(super) fn validate_message_content(input: &ResponsesInput) -> ExecutorResult<()> {
     let ResponsesInput::Items(items) = input else {
         return Ok(());
     };
-    let has_file = items.iter().any(|item| {
-        matches!(item, InputItem::Message(message)
-            if matches!(&message.content, InputMessageContent::Parts(parts)
-                if parts.iter().any(|part| matches!(part, InputContent::InputFile(_)))))
-    });
-    if has_file {
-        return Err(ExecutorError::InvalidRequest(
-            "input_file content in messages is not supported by the typed Responses executor; provide input_text or input_image content instead"
-                .to_owned(),
-        ));
+    for (item_index, item) in items.iter().enumerate() {
+        let InputItem::Message(message) = item else {
+            continue;
+        };
+        let InputMessageContent::Parts(parts) = &message.content else {
+            continue;
+        };
+        if parts.is_empty() {
+            return Err(ExecutorError::InvalidRequest(format!(
+                "input[{item_index}].content: a message must contain at least one content part"
+            )));
+        }
+        for (part_index, part) in parts.iter().enumerate() {
+            let unsupported = match part {
+                InputContent::InputFile(_) => "input_file",
+                InputContent::Unknown(kind) => kind.as_str(),
+                InputContent::InputText(_)
+                | InputContent::InputImage(_)
+                | InputContent::OutputText(_)
+                | InputContent::Refusal(_)
+                | InputContent::ReasoningText(_) => continue,
+            };
+            return Err(ExecutorError::InvalidRequest(format!(
+                "input[{item_index}].content[{part_index}]: message content part type `{unsupported}` is not \
+                 supported by the typed Responses executor; provide input_text or input_image content instead"
+            )));
+        }
     }
     Ok(())
 }
@@ -146,7 +168,7 @@ pub(crate) async fn rehydrate_with_continuation(
     continuation: Option<ResponseContinuation>,
 ) -> ExecutorResult<RequestContext> {
     // Fail before storage work for new files; check again once history is resolved.
-    validate_message_files(&request.input)?;
+    validate_message_content(&request.input)?;
     let response_id = uuid7_str("resp_");
     // Persistence keeps the public items. Tool lowering belongs to the enriched
     // inference copy, including when a later turn loads these items from storage.
@@ -181,7 +203,7 @@ pub(crate) async fn rehydrate_with_continuation(
         ctx.enriched_request.input = ResponsesInput::Items(Vec::from(&ctx.original_request.input));
     }
 
-    validate_message_files(&ctx.enriched_request.input)?;
+    validate_message_content(&ctx.enriched_request.input)?;
     Ok(ctx)
 }
 

crates/agentic-server-core/src/executor/upstream.rs

@@ -3,7 +3,7 @@ use crate::executor::error::{ExecutorError, ExecutorResult};
 use crate::executor::gateway_accumulator::StreamEvent;
 use crate::executor::inference::{call_inference, fetch_response_json};
 use crate::executor::pipeline::{AgentPipeline, StreamPayload};
-use crate::executor::rehydrate::validate_message_files;
+use crate::executor::rehydrate::validate_message_content;
 use crate::executor::request::{ExecutionContext, RequestContext};
 use crate::executor::response_budget::ExecutorResponseBudget;
 use crate::executor::translate::TranslationContext;
@@ -60,10 +60,10 @@ fn translation_context(registry: &ToolRegistry, agent: &AgentPipeline) -> Transl
 /// fields removed.
 ///
 /// # Errors
-/// Unsupported message files, a tool-configuration error, or a serialization failure.
+/// Unsupported message content, a tool-configuration error, or a serialization failure.
 pub fn upstream_request(ctx: &RequestContext, stream: bool) -> ExecutorResult<String> {
     // Composable callers may supply RequestContext without the rehydration step.
-    validate_message_files(&ctx.enriched_request.input)?;
+    validate_message_content(&ctx.enriched_request.input)?;
     let request = ctx.enriched_request.to_upstream_request(stream)?;
     serialize_to_string(&request).map_err(ExecutorError::JsonError)
 }

crates/agentic-server-core/src/lib.rs

@@ -26,11 +26,11 @@ pub use types::{
     InputFunctionToolCall, InputImageContent, InputItem, InputMessage, InputMessageContent, InputTextContent,
     InputTokenDetails, LocalShellEnvironment, McpCall, McpCallStatus, McpToolParam, NonEmptyToolName, OutputItem,
     OutputMessage, OutputTextContent, OutputTokenDetails, ReasoningConfig, ReasoningOutput, ReasoningTextContent,
-    RequestPayload, ResponsePayload, ResponseTextConfig, ResponseTextFormat, ResponseUsage, ResponsesInput,
-    ResponsesTool, ShellCall, ShellCallAction, ShellCallOutcome, ShellCallOutputContent, ShellCallOutputMessage,
-    ShellCallStatus, ShellEnvironment, ShellToolParam, ToolCallOutput, ToolChoice, ToolOutputContent, UpstreamRequest,
-    UpstreamTool, WebSearchAction, WebSearchActionFindInPage, WebSearchActionOpenPage, WebSearchActionSearch,
-    WebSearchCall, WebSearchCallStatus, WebSearchContextSize, WebSearchFilters, WebSearchSource, WebSearchToolParam,
-    WebSearchUserLocation,
+    RefusalContent, RequestPayload, ResponsePayload, ResponseTextConfig, ResponseTextFormat, ResponseUsage,
+    ResponsesInput, ResponsesTool, ShellCall, ShellCallAction, ShellCallOutcome, ShellCallOutputContent,
+    ShellCallOutputMessage, ShellCallStatus, ShellEnvironment, ShellToolParam, ToolCallOutput, ToolChoice,
+    ToolOutputContent, UpstreamRequest, UpstreamTool, WebSearchAction, WebSearchActionFindInPage,
+    WebSearchActionOpenPage, WebSearchActionSearch, WebSearchCall, WebSearchCallStatus, WebSearchContextSize,
+    WebSearchFilters, WebSearchSource, WebSearchToolParam, WebSearchUserLocation,
 };
 pub use utils::{utcnow_str, uuid7_str};