diff --git a/.rust-file-sizes.json b/.rust-file-sizes.json index 93e6012a..31cf9a29 100644 --- a/.rust-file-sizes.json +++ b/.rust-file-sizes.json @@ -3,7 +3,6 @@ "baseline": { "crates/agentic-server-core/src/executor/accumulator/mod.rs": 706, "crates/agentic-server-core/src/executor/accumulator/slot.rs": 679, - "crates/agentic-server-core/src/executor/compaction.rs": 530, "crates/agentic-server-core/src/executor/engine.rs": 823, "crates/agentic-server-core/src/executor/gateway.rs": 653, "crates/agentic-server-core/src/executor/messages_stream.rs": 520, @@ -12,7 +11,7 @@ "crates/agentic-server-core/src/tool/registry.rs": 515, "crates/agentic-server-core/src/tool/tool_search.rs": 1764, "crates/agentic-server-core/src/tool/web_search/mod.rs": 535, - "crates/agentic-server-core/src/types/io/input.rs": 678, + "crates/agentic-server-core/src/types/io/input.rs": 629, "crates/agentic-server-core/src/types/io/output.rs": 1110, "crates/agentic-server-core/src/types/request_response.rs": 582, "crates/agentic-server-core/src/types/tools/params.rs": 537, diff --git a/CHANGELOG.md b/CHANGELOG.md index e9360815..a564f0a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -84,6 +84,17 @@ All notable changes to Agentic API are documented here. stream, and bounded concurrency across streams (#240). - Added compile-time OpenAPI 3.1 schema generation and checked-in schema validation for the HTTP API (#229). - Added pinned SGLang conformance recordings, replay coverage, and launch and recording guidance (#267). +- Verified image preservation through the Responses gateway end to end (#253): integration coverage for mixed + text/image ordering, multiple images per turn, client-executed `view_image` tool output, `previous_response_id` + continuation, `conversation_id` rehydration, stateless `store: false` proxying, and compaction of retained + image-bearing user messages, over both the HTTP and WebSocket transports. +- Recorded paired image cassettes — client → OpenAI as the reference and client → gateway → vLLM serving + `Qwen/Qwen2.5-VL-3B-Instruct` — for a text-and-image message, two interleaved images, a `previous_response_id` + follow-up, and a client-executed tool returning an image through a structured `function_call_output`, each + streaming and non-streaming. Replay coverage compares request shape, completed-response structure, the streaming + event lifecycle, and the history the gateway forwards on continuation; model wording is never compared (#253). + The cassette recorder accepts `--input-file` for the first of several turns and sends a tool handler's list of + content parts as a structured output array. ### Changed @@ -99,6 +110,8 @@ All notable changes to Agentic API are documented here. architecture (#246). - Updated the execution architecture documentation to match the current scheduler and llm-d backend (#270). - Preserved the typed `ignore_eos` extension when forwarding Responses requests to vLLM (#268). +- Modeled `refusal` as an assistant-history content part so OpenAI-style history replays through the typed + Responses executor instead of being rejected as unmodeled (#253). ### Fixed @@ -110,6 +123,12 @@ All notable changes to Agentic API are documented here. - Required a healthy packaged gateway before `agentic-api doctor --mode local` reports success (#223). - Rebuilt workspace crates after `cargo-chef` dependency cooking so container binaries carry current source and package metadata (#208, #209). +- Rejected message content the typed Responses executor cannot convey — unmodeled part types and empty part arrays, + alongside the existing `input_file` rejection — with a `400` naming the offending part, instead of forwarding a + synthetic `{"type": "unknown"}` part or silently dropping it. Modeled parts keep their unmodeled extension fields + through the typed path, so a message is never mutated in transit, never means something different on the typed + path than on the raw `store: false` path, and is never persisted with content the client did not send (#253). +- Counted an image referenced by `file_id` as retained context during compaction, matching inline images (#253). - Hardened split execution with atomic duplicate persistence, strict relayed-response validation, independent secret validation, bounded hydrate and persist payloads, stable error envelopes, and graceful shutdown error propagation (#235). diff --git a/crates/agentic-server-core/src/executor/compaction.rs b/crates/agentic-server-core/src/executor/compaction.rs index a2045e77..9463bae4 100644 --- a/crates/agentic-server-core/src/executor/compaction.rs +++ b/crates/agentic-server-core/src/executor/compaction.rs @@ -1,3 +1,7 @@ +mod context; + +use context::item_has_meaningful_context; + use crate::executor::error::{ExecutorError, ExecutorResult}; use crate::executor::persist::persist_prepared_turn; use crate::executor::prepare::prepare_request_tools; @@ -128,47 +132,6 @@ fn response_output_text(output: &[OutputItem]) -> Option { (!text.is_empty()).then_some(text) } -fn value_has_content(value: &serde_json::Value) -> bool { - match value { - serde_json::Value::Null => false, - serde_json::Value::String(text) => !text.trim().is_empty(), - serde_json::Value::Array(values) => values.iter().any(value_has_content), - serde_json::Value::Object(values) => values.values().any(value_has_content), - serde_json::Value::Bool(_) | serde_json::Value::Number(_) => true, - } -} - -fn item_has_meaningful_context(item: &InputItem) -> bool { - match item { - InputItem::Message(message) => match &message.content { - InputMessageContent::Text(text) => !text.trim().is_empty(), - InputMessageContent::Parts(parts) => parts.iter().any(|part| match part { - InputContent::InputText(text) | InputContent::OutputText(text) | InputContent::ReasoningText(text) => { - !text.text.trim().is_empty() - } - InputContent::InputImage(image) => image.image_url.as_deref().is_some_and(|url| !url.trim().is_empty()), - // Message files are rejected during typed input validation. - InputContent::InputFile(_) | InputContent::Unknown => false, - }), - }, - InputItem::FunctionCall(call) => !call.name.trim().is_empty() || !call.arguments.trim().is_empty(), - InputItem::FunctionCallOutput(output) => output.output.has_content(), - InputItem::ToolSearchCall(call) => !call.call_id.trim().is_empty() || value_has_content(&call.arguments), - InputItem::ToolSearchOutput(output) => !output.call_id.trim().is_empty() || !output.tools.is_empty(), - InputItem::CustomToolCall(call) => !call.name.trim().is_empty() || !call.input.trim().is_empty(), - InputItem::CustomToolCallOutput(output) => output.output.has_content(), - InputItem::ShellCall(call) => !call.action.commands.is_empty(), - InputItem::ShellCallOutput(output) => !output.output.is_empty(), - InputItem::Reasoning(reasoning) => { - reasoning.content.iter().any(|content| !content.text.trim().is_empty()) - || reasoning.summary.iter().any(value_has_content) - || reasoning.encrypted_content.as_ref().is_some_and(value_has_content) - } - InputItem::Compaction(compaction) => !compaction.encrypted_content.trim().is_empty(), - InputItem::McpListTools(_) | InputItem::CompactionTrigger | InputItem::Unknown => false, - } -} - fn completed_summary_text(response: &ResponsePayload) -> ExecutorResult { if response.status != "completed" || response.error.is_some() { let details = response @@ -205,9 +168,13 @@ fn add_message_content(estimate: &mut InputTokenEstimate, content: &InputMessage estimate.add_tokens(ESTIMATED_CONTENT_PART_OVERHEAD_TOKENS); estimate.add_text(&text.text); } + InputContent::Refusal(refusal) => { + estimate.add_tokens(ESTIMATED_CONTENT_PART_OVERHEAD_TOKENS); + estimate.add_text(&refusal.refusal); + } InputContent::InputImage(_) => estimate.add_tokens(ESTIMATED_IMAGE_TOKENS), InputContent::InputFile(file) => add_file_content(estimate, file), - InputContent::Unknown => estimate.add_tokens(ESTIMATED_CONTENT_PART_OVERHEAD_TOKENS), + InputContent::Unknown(_) => estimate.add_tokens(ESTIMATED_CONTENT_PART_OVERHEAD_TOKENS), } } } @@ -565,9 +532,9 @@ mod tests { fn inline_image(encoded_bytes: usize) -> InputImageContent { InputImageContent { - file_id: None, image_url: Some(format!("data:image/png;base64,{}", "A".repeat(encoded_bytes))), detail: Some("auto".to_owned()), + ..InputImageContent::default() } } @@ -842,6 +809,29 @@ mod tests { } } + #[test] + fn an_image_referenced_by_file_id_is_meaningful_context() { + let image_by = |content: InputImageContent| { + InputItem::Message(InputMessage { + id: None, + role: "user".to_owned(), + status: None, + content: InputMessageContent::Parts(vec![InputContent::InputImage(content)]), + }) + }; + + assert!(super::item_has_meaningful_context(&image_message(1))); + assert!(super::item_has_meaningful_context(&image_by(InputImageContent { + file_id: Some("file_diagram".to_owned()), + ..InputImageContent::default() + }))); + assert!(!super::item_has_meaningful_context(&image_by(InputImageContent { + file_id: Some(" ".to_owned()), + image_url: Some(String::new()), + ..InputImageContent::default() + }))); + } + #[test] fn each_image_adds_the_fixed_image_budget() { let estimate_with_images = |count| { diff --git a/crates/agentic-server-core/src/executor/compaction/context.rs b/crates/agentic-server-core/src/executor/compaction/context.rs new file mode 100644 index 00000000..3cb5c5c1 --- /dev/null +++ b/crates/agentic-server-core/src/executor/compaction/context.rs @@ -0,0 +1,49 @@ +//! Identify whether resolved input contains context worth compacting. + +use crate::types::io::{InputContent, InputItem, InputMessageContent}; + +fn value_has_content(value: &serde_json::Value) -> bool { + match value { + serde_json::Value::Null => false, + serde_json::Value::String(text) => !text.trim().is_empty(), + serde_json::Value::Array(values) => values.iter().any(value_has_content), + serde_json::Value::Object(values) => values.values().any(value_has_content), + serde_json::Value::Bool(_) | serde_json::Value::Number(_) => true, + } +} + +pub(super) fn item_has_meaningful_context(item: &InputItem) -> bool { + match item { + InputItem::Message(message) => match &message.content { + InputMessageContent::Text(text) => !text.trim().is_empty(), + InputMessageContent::Parts(parts) => parts.iter().any(|part| match part { + InputContent::InputText(text) | InputContent::OutputText(text) | InputContent::ReasoningText(text) => { + !text.text.trim().is_empty() + } + // An image is context whether it is inline or a file reference. + InputContent::InputImage(image) => [image.image_url.as_deref(), image.file_id.as_deref()] + .into_iter() + .flatten() + .any(|reference| !reference.trim().is_empty()), + InputContent::Refusal(refusal) => !refusal.refusal.trim().is_empty(), + // Message files and unmodeled parts are rejected during typed input validation. + InputContent::InputFile(_) | InputContent::Unknown(_) => false, + }), + }, + InputItem::FunctionCall(call) => !call.name.trim().is_empty() || !call.arguments.trim().is_empty(), + InputItem::FunctionCallOutput(output) => output.output.has_content(), + InputItem::ToolSearchCall(call) => !call.call_id.trim().is_empty() || value_has_content(&call.arguments), + InputItem::ToolSearchOutput(output) => !output.call_id.trim().is_empty() || !output.tools.is_empty(), + InputItem::CustomToolCall(call) => !call.name.trim().is_empty() || !call.input.trim().is_empty(), + InputItem::CustomToolCallOutput(output) => output.output.has_content(), + InputItem::ShellCall(call) => !call.action.commands.is_empty(), + InputItem::ShellCallOutput(output) => !output.output.is_empty(), + InputItem::Reasoning(reasoning) => { + reasoning.content.iter().any(|content| !content.text.trim().is_empty()) + || reasoning.summary.iter().any(value_has_content) + || reasoning.encrypted_content.as_ref().is_some_and(value_has_content) + } + InputItem::Compaction(compaction) => !compaction.encrypted_content.trim().is_empty(), + InputItem::McpListTools(_) | InputItem::CompactionTrigger | InputItem::Unknown => false, + } +} diff --git a/crates/agentic-server-core/src/executor/rehydrate.rs b/crates/agentic-server-core/src/executor/rehydrate.rs index f4ebc215..fb1167e9 100644 --- a/crates/agentic-server-core/src/executor/rehydrate.rs +++ b/crates/agentic-server-core/src/executor/rehydrate.rs @@ -16,25 +16,47 @@ use crate::types::io::{ use crate::types::request_response::RequestPayload; use crate::utils::uuid7_str; -/// Reject unsupported message files on typed paths, including restored history. +/// Reject message content the typed executor cannot convey, including restored history. /// /// Keep this out of deserialization: eligible raw proxy requests must retain their /// original bytes and leave support decisions to the upstream. Structured tool /// call outputs have a separate content type and are deliberately not rejected. -pub(super) fn validate_message_files(input: &ResponsesInput) -> ExecutorResult<()> { +/// +/// A part is rejected rather than dropped so a message is never mutated in +/// transit: `input_file` because support is decided after routing, an unmodeled +/// type because it cannot be forwarded or persisted without inventing a +/// synthetic part, and an empty part array because the turn would carry nothing. +pub(super) fn validate_message_content(input: &ResponsesInput) -> ExecutorResult<()> { let ResponsesInput::Items(items) = input else { return Ok(()); }; - let has_file = items.iter().any(|item| { - matches!(item, InputItem::Message(message) - if matches!(&message.content, InputMessageContent::Parts(parts) - if parts.iter().any(|part| matches!(part, InputContent::InputFile(_))))) - }); - if has_file { - return Err(ExecutorError::InvalidRequest( - "input_file content in messages is not supported by the typed Responses executor; provide input_text or input_image content instead" - .to_owned(), - )); + for (item_index, item) in items.iter().enumerate() { + let InputItem::Message(message) = item else { + continue; + }; + let InputMessageContent::Parts(parts) = &message.content else { + continue; + }; + if parts.is_empty() { + return Err(ExecutorError::InvalidRequest(format!( + "input[{item_index}].content: a message must contain at least one content part" + ))); + } + for (part_index, part) in parts.iter().enumerate() { + let unsupported = match part { + InputContent::InputFile(_) => "input_file", + InputContent::Unknown(kind) => kind.as_str(), + InputContent::InputText(_) + | InputContent::InputImage(_) + | InputContent::OutputText(_) + | InputContent::Refusal(_) + | InputContent::ReasoningText(_) => continue, + }; + return Err(ExecutorError::InvalidRequest(format!( + "input[{item_index}].content[{part_index}]: message content part type `{unsupported}` is not \ + supported by the typed Responses executor; provide input_text or input_image content instead" + ))); + } } Ok(()) } @@ -146,7 +168,7 @@ pub(crate) async fn rehydrate_with_continuation( continuation: Option, ) -> ExecutorResult { // Fail before storage work for new files; check again once history is resolved. - validate_message_files(&request.input)?; + validate_message_content(&request.input)?; let response_id = uuid7_str("resp_"); // Persistence keeps the public items. Tool lowering belongs to the enriched // inference copy, including when a later turn loads these items from storage. @@ -181,7 +203,7 @@ pub(crate) async fn rehydrate_with_continuation( ctx.enriched_request.input = ResponsesInput::Items(Vec::from(&ctx.original_request.input)); } - validate_message_files(&ctx.enriched_request.input)?; + validate_message_content(&ctx.enriched_request.input)?; Ok(ctx) } diff --git a/crates/agentic-server-core/src/executor/upstream.rs b/crates/agentic-server-core/src/executor/upstream.rs index 7e903580..7aac69da 100644 --- a/crates/agentic-server-core/src/executor/upstream.rs +++ b/crates/agentic-server-core/src/executor/upstream.rs @@ -3,7 +3,7 @@ use crate::executor::error::{ExecutorError, ExecutorResult}; use crate::executor::gateway_accumulator::StreamEvent; use crate::executor::inference::{call_inference, fetch_response_json}; use crate::executor::pipeline::{AgentPipeline, StreamPayload}; -use crate::executor::rehydrate::validate_message_files; +use crate::executor::rehydrate::validate_message_content; use crate::executor::request::{ExecutionContext, RequestContext}; use crate::executor::response_budget::ExecutorResponseBudget; use crate::executor::translate::TranslationContext; @@ -60,10 +60,10 @@ fn translation_context(registry: &ToolRegistry, agent: &AgentPipeline) -> Transl /// fields removed. /// /// # Errors -/// Unsupported message files, a tool-configuration error, or a serialization failure. +/// Unsupported message content, a tool-configuration error, or a serialization failure. pub fn upstream_request(ctx: &RequestContext, stream: bool) -> ExecutorResult { // Composable callers may supply RequestContext without the rehydration step. - validate_message_files(&ctx.enriched_request.input)?; + validate_message_content(&ctx.enriched_request.input)?; let request = ctx.enriched_request.to_upstream_request(stream)?; serialize_to_string(&request).map_err(ExecutorError::JsonError) } diff --git a/crates/agentic-server-core/src/lib.rs b/crates/agentic-server-core/src/lib.rs index 47ff440b..95bcc64f 100644 --- a/crates/agentic-server-core/src/lib.rs +++ b/crates/agentic-server-core/src/lib.rs @@ -26,11 +26,11 @@ pub use types::{ InputFunctionToolCall, InputImageContent, InputItem, InputMessage, InputMessageContent, InputTextContent, InputTokenDetails, LocalShellEnvironment, McpCall, McpCallStatus, McpToolParam, NonEmptyToolName, OutputItem, OutputMessage, OutputTextContent, OutputTokenDetails, ReasoningConfig, ReasoningOutput, ReasoningTextContent, - RequestPayload, ResponsePayload, ResponseTextConfig, ResponseTextFormat, ResponseUsage, ResponsesInput, - ResponsesTool, ShellCall, ShellCallAction, ShellCallOutcome, ShellCallOutputContent, ShellCallOutputMessage, - ShellCallStatus, ShellEnvironment, ShellToolParam, ToolCallOutput, ToolChoice, ToolOutputContent, UpstreamRequest, - UpstreamTool, WebSearchAction, WebSearchActionFindInPage, WebSearchActionOpenPage, WebSearchActionSearch, - WebSearchCall, WebSearchCallStatus, WebSearchContextSize, WebSearchFilters, WebSearchSource, WebSearchToolParam, - WebSearchUserLocation, + RefusalContent, RequestPayload, ResponsePayload, ResponseTextConfig, ResponseTextFormat, ResponseUsage, + ResponsesInput, ResponsesTool, ShellCall, ShellCallAction, ShellCallOutcome, ShellCallOutputContent, + ShellCallOutputMessage, ShellCallStatus, ShellEnvironment, ShellToolParam, ToolCallOutput, ToolChoice, + ToolOutputContent, UpstreamRequest, UpstreamTool, WebSearchAction, WebSearchActionFindInPage, + WebSearchActionOpenPage, WebSearchActionSearch, WebSearchCall, WebSearchCallStatus, WebSearchContextSize, + WebSearchFilters, WebSearchSource, WebSearchToolParam, WebSearchUserLocation, }; pub use utils::{utcnow_str, uuid7_str}; diff --git a/crates/agentic-server-core/src/types/io/input.rs b/crates/agentic-server-core/src/types/io/input.rs index bd184138..1b5780f3 100644 --- a/crates/agentic-server-core/src/types/io/input.rs +++ b/crates/agentic-server-core/src/types/io/input.rs @@ -1,3 +1,7 @@ +mod content; + +pub use content::{InputContent, InputFileContent, InputImageContent, InputTextContent, RefusalContent}; + use std::borrow::Cow; use serde::{Deserialize, Serialize}; @@ -10,60 +14,6 @@ use crate::utils::common::deserialize_from_value; use super::output::{CustomToolCall, FunctionToolCall, McpListTools, ReasoningOutput, ToolSearchCall}; use super::shell::{ShellCall, ShellCallOutputMessage, ShellCallStatus}; -#[derive(Debug, Clone, Serialize, Deserialize)] -#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] -pub struct InputTextContent { - pub text: String, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] -pub struct InputImageContent { - #[serde(default, skip_serializing_if = "Option::is_none")] - pub file_id: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub image_url: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub detail: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] -pub struct InputFileContent { - #[serde(default, skip_serializing_if = "Option::is_none")] - pub file_data: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub file_id: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub file_url: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub filename: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub detail: Option, -} - -/// Content item inside a message input. -/// -/// Uses an internally-tagged enum — serde consumes `"type"` for the variant -/// discriminant so the inner structs must NOT redeclare a `type_` field. -/// `output_text` and `reasoning_text` reuse `InputTextContent` since they -/// carry only a `text` field; they are preserved so vLLM sees the full history. -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum InputContent { - InputText(InputTextContent), - InputImage(InputImageContent), - /// Preserved on the wire; support is validated after the routing decision. - InputFile(InputFileContent), - /// Assistant output text in rehydrated history. - OutputText(InputTextContent), - /// Reasoning step text in rehydrated history. - ReasoningText(InputTextContent), - /// Any other content type — drop silently. - #[serde(other)] - Unknown, -} - #[derive(Debug, Clone, Serialize, Deserialize)] #[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] pub struct InputMessage { @@ -206,6 +156,7 @@ mod openapi_schemas { ) .item(tagged_ref("input_file", "InputFileContent")) .item(tagged_text_variant("output_text")) + .item(tagged_ref("refusal", "RefusalContent")) .item(tagged_text_variant("reasoning_text")) .into() } @@ -651,9 +602,9 @@ impl ResponsesInput { id: None, role: "assistant".to_owned(), status: None, - content: InputMessageContent::Parts(vec![InputContent::OutputText(InputTextContent { - text: compaction.encrypted_content.clone(), - })]), + content: InputMessageContent::Parts(vec![InputContent::OutputText(InputTextContent::new( + compaction.encrypted_content.clone(), + ))]), }), other => other.clone(), }) @@ -958,6 +909,104 @@ mod tests { assert_eq!(value["output"], content); } + #[test] + fn structured_function_tool_output_preserves_image_array() { + let content = serde_json::json!([ + {"type": "input_text", "text": "attached local image path: diagram.png"}, + {"type": "input_image", "image_url": "data:image/png;base64,abc"} + ]); + let item: InputItem = serde_json::from_value(serde_json::json!({ + "type": "function_call_output", + "call_id": "call_view_image_1", + "output": content + })) + .expect("valid structured function-tool output"); + + let InputItem::FunctionCallOutput(output) = &item else { + panic!("expected function-tool output"); + }; + assert!(matches!(output.output, ToolCallOutput::Content(_))); + + let value = serde_json::to_value(&item).expect("output serializes"); + assert_eq!(value["output"], content, "structured output must not be stringified"); + } + + #[test] + fn unmodeled_message_content_part_keeps_its_type_name() { + let input: ResponsesInput = serde_json::from_value(serde_json::json!([{ + "role": "user", + "content": [ + {"type": "input_text", "text": "before"}, + {"type": "input_audio", "audio_url": "https://example.com/clip.wav"}, + {"type": "input_image", "image_url": "data:image/png;base64,abc", "detail": "low"} + ] + }])) + .expect("an unmodeled part must not fail deserialization"); + + let ResponsesInput::Items(items) = &input else { + panic!("expected items"); + }; + let InputItem::Message(message) = &items[0] else { + panic!("expected message item"); + }; + let InputMessageContent::Parts(parts) = &message.content else { + panic!("expected message parts"); + }; + assert!( + matches!(parts.as_slice(), [InputContent::InputText(_), InputContent::Unknown(kind), InputContent::InputImage(_)] + if kind == "input_audio"), + "the unmodeled part must keep its position and its type name" + ); + assert!( + serde_json::to_value(&input).is_err(), + "an unmodeled part must never serialize into a synthetic part" + ); + } + + #[test] + fn extension_fields_on_modeled_parts_round_trip() { + // The typed path must forward a known part exactly as the client sent + // it, unmodeled fields included, like the raw proxy path does. + let parts = serde_json::json!([ + {"type": "input_text", "text": "look", "x_future_text": true}, + {"type": "input_image", "image_url": "data:image/png;base64,abc", "detail": "low", "x_future_field": "kept"}, + {"type": "output_text", "text": "seen", "annotations": [], "logprobs": []}, + {"type": "refusal", "refusal": "no", "x_reason": "policy"} + ]); + let message: InputMessage = serde_json::from_value(serde_json::json!({"role": "user", "content": parts})) + .expect("modeled parts with extension fields deserialize"); + assert_eq!( + serde_json::to_value(&message).expect("message serializes")["content"], + parts, + "extension fields must survive the typed round trip" + ); + + let output: ToolCallOutput = serde_json::from_value(serde_json::json!([ + {"type": "input_image", "image_url": "data:image/png;base64,abc", "x_future_field": "kept"} + ])) + .expect("structured tool output deserializes"); + assert_eq!( + serde_json::to_value(&output).expect("output serializes")[0]["x_future_field"], + "kept", + "tool-output image parts keep extension fields too" + ); + } + + #[test] + fn message_content_part_without_a_type_is_rejected() { + let error = serde_json::from_value::(serde_json::json!({"text": "no type"})) + .expect_err("a part without a type has no wire meaning"); + assert!(error.to_string().contains("missing a string `type`"), "{error}"); + } + + #[test] + fn refusal_content_round_trips_in_assistant_history() { + let part = serde_json::json!({"type": "refusal", "refusal": "I can't help with that."}); + let content: InputContent = serde_json::from_value(part.clone()).expect("refusal is a modeled part"); + assert!(matches!(&content, InputContent::Refusal(refusal) if refusal.refusal == "I can't help with that.")); + assert_eq!(serde_json::to_value(&content).expect("refusal serializes"), part); + } + #[test] fn custom_tool_output_rejects_unsupported_shapes() { for output in [ diff --git a/crates/agentic-server-core/src/types/io/input/content.rs b/crates/agentic-server-core/src/types/io/input/content.rs new file mode 100644 index 00000000..43b71b9d --- /dev/null +++ b/crates/agentic-server-core/src/types/io/input/content.rs @@ -0,0 +1,134 @@ +//! Typed message content parts and their wire deserialization. + +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +use crate::utils::common::deserialize_from_value; + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] +pub struct InputTextContent { + pub text: String, + /// Unmodeled extension fields, preserved so the typed path forwards the + /// part exactly as the client sent it and leaves support decisions to the + /// upstream, like the raw proxy path does. + #[serde(default, flatten, skip_serializing_if = "Map::is_empty")] + pub extra: Map, +} + +impl InputTextContent { + #[must_use] + pub fn new(text: impl Into) -> Self { + Self { + text: text.into(), + extra: Map::new(), + } + } +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] +pub struct InputImageContent { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub file_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub image_url: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub detail: Option, + /// Unmodeled extension fields, preserved so the typed path forwards the + /// part exactly as the client sent it and leaves support decisions to the + /// upstream, like the raw proxy path does. + #[serde(default, flatten, skip_serializing_if = "Map::is_empty")] + pub extra: Map, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] +pub struct InputFileContent { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub file_data: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub file_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub file_url: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub filename: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub detail: Option, + /// Unmodeled extension fields, preserved so the typed path forwards the + /// part exactly as the client sent it and leaves support decisions to the + /// upstream, like the raw proxy path does. + #[serde(default, flatten, skip_serializing_if = "Map::is_empty")] + pub extra: Map, +} + +/// A refusal in rehydrated assistant history. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] +pub struct RefusalContent { + pub refusal: String, + /// Unmodeled extension fields, preserved so the typed path forwards the + /// part exactly as the client sent it and leaves support decisions to the + /// upstream, like the raw proxy path does. + #[serde(default, flatten, skip_serializing_if = "Map::is_empty")] + pub extra: Map, +} + +/// Content item inside a message input. +/// +/// Serialized as an internally-tagged enum — `"type"` is the variant +/// discriminant so the inner structs must NOT redeclare a `type_` field. +/// `output_text` and `reasoning_text` reuse [`InputTextContent`] since they +/// carry only a `text` field; they and `refusal` are preserved so the upstream +/// sees the full assistant history. +/// +/// Deserialization is hand-written so a part of a type the gateway does not +/// model keeps its type name in [`InputContent::Unknown`]. That variant never +/// serializes: typed paths reject it before the request reaches storage or the +/// upstream, so no synthetic part is ever forwarded or persisted in place of +/// what the client sent. +#[derive(Debug, Clone, Serialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum InputContent { + InputText(InputTextContent), + InputImage(InputImageContent), + /// Preserved on the wire; support is validated after the routing decision. + InputFile(InputFileContent), + /// Assistant output text in rehydrated history. + OutputText(InputTextContent), + /// Assistant refusal in rehydrated history. + Refusal(RefusalContent), + /// Reasoning step text in rehydrated history. + ReasoningText(InputTextContent), + /// A content type this gateway does not model, carrying the type name the + /// client sent so the rejection can name it. + #[serde(skip)] + Unknown(String), +} + +impl<'de> Deserialize<'de> for InputContent { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let mut value = Value::deserialize(deserializer)?; + let kind = value + .as_object_mut() + .and_then(|object| object.remove("type")) + .and_then(|kind| match kind { + Value::String(kind) => Some(kind), + _ => None, + }) + .ok_or_else(|| serde::de::Error::custom("message content part is missing a string `type`"))?; + let part = match kind.as_str() { + "input_text" => deserialize_from_value(value).map(Self::InputText), + "input_image" => deserialize_from_value(value).map(Self::InputImage), + "input_file" => deserialize_from_value(value).map(Self::InputFile), + "output_text" => deserialize_from_value(value).map(Self::OutputText), + "refusal" => deserialize_from_value(value).map(Self::Refusal), + "reasoning_text" => deserialize_from_value(value).map(Self::ReasoningText), + _ => return Ok(Self::Unknown(kind)), + }; + part.map_err(serde::de::Error::custom) + } +} diff --git a/crates/agentic-server-core/src/types/io/mod.rs b/crates/agentic-server-core/src/types/io/mod.rs index 62549f8b..7ca03021 100644 --- a/crates/agentic-server-core/src/types/io/mod.rs +++ b/crates/agentic-server-core/src/types/io/mod.rs @@ -7,7 +7,7 @@ pub mod usage; pub use input::{ CompactionItem, CustomToolCallOutputMessage, FunctionToolResultMessage, InputContent, InputFileContent, InputFunctionToolCall, InputImageContent, InputItem, InputMessage, InputMessageContent, InputTextContent, - InputToolSearchCall, ResponsesInput, ToolCallOutput, ToolOutputContent, ToolSearchOutputMessage, + InputToolSearchCall, RefusalContent, ResponsesInput, ToolCallOutput, ToolOutputContent, ToolSearchOutputMessage, }; pub use output::{ ApplyDone, CustomToolCall, FunctionToolCall, GatewayCallStatus, McpCall, McpCallError, McpCallStatus, McpListTool, diff --git a/crates/agentic-server-core/src/types/io/output.rs b/crates/agentic-server-core/src/types/io/output.rs index 26bc72af..c9df86dc 100644 --- a/crates/agentic-server-core/src/types/io/output.rs +++ b/crates/agentic-server-core/src/types/io/output.rs @@ -78,7 +78,7 @@ impl From for InputMessage { let parts = msg .content .into_iter() - .map(|c| InputContent::OutputText(InputTextContent { text: c.text })) + .map(|c| InputContent::OutputText(InputTextContent::new(c.text))) .collect(); Self { id: Some(msg.id), @@ -1690,6 +1690,6 @@ mod tests { let InputMessageContent::Parts(parts) = &message.content else { panic!("expected message parts"); }; - assert!(matches!(parts.as_slice(), [InputContent::Unknown])); + assert!(matches!(parts.as_slice(), [InputContent::Unknown(kind)] if kind == "future_content")); } } diff --git a/crates/agentic-server-core/src/types/mod.rs b/crates/agentic-server-core/src/types/mod.rs index bff79f6f..b19bb9bd 100644 --- a/crates/agentic-server-core/src/types/mod.rs +++ b/crates/agentic-server-core/src/types/mod.rs @@ -10,7 +10,7 @@ pub use io::{ InputFunctionToolCall, InputImageContent, InputItem, InputMessage, InputMessageContent, InputTextContent, InputTokenDetails, InputToolSearchCall, McpCall, McpCallError, McpCallStatus, McpToolExecutionError, McpToolExecutionErrorContent, OutputItem, OutputMessage, OutputTextContent, OutputTokenDetails, ReasoningOutput, - ReasoningTextContent, ResponseUsage, ResponsesInput, ShellCall, ShellCallAction, ShellCallOutcome, + ReasoningTextContent, RefusalContent, ResponseUsage, ResponsesInput, ShellCall, ShellCallAction, ShellCallOutcome, ShellCallOutputContent, ShellCallOutputMessage, ShellCallStatus, ToolCallOutput, ToolChoice, ToolOutputContent, ToolSearchCall, ToolSearchOutputMessage, WebSearchAction, WebSearchActionError, WebSearchActionFindInPage, WebSearchActionOpenPage, WebSearchActionSearch, WebSearchCall, WebSearchCallStatus, WebSearchSource, diff --git a/crates/agentic-server-core/tests/cassettes/README.md b/crates/agentic-server-core/tests/cassettes/README.md index 2b35a3fa..69cdc506 100644 --- a/crates/agentic-server-core/tests/cassettes/README.md +++ b/crates/agentic-server-core/tests/cassettes/README.md @@ -25,6 +25,9 @@ printf 'Use web search to look up potato, then summarize in one sentence.\n' | p # structured single-turn input -- sends the JSON string or item array from input.json python tests/cassettes/record_cassette.py --mode responses --turns 1 --no-stream --no-store --max-output-tokens 0 --input-file input.json --model gpt-4o --output out.yaml + +# structured opening turn, then a typed follow-up chained by previous_response_id +printf 'What did I just show you?\n' | python tests/cassettes/record_cassette.py --mode responses --turns 2 --no-stream --input-file input.json --model gpt-4o --output out.yaml ``` The recorder scripts (`record_reasoning_cassettes.sh`, `record_tool_call_cassettes.sh`, etc.) use `printf` to feed fixed prompts per test so no manual input is needed. @@ -95,7 +98,7 @@ model requested by Codex 0.149.1. Effective tools after normalized direct-vLLM search --manual-item-replay Replay accumulated items with store=false for direct-vLLM or gateway tool search --reasoning JSON JSON object containing Responses reasoning settings ---input-file FILE JSON string or item array for one HTTP Responses turn +--input-file FILE JSON string or item array for turn 1 of an HTTP Responses recording; later turns are prompted --max-output-tokens N max_output_tokens for Responses requests (default 1024; use 0 to omit) --proxy-port PORT Local proxy port (default 7070) --branch-from TURN Branch from this turn's response id (repeatable) @@ -212,6 +215,7 @@ turns: | `record_mcp_cassettes.sh` | Native MCP counter tool discovery and calls (streaming + non-streaming) | gateway and OpenAI reference | | `record_web_search_cassettes.sh` | Matching web-search calls (streaming + non-streaming) | gateway and OpenAI reference | | `record_messages_tool_choice.py` | Forced `any` and named Messages searches followed by an automatic answer (JSON + SSE) | gateway's upstream traffic to vLLM | +| `record_image_input_cassettes.sh` | Matching two-turn image-input conversations (streaming + non-streaming) | gateway and OpenAI reference | | `record_dynamo_cassettes.sh` | Stateful two-turn and client-executed function tool call cassettes (streaming + non-streaming) | NVIDIA Dynamo frontend | | `record_dynamo_messages_cassettes.sh` | Two-round Messages web-search cassettes (streaming + non-streaming), validated before replacement | Existing NVIDIA Dynamo frontend | | `record_sglang_cassettes.sh` | Same shared executor scenarios as Dynamo, staged validation and sanitized provenance | SGLang | @@ -399,16 +403,61 @@ OPENAI_API_KEY=sk-... \ bash crates/agentic-server-core/tests/cassettes/record_web_search_cassettes.sh ``` -The typed You.com normalization tests replay sanitized provider responses in `tests/fixtures/you_search_response*.json`. -Regenerate them from a recorded cassette instead of editing them by hand: +### Image input (gateway → vLLM vision model, and OpenAI) + +The reference path is client → OpenAI Responses API. The gateway path is client → Agentic API → vLLM hosting an +open-source vision model. Both paths receive the same image bytes, prompts, and tool definitions; only the model name +differs. `image_input_test.rs` replays every pair and compares request shape, completed-response structure, the +streaming event lifecycle, and the history the gateway forwards on continuation — never the model's wording or token +counts. + +| Scenario | Turns | What it proves | +|---|---|---| +| `single-image` | 1 | `input_text` + inline PNG (`images/inputs/single-image.json`) reach the model unchanged | +| `multi-image` | 1 | text and two different PNGs interleave in order (`images/inputs/multi-image.json`) | +| `continuation` | 2 | a text follow-up by `previous_response_id` rehydrates the earlier image into context | +| `tool-image` | 2 | the model calls `view_image`, the client returns a `function_call_output` whose `output` is a content array carrying the PNG, and the model answers from it | + +Each scenario is recorded streaming and non-streaming per provider (16 cassettes). Every recording is validated +(fixture bytes preserved, `previous_response_id` chained, exactly one `view_image` call answered by a structured +output) and staged before any final fixture is replaced. To change an image, replace the PNG and regenerate the JSON +turns from it; the script refuses to record when they disagree. + +**Recorded configuration.** vLLM 0.29.0 serving `Qwen/Qwen2.5-VL-3B-Instruct` in bfloat16 on one 12 GB GPU +(RTX 4080 Laptop, WSL2). The stock Qwen2.5-VL chat template renders images but has no `tools` block, so with +`tool_choice: auto` the model never sees declared functions; `images/qwen2.5-vl-hermes-tools.jinja` adds the +Hermes-style tools prompt and `` history rendering from Qwen2.5-Instruct while keeping the multimodal +rendering, including images inside tool responses. It must be passed with `--chat-template`. ```bash -python crates/agentic-server-core/tests/cassettes/extract_you_search_fixture.py \ - --cassette crates/agentic-server-core/tests/cassettes/messages_multiround/sequential-web-search-qwen3-nonstreaming.yaml \ - --query "latest stable Rust version number" --web 3 --news 2 \ - --output crates/agentic-server-core/tests/fixtures/you_search_response.json +# 1. Serve the vision model. VLLM_WSL2_ENABLE_PIN_MEMORY is needed under WSL2 only; +# VLLM_USE_FLASHINFER_SAMPLER=0 avoids a JIT build when no CUDA toolkit (nvcc) is installed. +VLLM_WSL2_ENABLE_PIN_MEMORY=1 VLLM_USE_FLASHINFER_SAMPLER=0 \ +vllm serve Qwen/Qwen2.5-VL-3B-Instruct \ + --dtype bfloat16 --max-model-len 8192 --max-num-seqs 2 \ + --gpu-memory-utilization 0.82 --enforce-eager \ + --limit-mm-per-prompt '{"image": 4}' \ + --mm-processor-kwargs '{"max_pixels": 200704}' \ + --enable-auto-tool-choice --tool-call-parser hermes \ + --chat-template crates/agentic-server-core/tests/cassettes/images/qwen2.5-vl-hermes-tools.jinja \ + --port 8000 + +# 2. Start the gateway against it. +cargo run -p agentic-server -- --llm-api-base http://127.0.0.1:8000 + +# 3. Record the OpenAI reference and the gateway set. +OPENAI_API_KEY=sk-... \ +GATEWAY_URL=http://localhost:9000 \ +MODEL=Qwen/Qwen2.5-VL-3B-Instruct \ +bash crates/agentic-server-core/tests/cassettes/record_image_input_cassettes.sh ``` +Use `IMAGE_RECORD_SET=gateway` or `IMAGE_RECORD_SET=openai` to record one provider, `IMAGE_SCENARIOS="tool-image"` +(space-separated) to record a subset, and `OPENAI_MODEL` to change the reference model. A different gateway model +changes the cassette file names; update `GATEWAY_MODEL` and `GATEWAY_MODEL_SLUG` in `image_input_test.rs` to match. +The `tool-image` scenario uses `tool_choice: auto` so the recording proves the model chose to call the tool; if a +small model answers without calling it, validation fails and the scenario can simply be re-run. + ### Custom tool (gateway and OpenAI) This records an unformatted freeform custom tool, including the diff --git a/crates/agentic-server-core/tests/cassettes/images/inputs/green-yellow-64.png b/crates/agentic-server-core/tests/cassettes/images/inputs/green-yellow-64.png new file mode 100644 index 00000000..007f726a Binary files /dev/null and b/crates/agentic-server-core/tests/cassettes/images/inputs/green-yellow-64.png differ diff --git a/crates/agentic-server-core/tests/cassettes/images/inputs/multi-image.json b/crates/agentic-server-core/tests/cassettes/images/inputs/multi-image.json new file mode 100644 index 00000000..eb6149b8 --- /dev/null +++ b/crates/agentic-server-core/tests/cassettes/images/inputs/multi-image.json @@ -0,0 +1,30 @@ +[ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Two images follow. For each, name the color on its left half." + }, + { + "type": "input_image", + "image_url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAT0lEQVR42u3PsQkAAAzDsPz/dHpCp2wCzwalybTxvgEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA+Dq47PDiz8p6GQAAAABJRU5ErkJggg==", + "detail": "low" + }, + { + "type": "input_text", + "text": "That was the first image. Here is the second." + }, + { + "type": "input_image", + "image_url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAATUlEQVR42u3PMQ0AAAwDoPo33UnY1Y8EAyTNVMciICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIPA5X1jpWlc21NUAAAAASUVORK5CYII=", + "detail": "low" + }, + { + "type": "input_text", + "text": "Reply with exactly two words: the first image's left color, then the second image's left color." + } + ] + } +] diff --git a/crates/agentic-server-core/tests/cassettes/images/inputs/red-blue-64.png b/crates/agentic-server-core/tests/cassettes/images/inputs/red-blue-64.png new file mode 100644 index 00000000..37ebc32b Binary files /dev/null and b/crates/agentic-server-core/tests/cassettes/images/inputs/red-blue-64.png differ diff --git a/crates/agentic-server-core/tests/cassettes/images/inputs/single-image.json b/crates/agentic-server-core/tests/cassettes/images/inputs/single-image.json new file mode 100644 index 00000000..12dae163 --- /dev/null +++ b/crates/agentic-server-core/tests/cassettes/images/inputs/single-image.json @@ -0,0 +1,17 @@ +[ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Reply with exactly two words: the color on the left half of this image, then the color on the right half." + }, + { + "type": "input_image", + "image_url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAT0lEQVR42u3PsQkAAAzDsPz/dHpCp2wCzwalybTxvgEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA+Dq47PDiz8p6GQAAAABJRU5ErkJggg==", + "detail": "low" + } + ] + } +] diff --git a/crates/agentic-server-core/tests/cassettes/images/inputs/view_image_outputs.py b/crates/agentic-server-core/tests/cassettes/images/inputs/view_image_outputs.py new file mode 100644 index 00000000..59c0bc99 --- /dev/null +++ b/crates/agentic-server-core/tests/cassettes/images/inputs/view_image_outputs.py @@ -0,0 +1,20 @@ +"""Client-executed `view_image` handler for the recorder's --tool-outputs. + +Returns structured content parts, so the recorder submits a +`function_call_output` whose `output` is an array carrying the committed +red|blue PNG as an inline `input_image` -- the shape Codex uses when a tool +hands an image back to the model. +""" + +import base64 +from pathlib import Path + +_IMAGE = Path(__file__).with_name("red-blue-64.png").read_bytes() +_IMAGE_URL = "data:image/png;base64," + base64.b64encode(_IMAGE).decode("ascii") + + +def view_image(path: str = "") -> list[dict]: + return [ + {"type": "input_text", "text": f"Loaded {path or 'diagram.png'}:"}, + {"type": "input_image", "image_url": _IMAGE_URL, "detail": "low"}, + ] diff --git a/crates/agentic-server-core/tests/cassettes/images/inputs/view_image_tool.json b/crates/agentic-server-core/tests/cassettes/images/inputs/view_image_tool.json new file mode 100644 index 00000000..64e866bf --- /dev/null +++ b/crates/agentic-server-core/tests/cassettes/images/inputs/view_image_tool.json @@ -0,0 +1,21 @@ +[ + { + "type": "function", + "name": "view_image", + "description": "Load an image from the local workspace so you can look at it. Call it before describing any image the user names.", + "parameters": { + "type": "object", + "properties": { + "path": { + "type": "string", + "description": "Path of the image to load." + } + }, + "required": [ + "path" + ], + "additionalProperties": false + }, + "strict": true + } +] diff --git a/crates/agentic-server-core/tests/cassettes/images/qwen2.5-vl-hermes-tools.jinja b/crates/agentic-server-core/tests/cassettes/images/qwen2.5-vl-hermes-tools.jinja new file mode 100644 index 00000000..ad47c984 --- /dev/null +++ b/crates/agentic-server-core/tests/cassettes/images/qwen2.5-vl-hermes-tools.jinja @@ -0,0 +1,83 @@ +{#- Qwen2.5-VL chat template with Hermes-style tool calling. + + Qwen/Qwen2.5-VL-*-Instruct ships a template that renders images but has no + `tools` block, so with `tool_choice: auto` the model never sees declared + functions. This template keeps the stock multimodal rendering (any role, + including tool responses that carry an image) and adds the tools system + prompt, `` history rendering, and `` wrapping + from Qwen/Qwen2.5-*-Instruct, which vLLM's `hermes` tool-call parser + understands. Pass it to vLLM with `--chat-template`. -#} +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content) -%} + {%- if content is string -%} + {{- content }} + {%- else -%} + {%- for part in content -%} + {%- if part['type'] == 'image' or 'image' in part or 'image_url' in part -%} + {%- set image_count.value = image_count.value + 1 -%} + {%- if add_vision_id -%}Picture {{ image_count.value }}: {% endif -%} + <|vision_start|><|image_pad|><|vision_end|> + {%- elif part['type'] == 'video' or 'video' in part -%} + {%- set video_count.value = video_count.value + 1 -%} + {%- if add_vision_id -%}Video {{ video_count.value }}: {% endif -%} + <|vision_start|><|video_pad|><|vision_end|> + {%- elif 'text' in part -%} + {{- part['text'] }} + {%- endif -%} + {%- endfor -%} + {%- endif -%} +{%- endmacro -%} +{%- if messages[0]['role'] == 'system' %} + {%- set system_text = render_content(messages[0]['content']) %} +{%- else %} + {%- set system_text = 'You are a helpful assistant.' %} +{%- endif %} +{%- if tools %} + {{- '<|im_start|>system\n' + system_text }} + {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {{- '<|im_start|>system\n' + system_text + '<|im_end|>\n' }} +{%- endif %} +{%- for message in messages %} + {%- if message['role'] == 'system' and loop.first %} + {%- elif message['role'] == 'assistant' and message.tool_calls %} + {{- '<|im_start|>assistant' }} + {%- if message['content'] %} + {{- '\n' + render_content(message['content']) }} + {%- endif %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n\n{"name": ' }} + {{- tool_call.name | tojson }} + {{- ', "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {{- '<|im_end|>\n' }} + {%- elif message['role'] == 'tool' %} + {%- if loop.index0 == 0 or messages[loop.index0 - 1]['role'] != 'tool' %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' + render_content(message['content']) + '\n' }} + {%- if loop.last or messages[loop.index0 + 1]['role'] != 'tool' %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message['role'] + '\n' + render_content(message['content']) + '<|im_end|>\n' }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} +{%- endif %} diff --git a/crates/agentic-server-core/tests/cassettes/images/responses/image-continuation-gateway-Qwen-Qwen2.5-VL-3B-Instruct-nonstreaming.yaml b/crates/agentic-server-core/tests/cassettes/images/responses/image-continuation-gateway-Qwen-Qwen2.5-VL-3B-Instruct-nonstreaming.yaml new file mode 100644 index 00000000..8f925bc7 --- /dev/null +++ b/crates/agentic-server-core/tests/cassettes/images/responses/image-continuation-gateway-Qwen-Qwen2.5-VL-3B-Instruct-nonstreaming.yaml @@ -0,0 +1,106 @@ +turns: +- filename: t1 + request: + body: + input: + - content: + - text: 'Reply with exactly two words: the color on the left half of this + image, then the color on the right half.' + type: input_text + - detail: low + image_url: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAT0lEQVR42u3PsQkAAAzDsPz/dHpCp2wCzwalybTxvgEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA+Dq47PDiz8p6GQAAAABJRU5ErkJggg== + type: input_image + role: user + type: message + max_output_tokens: 64 + model: Qwen/Qwen2.5-VL-3B-Instruct + store: true + stream: false + headers: + accept: '*/*' + content-type: application/json + user-agent: python-httpx/0.28.1 + method: POST + path: /v1/responses + query_params: {} + response: + body: + conversation_id: null + created_at: 1789656353 + error: null + id: resp_01a0afd4-c939-7c81-8595-8fb4d94eab9a + incomplete_details: null + instructions: null + model: Qwen/Qwen2.5-VL-3B-Instruct + object: response + output: + - content: + - annotations: [] + text: Red, Blue + type: output_text + id: msg_bad382e40d62feca + role: assistant + status: completed + type: message + previous_response_id: null + status: completed + usage: + input_tokens: 49 + input_tokens_details: + cached_tokens: 48 + output_tokens: 4 + output_tokens_details: + reasoning_tokens: 0 + total_tokens: 53 + headers: + content-type: application/json + status_code: 200 +- filename: t2 + request: + body: + input: 'Without repeating the colors, reply with exactly one word: did my previous + message include an image? YES or NO.' + max_output_tokens: 64 + model: Qwen/Qwen2.5-VL-3B-Instruct + previous_response_id: resp_01a0afd4-c939-7c81-8595-8fb4d94eab9a + store: true + stream: false + headers: + accept: '*/*' + content-type: application/json + user-agent: python-httpx/0.28.1 + method: POST + path: /v1/responses + query_params: {} + response: + body: + conversation_id: null + created_at: 1789656353 + error: null + id: resp_01a0afd4-c9b4-7ab0-841d-da69c2ff3d4a + incomplete_details: null + instructions: null + model: Qwen/Qwen2.5-VL-3B-Instruct + object: response + output: + - content: + - annotations: [] + text: 'NO' + type: output_text + id: msg_8701f7e574d79382 + role: assistant + status: completed + type: message + previous_response_id: resp_01a0afd4-c939-7c81-8595-8fb4d94eab9a + status: completed + usage: + input_tokens: 85 + input_tokens_details: + cached_tokens: 80 + output_tokens: 2 + output_tokens_details: + reasoning_tokens: 0 + total_tokens: 87 + headers: + content-type: application/json + status_code: 200 diff --git a/crates/agentic-server-core/tests/cassettes/images/responses/image-continuation-gateway-Qwen-Qwen2.5-VL-3B-Instruct-streaming.yaml b/crates/agentic-server-core/tests/cassettes/images/responses/image-continuation-gateway-Qwen-Qwen2.5-VL-3B-Instruct-streaming.yaml new file mode 100644 index 00000000..9634d939 --- /dev/null +++ b/crates/agentic-server-core/tests/cassettes/images/responses/image-continuation-gateway-Qwen-Qwen2.5-VL-3B-Instruct-streaming.yaml @@ -0,0 +1,249 @@ +turns: +- filename: t1 + request: + body: + input: + - content: + - text: 'Reply with exactly two words: the color on the left half of this + image, then the color on the right half.' + type: input_text + - detail: low + image_url: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAT0lEQVR42u3PsQkAAAzDsPz/dHpCp2wCzwalybTxvgEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA+Dq47PDiz8p6GQAAAABJRU5ErkJggg== + type: input_image + role: user + type: message + max_output_tokens: 64 + model: Qwen/Qwen2.5-VL-3B-Instruct + store: true + stream: true + headers: + accept: '*/*' + content-type: application/json + user-agent: python-httpx/0.28.1 + method: POST + path: /v1/responses + query_params: {} + response: + headers: + content-type: text/event-stream; charset=utf-8 + sse: + - 'event: response.created + + ' + - 'data: {"type":"response.created","sequence_number":0,"response":{"id":"resp_01a0afd4-c40c-7c52-8357-c4b4b08aa4ef","created_at":1789656351,"incomplete_details":null,"instructions":null,"metadata":null,"model":"Qwen/Qwen2.5-VL-3B-Instruct","object":"response","output":[],"parallel_tool_calls":false,"temperature":0.01,"tool_choice":"none","tools":[],"top_p":1.0,"background":false,"max_output_tokens":64,"max_tool_calls":null,"previous_response_id":null,"prompt":null,"reasoning":null,"service_tier":"auto","status":"in_progress","text":null,"top_logprobs":null,"truncation":"disabled","usage":null,"user":null,"presence_penalty":0.0,"frequency_penalty":0.0,"kv_transfer_params":null,"ec_transfer_params":null,"input_messages":null,"output_messages":null}} + + ' + - ' + + ' + - 'event: response.in_progress + + ' + - 'data: {"type":"response.in_progress","sequence_number":1,"response":{"id":"resp_01a0afd4-c40c-7c52-8357-c4b4b08aa4ef","created_at":1789656351,"incomplete_details":null,"instructions":null,"metadata":null,"model":"Qwen/Qwen2.5-VL-3B-Instruct","object":"response","output":[],"parallel_tool_calls":false,"temperature":0.01,"tool_choice":"none","tools":[],"top_p":1.0,"background":false,"max_output_tokens":64,"max_tool_calls":null,"previous_response_id":null,"prompt":null,"reasoning":null,"service_tier":"auto","status":"in_progress","text":null,"top_logprobs":null,"truncation":"disabled","usage":null,"user":null,"presence_penalty":0.0,"frequency_penalty":0.0,"kv_transfer_params":null,"ec_transfer_params":null,"input_messages":null,"output_messages":null}} + + ' + - ' + + ' + - 'event: response.output_item.added + + ' + - 'data: {"type":"response.output_item.added","sequence_number":2,"output_index":0,"item":{"id":"9c789b2ec3cb931a","content":[],"role":"assistant","status":"in_progress","type":"message","phase":null}} + + ' + - ' + + ' + - 'event: response.content_part.added + + ' + - 'data: {"type":"response.content_part.added","sequence_number":3,"output_index":0,"content_index":0,"item_id":"9c789b2ec3cb931a","part":{"annotations":[],"text":"","type":"output_text","logprobs":[]}} + + ' + - ' + + ' + - 'event: response.output_text.delta + + ' + - 'data: {"type":"response.output_text.delta","sequence_number":4,"output_index":0,"content_index":0,"delta":"Red","item_id":"9c789b2ec3cb931a","logprobs":[]} + + ' + - ' + + ' + - 'event: response.output_text.delta + + ' + - 'data: {"type":"response.output_text.delta","sequence_number":5,"output_index":0,"content_index":0,"delta":",","item_id":"9c789b2ec3cb931a","logprobs":[]} + + ' + - ' + + ' + - 'event: response.output_text.delta + + ' + - 'data: {"type":"response.output_text.delta","sequence_number":6,"output_index":0,"content_index":0,"delta":" + Blue","item_id":"9c789b2ec3cb931a","logprobs":[]} + + ' + - ' + + ' + - 'event: response.output_text.done + + ' + - 'data: {"type":"response.output_text.done","sequence_number":7,"output_index":0,"content_index":0,"item_id":"9c789b2ec3cb931a","logprobs":[],"text":"Red, + Blue"} + + ' + - ' + + ' + - 'event: response.content_part.done + + ' + - 'data: {"type":"response.content_part.done","sequence_number":8,"output_index":0,"content_index":0,"item_id":"9c789b2ec3cb931a","part":{"annotations":[],"text":"Red, + Blue","type":"output_text","logprobs":null}} + + ' + - ' + + ' + - 'event: response.output_item.done + + ' + - 'data: {"type":"response.output_item.done","sequence_number":9,"output_index":0,"item":{"id":"9c789b2ec3cb931a","content":[{"annotations":[],"text":"Red, + Blue","type":"output_text","logprobs":null}],"role":"assistant","status":"completed","type":"message","phase":null,"summary":[]}} + + ' + - ' + + ' + - 'event: response.completed + + ' + - 'data: {"type":"response.completed","sequence_number":10,"response":{"id":"resp_01a0afd4-c40c-7c52-8357-c4b4b08aa4ef","object":"response","created_at":1789656351,"model":"Qwen/Qwen2.5-VL-3B-Instruct","status":"completed","output":[{"type":"message","id":"9c789b2ec3cb931a","role":"assistant","status":"completed","content":[{"type":"output_text","text":"Red, + Blue","annotations":[]}]}],"usage":{"input_tokens":49,"output_tokens":4,"total_tokens":53,"input_tokens_details":{"cached_tokens":48},"output_tokens_details":{"reasoning_tokens":0}},"incomplete_details":null,"error":null,"previous_response_id":null,"conversation_id":null,"instructions":null}} + + ' + - ' + + ' + - 'data: [DONE] + + ' + - ' + + ' + status_code: 200 +- filename: t2 + request: + body: + input: 'Without repeating the colors, reply with exactly one word: did my previous + message include an image? YES or NO.' + max_output_tokens: 64 + model: Qwen/Qwen2.5-VL-3B-Instruct + previous_response_id: resp_01a0afd4-c40c-7c52-8357-c4b4b08aa4ef + store: true + stream: true + headers: + accept: '*/*' + content-type: application/json + user-agent: python-httpx/0.28.1 + method: POST + path: /v1/responses + query_params: {} + response: + headers: + content-type: text/event-stream; charset=utf-8 + sse: + - 'event: response.created + + ' + - 'data: {"type":"response.created","sequence_number":0,"response":{"id":"resp_01a0afd4-c487-7c82-b0c7-d084da41ac2f","created_at":1789656351,"incomplete_details":null,"instructions":null,"metadata":null,"model":"Qwen/Qwen2.5-VL-3B-Instruct","object":"response","output":[],"parallel_tool_calls":false,"temperature":0.01,"tool_choice":"none","tools":[],"top_p":1.0,"background":false,"max_output_tokens":64,"max_tool_calls":null,"previous_response_id":"resp_01a0afd4-c40c-7c52-8357-c4b4b08aa4ef","prompt":null,"reasoning":null,"service_tier":"auto","status":"in_progress","text":null,"top_logprobs":null,"truncation":"disabled","usage":null,"user":null,"presence_penalty":0.0,"frequency_penalty":0.0,"kv_transfer_params":null,"ec_transfer_params":null,"input_messages":null,"output_messages":null}} + + ' + - ' + + ' + - 'event: response.in_progress + + ' + - 'data: {"type":"response.in_progress","sequence_number":1,"response":{"id":"resp_01a0afd4-c487-7c82-b0c7-d084da41ac2f","created_at":1789656351,"incomplete_details":null,"instructions":null,"metadata":null,"model":"Qwen/Qwen2.5-VL-3B-Instruct","object":"response","output":[],"parallel_tool_calls":false,"temperature":0.01,"tool_choice":"none","tools":[],"top_p":1.0,"background":false,"max_output_tokens":64,"max_tool_calls":null,"previous_response_id":"resp_01a0afd4-c40c-7c52-8357-c4b4b08aa4ef","prompt":null,"reasoning":null,"service_tier":"auto","status":"in_progress","text":null,"top_logprobs":null,"truncation":"disabled","usage":null,"user":null,"presence_penalty":0.0,"frequency_penalty":0.0,"kv_transfer_params":null,"ec_transfer_params":null,"input_messages":null,"output_messages":null}} + + ' + - ' + + ' + - 'event: response.output_item.added + + ' + - 'data: {"type":"response.output_item.added","sequence_number":2,"output_index":0,"item":{"id":"8cfa869dcbb87d3c","content":[],"role":"assistant","status":"in_progress","type":"message","phase":null}} + + ' + - ' + + ' + - 'event: response.content_part.added + + ' + - 'data: {"type":"response.content_part.added","sequence_number":3,"output_index":0,"content_index":0,"item_id":"8cfa869dcbb87d3c","part":{"annotations":[],"text":"","type":"output_text","logprobs":[]}} + + ' + - ' + + ' + - 'event: response.output_text.delta + + ' + - 'data: {"type":"response.output_text.delta","sequence_number":4,"output_index":0,"content_index":0,"delta":"NO","item_id":"8cfa869dcbb87d3c","logprobs":[]} + + ' + - ' + + ' + - 'event: response.output_text.done + + ' + - 'data: {"type":"response.output_text.done","sequence_number":5,"output_index":0,"content_index":0,"item_id":"8cfa869dcbb87d3c","logprobs":[],"text":"NO"} + + ' + - ' + + ' + - 'event: response.content_part.done + + ' + - 'data: {"type":"response.content_part.done","sequence_number":6,"output_index":0,"content_index":0,"item_id":"8cfa869dcbb87d3c","part":{"annotations":[],"text":"NO","type":"output_text","logprobs":null}} + + ' + - ' + + ' + - 'event: response.output_item.done + + ' + - 'data: {"type":"response.output_item.done","sequence_number":7,"output_index":0,"item":{"id":"8cfa869dcbb87d3c","content":[{"annotations":[],"text":"NO","type":"output_text","logprobs":null}],"role":"assistant","status":"completed","type":"message","phase":null,"summary":[]}} + + ' + - ' + + ' + - 'event: response.completed + + ' + - 'data: {"type":"response.completed","sequence_number":8,"response":{"id":"resp_01a0afd4-c487-7c82-b0c7-d084da41ac2f","object":"response","created_at":1789656351,"model":"Qwen/Qwen2.5-VL-3B-Instruct","status":"completed","output":[{"type":"message","id":"8cfa869dcbb87d3c","role":"assistant","status":"completed","content":[{"type":"output_text","text":"NO","annotations":[]}]}],"usage":{"input_tokens":85,"output_tokens":2,"total_tokens":87,"input_tokens_details":{"cached_tokens":48},"output_tokens_details":{"reasoning_tokens":0}},"incomplete_details":null,"error":null,"previous_response_id":"resp_01a0afd4-c40c-7c52-8357-c4b4b08aa4ef","conversation_id":null,"instructions":null}} + + ' + - ' + + ' + - 'data: [DONE] + + ' + - ' + + ' + status_code: 200 diff --git a/crates/agentic-server-core/tests/cassettes/images/responses/image-continuation-openai-gpt-4o-nonstreaming.yaml b/crates/agentic-server-core/tests/cassettes/images/responses/image-continuation-openai-gpt-4o-nonstreaming.yaml new file mode 100644 index 00000000..ef25128d --- /dev/null +++ b/crates/agentic-server-core/tests/cassettes/images/responses/image-continuation-openai-gpt-4o-nonstreaming.yaml @@ -0,0 +1,198 @@ +turns: +- filename: t1 + request: + body: + input: + - content: + - text: 'Reply with exactly two words: the color on the left half of this + image, then the color on the right half.' + type: input_text + - detail: low + image_url: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAT0lEQVR42u3PsQkAAAzDsPz/dHpCp2wCzwalybTxvgEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA+Dq47PDiz8p6GQAAAABJRU5ErkJggg== + type: input_image + role: user + type: message + max_output_tokens: 64 + model: gpt-4o + store: true + stream: false + headers: + accept: '*/*' + authorization: Bearer *** + content-type: application/json + user-agent: python-httpx/0.28.1 + method: POST + path: /v1/responses + query_params: {} + response: + body: + background: false + billing: + payer: developer + completed_at: 1789656303 + created_at: 1789656301 + error: null + frequency_penalty: 0.0 + id: resp_0a456069ca488b75006aabfced8c3087d28a88824284870b87 + incomplete_details: null + instructions: null + max_output_tokens: 64 + max_tool_calls: null + metadata: {} + model: gpt-4o-2024-08-06 + moderation: null + object: response + output: + - content: + - annotations: [] + logprobs: [] + text: Red blue + type: output_text + id: msg_0a456069ca488b75006aabfcef64f487d2adcc51d19cb6303c + role: assistant + status: completed + type: message + parallel_tool_calls: true + presence_penalty: 0.0 + previous_response_id: null + prompt_cache_key: null + prompt_cache_retention: in_memory + reasoning: + context: null + effort: null + summary: null + safety_identifier: null + service_tier: default + status: completed + store: true + temperature: 1.0 + text: + format: + type: text + verbosity: medium + tool_choice: auto + tool_usage: + image_gen: + input_tokens: 0 + input_tokens_details: + image_tokens: 0 + text_tokens: 0 + output_tokens: 0 + output_tokens_details: + image_tokens: 0 + text_tokens: 0 + total_tokens: 0 + web_search: + num_requests: 0 + tools: [] + top_logprobs: 0 + top_p: 1.0 + truncation: disabled + usage: + input_tokens: 116 + input_tokens_details: + cache_write_tokens: 0 + cached_tokens: 0 + output_tokens: 3 + output_tokens_details: + reasoning_tokens: 0 + total_tokens: 119 + user: null + headers: + content-type: application/json + status_code: 200 +- filename: t2 + request: + body: + input: 'Without repeating the colors, reply with exactly one word: did my previous + message include an image? YES or NO.' + max_output_tokens: 64 + model: gpt-4o + previous_response_id: resp_0a456069ca488b75006aabfced8c3087d28a88824284870b87 + store: true + stream: false + headers: + accept: '*/*' + authorization: Bearer *** + content-type: application/json + user-agent: python-httpx/0.28.1 + method: POST + path: /v1/responses + query_params: {} + response: + body: + background: false + billing: + payer: developer + completed_at: 1789656305 + created_at: 1789656303 + error: null + frequency_penalty: 0.0 + id: resp_0a456069ca488b75006aabfcefcefc87d2a6b2ea02c4443774 + incomplete_details: null + instructions: null + max_output_tokens: 64 + max_tool_calls: null + metadata: {} + model: gpt-4o-2024-08-06 + moderation: null + object: response + output: + - content: + - annotations: [] + logprobs: [] + text: 'YES' + type: output_text + id: msg_0a456069ca488b75006aabfcf126d887d2a63e3d0256ad48c9 + role: assistant + status: completed + type: message + parallel_tool_calls: true + presence_penalty: 0.0 + previous_response_id: resp_0a456069ca488b75006aabfced8c3087d28a88824284870b87 + prompt_cache_key: null + prompt_cache_retention: in_memory + reasoning: + context: null + effort: null + summary: null + safety_identifier: null + service_tier: default + status: completed + store: true + temperature: 1.0 + text: + format: + type: text + verbosity: medium + tool_choice: auto + tool_usage: + image_gen: + input_tokens: 0 + input_tokens_details: + image_tokens: 0 + text_tokens: 0 + output_tokens: 0 + output_tokens_details: + image_tokens: 0 + text_tokens: 0 + total_tokens: 0 + web_search: + num_requests: 0 + tools: [] + top_logprobs: 0 + top_p: 1.0 + truncation: disabled + usage: + input_tokens: 149 + input_tokens_details: + cache_write_tokens: 0 + cached_tokens: 0 + output_tokens: 2 + output_tokens_details: + reasoning_tokens: 0 + total_tokens: 151 + user: null + headers: + content-type: application/json + status_code: 200 diff --git a/crates/agentic-server-core/tests/cassettes/images/responses/image-continuation-openai-gpt-4o-streaming.yaml b/crates/agentic-server-core/tests/cassettes/images/responses/image-continuation-openai-gpt-4o-streaming.yaml new file mode 100644 index 00000000..2a305e83 --- /dev/null +++ b/crates/agentic-server-core/tests/cassettes/images/responses/image-continuation-openai-gpt-4o-streaming.yaml @@ -0,0 +1,229 @@ +turns: +- filename: t1 + request: + body: + input: + - content: + - text: 'Reply with exactly two words: the color on the left half of this + image, then the color on the right half.' + type: input_text + - detail: low + image_url: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAT0lEQVR42u3PsQkAAAzDsPz/dHpCp2wCzwalybTxvgEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA+Dq47PDiz8p6GQAAAABJRU5ErkJggg== + type: input_image + role: user + type: message + max_output_tokens: 64 + model: gpt-4o + store: true + stream: true + headers: + accept: '*/*' + authorization: Bearer *** + content-type: application/json + user-agent: python-httpx/0.28.1 + method: POST + path: /v1/responses + query_params: {} + response: + headers: + content-type: text/event-stream; charset=utf-8 + sse: + - 'event: response.created + + ' + - 'data: {"type":"response.created","response":{"id":"resp_02f783bc2b00e7ec006aabfce78a2887d2b805d46717eac87a","object":"response","created_at":1789656295,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":64,"max_tool_calls":null,"model":"gpt-4o-2024-08-06","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} + + ' + - ' + + ' + - 'event: response.in_progress + + ' + - 'data: {"type":"response.in_progress","response":{"id":"resp_02f783bc2b00e7ec006aabfce78a2887d2b805d46717eac87a","object":"response","created_at":1789656295,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":64,"max_tool_calls":null,"model":"gpt-4o-2024-08-06","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} + + ' + - ' + + ' + - 'event: response.output_item.added + + ' + - 'data: {"type":"response.output_item.added","item":{"id":"msg_02f783bc2b00e7ec006aabfce91f8887d2ac8d0b57698b3139","type":"message","status":"in_progress","content":[],"role":"assistant"},"output_index":0,"sequence_number":2} + + ' + - ' + + ' + - 'event: response.content_part.added + + ' + - 'data: {"type":"response.content_part.added","content_index":0,"item_id":"msg_02f783bc2b00e7ec006aabfce91f8887d2ac8d0b57698b3139","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":""},"sequence_number":3} + + ' + - ' + + ' + - 'event: response.output_text.delta + + ' + - 'data: {"type":"response.output_text.delta","content_index":0,"delta":"Red","item_id":"msg_02f783bc2b00e7ec006aabfce91f8887d2ac8d0b57698b3139","logprobs":[],"obfuscation":"4m1nEabJoVSsv","output_index":0,"sequence_number":4} + + ' + - ' + + ' + - 'event: response.output_text.delta + + ' + - 'data: {"type":"response.output_text.delta","content_index":0,"delta":" blue","item_id":"msg_02f783bc2b00e7ec006aabfce91f8887d2ac8d0b57698b3139","logprobs":[],"obfuscation":"ufoLCwWQSiM","output_index":0,"sequence_number":5} + + ' + - ' + + ' + - 'event: response.output_text.done + + ' + - 'data: {"type":"response.output_text.done","content_index":0,"item_id":"msg_02f783bc2b00e7ec006aabfce91f8887d2ac8d0b57698b3139","logprobs":[],"output_index":0,"sequence_number":6,"text":"Red + blue"} + + ' + - ' + + ' + - 'event: response.content_part.done + + ' + - 'data: {"type":"response.content_part.done","content_index":0,"item_id":"msg_02f783bc2b00e7ec006aabfce91f8887d2ac8d0b57698b3139","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":"Red + blue"},"sequence_number":7} + + ' + - ' + + ' + - 'event: response.output_item.done + + ' + - 'data: {"type":"response.output_item.done","item":{"id":"msg_02f783bc2b00e7ec006aabfce91f8887d2ac8d0b57698b3139","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"Red + blue"}],"role":"assistant"},"output_index":0,"sequence_number":8} + + ' + - ' + + ' + - 'event: response.completed + + ' + - 'data: {"type":"response.completed","response":{"id":"resp_02f783bc2b00e7ec006aabfce78a2887d2b805d46717eac87a","object":"response","created_at":1789656295,"status":"completed","background":false,"completed_at":1789656297,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":64,"max_tool_calls":null,"model":"gpt-4o-2024-08-06","moderation":null,"output":[{"id":"msg_02f783bc2b00e7ec006aabfce91f8887d2ac8d0b57698b3139","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"Red + blue"}],"role":"assistant"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"default","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":116,"input_tokens_details":{"cache_write_tokens":0,"cached_tokens":0},"output_tokens":3,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":119},"user":null,"metadata":{}},"sequence_number":9} + + ' + - ' + + ' + status_code: 200 +- filename: t2 + request: + body: + input: 'Without repeating the colors, reply with exactly one word: did my previous + message include an image? YES or NO.' + max_output_tokens: 64 + model: gpt-4o + previous_response_id: resp_02f783bc2b00e7ec006aabfce78a2887d2b805d46717eac87a + store: true + stream: true + headers: + accept: '*/*' + authorization: Bearer *** + content-type: application/json + user-agent: python-httpx/0.28.1 + method: POST + path: /v1/responses + query_params: {} + response: + headers: + content-type: text/event-stream; charset=utf-8 + sse: + - 'event: response.created + + ' + - 'data: {"type":"response.created","response":{"id":"resp_02f783bc2b00e7ec006aabfce98cd887d2a8ad5aa37c1aa1c4","object":"response","created_at":1789656297,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":64,"max_tool_calls":null,"model":"gpt-4o-2024-08-06","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":"resp_02f783bc2b00e7ec006aabfce78a2887d2b805d46717eac87a","prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} + + ' + - ' + + ' + - 'event: response.in_progress + + ' + - 'data: {"type":"response.in_progress","response":{"id":"resp_02f783bc2b00e7ec006aabfce98cd887d2a8ad5aa37c1aa1c4","object":"response","created_at":1789656297,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":64,"max_tool_calls":null,"model":"gpt-4o-2024-08-06","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":"resp_02f783bc2b00e7ec006aabfce78a2887d2b805d46717eac87a","prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} + + ' + - ' + + ' + - 'event: response.output_item.added + + ' + - 'data: {"type":"response.output_item.added","item":{"id":"msg_02f783bc2b00e7ec006aabfcebaeac87d2a3ca1143d81e5f37","type":"message","status":"in_progress","content":[],"role":"assistant"},"output_index":0,"sequence_number":2} + + ' + - ' + + ' + - 'event: response.content_part.added + + ' + - 'data: {"type":"response.content_part.added","content_index":0,"item_id":"msg_02f783bc2b00e7ec006aabfcebaeac87d2a3ca1143d81e5f37","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":""},"sequence_number":3} + + ' + - ' + + ' + - 'event: response.output_text.delta + + ' + - 'data: {"type":"response.output_text.delta","content_index":0,"delta":"Yes","item_id":"msg_02f783bc2b00e7ec006aabfcebaeac87d2a3ca1143d81e5f37","logprobs":[],"obfuscation":"nz976XSTSfXCI","output_index":0,"sequence_number":4} + + ' + - ' + + ' + - 'event: response.output_text.done + + ' + - 'data: {"type":"response.output_text.done","content_index":0,"item_id":"msg_02f783bc2b00e7ec006aabfcebaeac87d2a3ca1143d81e5f37","logprobs":[],"output_index":0,"sequence_number":5,"text":"Yes"} + + ' + - ' + + ' + - 'event: response.content_part.done + + ' + - 'data: {"type":"response.content_part.done","content_index":0,"item_id":"msg_02f783bc2b00e7ec006aabfcebaeac87d2a3ca1143d81e5f37","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":"Yes"},"sequence_number":6} + + ' + - ' + + ' + - 'event: response.output_item.done + + ' + - 'data: {"type":"response.output_item.done","item":{"id":"msg_02f783bc2b00e7ec006aabfcebaeac87d2a3ca1143d81e5f37","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"Yes"}],"role":"assistant"},"output_index":0,"sequence_number":7} + + ' + - ' + + ' + - 'event: response.completed + + ' + - 'data: {"type":"response.completed","response":{"id":"resp_02f783bc2b00e7ec006aabfce98cd887d2a8ad5aa37c1aa1c4","object":"response","created_at":1789656297,"status":"completed","background":false,"completed_at":1789656299,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":64,"max_tool_calls":null,"model":"gpt-4o-2024-08-06","moderation":null,"output":[{"id":"msg_02f783bc2b00e7ec006aabfcebaeac87d2a3ca1143d81e5f37","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"Yes"}],"role":"assistant"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":"resp_02f783bc2b00e7ec006aabfce78a2887d2b805d46717eac87a","prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"default","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":149,"input_tokens_details":{"cache_write_tokens":0,"cached_tokens":0},"output_tokens":2,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":151},"user":null,"metadata":{}},"sequence_number":8} + + ' + - ' + + ' + status_code: 200 diff --git a/crates/agentic-server-core/tests/cassettes/images/responses/image-multi-image-gateway-Qwen-Qwen2.5-VL-3B-Instruct-nonstreaming.yaml b/crates/agentic-server-core/tests/cassettes/images/responses/image-multi-image-gateway-Qwen-Qwen2.5-VL-3B-Instruct-nonstreaming.yaml new file mode 100644 index 00000000..283eb67f --- /dev/null +++ b/crates/agentic-server-core/tests/cassettes/images/responses/image-multi-image-gateway-Qwen-Qwen2.5-VL-3B-Instruct-nonstreaming.yaml @@ -0,0 +1,64 @@ +turns: +- filename: t1 + request: + body: + input: + - content: + - text: Two images follow. For each, name the color on its left half. + type: input_text + - detail: low + image_url: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAT0lEQVR42u3PsQkAAAzDsPz/dHpCp2wCzwalybTxvgEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA+Dq47PDiz8p6GQAAAABJRU5ErkJggg== + type: input_image + - text: That was the first image. Here is the second. + type: input_text + - detail: low + image_url: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAATUlEQVR42u3PMQ0AAAwDoPo33UnY1Y8EAyTNVMciICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIPA5X1jpWlc21NUAAAAASUVORK5CYII= + type: input_image + - text: 'Reply with exactly two words: the first image''s left color, then + the second image''s left color.' + type: input_text + role: user + type: message + max_output_tokens: 64 + model: Qwen/Qwen2.5-VL-3B-Instruct + store: true + stream: false + headers: + accept: '*/*' + content-type: application/json + user-agent: python-httpx/0.28.1 + method: POST + path: /v1/responses + query_params: {} + response: + body: + conversation_id: null + created_at: 1789656350 + error: null + id: resp_01a0afd4-bf4a-7e73-9ddc-e416a4a13e9d + incomplete_details: null + instructions: null + model: Qwen/Qwen2.5-VL-3B-Instruct + object: response + output: + - content: + - annotations: [] + text: Red, Green + type: output_text + id: msg_8654e3545428658d + role: assistant + status: completed + type: message + previous_response_id: null + status: completed + usage: + input_tokens: 78 + input_tokens_details: + cached_tokens: 64 + output_tokens: 4 + output_tokens_details: + reasoning_tokens: 0 + total_tokens: 82 + headers: + content-type: application/json + status_code: 200 diff --git a/crates/agentic-server-core/tests/cassettes/images/responses/image-multi-image-gateway-Qwen-Qwen2.5-VL-3B-Instruct-streaming.yaml b/crates/agentic-server-core/tests/cassettes/images/responses/image-multi-image-gateway-Qwen-Qwen2.5-VL-3B-Instruct-streaming.yaml new file mode 100644 index 00000000..9b9bd849 --- /dev/null +++ b/crates/agentic-server-core/tests/cassettes/images/responses/image-multi-image-gateway-Qwen-Qwen2.5-VL-3B-Instruct-streaming.yaml @@ -0,0 +1,147 @@ +turns: +- filename: t1 + request: + body: + input: + - content: + - text: Two images follow. For each, name the color on its left half. + type: input_text + - detail: low + image_url: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAT0lEQVR42u3PsQkAAAzDsPz/dHpCp2wCzwalybTxvgEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA+Dq47PDiz8p6GQAAAABJRU5ErkJggg== + type: input_image + - text: That was the first image. Here is the second. + type: input_text + - detail: low + image_url: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAATUlEQVR42u3PMQ0AAAwDoPo33UnY1Y8EAyTNVMciICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIPA5X1jpWlc21NUAAAAASUVORK5CYII= + type: input_image + - text: 'Reply with exactly two words: the first image''s left color, then + the second image''s left color.' + type: input_text + role: user + type: message + max_output_tokens: 64 + model: Qwen/Qwen2.5-VL-3B-Instruct + store: true + stream: true + headers: + accept: '*/*' + content-type: application/json + user-agent: python-httpx/0.28.1 + method: POST + path: /v1/responses + query_params: {} + response: + headers: + content-type: text/event-stream; charset=utf-8 + sse: + - 'event: response.created + + ' + - 'data: {"type":"response.created","sequence_number":0,"response":{"id":"resp_01a0afd4-ba71-7582-bc82-c3bfaa7eaadf","created_at":1789656349,"incomplete_details":null,"instructions":null,"metadata":null,"model":"Qwen/Qwen2.5-VL-3B-Instruct","object":"response","output":[],"parallel_tool_calls":false,"temperature":0.01,"tool_choice":"none","tools":[],"top_p":1.0,"background":false,"max_output_tokens":64,"max_tool_calls":null,"previous_response_id":null,"prompt":null,"reasoning":null,"service_tier":"auto","status":"in_progress","text":null,"top_logprobs":null,"truncation":"disabled","usage":null,"user":null,"presence_penalty":0.0,"frequency_penalty":0.0,"kv_transfer_params":null,"ec_transfer_params":null,"input_messages":null,"output_messages":null}} + + ' + - ' + + ' + - 'event: response.in_progress + + ' + - 'data: {"type":"response.in_progress","sequence_number":1,"response":{"id":"resp_01a0afd4-ba71-7582-bc82-c3bfaa7eaadf","created_at":1789656349,"incomplete_details":null,"instructions":null,"metadata":null,"model":"Qwen/Qwen2.5-VL-3B-Instruct","object":"response","output":[],"parallel_tool_calls":false,"temperature":0.01,"tool_choice":"none","tools":[],"top_p":1.0,"background":false,"max_output_tokens":64,"max_tool_calls":null,"previous_response_id":null,"prompt":null,"reasoning":null,"service_tier":"auto","status":"in_progress","text":null,"top_logprobs":null,"truncation":"disabled","usage":null,"user":null,"presence_penalty":0.0,"frequency_penalty":0.0,"kv_transfer_params":null,"ec_transfer_params":null,"input_messages":null,"output_messages":null}} + + ' + - ' + + ' + - 'event: response.output_item.added + + ' + - 'data: {"type":"response.output_item.added","sequence_number":2,"output_index":0,"item":{"id":"a86fff7e5612abab","content":[],"role":"assistant","status":"in_progress","type":"message","phase":null}} + + ' + - ' + + ' + - 'event: response.content_part.added + + ' + - 'data: {"type":"response.content_part.added","sequence_number":3,"output_index":0,"content_index":0,"item_id":"a86fff7e5612abab","part":{"annotations":[],"text":"","type":"output_text","logprobs":[]}} + + ' + - ' + + ' + - 'event: response.output_text.delta + + ' + - 'data: {"type":"response.output_text.delta","sequence_number":4,"output_index":0,"content_index":0,"delta":"Red","item_id":"a86fff7e5612abab","logprobs":[]} + + ' + - ' + + ' + - 'event: response.output_text.delta + + ' + - 'data: {"type":"response.output_text.delta","sequence_number":5,"output_index":0,"content_index":0,"delta":",","item_id":"a86fff7e5612abab","logprobs":[]} + + ' + - ' + + ' + - 'event: response.output_text.delta + + ' + - 'data: {"type":"response.output_text.delta","sequence_number":6,"output_index":0,"content_index":0,"delta":" + Green","item_id":"a86fff7e5612abab","logprobs":[]} + + ' + - ' + + ' + - 'event: response.output_text.done + + ' + - 'data: {"type":"response.output_text.done","sequence_number":7,"output_index":0,"content_index":0,"item_id":"a86fff7e5612abab","logprobs":[],"text":"Red, + Green"} + + ' + - ' + + ' + - 'event: response.content_part.done + + ' + - 'data: {"type":"response.content_part.done","sequence_number":8,"output_index":0,"content_index":0,"item_id":"a86fff7e5612abab","part":{"annotations":[],"text":"Red, + Green","type":"output_text","logprobs":null}} + + ' + - ' + + ' + - 'event: response.output_item.done + + ' + - 'data: {"type":"response.output_item.done","sequence_number":9,"output_index":0,"item":{"id":"a86fff7e5612abab","content":[{"annotations":[],"text":"Red, + Green","type":"output_text","logprobs":null}],"role":"assistant","status":"completed","type":"message","phase":null,"summary":[]}} + + ' + - ' + + ' + - 'event: response.completed + + ' + - 'data: {"type":"response.completed","sequence_number":10,"response":{"id":"resp_01a0afd4-ba71-7582-bc82-c3bfaa7eaadf","object":"response","created_at":1789656349,"model":"Qwen/Qwen2.5-VL-3B-Instruct","status":"completed","output":[{"type":"message","id":"a86fff7e5612abab","role":"assistant","status":"completed","content":[{"type":"output_text","text":"Red, + Green","annotations":[]}]}],"usage":{"input_tokens":78,"output_tokens":4,"total_tokens":82,"input_tokens_details":{"cached_tokens":0},"output_tokens_details":{"reasoning_tokens":0}},"incomplete_details":null,"error":null,"previous_response_id":null,"conversation_id":null,"instructions":null}} + + ' + - ' + + ' + - 'data: [DONE] + + ' + - ' + + ' + status_code: 200 diff --git a/crates/agentic-server-core/tests/cassettes/images/responses/image-multi-image-openai-gpt-4o-nonstreaming.yaml b/crates/agentic-server-core/tests/cassettes/images/responses/image-multi-image-openai-gpt-4o-nonstreaming.yaml new file mode 100644 index 00000000..44ceeffb --- /dev/null +++ b/crates/agentic-server-core/tests/cassettes/images/responses/image-multi-image-openai-gpt-4o-nonstreaming.yaml @@ -0,0 +1,110 @@ +turns: +- filename: t1 + request: + body: + input: + - content: + - text: Two images follow. For each, name the color on its left half. + type: input_text + - detail: low + image_url: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAT0lEQVR42u3PsQkAAAzDsPz/dHpCp2wCzwalybTxvgEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA+Dq47PDiz8p6GQAAAABJRU5ErkJggg== + type: input_image + - text: That was the first image. Here is the second. + type: input_text + - detail: low + image_url: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAATUlEQVR42u3PMQ0AAAwDoPo33UnY1Y8EAyTNVMciICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIPA5X1jpWlc21NUAAAAASUVORK5CYII= + type: input_image + - text: 'Reply with exactly two words: the first image''s left color, then + the second image''s left color.' + type: input_text + role: user + type: message + max_output_tokens: 64 + model: gpt-4o + store: true + stream: false + headers: + accept: '*/*' + authorization: Bearer *** + content-type: application/json + user-agent: python-httpx/0.28.1 + method: POST + path: /v1/responses + query_params: {} + response: + body: + background: false + billing: + payer: developer + completed_at: 1789656293 + created_at: 1789656293 + error: null + frequency_penalty: 0.0 + id: resp_093accbe17fa585d006aabfce508d887d29f3ed6e8676140a3 + incomplete_details: null + instructions: null + max_output_tokens: 64 + max_tool_calls: null + metadata: {} + model: gpt-4o-2024-08-06 + moderation: null + object: response + output: + - content: + - annotations: [] + logprobs: [] + text: Red Green + type: output_text + id: msg_093accbe17fa585d006aabfce5a0ec87d2a60f4ce73382a8a1 + role: assistant + status: completed + type: message + parallel_tool_calls: true + presence_penalty: 0.0 + previous_response_id: null + prompt_cache_key: null + prompt_cache_retention: in_memory + reasoning: + context: null + effort: null + summary: null + safety_identifier: null + service_tier: default + status: completed + store: true + temperature: 1.0 + text: + format: + type: text + verbosity: medium + tool_choice: auto + tool_usage: + image_gen: + input_tokens: 0 + input_tokens_details: + image_tokens: 0 + text_tokens: 0 + output_tokens: 0 + output_tokens_details: + image_tokens: 0 + text_tokens: 0 + total_tokens: 0 + web_search: + num_requests: 0 + tools: [] + top_logprobs: 0 + top_p: 1.0 + truncation: disabled + usage: + input_tokens: 224 + input_tokens_details: + cache_write_tokens: 0 + cached_tokens: 0 + output_tokens: 3 + output_tokens_details: + reasoning_tokens: 0 + total_tokens: 227 + user: null + headers: + content-type: application/json + status_code: 200 diff --git a/crates/agentic-server-core/tests/cassettes/images/responses/image-multi-image-openai-gpt-4o-streaming.yaml b/crates/agentic-server-core/tests/cassettes/images/responses/image-multi-image-openai-gpt-4o-streaming.yaml new file mode 100644 index 00000000..e353e9a2 --- /dev/null +++ b/crates/agentic-server-core/tests/cassettes/images/responses/image-multi-image-openai-gpt-4o-streaming.yaml @@ -0,0 +1,132 @@ +turns: +- filename: t1 + request: + body: + input: + - content: + - text: Two images follow. For each, name the color on its left half. + type: input_text + - detail: low + image_url: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAT0lEQVR42u3PsQkAAAzDsPz/dHpCp2wCzwalybTxvgEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA+Dq47PDiz8p6GQAAAABJRU5ErkJggg== + type: input_image + - text: That was the first image. Here is the second. + type: input_text + - detail: low + image_url: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAATUlEQVR42u3PMQ0AAAwDoPo33UnY1Y8EAyTNVMciICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIPA5X1jpWlc21NUAAAAASUVORK5CYII= + type: input_image + - text: 'Reply with exactly two words: the first image''s left color, then + the second image''s left color.' + type: input_text + role: user + type: message + max_output_tokens: 64 + model: gpt-4o + store: true + stream: true + headers: + accept: '*/*' + authorization: Bearer *** + content-type: application/json + user-agent: python-httpx/0.28.1 + method: POST + path: /v1/responses + query_params: {} + response: + headers: + content-type: text/event-stream; charset=utf-8 + sse: + - 'event: response.created + + ' + - 'data: {"type":"response.created","response":{"id":"resp_0876791aeda66de7006aabfcdf8f5087d2abee282fc8ab6e67","object":"response","created_at":1789656287,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":64,"max_tool_calls":null,"model":"gpt-4o-2024-08-06","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} + + ' + - ' + + ' + - 'event: response.in_progress + + ' + - 'data: {"type":"response.in_progress","response":{"id":"resp_0876791aeda66de7006aabfcdf8f5087d2abee282fc8ab6e67","object":"response","created_at":1789656287,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":64,"max_tool_calls":null,"model":"gpt-4o-2024-08-06","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} + + ' + - ' + + ' + - 'event: response.output_item.added + + ' + - 'data: {"type":"response.output_item.added","item":{"id":"msg_0876791aeda66de7006aabfce2e79087d29beef21fab15e3f5","type":"message","status":"in_progress","content":[],"role":"assistant"},"output_index":0,"sequence_number":2} + + ' + - ' + + ' + - 'event: response.content_part.added + + ' + - 'data: {"type":"response.content_part.added","content_index":0,"item_id":"msg_0876791aeda66de7006aabfce2e79087d29beef21fab15e3f5","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":""},"sequence_number":3} + + ' + - ' + + ' + - 'event: response.output_text.delta + + ' + - 'data: {"type":"response.output_text.delta","content_index":0,"delta":"Red","item_id":"msg_0876791aeda66de7006aabfce2e79087d29beef21fab15e3f5","logprobs":[],"obfuscation":"CN3uaAauxANnl","output_index":0,"sequence_number":4} + + ' + - ' + + ' + - 'event: response.output_text.delta + + ' + - 'data: {"type":"response.output_text.delta","content_index":0,"delta":" Green","item_id":"msg_0876791aeda66de7006aabfce2e79087d29beef21fab15e3f5","logprobs":[],"obfuscation":"qNR6NTTED7","output_index":0,"sequence_number":5} + + ' + - ' + + ' + - 'event: response.output_text.done + + ' + - 'data: {"type":"response.output_text.done","content_index":0,"item_id":"msg_0876791aeda66de7006aabfce2e79087d29beef21fab15e3f5","logprobs":[],"output_index":0,"sequence_number":6,"text":"Red + Green"} + + ' + - ' + + ' + - 'event: response.content_part.done + + ' + - 'data: {"type":"response.content_part.done","content_index":0,"item_id":"msg_0876791aeda66de7006aabfce2e79087d29beef21fab15e3f5","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":"Red + Green"},"sequence_number":7} + + ' + - ' + + ' + - 'event: response.output_item.done + + ' + - 'data: {"type":"response.output_item.done","item":{"id":"msg_0876791aeda66de7006aabfce2e79087d29beef21fab15e3f5","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"Red + Green"}],"role":"assistant"},"output_index":0,"sequence_number":8} + + ' + - ' + + ' + - 'event: response.completed + + ' + - 'data: {"type":"response.completed","response":{"id":"resp_0876791aeda66de7006aabfcdf8f5087d2abee282fc8ab6e67","object":"response","created_at":1789656287,"status":"completed","background":false,"completed_at":1789656290,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":64,"max_tool_calls":null,"model":"gpt-4o-2024-08-06","moderation":null,"output":[{"id":"msg_0876791aeda66de7006aabfce2e79087d29beef21fab15e3f5","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"Red + Green"}],"role":"assistant"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"default","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":224,"input_tokens_details":{"cache_write_tokens":0,"cached_tokens":0},"output_tokens":3,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":227},"user":null,"metadata":{}},"sequence_number":9} + + ' + - ' + + ' + status_code: 200 diff --git a/crates/agentic-server-core/tests/cassettes/images/responses/image-single-image-gateway-Qwen-Qwen2.5-VL-3B-Instruct-nonstreaming.yaml b/crates/agentic-server-core/tests/cassettes/images/responses/image-single-image-gateway-Qwen-Qwen2.5-VL-3B-Instruct-nonstreaming.yaml new file mode 100644 index 00000000..f5293564 --- /dev/null +++ b/crates/agentic-server-core/tests/cassettes/images/responses/image-single-image-gateway-Qwen-Qwen2.5-VL-3B-Instruct-nonstreaming.yaml @@ -0,0 +1,57 @@ +turns: +- filename: t1 + request: + body: + input: + - content: + - text: 'Reply with exactly two words: the color on the left half of this + image, then the color on the right half.' + type: input_text + - detail: low + image_url: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAT0lEQVR42u3PsQkAAAzDsPz/dHpCp2wCzwalybTxvgEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA+Dq47PDiz8p6GQAAAABJRU5ErkJggg== + type: input_image + role: user + type: message + max_output_tokens: 64 + model: Qwen/Qwen2.5-VL-3B-Instruct + store: true + stream: false + headers: + accept: '*/*' + content-type: application/json + user-agent: python-httpx/0.28.1 + method: POST + path: /v1/responses + query_params: {} + response: + body: + conversation_id: null + created_at: 1789656348 + error: null + id: resp_01a0afd4-b583-71c1-bb87-1bc13c868d49 + incomplete_details: null + instructions: null + model: Qwen/Qwen2.5-VL-3B-Instruct + object: response + output: + - content: + - annotations: [] + text: Red, Blue + type: output_text + id: msg_b1de66413b8bd767 + role: assistant + status: completed + type: message + previous_response_id: null + status: completed + usage: + input_tokens: 49 + input_tokens_details: + cached_tokens: 48 + output_tokens: 4 + output_tokens_details: + reasoning_tokens: 0 + total_tokens: 53 + headers: + content-type: application/json + status_code: 200 diff --git a/crates/agentic-server-core/tests/cassettes/images/responses/image-single-image-gateway-Qwen-Qwen2.5-VL-3B-Instruct-streaming.yaml b/crates/agentic-server-core/tests/cassettes/images/responses/image-single-image-gateway-Qwen-Qwen2.5-VL-3B-Instruct-streaming.yaml new file mode 100644 index 00000000..526fcd29 --- /dev/null +++ b/crates/agentic-server-core/tests/cassettes/images/responses/image-single-image-gateway-Qwen-Qwen2.5-VL-3B-Instruct-streaming.yaml @@ -0,0 +1,140 @@ +turns: +- filename: t1 + request: + body: + input: + - content: + - text: 'Reply with exactly two words: the color on the left half of this + image, then the color on the right half.' + type: input_text + - detail: low + image_url: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAT0lEQVR42u3PsQkAAAzDsPz/dHpCp2wCzwalybTxvgEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA+Dq47PDiz8p6GQAAAABJRU5ErkJggg== + type: input_image + role: user + type: message + max_output_tokens: 64 + model: Qwen/Qwen2.5-VL-3B-Instruct + store: true + stream: true + headers: + accept: '*/*' + content-type: application/json + user-agent: python-httpx/0.28.1 + method: POST + path: /v1/responses + query_params: {} + response: + headers: + content-type: text/event-stream; charset=utf-8 + sse: + - 'event: response.created + + ' + - 'data: {"type":"response.created","sequence_number":0,"response":{"id":"resp_01a0afd4-aece-7360-bd12-cdee949aff98","created_at":1789656346,"incomplete_details":null,"instructions":null,"metadata":null,"model":"Qwen/Qwen2.5-VL-3B-Instruct","object":"response","output":[],"parallel_tool_calls":false,"temperature":0.01,"tool_choice":"none","tools":[],"top_p":1.0,"background":false,"max_output_tokens":64,"max_tool_calls":null,"previous_response_id":null,"prompt":null,"reasoning":null,"service_tier":"auto","status":"in_progress","text":null,"top_logprobs":null,"truncation":"disabled","usage":null,"user":null,"presence_penalty":0.0,"frequency_penalty":0.0,"kv_transfer_params":null,"ec_transfer_params":null,"input_messages":null,"output_messages":null}} + + ' + - ' + + ' + - 'event: response.in_progress + + ' + - 'data: {"type":"response.in_progress","sequence_number":1,"response":{"id":"resp_01a0afd4-aece-7360-bd12-cdee949aff98","created_at":1789656346,"incomplete_details":null,"instructions":null,"metadata":null,"model":"Qwen/Qwen2.5-VL-3B-Instruct","object":"response","output":[],"parallel_tool_calls":false,"temperature":0.01,"tool_choice":"none","tools":[],"top_p":1.0,"background":false,"max_output_tokens":64,"max_tool_calls":null,"previous_response_id":null,"prompt":null,"reasoning":null,"service_tier":"auto","status":"in_progress","text":null,"top_logprobs":null,"truncation":"disabled","usage":null,"user":null,"presence_penalty":0.0,"frequency_penalty":0.0,"kv_transfer_params":null,"ec_transfer_params":null,"input_messages":null,"output_messages":null}} + + ' + - ' + + ' + - 'event: response.output_item.added + + ' + - 'data: {"type":"response.output_item.added","sequence_number":2,"output_index":0,"item":{"id":"89e9943f7ce28e24","content":[],"role":"assistant","status":"in_progress","type":"message","phase":null}} + + ' + - ' + + ' + - 'event: response.content_part.added + + ' + - 'data: {"type":"response.content_part.added","sequence_number":3,"output_index":0,"content_index":0,"item_id":"89e9943f7ce28e24","part":{"annotations":[],"text":"","type":"output_text","logprobs":[]}} + + ' + - ' + + ' + - 'event: response.output_text.delta + + ' + - 'data: {"type":"response.output_text.delta","sequence_number":4,"output_index":0,"content_index":0,"delta":"Red","item_id":"89e9943f7ce28e24","logprobs":[]} + + ' + - ' + + ' + - 'event: response.output_text.delta + + ' + - 'data: {"type":"response.output_text.delta","sequence_number":5,"output_index":0,"content_index":0,"delta":",","item_id":"89e9943f7ce28e24","logprobs":[]} + + ' + - ' + + ' + - 'event: response.output_text.delta + + ' + - 'data: {"type":"response.output_text.delta","sequence_number":6,"output_index":0,"content_index":0,"delta":" + Blue","item_id":"89e9943f7ce28e24","logprobs":[]} + + ' + - ' + + ' + - 'event: response.output_text.done + + ' + - 'data: {"type":"response.output_text.done","sequence_number":7,"output_index":0,"content_index":0,"item_id":"89e9943f7ce28e24","logprobs":[],"text":"Red, + Blue"} + + ' + - ' + + ' + - 'event: response.content_part.done + + ' + - 'data: {"type":"response.content_part.done","sequence_number":8,"output_index":0,"content_index":0,"item_id":"89e9943f7ce28e24","part":{"annotations":[],"text":"Red, + Blue","type":"output_text","logprobs":null}} + + ' + - ' + + ' + - 'event: response.output_item.done + + ' + - 'data: {"type":"response.output_item.done","sequence_number":9,"output_index":0,"item":{"id":"89e9943f7ce28e24","content":[{"annotations":[],"text":"Red, + Blue","type":"output_text","logprobs":null}],"role":"assistant","status":"completed","type":"message","phase":null,"summary":[]}} + + ' + - ' + + ' + - 'event: response.completed + + ' + - 'data: {"type":"response.completed","sequence_number":10,"response":{"id":"resp_01a0afd4-aece-7360-bd12-cdee949aff98","object":"response","created_at":1789656346,"model":"Qwen/Qwen2.5-VL-3B-Instruct","status":"completed","output":[{"type":"message","id":"89e9943f7ce28e24","role":"assistant","status":"completed","content":[{"type":"output_text","text":"Red, + Blue","annotations":[]}]}],"usage":{"input_tokens":49,"output_tokens":4,"total_tokens":53,"input_tokens_details":{"cached_tokens":0},"output_tokens_details":{"reasoning_tokens":0}},"incomplete_details":null,"error":null,"previous_response_id":null,"conversation_id":null,"instructions":null}} + + ' + - ' + + ' + - 'data: [DONE] + + ' + - ' + + ' + status_code: 200 diff --git a/crates/agentic-server-core/tests/cassettes/images/responses/image-single-image-openai-gpt-4o-nonstreaming.yaml b/crates/agentic-server-core/tests/cassettes/images/responses/image-single-image-openai-gpt-4o-nonstreaming.yaml new file mode 100644 index 00000000..bd1e5a25 --- /dev/null +++ b/crates/agentic-server-core/tests/cassettes/images/responses/image-single-image-openai-gpt-4o-nonstreaming.yaml @@ -0,0 +1,103 @@ +turns: +- filename: t1 + request: + body: + input: + - content: + - text: 'Reply with exactly two words: the color on the left half of this + image, then the color on the right half.' + type: input_text + - detail: low + image_url: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAT0lEQVR42u3PsQkAAAzDsPz/dHpCp2wCzwalybTxvgEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA+Dq47PDiz8p6GQAAAABJRU5ErkJggg== + type: input_image + role: user + type: message + max_output_tokens: 64 + model: gpt-4o + store: true + stream: false + headers: + accept: '*/*' + authorization: Bearer *** + content-type: application/json + user-agent: python-httpx/0.28.1 + method: POST + path: /v1/responses + query_params: {} + response: + body: + background: false + billing: + payer: developer + completed_at: 1789656284 + created_at: 1789656282 + error: null + frequency_penalty: 0.0 + id: resp_05153e3f4405ef7d006aabfcda91b887d282ac53632c2d90e2 + incomplete_details: null + instructions: null + max_output_tokens: 64 + max_tool_calls: null + metadata: {} + model: gpt-4o-2024-08-06 + moderation: null + object: response + output: + - content: + - annotations: [] + logprobs: [] + text: Red Blue + type: output_text + id: msg_05153e3f4405ef7d006aabfcdce50887d2b4ef08f5f7c0f498 + role: assistant + status: completed + type: message + parallel_tool_calls: true + presence_penalty: 0.0 + previous_response_id: null + prompt_cache_key: null + prompt_cache_retention: in_memory + reasoning: + context: null + effort: null + summary: null + safety_identifier: null + service_tier: default + status: completed + store: true + temperature: 1.0 + text: + format: + type: text + verbosity: medium + tool_choice: auto + tool_usage: + image_gen: + input_tokens: 0 + input_tokens_details: + image_tokens: 0 + text_tokens: 0 + output_tokens: 0 + output_tokens_details: + image_tokens: 0 + text_tokens: 0 + total_tokens: 0 + web_search: + num_requests: 0 + tools: [] + top_logprobs: 0 + top_p: 1.0 + truncation: disabled + usage: + input_tokens: 116 + input_tokens_details: + cache_write_tokens: 0 + cached_tokens: 0 + output_tokens: 3 + output_tokens_details: + reasoning_tokens: 0 + total_tokens: 119 + user: null + headers: + content-type: application/json + status_code: 200 diff --git a/crates/agentic-server-core/tests/cassettes/images/responses/image-single-image-openai-gpt-4o-streaming.yaml b/crates/agentic-server-core/tests/cassettes/images/responses/image-single-image-openai-gpt-4o-streaming.yaml new file mode 100644 index 00000000..23be41a4 --- /dev/null +++ b/crates/agentic-server-core/tests/cassettes/images/responses/image-single-image-openai-gpt-4o-streaming.yaml @@ -0,0 +1,125 @@ +turns: +- filename: t1 + request: + body: + input: + - content: + - text: 'Reply with exactly two words: the color on the left half of this + image, then the color on the right half.' + type: input_text + - detail: low + image_url: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAT0lEQVR42u3PsQkAAAzDsPz/dHpCp2wCzwalybTxvgEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA+Dq47PDiz8p6GQAAAABJRU5ErkJggg== + type: input_image + role: user + type: message + max_output_tokens: 64 + model: gpt-4o + store: true + stream: true + headers: + accept: '*/*' + authorization: Bearer *** + content-type: application/json + user-agent: python-httpx/0.28.1 + method: POST + path: /v1/responses + query_params: {} + response: + headers: + content-type: text/event-stream; charset=utf-8 + sse: + - 'event: response.created + + ' + - 'data: {"type":"response.created","response":{"id":"resp_05acf28236bc958f006aabfcd5b3c887d2aadc92267d7c2d29","object":"response","created_at":1789656277,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":64,"max_tool_calls":null,"model":"gpt-4o-2024-08-06","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} + + ' + - ' + + ' + - 'event: response.in_progress + + ' + - 'data: {"type":"response.in_progress","response":{"id":"resp_05acf28236bc958f006aabfcd5b3c887d2aadc92267d7c2d29","object":"response","created_at":1789656277,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":64,"max_tool_calls":null,"model":"gpt-4o-2024-08-06","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} + + ' + - ' + + ' + - 'event: response.output_item.added + + ' + - 'data: {"type":"response.output_item.added","item":{"id":"msg_05acf28236bc958f006aabfcd8013c87d2a05e7400451cd00e","type":"message","status":"in_progress","content":[],"role":"assistant"},"output_index":0,"sequence_number":2} + + ' + - ' + + ' + - 'event: response.content_part.added + + ' + - 'data: {"type":"response.content_part.added","content_index":0,"item_id":"msg_05acf28236bc958f006aabfcd8013c87d2a05e7400451cd00e","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":""},"sequence_number":3} + + ' + - ' + + ' + - 'event: response.output_text.delta + + ' + - 'data: {"type":"response.output_text.delta","content_index":0,"delta":"Red","item_id":"msg_05acf28236bc958f006aabfcd8013c87d2a05e7400451cd00e","logprobs":[],"obfuscation":"Ch97z9El8bQrg","output_index":0,"sequence_number":4} + + ' + - ' + + ' + - 'event: response.output_text.delta + + ' + - 'data: {"type":"response.output_text.delta","content_index":0,"delta":" blue","item_id":"msg_05acf28236bc958f006aabfcd8013c87d2a05e7400451cd00e","logprobs":[],"obfuscation":"WwBaugi66xs","output_index":0,"sequence_number":5} + + ' + - ' + + ' + - 'event: response.output_text.done + + ' + - 'data: {"type":"response.output_text.done","content_index":0,"item_id":"msg_05acf28236bc958f006aabfcd8013c87d2a05e7400451cd00e","logprobs":[],"output_index":0,"sequence_number":6,"text":"Red + blue"} + + ' + - ' + + ' + - 'event: response.content_part.done + + ' + - 'data: {"type":"response.content_part.done","content_index":0,"item_id":"msg_05acf28236bc958f006aabfcd8013c87d2a05e7400451cd00e","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":"Red + blue"},"sequence_number":7} + + ' + - ' + + ' + - 'event: response.output_item.done + + ' + - 'data: {"type":"response.output_item.done","item":{"id":"msg_05acf28236bc958f006aabfcd8013c87d2a05e7400451cd00e","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"Red + blue"}],"role":"assistant"},"output_index":0,"sequence_number":8} + + ' + - ' + + ' + - 'event: response.completed + + ' + - 'data: {"type":"response.completed","response":{"id":"resp_05acf28236bc958f006aabfcd5b3c887d2aadc92267d7c2d29","object":"response","created_at":1789656277,"status":"completed","background":false,"completed_at":1789656280,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":64,"max_tool_calls":null,"model":"gpt-4o-2024-08-06","moderation":null,"output":[{"id":"msg_05acf28236bc958f006aabfcd8013c87d2a05e7400451cd00e","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"Red + blue"}],"role":"assistant"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"default","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":116,"input_tokens_details":{"cache_write_tokens":0,"cached_tokens":0},"output_tokens":3,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":119},"user":null,"metadata":{}},"sequence_number":9} + + ' + - ' + + ' + status_code: 200 diff --git a/crates/agentic-server-core/tests/cassettes/images/responses/image-tool-image-gateway-Qwen-Qwen2.5-VL-3B-Instruct-nonstreaming.yaml b/crates/agentic-server-core/tests/cassettes/images/responses/image-tool-image-gateway-Qwen-Qwen2.5-VL-3B-Instruct-nonstreaming.yaml new file mode 100644 index 00000000..3be6e81c --- /dev/null +++ b/crates/agentic-server-core/tests/cassettes/images/responses/image-tool-image-gateway-Qwen-Qwen2.5-VL-3B-Instruct-nonstreaming.yaml @@ -0,0 +1,136 @@ +turns: +- filename: t1 + request: + body: + input: 'Call the view_image tool exactly once, with path "diagram.png". After + you see the image, reply with exactly two words: the color on its left half, + then the color on its right half.' + max_output_tokens: 64 + model: Qwen/Qwen2.5-VL-3B-Instruct + store: true + stream: false + tool_choice: auto + tools: + - description: Load an image from the local workspace so you can look at it. + Call it before describing any image the user names. + name: view_image + parameters: + additionalProperties: false + properties: + path: + description: Path of the image to load. + type: string + required: + - path + type: object + strict: true + type: function + headers: + accept: '*/*' + content-type: application/json + user-agent: python-httpx/0.28.1 + method: POST + path: /v1/responses + query_params: {} + response: + body: + conversation_id: null + created_at: 1789656356 + error: null + id: resp_01a0afd4-d5d2-72c3-b893-5d3b27f29423 + incomplete_details: null + instructions: null + model: Qwen/Qwen2.5-VL-3B-Instruct + object: response + output: + - arguments: '{"path": "diagram.png"}' + call_id: chatcmpl-tool-b5ff12811cd3a37e + id: fc_8a62adfd9557534b + name: view_image + status: completed + type: function_call + previous_response_id: null + status: completed + usage: + input_tokens: 232 + input_tokens_details: + cached_tokens: 224 + output_tokens: 22 + output_tokens_details: + reasoning_tokens: 0 + total_tokens: 254 + headers: + content-type: application/json + status_code: 200 +- filename: t2 + request: + body: + input: + - call_id: chatcmpl-tool-b5ff12811cd3a37e + output: + - text: 'Loaded diagram.png:' + type: input_text + - detail: low + image_url: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAT0lEQVR42u3PsQkAAAzDsPz/dHpCp2wCzwalybTxvgEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA+Dq47PDiz8p6GQAAAABJRU5ErkJggg== + type: input_image + type: function_call_output + max_output_tokens: 64 + model: Qwen/Qwen2.5-VL-3B-Instruct + previous_response_id: resp_01a0afd4-d5d2-72c3-b893-5d3b27f29423 + store: true + stream: false + tool_choice: auto + tools: + - description: Load an image from the local workspace so you can look at it. + Call it before describing any image the user names. + name: view_image + parameters: + additionalProperties: false + properties: + path: + description: Path of the image to load. + type: string + required: + - path + type: object + strict: true + type: function + headers: + accept: '*/*' + content-type: application/json + user-agent: python-httpx/0.28.1 + method: POST + path: /v1/responses + query_params: {} + response: + body: + conversation_id: null + created_at: 1789656357 + error: null + id: resp_01a0afd4-d84b-7312-969d-6b60d242b9d7 + incomplete_details: null + instructions: null + model: Qwen/Qwen2.5-VL-3B-Instruct + object: response + output: + - content: + - annotations: [] + text: Red, Blue + type: output_text + id: msg_8dd50379021d4340 + role: assistant + status: completed + type: message + previous_response_id: resp_01a0afd4-d5d2-72c3-b893-5d3b27f29423 + status: completed + usage: + input_tokens: 282 + input_tokens_details: + cached_tokens: 272 + output_tokens: 4 + output_tokens_details: + reasoning_tokens: 0 + total_tokens: 286 + headers: + content-type: application/json + status_code: 200 diff --git a/crates/agentic-server-core/tests/cassettes/images/responses/image-tool-image-gateway-Qwen-Qwen2.5-VL-3B-Instruct-streaming.yaml b/crates/agentic-server-core/tests/cassettes/images/responses/image-tool-image-gateway-Qwen-Qwen2.5-VL-3B-Instruct-streaming.yaml new file mode 100644 index 00000000..7aa92ded --- /dev/null +++ b/crates/agentic-server-core/tests/cassettes/images/responses/image-tool-image-gateway-Qwen-Qwen2.5-VL-3B-Instruct-streaming.yaml @@ -0,0 +1,342 @@ +turns: +- filename: t1 + request: + body: + input: 'Call the view_image tool exactly once, with path "diagram.png". After + you see the image, reply with exactly two words: the color on its left half, + then the color on its right half.' + max_output_tokens: 64 + model: Qwen/Qwen2.5-VL-3B-Instruct + store: true + stream: true + tool_choice: auto + tools: + - description: Load an image from the local workspace so you can look at it. + Call it before describing any image the user names. + name: view_image + parameters: + additionalProperties: false + properties: + path: + description: Path of the image to load. + type: string + required: + - path + type: object + strict: true + type: function + headers: + accept: '*/*' + content-type: application/json + user-agent: python-httpx/0.28.1 + method: POST + path: /v1/responses + query_params: {} + response: + headers: + content-type: text/event-stream; charset=utf-8 + sse: + - 'event: response.created + + ' + - 'data: {"type":"response.created","sequence_number":0,"response":{"id":"resp_01a0afd4-ce6e-7981-acd2-fdef30678229","created_at":1789656354,"incomplete_details":null,"instructions":null,"metadata":null,"model":"Qwen/Qwen2.5-VL-3B-Instruct","object":"response","output":[],"parallel_tool_calls":false,"temperature":0.01,"tool_choice":"auto","tools":[{"name":"view_image","parameters":{"type":"object","properties":{"path":{"type":"string","description":"Path + of the image to load."}},"required":["path"],"additionalProperties":false},"strict":true,"type":"function","allowed_callers":null,"async":null,"defer_loading":null,"description":"Load + an image from the local workspace so you can look at it. Call it before describing + any image the user names.","output_schema":null}],"top_p":1.0,"background":false,"max_output_tokens":64,"max_tool_calls":null,"previous_response_id":null,"prompt":null,"reasoning":null,"service_tier":"auto","status":"in_progress","text":null,"top_logprobs":null,"truncation":"disabled","usage":null,"user":null,"presence_penalty":0.0,"frequency_penalty":0.0,"kv_transfer_params":null,"ec_transfer_params":null,"input_messages":null,"output_messages":null}} + + ' + - ' + + ' + - 'event: response.in_progress + + ' + - 'data: {"type":"response.in_progress","sequence_number":1,"response":{"id":"resp_01a0afd4-ce6e-7981-acd2-fdef30678229","created_at":1789656354,"incomplete_details":null,"instructions":null,"metadata":null,"model":"Qwen/Qwen2.5-VL-3B-Instruct","object":"response","output":[],"parallel_tool_calls":false,"temperature":0.01,"tool_choice":"auto","tools":[{"name":"view_image","parameters":{"type":"object","properties":{"path":{"type":"string","description":"Path + of the image to load."}},"required":["path"],"additionalProperties":false},"strict":true,"type":"function","allowed_callers":null,"async":null,"defer_loading":null,"description":"Load + an image from the local workspace so you can look at it. Call it before describing + any image the user names.","output_schema":null}],"top_p":1.0,"background":false,"max_output_tokens":64,"max_tool_calls":null,"previous_response_id":null,"prompt":null,"reasoning":null,"service_tier":"auto","status":"in_progress","text":null,"top_logprobs":null,"truncation":"disabled","usage":null,"user":null,"presence_penalty":0.0,"frequency_penalty":0.0,"kv_transfer_params":null,"ec_transfer_params":null,"input_messages":null,"output_messages":null}} + + ' + - ' + + ' + - 'event: response.output_item.added + + ' + - 'data: {"type":"response.output_item.added","sequence_number":2,"output_index":0,"item":{"arguments":"","call_id":"call_a147b69426eaea7e","name":"view_image","type":"function_call","id":"854852885ad95f75","async":null,"caller":null,"namespace":null,"status":"in_progress"}} + + ' + - ' + + ' + - 'event: response.function_call_arguments.delta + + ' + - 'data: {"type":"response.function_call_arguments.delta","sequence_number":3,"output_index":0,"delta":"{\"","item_id":"854852885ad95f75"} + + ' + - ' + + ' + - 'event: response.function_call_arguments.delta + + ' + - 'data: {"type":"response.function_call_arguments.delta","sequence_number":4,"output_index":0,"delta":"path","item_id":"854852885ad95f75"} + + ' + - ' + + ' + - 'event: response.function_call_arguments.delta + + ' + - 'data: {"type":"response.function_call_arguments.delta","sequence_number":5,"output_index":0,"delta":"\":","item_id":"854852885ad95f75"} + + ' + - ' + + ' + - 'event: response.function_call_arguments.delta + + ' + - 'data: {"type":"response.function_call_arguments.delta","sequence_number":6,"output_index":0,"delta":" + \"","item_id":"854852885ad95f75"} + + ' + - ' + + ' + - 'event: response.function_call_arguments.delta + + ' + - 'data: {"type":"response.function_call_arguments.delta","sequence_number":7,"output_index":0,"delta":"di","item_id":"854852885ad95f75"} + + ' + - ' + + ' + - 'event: response.function_call_arguments.delta + + ' + - 'data: {"type":"response.function_call_arguments.delta","sequence_number":8,"output_index":0,"delta":"agram","item_id":"854852885ad95f75"} + + ' + - ' + + ' + - 'event: response.function_call_arguments.delta + + ' + - 'data: {"type":"response.function_call_arguments.delta","sequence_number":9,"output_index":0,"delta":".png","item_id":"854852885ad95f75"} + + ' + - ' + + ' + - 'event: response.function_call_arguments.delta + + ' + - 'data: {"type":"response.function_call_arguments.delta","sequence_number":10,"output_index":0,"delta":"\"}","item_id":"854852885ad95f75"} + + ' + - ' + + ' + - 'event: response.function_call_arguments.done + + ' + - 'data: {"type":"response.function_call_arguments.done","sequence_number":11,"output_index":0,"arguments":"{\"path\": + \"diagram.png\"}","item_id":"854852885ad95f75","name":"view_image"} + + ' + - ' + + ' + - 'event: response.output_item.done + + ' + - 'data: {"type":"response.output_item.done","sequence_number":12,"output_index":0,"item":{"arguments":"{\"path\": + \"diagram.png\"}","call_id":"call_a147b69426eaea7e","name":"view_image","type":"function_call","id":"854852885ad95f75","async":null,"caller":null,"namespace":null,"status":"completed"}} + + ' + - ' + + ' + - 'event: response.completed + + ' + - 'data: {"type":"response.completed","sequence_number":13,"response":{"id":"resp_01a0afd4-ce6e-7981-acd2-fdef30678229","object":"response","created_at":1789656355,"model":"Qwen/Qwen2.5-VL-3B-Instruct","status":"completed","output":[{"type":"function_call","id":"854852885ad95f75","call_id":"call_a147b69426eaea7e","name":"view_image","arguments":"{\"path\": + \"diagram.png\"}","status":"completed"}],"usage":{"input_tokens":232,"output_tokens":22,"total_tokens":254,"input_tokens_details":{"cached_tokens":96},"output_tokens_details":{"reasoning_tokens":0}},"incomplete_details":null,"error":null,"previous_response_id":null,"conversation_id":null,"instructions":null}} + + ' + - ' + + ' + - 'data: [DONE] + + ' + - ' + + ' + status_code: 200 +- filename: t2 + request: + body: + input: + - call_id: call_a147b69426eaea7e + output: + - text: 'Loaded diagram.png:' + type: input_text + - detail: low + image_url: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAT0lEQVR42u3PsQkAAAzDsPz/dHpCp2wCzwalybTxvgEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA+Dq47PDiz8p6GQAAAABJRU5ErkJggg== + type: input_image + type: function_call_output + max_output_tokens: 64 + model: Qwen/Qwen2.5-VL-3B-Instruct + previous_response_id: resp_01a0afd4-ce6e-7981-acd2-fdef30678229 + store: true + stream: true + tool_choice: auto + tools: + - description: Load an image from the local workspace so you can look at it. + Call it before describing any image the user names. + name: view_image + parameters: + additionalProperties: false + properties: + path: + description: Path of the image to load. + type: string + required: + - path + type: object + strict: true + type: function + headers: + accept: '*/*' + content-type: application/json + user-agent: python-httpx/0.28.1 + method: POST + path: /v1/responses + query_params: {} + response: + headers: + content-type: text/event-stream; charset=utf-8 + sse: + - 'event: response.created + + ' + - 'data: {"type":"response.created","sequence_number":0,"response":{"id":"resp_01a0afd4-d0d6-7cc0-877e-69ffe274dcb9","created_at":1789656355,"incomplete_details":null,"instructions":null,"metadata":null,"model":"Qwen/Qwen2.5-VL-3B-Instruct","object":"response","output":[],"parallel_tool_calls":false,"temperature":0.01,"tool_choice":"auto","tools":[{"name":"view_image","parameters":{"type":"object","properties":{"path":{"type":"string","description":"Path + of the image to load."}},"required":["path"],"additionalProperties":false},"strict":true,"type":"function","allowed_callers":null,"async":null,"defer_loading":null,"description":"Load + an image from the local workspace so you can look at it. Call it before describing + any image the user names.","output_schema":null}],"top_p":1.0,"background":false,"max_output_tokens":64,"max_tool_calls":null,"previous_response_id":"resp_01a0afd4-ce6e-7981-acd2-fdef30678229","prompt":null,"reasoning":null,"service_tier":"auto","status":"in_progress","text":null,"top_logprobs":null,"truncation":"disabled","usage":null,"user":null,"presence_penalty":0.0,"frequency_penalty":0.0,"kv_transfer_params":null,"ec_transfer_params":null,"input_messages":null,"output_messages":null}} + + ' + - ' + + ' + - 'event: response.in_progress + + ' + - 'data: {"type":"response.in_progress","sequence_number":1,"response":{"id":"resp_01a0afd4-d0d6-7cc0-877e-69ffe274dcb9","created_at":1789656355,"incomplete_details":null,"instructions":null,"metadata":null,"model":"Qwen/Qwen2.5-VL-3B-Instruct","object":"response","output":[],"parallel_tool_calls":false,"temperature":0.01,"tool_choice":"auto","tools":[{"name":"view_image","parameters":{"type":"object","properties":{"path":{"type":"string","description":"Path + of the image to load."}},"required":["path"],"additionalProperties":false},"strict":true,"type":"function","allowed_callers":null,"async":null,"defer_loading":null,"description":"Load + an image from the local workspace so you can look at it. Call it before describing + any image the user names.","output_schema":null}],"top_p":1.0,"background":false,"max_output_tokens":64,"max_tool_calls":null,"previous_response_id":"resp_01a0afd4-ce6e-7981-acd2-fdef30678229","prompt":null,"reasoning":null,"service_tier":"auto","status":"in_progress","text":null,"top_logprobs":null,"truncation":"disabled","usage":null,"user":null,"presence_penalty":0.0,"frequency_penalty":0.0,"kv_transfer_params":null,"ec_transfer_params":null,"input_messages":null,"output_messages":null}} + + ' + - ' + + ' + - 'event: response.output_item.added + + ' + - 'data: {"type":"response.output_item.added","sequence_number":2,"output_index":0,"item":{"id":"8a3451ab70d9c190","content":[],"role":"assistant","status":"in_progress","type":"message","phase":null}} + + ' + - ' + + ' + - 'event: response.content_part.added + + ' + - 'data: {"type":"response.content_part.added","sequence_number":3,"output_index":0,"content_index":0,"item_id":"8a3451ab70d9c190","part":{"annotations":[],"text":"","type":"output_text","logprobs":[]}} + + ' + - ' + + ' + - 'event: response.output_text.delta + + ' + - 'data: {"type":"response.output_text.delta","sequence_number":4,"output_index":0,"content_index":0,"delta":"Red","item_id":"8a3451ab70d9c190","logprobs":[]} + + ' + - ' + + ' + - 'event: response.output_text.delta + + ' + - 'data: {"type":"response.output_text.delta","sequence_number":5,"output_index":0,"content_index":0,"delta":",","item_id":"8a3451ab70d9c190","logprobs":[]} + + ' + - ' + + ' + - 'event: response.output_text.delta + + ' + - 'data: {"type":"response.output_text.delta","sequence_number":6,"output_index":0,"content_index":0,"delta":" + Blue","item_id":"8a3451ab70d9c190","logprobs":[]} + + ' + - ' + + ' + - 'event: response.output_text.done + + ' + - 'data: {"type":"response.output_text.done","sequence_number":7,"output_index":0,"content_index":0,"item_id":"8a3451ab70d9c190","logprobs":[],"text":"Red, + Blue"} + + ' + - ' + + ' + - 'event: response.content_part.done + + ' + - 'data: {"type":"response.content_part.done","sequence_number":8,"output_index":0,"content_index":0,"item_id":"8a3451ab70d9c190","part":{"annotations":[],"text":"Red, + Blue","type":"output_text","logprobs":null}} + + ' + - ' + + ' + - 'event: response.output_item.done + + ' + - 'data: {"type":"response.output_item.done","sequence_number":9,"output_index":0,"item":{"id":"8a3451ab70d9c190","content":[{"annotations":[],"text":"Red, + Blue","type":"output_text","logprobs":null}],"role":"assistant","status":"completed","type":"message","phase":null,"summary":[]}} + + ' + - ' + + ' + - 'event: response.completed + + ' + - 'data: {"type":"response.completed","sequence_number":10,"response":{"id":"resp_01a0afd4-d0d6-7cc0-877e-69ffe274dcb9","object":"response","created_at":1789656355,"model":"Qwen/Qwen2.5-VL-3B-Instruct","status":"completed","output":[{"type":"message","id":"8a3451ab70d9c190","role":"assistant","status":"completed","content":[{"type":"output_text","text":"Red, + Blue","annotations":[]}]}],"usage":{"input_tokens":282,"output_tokens":4,"total_tokens":286,"input_tokens_details":{"cached_tokens":240},"output_tokens_details":{"reasoning_tokens":0}},"incomplete_details":null,"error":null,"previous_response_id":"resp_01a0afd4-ce6e-7981-acd2-fdef30678229","conversation_id":null,"instructions":null}} + + ' + - ' + + ' + - 'data: [DONE] + + ' + - ' + + ' + status_code: 200 diff --git a/crates/agentic-server-core/tests/cassettes/images/responses/image-tool-image-openai-gpt-4o-nonstreaming.yaml b/crates/agentic-server-core/tests/cassettes/images/responses/image-tool-image-openai-gpt-4o-nonstreaming.yaml new file mode 100644 index 00000000..693a3c2e --- /dev/null +++ b/crates/agentic-server-core/tests/cassettes/images/responses/image-tool-image-openai-gpt-4o-nonstreaming.yaml @@ -0,0 +1,257 @@ +turns: +- filename: t1 + request: + body: + input: 'Call the view_image tool exactly once, with path "diagram.png". After + you see the image, reply with exactly two words: the color on its left half, + then the color on its right half.' + max_output_tokens: 64 + model: gpt-4o + store: true + stream: false + tool_choice: auto + tools: + - description: Load an image from the local workspace so you can look at it. + Call it before describing any image the user names. + name: view_image + parameters: + additionalProperties: false + properties: + path: + description: Path of the image to load. + type: string + required: + - path + type: object + strict: true + type: function + headers: + accept: '*/*' + authorization: Bearer *** + content-type: application/json + user-agent: python-httpx/0.28.1 + method: POST + path: /v1/responses + query_params: {} + response: + body: + background: false + billing: + payer: developer + completed_at: 1789656312 + created_at: 1789656310 + error: null + frequency_penalty: 0.0 + id: resp_025a8758bbf2b3ef006aabfcf6f84487d2b216c3ca874763f2 + incomplete_details: null + instructions: null + max_output_tokens: 64 + max_tool_calls: null + metadata: {} + model: gpt-4o-2024-08-06 + moderation: null + object: response + output: + - arguments: '{"path":"diagram.png"}' + call_id: call_q080X8fzMRemjc05iERjk71W + id: fc_025a8758bbf2b3ef006aabfcf7ead887d2b58635be202c75d7 + name: view_image + status: completed + type: function_call + parallel_tool_calls: true + presence_penalty: 0.0 + previous_response_id: null + prompt_cache_key: null + prompt_cache_retention: in_memory + reasoning: + context: null + effort: null + summary: null + safety_identifier: null + service_tier: default + status: completed + store: true + temperature: 1.0 + text: + format: + type: text + verbosity: medium + tool_choice: auto + tool_usage: + image_gen: + input_tokens: 0 + input_tokens_details: + image_tokens: 0 + text_tokens: 0 + output_tokens: 0 + output_tokens_details: + image_tokens: 0 + text_tokens: 0 + total_tokens: 0 + web_search: + num_requests: 0 + tools: + - description: Load an image from the local workspace so you can look at it. + Call it before describing any image the user names. + name: view_image + output_schema: null + parameters: + additionalProperties: false + properties: + path: + description: Path of the image to load. + type: string + required: + - path + type: object + strict: true + type: function + top_logprobs: 0 + top_p: 1.0 + truncation: disabled + usage: + input_tokens: 104 + input_tokens_details: + cache_write_tokens: 0 + cached_tokens: 0 + output_tokens: 16 + output_tokens_details: + reasoning_tokens: 0 + total_tokens: 120 + user: null + headers: + content-type: application/json + status_code: 200 +- filename: t2 + request: + body: + input: + - call_id: call_q080X8fzMRemjc05iERjk71W + output: + - text: 'Loaded diagram.png:' + type: input_text + - detail: low + image_url: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAT0lEQVR42u3PsQkAAAzDsPz/dHpCp2wCzwalybTxvgEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA+Dq47PDiz8p6GQAAAABJRU5ErkJggg== + type: input_image + type: function_call_output + max_output_tokens: 64 + model: gpt-4o + previous_response_id: resp_025a8758bbf2b3ef006aabfcf6f84487d2b216c3ca874763f2 + store: true + stream: false + tool_choice: auto + tools: + - description: Load an image from the local workspace so you can look at it. + Call it before describing any image the user names. + name: view_image + parameters: + additionalProperties: false + properties: + path: + description: Path of the image to load. + type: string + required: + - path + type: object + strict: true + type: function + headers: + accept: '*/*' + authorization: Bearer *** + content-type: application/json + user-agent: python-httpx/0.28.1 + method: POST + path: /v1/responses + query_params: {} + response: + body: + background: false + billing: + payer: developer + completed_at: 1789656313 + created_at: 1789656312 + error: null + frequency_penalty: 0.0 + id: resp_025a8758bbf2b3ef006aabfcf864b887d2aa56cd056d36500e + incomplete_details: null + instructions: null + max_output_tokens: 64 + max_tool_calls: null + metadata: {} + model: gpt-4o-2024-08-06 + moderation: null + object: response + output: + - content: + - annotations: [] + logprobs: [] + text: red blue + type: output_text + id: msg_025a8758bbf2b3ef006aabfcf9364c87d29d0d588ef8cabb6f + role: assistant + status: completed + type: message + parallel_tool_calls: true + presence_penalty: 0.0 + previous_response_id: resp_025a8758bbf2b3ef006aabfcf6f84487d2b216c3ca874763f2 + prompt_cache_key: null + prompt_cache_retention: in_memory + reasoning: + context: null + effort: null + summary: null + safety_identifier: null + service_tier: default + status: completed + store: true + temperature: 1.0 + text: + format: + type: text + verbosity: medium + tool_choice: auto + tool_usage: + image_gen: + input_tokens: 0 + input_tokens_details: + image_tokens: 0 + text_tokens: 0 + output_tokens: 0 + output_tokens_details: + image_tokens: 0 + text_tokens: 0 + total_tokens: 0 + web_search: + num_requests: 0 + tools: + - description: Load an image from the local workspace so you can look at it. + Call it before describing any image the user names. + name: view_image + output_schema: null + parameters: + additionalProperties: false + properties: + path: + description: Path of the image to load. + type: string + required: + - path + type: object + strict: true + type: function + top_logprobs: 0 + top_p: 1.0 + truncation: disabled + usage: + input_tokens: 217 + input_tokens_details: + cache_write_tokens: 0 + cached_tokens: 0 + output_tokens: 4 + output_tokens_details: + reasoning_tokens: 0 + total_tokens: 221 + user: null + headers: + content-type: application/json + status_code: 200 diff --git a/crates/agentic-server-core/tests/cassettes/images/responses/image-tool-image-openai-gpt-4o-streaming.yaml b/crates/agentic-server-core/tests/cassettes/images/responses/image-tool-image-openai-gpt-4o-streaming.yaml new file mode 100644 index 00000000..e1d7a021 --- /dev/null +++ b/crates/agentic-server-core/tests/cassettes/images/responses/image-tool-image-openai-gpt-4o-streaming.yaml @@ -0,0 +1,306 @@ +turns: +- filename: t1 + request: + body: + input: 'Call the view_image tool exactly once, with path "diagram.png". After + you see the image, reply with exactly two words: the color on its left half, + then the color on its right half.' + max_output_tokens: 64 + model: gpt-4o + store: true + stream: true + tool_choice: auto + tools: + - description: Load an image from the local workspace so you can look at it. + Call it before describing any image the user names. + name: view_image + parameters: + additionalProperties: false + properties: + path: + description: Path of the image to load. + type: string + required: + - path + type: object + strict: true + type: function + headers: + accept: '*/*' + authorization: Bearer *** + content-type: application/json + user-agent: python-httpx/0.28.1 + method: POST + path: /v1/responses + query_params: {} + response: + headers: + content-type: text/event-stream; charset=utf-8 + sse: + - 'event: response.created + + ' + - 'data: {"type":"response.created","response":{"id":"resp_07d3c2a7c3c1af47006aabfcf32c6487d2bb4e6816a6e83545","object":"response","created_at":1789656307,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":64,"max_tool_calls":null,"model":"gpt-4o-2024-08-06","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[{"type":"function","description":"Load + an image from the local workspace so you can look at it. Call it before describing + any image the user names.","name":"view_image","output_schema":null,"parameters":{"type":"object","properties":{"path":{"type":"string","description":"Path + of the image to load."}},"required":["path"],"additionalProperties":false},"strict":true}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} + + ' + - ' + + ' + - 'event: response.in_progress + + ' + - 'data: {"type":"response.in_progress","response":{"id":"resp_07d3c2a7c3c1af47006aabfcf32c6487d2bb4e6816a6e83545","object":"response","created_at":1789656307,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":64,"max_tool_calls":null,"model":"gpt-4o-2024-08-06","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[{"type":"function","description":"Load + an image from the local workspace so you can look at it. Call it before describing + any image the user names.","name":"view_image","output_schema":null,"parameters":{"type":"object","properties":{"path":{"type":"string","description":"Path + of the image to load."}},"required":["path"],"additionalProperties":false},"strict":true}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} + + ' + - ' + + ' + - 'event: response.output_item.added + + ' + - 'data: {"type":"response.output_item.added","item":{"id":"fc_07d3c2a7c3c1af47006aabfcf3c7c487d2a0c67b5f441ef0d6","type":"function_call","status":"in_progress","arguments":"","call_id":"call_mizKIwPAjae7oHtaaJHKaiWd","name":"view_image"},"output_index":0,"sequence_number":2} + + ' + - ' + + ' + - 'event: response.function_call_arguments.delta + + ' + - 'data: {"type":"response.function_call_arguments.delta","delta":"{\"","item_id":"fc_07d3c2a7c3c1af47006aabfcf3c7c487d2a0c67b5f441ef0d6","obfuscation":"peAzLWWCpAZmdl","output_index":0,"sequence_number":3} + + ' + - ' + + ' + - 'event: response.function_call_arguments.delta + + ' + - 'data: {"type":"response.function_call_arguments.delta","delta":"path","item_id":"fc_07d3c2a7c3c1af47006aabfcf3c7c487d2a0c67b5f441ef0d6","obfuscation":"T1fj8AGzRVwS","output_index":0,"sequence_number":4} + + ' + - ' + + ' + - 'event: response.function_call_arguments.delta + + ' + - 'data: {"type":"response.function_call_arguments.delta","delta":"\":\"","item_id":"fc_07d3c2a7c3c1af47006aabfcf3c7c487d2a0c67b5f441ef0d6","obfuscation":"4pF3B2KwnRqgq","output_index":0,"sequence_number":5} + + ' + - ' + + ' + - 'event: response.function_call_arguments.delta + + ' + - 'data: {"type":"response.function_call_arguments.delta","delta":"diagram","item_id":"fc_07d3c2a7c3c1af47006aabfcf3c7c487d2a0c67b5f441ef0d6","obfuscation":"VYKEBIOUy","output_index":0,"sequence_number":6} + + ' + - ' + + ' + - 'event: response.function_call_arguments.delta + + ' + - 'data: {"type":"response.function_call_arguments.delta","delta":".png","item_id":"fc_07d3c2a7c3c1af47006aabfcf3c7c487d2a0c67b5f441ef0d6","obfuscation":"mVczO47dzwE2","output_index":0,"sequence_number":7} + + ' + - ' + + ' + - 'event: response.function_call_arguments.delta + + ' + - 'data: {"type":"response.function_call_arguments.delta","delta":"\"}","item_id":"fc_07d3c2a7c3c1af47006aabfcf3c7c487d2a0c67b5f441ef0d6","obfuscation":"KkukuHfRbm1Fci","output_index":0,"sequence_number":8} + + ' + - ' + + ' + - 'event: response.function_call_arguments.done + + ' + - 'data: {"type":"response.function_call_arguments.done","arguments":"{\"path\":\"diagram.png\"}","item_id":"fc_07d3c2a7c3c1af47006aabfcf3c7c487d2a0c67b5f441ef0d6","output_index":0,"sequence_number":9} + + ' + - ' + + ' + - 'event: response.output_item.done + + ' + - 'data: {"type":"response.output_item.done","item":{"id":"fc_07d3c2a7c3c1af47006aabfcf3c7c487d2a0c67b5f441ef0d6","type":"function_call","status":"completed","arguments":"{\"path\":\"diagram.png\"}","call_id":"call_mizKIwPAjae7oHtaaJHKaiWd","name":"view_image"},"output_index":0,"sequence_number":10} + + ' + - ' + + ' + - 'event: response.completed + + ' + - 'data: {"type":"response.completed","response":{"id":"resp_07d3c2a7c3c1af47006aabfcf32c6487d2bb4e6816a6e83545","object":"response","created_at":1789656307,"status":"completed","background":false,"completed_at":1789656307,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":64,"max_tool_calls":null,"model":"gpt-4o-2024-08-06","moderation":null,"output":[{"id":"fc_07d3c2a7c3c1af47006aabfcf3c7c487d2a0c67b5f441ef0d6","type":"function_call","status":"completed","arguments":"{\"path\":\"diagram.png\"}","call_id":"call_mizKIwPAjae7oHtaaJHKaiWd","name":"view_image"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":null,"prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"default","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[{"type":"function","description":"Load + an image from the local workspace so you can look at it. Call it before describing + any image the user names.","name":"view_image","output_schema":null,"parameters":{"type":"object","properties":{"path":{"type":"string","description":"Path + of the image to load."}},"required":["path"],"additionalProperties":false},"strict":true}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":104,"input_tokens_details":{"cache_write_tokens":0,"cached_tokens":0},"output_tokens":16,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":120},"user":null,"metadata":{}},"sequence_number":11} + + ' + - ' + + ' + status_code: 200 +- filename: t2 + request: + body: + input: + - call_id: call_mizKIwPAjae7oHtaaJHKaiWd + output: + - text: 'Loaded diagram.png:' + type: input_text + - detail: low + image_url: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAT0lEQVR42u3PsQkAAAzDsPz/dHpCp2wCzwalybTxvgEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA+Dq47PDiz8p6GQAAAABJRU5ErkJggg== + type: input_image + type: function_call_output + max_output_tokens: 64 + model: gpt-4o + previous_response_id: resp_07d3c2a7c3c1af47006aabfcf32c6487d2bb4e6816a6e83545 + store: true + stream: true + tool_choice: auto + tools: + - description: Load an image from the local workspace so you can look at it. + Call it before describing any image the user names. + name: view_image + parameters: + additionalProperties: false + properties: + path: + description: Path of the image to load. + type: string + required: + - path + type: object + strict: true + type: function + headers: + accept: '*/*' + authorization: Bearer *** + content-type: application/json + user-agent: python-httpx/0.28.1 + method: POST + path: /v1/responses + query_params: {} + response: + headers: + content-type: text/event-stream; charset=utf-8 + sse: + - 'event: response.created + + ' + - 'data: {"type":"response.created","response":{"id":"resp_07d3c2a7c3c1af47006aabfcf4388887d2aefd5eedda139f34","object":"response","created_at":1789656308,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":64,"max_tool_calls":null,"model":"gpt-4o-2024-08-06","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":"resp_07d3c2a7c3c1af47006aabfcf32c6487d2bb4e6816a6e83545","prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[{"type":"function","description":"Load + an image from the local workspace so you can look at it. Call it before describing + any image the user names.","name":"view_image","output_schema":null,"parameters":{"type":"object","properties":{"path":{"type":"string","description":"Path + of the image to load."}},"required":["path"],"additionalProperties":false},"strict":true}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0} + + ' + - ' + + ' + - 'event: response.in_progress + + ' + - 'data: {"type":"response.in_progress","response":{"id":"resp_07d3c2a7c3c1af47006aabfcf4388887d2aefd5eedda139f34","object":"response","created_at":1789656308,"status":"in_progress","background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":64,"max_tool_calls":null,"model":"gpt-4o-2024-08-06","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":"resp_07d3c2a7c3c1af47006aabfcf32c6487d2bb4e6816a6e83545","prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[{"type":"function","description":"Load + an image from the local workspace so you can look at it. Call it before describing + any image the user names.","name":"view_image","output_schema":null,"parameters":{"type":"object","properties":{"path":{"type":"string","description":"Path + of the image to load."}},"required":["path"],"additionalProperties":false},"strict":true}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1} + + ' + - ' + + ' + - 'event: response.output_item.added + + ' + - 'data: {"type":"response.output_item.added","item":{"id":"msg_07d3c2a7c3c1af47006aabfcf53fcc87d2ae9f2d33fe00c492","type":"message","status":"in_progress","content":[],"role":"assistant"},"output_index":0,"sequence_number":2} + + ' + - ' + + ' + - 'event: response.content_part.added + + ' + - 'data: {"type":"response.content_part.added","content_index":0,"item_id":"msg_07d3c2a7c3c1af47006aabfcf53fcc87d2ae9f2d33fe00c492","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":""},"sequence_number":3} + + ' + - ' + + ' + - 'event: response.output_text.delta + + ' + - 'data: {"type":"response.output_text.delta","content_index":0,"delta":"Red","item_id":"msg_07d3c2a7c3c1af47006aabfcf53fcc87d2ae9f2d33fe00c492","logprobs":[],"obfuscation":"rvUSIoOpRqOda","output_index":0,"sequence_number":4} + + ' + - ' + + ' + - 'event: response.output_text.delta + + ' + - 'data: {"type":"response.output_text.delta","content_index":0,"delta":" Blue","item_id":"msg_07d3c2a7c3c1af47006aabfcf53fcc87d2ae9f2d33fe00c492","logprobs":[],"obfuscation":"bomey9nK5ac","output_index":0,"sequence_number":5} + + ' + - ' + + ' + - 'event: response.output_text.done + + ' + - 'data: {"type":"response.output_text.done","content_index":0,"item_id":"msg_07d3c2a7c3c1af47006aabfcf53fcc87d2ae9f2d33fe00c492","logprobs":[],"output_index":0,"sequence_number":6,"text":"Red + Blue"} + + ' + - ' + + ' + - 'event: response.content_part.done + + ' + - 'data: {"type":"response.content_part.done","content_index":0,"item_id":"msg_07d3c2a7c3c1af47006aabfcf53fcc87d2ae9f2d33fe00c492","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":"Red + Blue"},"sequence_number":7} + + ' + - ' + + ' + - 'event: response.output_item.done + + ' + - 'data: {"type":"response.output_item.done","item":{"id":"msg_07d3c2a7c3c1af47006aabfcf53fcc87d2ae9f2d33fe00c492","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"Red + Blue"}],"role":"assistant"},"output_index":0,"sequence_number":8} + + ' + - ' + + ' + - 'event: response.completed + + ' + - 'data: {"type":"response.completed","response":{"id":"resp_07d3c2a7c3c1af47006aabfcf4388887d2aefd5eedda139f34","object":"response","created_at":1789656308,"status":"completed","background":false,"completed_at":1789656309,"error":null,"frequency_penalty":0.0,"incomplete_details":null,"instructions":null,"max_output_tokens":64,"max_tool_calls":null,"model":"gpt-4o-2024-08-06","moderation":null,"output":[{"id":"msg_07d3c2a7c3c1af47006aabfcf53fcc87d2ae9f2d33fe00c492","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"Red + Blue"}],"role":"assistant"}],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":"resp_07d3c2a7c3c1af47006aabfcf32c6487d2bb4e6816a6e83545","prompt_cache_key":null,"prompt_cache_retention":"in_memory","reasoning":{"context":null,"effort":null,"summary":null},"safety_identifier":null,"service_tier":"default","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[{"type":"function","description":"Load + an image from the local workspace so you can look at it. Call it before describing + any image the user names.","name":"view_image","output_schema":null,"parameters":{"type":"object","properties":{"path":{"type":"string","description":"Path + of the image to load."}},"required":["path"],"additionalProperties":false},"strict":true}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":217,"input_tokens_details":{"cache_write_tokens":0,"cached_tokens":0},"output_tokens":4,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":221},"user":null,"metadata":{}},"sequence_number":9} + + ' + - ' + + ' + status_code: 200 diff --git a/crates/agentic-server-core/tests/cassettes/record_cassette.py b/crates/agentic-server-core/tests/cassettes/record_cassette.py index 60eeb6a8..9156c7da 100644 --- a/crates/agentic-server-core/tests/cassettes/record_cassette.py +++ b/crates/agentic-server-core/tests/cassettes/record_cassette.py @@ -706,6 +706,27 @@ def _extract_tool_calls(response_data: dict | None) -> list[dict]: ] +STRUCTURED_OUTPUT_PART_TYPES = frozenset({"input_text", "input_image", "input_file"}) + + +def _is_content_part_list(value: Any) -> bool: + """Whether a tool handler returned structured Responses content parts. + + Only the part types the Responses API accepts in a structured tool output + qualify; any other list -- including dicts that merely carry a `type` key + such as `[{"type": "product", ...}]` -- stays an ordinary JSON result and + is stringified as before. + """ + return ( + isinstance(value, list) + and bool(value) + and all( + isinstance(part, dict) and part.get("type") in STRUCTURED_OUTPUT_PART_TYPES + for part in value + ) + ) + + def _build_tool_output_input( tool_calls: list[dict], tool_outputs: "dict[str, Any] | types.ModuleType", @@ -724,7 +745,10 @@ def _build_tool_output_input( matching function with its actual parsed `arguments` as keyword arguments -- naturally handling whatever argument types the model used (string, number, ...) -- and the JSON-serialized return value - becomes the output. A function returning `None` omits that call's + becomes the output. A returned list of typed content parts (for + example `input_text` and `input_image` dicts) is sent as the + structured `output` array instead of a string, which is how a + cassette records a tool returning an image. A function returning `None` omits that call's output item entirely, which is how a cassette deliberately tests a provider's behavior when the client leaves one specific pending call unresolved (e.g. one of two parallel calls to the same tool @@ -815,7 +839,13 @@ def _build_tool_output_input( result = fn(**kwargs) if result is None: continue - output = result if isinstance(result, str) else json.dumps(result) + if isinstance(result, str) or _is_content_part_list(result): + # A string is the plain tool result; a list of typed content + # parts (input_text / input_image / input_file) is the + # structured Responses output and must stay an array. + output = result + else: + output = json.dumps(result) else: if name not in tool_outputs: continue @@ -1093,7 +1123,10 @@ def run_responses( click.echo( f"\n[Branch] turn {turn} chains from turn {branch_from} (response_id={previous_response_id})" ) - if preset_input is not None: + if preset_input is not None and turn == 1: + # The preset value replaces the first prompt only; later turns are + # typed as usual so a structured opening turn (for example an + # input_image item array) can still be continued by previous_response_id. input_value: Any = preset_input else: prompt = _prompt(f"Turn {turn}/{turns} — enter prompt: ") @@ -1373,7 +1406,10 @@ def run_responses( @click.option( "--input-file", type=click.Path(exists=True, dir_okay=False), - help="JSON file containing one Responses input value; requires HTTP --mode responses --turns 1.", + help=( + "JSON file containing the Responses input value for turn 1; later turns are prompted. " + "Requires HTTP --mode responses without branches." + ), ) @click.option( "--reasoning", @@ -1502,9 +1538,9 @@ def main( ) if max_output_tokens < 0: raise click.UsageError("--max-output-tokens must be >= 0.") - if input_file and (mode != "responses" or turns != 1 or branches or transport != "http"): + if input_file and (mode != "responses" or branches or transport != "http"): raise click.UsageError( - "--input-file requires HTTP --mode responses --turns 1 without branches." + "--input-file requires HTTP --mode responses without branches." ) if reasoning_raw is not None and mode != "responses": raise click.UsageError("--reasoning is only supported with --mode responses.") diff --git a/crates/agentic-server-core/tests/cassettes/record_image_input_cassettes.sh b/crates/agentic-server-core/tests/cassettes/record_image_input_cassettes.sh new file mode 100755 index 00000000..67cfe25c --- /dev/null +++ b/crates/agentic-server-core/tests/cassettes/record_image_input_cassettes.sh @@ -0,0 +1,354 @@ +#!/usr/bin/env bash +# Records paired image-input conversations against OpenAI (the reference path, +# client -> OpenAI Responses API) and the gateway (client -> Agentic API -> +# vLLM hosting an open-source vision model). Both paths receive the same image +# bytes, prompts, and tool definitions; only the model name differs. +# +# Scenarios, each recorded streaming and non-streaming per provider: +# single-image one user message with text and an inline PNG +# multi-image text and two different PNGs interleaved, order preserved +# continuation the single-image turn, then a text follow-up chained by +# previous_response_id +# tool-image the model calls the client-executed `view_image` function, +# the client submits a function_call_output whose output is a +# content array carrying the PNG, and the model answers +# +# Usage from the repository root (see README.md "Image input" for the vLLM and +# gateway launch commands): +# OPENAI_API_KEY=sk-... GATEWAY_URL=http://localhost:9000 MODEL=Qwen/Qwen2.5-VL-3B-Instruct \ +# bash crates/agentic-server-core/tests/cassettes/record_image_input_cassettes.sh +# IMAGE_RECORD_SET=gateway IMAGE_SCENARIOS=tool-image ... (one provider, one scenario) + +set -euo pipefail + +SCRIPTS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BASE_DIR="${IMAGE_OUTPUT_DIR:-$SCRIPTS_DIR/images/responses}" +INPUT_DIR="$SCRIPTS_DIR/images/inputs" +RECORDER="${RECORDER:-$SCRIPTS_DIR/record_cassette.py}" +GATEWAY_URL="${GATEWAY_URL:-http://localhost:9000}" +MODEL="${MODEL:-Qwen/Qwen2.5-VL-3B-Instruct}" +MODEL_SLUG="$(echo "$MODEL" | tr '/: ' '---')" +OPENAI_MODEL="${OPENAI_MODEL:-gpt-4o}" +OPENAI_MODEL_SLUG="$(echo "$OPENAI_MODEL" | tr '/: ' '---')" +IMAGE_RECORD_SET="${IMAGE_RECORD_SET:-all}" +IMAGE_SCENARIOS="${IMAGE_SCENARIOS:-single-image multi-image continuation tool-image}" +MAX_OUTPUT_TOKENS="${MAX_OUTPUT_TOKENS:-64}" +FOLLOW_UP_PROMPT='Without repeating the colors, reply with exactly one word: did my previous message include an image? YES or NO.' +TOOL_PROMPT='Call the view_image tool exactly once, with path "diagram.png". After you see the image, reply with exactly two words: the color on its left half, then the color on its right half.' +STAGING_DIR="" +STAGED_OUTPUTS=() +FINAL_OUTPUTS=() + +green() { printf '\033[32m%s\033[0m\n' "$*"; } +bold() { printf '\033[1m%s\033[0m\n' "$*"; } + +cleanup_staging() { + if [[ -n "$STAGING_DIR" ]]; then + rm -rf -- "$STAGING_DIR" + fi +} + +trap cleanup_staging EXIT + +# The committed JSON turns must embed exactly the committed PNGs, so a +# recording can never drift from the fixtures the replay tests compare against. +validate_input_fixtures() { + python - "$INPUT_DIR" <<'PY' +import base64 +import json +import sys +from pathlib import Path + +inputs = Path(sys.argv[1]) + + +def data_url(name: str) -> str: + return "data:image/png;base64," + base64.b64encode((inputs / name).read_bytes()).decode("ascii") + + +def image_urls(turn_file: str) -> list[str]: + turn = json.loads((inputs / turn_file).read_text(encoding="utf-8")) + return [part["image_url"] for part in turn[0]["content"] if part.get("type") == "input_image"] + + +if image_urls("single-image.json") != [data_url("red-blue-64.png")]: + raise SystemExit("ERROR: single-image.json does not embed red-blue-64.png as its single input_image part") +if image_urls("multi-image.json") != [data_url("red-blue-64.png"), data_url("green-yellow-64.png")]: + raise SystemExit("ERROR: multi-image.json must embed red-blue-64.png then green-yellow-64.png") +tools = json.loads((inputs / "view_image_tool.json").read_text(encoding="utf-8")) +if [tool.get("name") for tool in tools] != ["view_image"]: + raise SystemExit("ERROR: view_image_tool.json must declare exactly the view_image function") +PY +} + +validate_recorded_scenario() { + local scenario="$1" + local file="$2" + local stream_flag="$3" + + python - "$scenario" "$file" "$stream_flag" "$INPUT_DIR" "$FOLLOW_UP_PROMPT" "$TOOL_PROMPT" <<'PY' +import base64 +import json +import sys +from pathlib import Path + +import yaml + +scenario = sys.argv[1] +path = Path(sys.argv[2]) +streaming = sys.argv[3] == "--stream" +inputs = Path(sys.argv[4]) +follow_up = sys.argv[5] +tool_prompt = sys.argv[6] +document = yaml.safe_load(path.read_text(encoding="utf-8")) or {} +turns = document.get("turns") or [] +expected_turns = 2 if scenario in {"continuation", "tool-image"} else 1 +if len(turns) != expected_turns: + raise SystemExit(f"ERROR: expected {expected_turns} recorded turn(s) in {path}, found {len(turns)}") + + +def fixture(name: str): + return json.loads((inputs / name).read_text(encoding="utf-8")) + + +def request_body(turn: dict) -> dict: + return (turn.get("request") or {}).get("body") or {} + + +def terminal_response(turn: dict) -> dict: + response = turn.get("response") or {} + status_code = response.get("status_code") + if status_code != 200: + raise SystemExit(f"ERROR: recording returned HTTP {status_code}: {response.get('body')}") + if not streaming: + return response.get("body") or {} + events = [] + for raw in response.get("sse") or []: + for line in raw.splitlines(): + if not line.startswith("data: ") or line == "data: [DONE]": + continue + try: + events.append(json.loads(line.removeprefix("data: "))) + except json.JSONDecodeError: + continue + errors = [event.get("error") for event in events if event.get("type") == "error"] + if errors: + raise SystemExit(f"ERROR: streaming recording returned an error event: {errors[0]}") + return next( + (event.get("response") for event in reversed(events) if event.get("type") == "response.completed"), + None, + ) or {} + + +def check_completed(terminal: dict, label: str) -> None: + if terminal.get("status") != "completed": + raise SystemExit(f"ERROR: {label} did not complete: {terminal}") + + +def check_message(terminal: dict, label: str) -> None: + check_completed(terminal, label) + output = terminal.get("output") or [] + text = "".join( + part.get("text", "") + for item in output + if item.get("type") == "message" + for part in item.get("content") or [] + if part.get("type") == "output_text" + ) + if not text.strip(): + raise SystemExit(f"ERROR: {label} produced no message text: {[item.get('type') for item in output]}") + + +for index, turn in enumerate(turns, start=1): + if request_body(turn).get("stream") is not streaming: + raise SystemExit(f"ERROR: turn {index} stream mode differs from {streaming}") + +first = turns[0] +first_terminal = terminal_response(first) +calls: list[dict] = [] + +if scenario in {"single-image", "continuation"}: + if request_body(first).get("input") != fixture("single-image.json"): + raise SystemExit("ERROR: turn 1 request input differs from single-image.json") + check_message(first_terminal, "turn 1") +elif scenario == "multi-image": + if request_body(first).get("input") != fixture("multi-image.json"): + raise SystemExit("ERROR: turn 1 request input differs from multi-image.json") + check_message(first_terminal, "turn 1") +elif scenario == "tool-image": + if request_body(first).get("input") != tool_prompt: + raise SystemExit("ERROR: turn 1 request input is not the tool prompt") + if request_body(first).get("tools") != fixture("view_image_tool.json"): + raise SystemExit("ERROR: turn 1 tools differ from view_image_tool.json") + check_completed(first_terminal, "turn 1") + calls = [item for item in first_terminal.get("output") or [] if item.get("type") == "function_call"] + if [call.get("name") for call in calls] != ["view_image"]: + raise SystemExit( + "ERROR: turn 1 must end with exactly one view_image function_call; " + f"got {[item.get('type') for item in first_terminal.get('output') or []]} -- re-run the recording" + ) +else: + raise SystemExit(f"ERROR: unknown scenario {scenario}") + +if expected_turns == 2: + second = turns[1] + second_request = request_body(second) + if second_request.get("previous_response_id") != first_terminal.get("id"): + raise SystemExit( + "ERROR: turn 2 previous_response_id does not reference turn 1: " + f"{second_request.get('previous_response_id')!r} != {first_terminal.get('id')!r}" + ) + if scenario == "continuation": + if second_request.get("input") != follow_up: + raise SystemExit(f"ERROR: turn 2 request input is not the follow-up prompt: {second_request.get('input')!r}") + else: + call = calls[0] + expected_url = "data:image/png;base64," + base64.b64encode((inputs / "red-blue-64.png").read_bytes()).decode("ascii") + items = second_request.get("input") + if not isinstance(items, list) or len(items) != 1 or items[0].get("type") != "function_call_output": + raise SystemExit(f"ERROR: turn 2 must submit exactly one function_call_output: {items!r}"[:400]) + output_item = items[0] + if output_item.get("call_id") != call.get("call_id"): + raise SystemExit("ERROR: turn 2 function_call_output does not answer the recorded call_id") + parts = output_item.get("output") + if not isinstance(parts, list) or [part.get("type") for part in parts] != ["input_text", "input_image"]: + raise SystemExit(f"ERROR: turn 2 output must be an [input_text, input_image] array: {parts!r}"[:400]) + if parts[1].get("image_url") != expected_url: + raise SystemExit("ERROR: turn 2 tool output does not carry red-blue-64.png") + if second_request.get("tools") != fixture("view_image_tool.json"): + raise SystemExit("ERROR: turn 2 tools differ from view_image_tool.json") + check_message(terminal_response(second), "turn 2") +PY +} + +record_scenario() { + local scenario="$1" + local endpoint_flag="$2" + local endpoint="$3" + local model="$4" + local output="$5" + local stream_flag="$6" + local staged_output + local temporary_output + local -a recorder_args + local stdin_lines + + staged_output="$STAGING_DIR/$(basename "$output")" + temporary_output="$(mktemp "$STAGING_DIR/.image-cassette.XXXXXX")" + + recorder_args=( + --mode responses + "$stream_flag" + --model "$model" + "$endpoint_flag" "$endpoint" + --max-output-tokens "$MAX_OUTPUT_TOKENS" + --output "$temporary_output" + ) + case "$scenario" in + single-image) + recorder_args+=(--turns 1 --input-file "$INPUT_DIR/single-image.json") + stdin_lines='' + ;; + multi-image) + recorder_args+=(--turns 1 --input-file "$INPUT_DIR/multi-image.json") + stdin_lines='' + ;; + continuation) + recorder_args+=(--turns 2 --input-file "$INPUT_DIR/single-image.json") + stdin_lines="$FOLLOW_UP_PROMPT"$'\n' + ;; + tool-image) + recorder_args+=( + --turns 2 + --tools "$INPUT_DIR/view_image_tool.json" + --tool-choice auto + --tool-outputs "$INPUT_DIR/view_image_outputs.py" + ) + # Turn 1 is the tool prompt; the empty second line makes turn 2 a + # tool-output-only turn with no user message. + stdin_lines="$TOOL_PROMPT"$'\n\n' + ;; + *) + echo "ERROR: unknown scenario $scenario" >&2 + return 1 + ;; + esac + + if ! printf '%s' "$stdin_lines" | python "$RECORDER" "${recorder_args[@]}"; then + rm -f -- "$temporary_output" + return 1 + fi + + if ! validate_recorded_scenario "$scenario" "$temporary_output" "$stream_flag"; then + rm -f -- "$temporary_output" + return 1 + fi + mv -- "$temporary_output" "$staged_output" + STAGED_OUTPUTS+=("$staged_output") + FINAL_OUTPUTS+=("$output") + green "✓ $scenario cassette validated -> $output" +} + +promote_recorded_suite() { + local index + + for index in "${!STAGED_OUTPUTS[@]}"; do + mv -- "${STAGED_OUTPUTS[$index]}" "${FINAL_OUTPUTS[$index]}" + green "✓ image cassette promoted -> ${FINAL_OUTPUTS[$index]}" + done +} + +record_provider_suite() { + local provider="$1" + local endpoint_flag="$2" + local endpoint="$3" + local model="$4" + local model_slug="$5" + local scenario + + bold "$provider image-input cassettes" + bold "Endpoint: $endpoint" + bold "Model: $model" + bold "Scenarios: $IMAGE_SCENARIOS" + + for scenario in $IMAGE_SCENARIOS; do + bold "$provider $scenario (streaming)" + record_scenario "$scenario" "$endpoint_flag" "$endpoint" "$model" \ + "$BASE_DIR/image-${scenario}-${provider,,}-${model_slug}-streaming.yaml" --stream + bold "$provider $scenario (non-streaming)" + record_scenario "$scenario" "$endpoint_flag" "$endpoint" "$model" \ + "$BASE_DIR/image-${scenario}-${provider,,}-${model_slug}-nonstreaming.yaml" --no-stream + done +} + +case "$IMAGE_RECORD_SET" in + gateway|openai|all) ;; + *) + echo "ERROR: IMAGE_RECORD_SET must be gateway, openai, or all" >&2 + exit 1 + ;; +esac + +validate_input_fixtures + +# Validate OpenAI requirements before making any live requests. Final fixtures +# remain unchanged until every selected recording has completed validation. +if [[ "$IMAGE_RECORD_SET" == "openai" || "$IMAGE_RECORD_SET" == "all" ]]; then + if [[ -z "${OPENAI_API_KEY:-}" ]]; then + echo "ERROR: OPENAI_API_KEY must be set for IMAGE_RECORD_SET=$IMAGE_RECORD_SET" >&2 + exit 1 + fi +fi + +mkdir -p "$BASE_DIR" +STAGING_DIR="$(mktemp -d "$BASE_DIR/.image-suite.XXXXXX")" + +if [[ "$IMAGE_RECORD_SET" == "openai" || "$IMAGE_RECORD_SET" == "all" ]]; then + record_provider_suite OpenAI --openai https://api.openai.com "$OPENAI_MODEL" "$OPENAI_MODEL_SLUG" +fi + +if [[ "$IMAGE_RECORD_SET" == "gateway" || "$IMAGE_RECORD_SET" == "all" ]]; then + record_provider_suite Gateway --gateway "$GATEWAY_URL" "$MODEL" "$MODEL_SLUG" +fi + +promote_recorded_suite diff --git a/crates/agentic-server-core/tests/image_input_test.rs b/crates/agentic-server-core/tests/image_input_test.rs new file mode 100644 index 00000000..5ab170fd --- /dev/null +++ b/crates/agentic-server-core/tests/image_input_test.rs @@ -0,0 +1,808 @@ +//! Replays the recorded image-input scenarios and checks that the gateway path +//! (client -> Agentic API -> vLLM hosting an open-source vision model) preserves +//! the `OpenAI` Responses contract of the reference path (client -> `OpenAI`): +//! the request an image travels in, the shape of the completed response, the +//! streaming event lifecycle, continuation by `previous_response_id`, and a +//! client-executed tool returning an image. Both providers receive the same +//! image bytes, prompts, and tool definitions; only the model name differs, and +//! model wording and token counts are never compared. + +use agentic_core::executor::ExecuteRequest; +use agentic_core::types::io::OutputItem; +use agentic_core::types::request_response::{RequestPayload, ResponsePayload}; +use either::Either; +use futures::StreamExt; +use serde_json::{Value, json}; + +mod support; + +const CASSETTE_DIR: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/cassettes/images/responses"); +const RED_BLUE_PNG: &[u8] = include_bytes!("cassettes/images/inputs/red-blue-64.png"); +const GREEN_YELLOW_PNG: &[u8] = include_bytes!("cassettes/images/inputs/green-yellow-64.png"); +const SINGLE_IMAGE_TURN: &str = include_str!("cassettes/images/inputs/single-image.json"); +const MULTI_IMAGE_TURN: &str = include_str!("cassettes/images/inputs/multi-image.json"); +const VIEW_IMAGE_TOOL: &str = include_str!("cassettes/images/inputs/view_image_tool.json"); +const OPENAI_MODEL: &str = "gpt-4o"; +const OPENAI_MODEL_SLUG: &str = "gpt-4o"; +const GATEWAY_MODEL: &str = "Qwen/Qwen2.5-VL-3B-Instruct"; +const GATEWAY_MODEL_SLUG: &str = "Qwen-Qwen2.5-VL-3B-Instruct"; +const FOLLOW_UP_PROMPT: &str = + "Without repeating the colors, reply with exactly one word: did my previous message include an image? YES or NO."; +const TOOL_PROMPT: &str = "Call the view_image tool exactly once, with path \"diagram.png\". After you see the image, \ + reply with exactly two words: the color on its left half, then the color on its right half."; +const MAX_OUTPUT_TOKENS: u64 = 64; +const MESSAGE_LIFECYCLE: &[&str] = &[ + "response.created", + "response.in_progress", + "response.output_item.added:message", + "response.content_part.added", + "response.output_text.delta", + "response.output_text.done", + "response.content_part.done", + "response.output_item.done:message", + "response.completed", +]; +const FUNCTION_CALL_LIFECYCLE: &[&str] = &[ + "response.created", + "response.in_progress", + "response.output_item.added:function_call", + "response.function_call_arguments.delta", + "response.function_call_arguments.done", + "response.output_item.done:function_call", + "response.completed", +]; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Scenario { + SingleImage, + MultiImage, + Continuation, + ToolImage, +} + +impl Scenario { + const ALL: [Self; 4] = [Self::SingleImage, Self::MultiImage, Self::Continuation, Self::ToolImage]; + + const fn name(self) -> &'static str { + match self { + Self::SingleImage => "single-image", + Self::MultiImage => "multi-image", + Self::Continuation => "continuation", + Self::ToolImage => "tool-image", + } + } + + const fn turns(self) -> usize { + match self { + Self::SingleImage | Self::MultiImage => 1, + Self::Continuation | Self::ToolImage => 2, + } + } + + /// The output item each turn must produce, in order. + const fn expected_items(self) -> &'static [&'static str] { + match self { + Self::SingleImage | Self::MultiImage => &["message"], + Self::Continuation => &["message", "message"], + Self::ToolImage => &["function_call", "message"], + } + } + + /// The image data URLs the first turn carries, in order. + fn image_urls(self) -> Vec { + match self { + Self::SingleImage | Self::Continuation => vec![data_url(RED_BLUE_PNG)], + Self::MultiImage => vec![data_url(RED_BLUE_PNG), data_url(GREEN_YELLOW_PNG)], + Self::ToolImage => Vec::new(), + } + } +} + +#[derive(Clone, Copy, Debug)] +enum Provider { + OpenAi, + Gateway, +} + +impl Provider { + const fn model(self) -> &'static str { + match self { + Self::OpenAi => OPENAI_MODEL, + Self::Gateway => GATEWAY_MODEL, + } + } + + fn cassette_path(self, scenario: Scenario, streaming: bool) -> String { + let (name, slug) = match self { + Self::OpenAi => ("openai", OPENAI_MODEL_SLUG), + Self::Gateway => ("gateway", GATEWAY_MODEL_SLUG), + }; + let mode = if streaming { "streaming" } else { "nonstreaming" }; + format!("{CASSETTE_DIR}/image-{}-{name}-{slug}-{mode}.yaml", scenario.name()) + } +} + +/// Standard base64 without a dependency: the fixtures are a few hundred bytes, +/// and the test only needs to prove the committed JSON turns embed the +/// committed PNGs. +fn base64_encode(bytes: &[u8]) -> String { + const ALPHABET: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + let mut out = String::with_capacity(bytes.len().div_ceil(3) * 4); + for chunk in bytes.chunks(3) { + let mut word = 0u32; + for (index, byte) in chunk.iter().enumerate() { + word |= u32::from(*byte) << (16 - 8 * index); + } + for index in 0..4 { + if index <= chunk.len() { + let sextet = usize::try_from((word >> (18 - 6 * index)) & 0x3f).expect("sextet fits usize"); + out.push(char::from(ALPHABET[sextet])); + } else { + out.push('='); + } + } + } + out +} + +fn data_url(png: &[u8]) -> String { + format!("data:image/png;base64,{}", base64_encode(png)) +} + +fn fixture_json(text: &str) -> Value { + serde_json::from_str(text).expect("fixture should be valid JSON") +} + +fn first_turn_input(scenario: Scenario) -> Value { + match scenario { + Scenario::SingleImage | Scenario::Continuation => fixture_json(SINGLE_IMAGE_TURN), + Scenario::MultiImage => fixture_json(MULTI_IMAGE_TURN), + Scenario::ToolImage => Value::String(TOOL_PROMPT.to_owned()), + } +} + +fn view_image_tools() -> Vec { + fixture_json(VIEW_IMAGE_TOOL) + .as_array() + .expect("tool fixture should be an array") + .clone() +} + +/// The `input_image` URLs inside a message's content parts, in order. +fn image_urls_of(content: &Value) -> Vec { + content + .as_array() + .into_iter() + .flatten() + .filter(|part| part["type"] == "input_image") + .filter_map(|part| part["image_url"].as_str().map(str::to_owned)) + .collect() +} + +fn terminal_event_response(events: &[Value]) -> Value { + events + .iter() + .rev() + .find_map(|event| { + (event["type"] == "response.completed") + .then(|| event.get("response").cloned()) + .flatten() + }) + .expect("stream should contain response.completed") +} + +fn terminal_response(turn: &support::Turn) -> Value { + if let Some(body) = &turn.response.body { + return body.clone(); + } + terminal_event_response(&support::recorded_named_sse_events(turn)) +} + +fn output_types(response: &Value) -> Vec { + response["output"] + .as_array() + .expect("completed response should contain output") + .iter() + .filter_map(|item| item["type"].as_str().map(str::to_owned)) + .collect() +} + +fn message_text(response: &Value) -> String { + response["output"] + .as_array() + .expect("completed response should contain output") + .iter() + .filter(|item| item["type"] == "message") + .flat_map(|item| item["content"].as_array().into_iter().flatten()) + .filter(|part| part["type"] == "output_text") + .filter_map(|part| part["text"].as_str()) + .collect() +} + +fn function_call(response: &Value) -> &Value { + let calls = response["output"] + .as_array() + .expect("completed response should contain output") + .iter() + .filter(|item| item["type"] == "function_call") + .collect::>(); + assert_eq!(calls.len(), 1, "the tool turn should produce exactly one function call"); + calls[0] +} + +/// A recorded tool-output turn with the provider-specific `call_id` removed, +/// so both providers' second turns can be compared as one input. +fn without_call_ids(input: &Value) -> Value { + let mut input = input.clone(); + if let Some(items) = input.as_array_mut() { + for item in items { + if let Some(object) = item.as_object_mut() { + object.remove("call_id"); + } + } + } + input +} + +struct Pair { + scenario: Scenario, + streaming: bool, + openai: support::Cassette, + gateway: support::Cassette, +} + +fn load_pair(scenario: Scenario, streaming: bool) -> Pair { + Pair { + scenario, + streaming, + openai: support::load_cassette(&Provider::OpenAi.cassette_path(scenario, streaming)), + gateway: support::load_cassette(&Provider::Gateway.cassette_path(scenario, streaming)), + } +} + +/// Both providers must have received equivalent requests: the same image +/// bytes, prompts, tool declarations, and Responses settings. Only the model +/// name and the provider-assigned identifiers may differ. +fn assert_request_contract(pair: &Pair) { + let Pair { + scenario, + streaming, + openai, + gateway, + } = pair; + assert_eq!( + openai.turns.len(), + scenario.turns(), + "OpenAI turn count for {}", + scenario.name() + ); + assert_eq!( + gateway.turns.len(), + scenario.turns(), + "gateway turn count for {}", + scenario.name() + ); + + for (index, (openai_turn, gateway_turn)) in openai.turns.iter().zip(&gateway.turns).enumerate() { + let openai_body = &openai_turn.request.body; + let gateway_body = &gateway_turn.request.body; + assert_eq!(openai_turn.request.path, "/v1/responses"); + assert_eq!(gateway_turn.request.path, openai_turn.request.path); + assert_eq!(openai_body.model.as_deref(), Some(Provider::OpenAi.model())); + assert_eq!(gateway_body.model.as_deref(), Some(Provider::Gateway.model())); + assert!(openai_body.store, "stored turns are needed for continuation"); + assert_eq!(gateway_body.store, openai_body.store); + assert_eq!(openai_body.stream, *streaming); + assert_eq!(gateway_body.stream, openai_body.stream); + assert_eq!(openai_body.max_output_tokens, Some(MAX_OUTPUT_TOKENS)); + assert_eq!(gateway_body.max_output_tokens, openai_body.max_output_tokens); + assert_eq!( + without_call_ids(&gateway_body.input), + without_call_ids(&openai_body.input), + "{} turn {} must carry the same input to both providers", + scenario.name(), + index + 1 + ); + assert_eq!(gateway_body.tools, openai_body.tools); + assert_eq!(gateway_body.tool_choice, openai_body.tool_choice); + assert_eq!(gateway_body.extra, openai_body.extra); + } + + for cassette in [openai, gateway] { + let first = &cassette.turns[0]; + assert_eq!(first.request.body.input, first_turn_input(*scenario)); + assert_eq!(first.request.body.previous_response_id, None); + assert_eq!( + image_urls_of(&first.request.body.input[0]["content"]), + scenario.image_urls() + ); + if *scenario == Scenario::ToolImage { + assert_eq!(first.request.body.tools, view_image_tools()); + assert_eq!(first.request.body.tool_choice, Some(json!("auto"))); + } else { + assert!(first.request.body.tools.is_empty()); + } + + if scenario.turns() == 2 { + let first_response = terminal_response(first); + let second = &cassette.turns[1]; + assert_eq!( + second.request.body.previous_response_id.as_deref(), + first_response["id"].as_str(), + "the second turn must continue the first by previous_response_id" + ); + match scenario { + Scenario::Continuation => { + assert_eq!(second.request.body.input, Value::String(FOLLOW_UP_PROMPT.to_owned())); + } + Scenario::ToolImage => { + let call = function_call(&first_response); + let items = second.request.body.input.as_array().expect("tool output items"); + assert_eq!(items.len(), 1, "the client submits only the tool output"); + assert_eq!(items[0]["type"], "function_call_output"); + assert_eq!(items[0]["call_id"], call["call_id"]); + assert_tool_output_carries_image(&items[0]["output"]); + assert_eq!(second.request.body.tools, view_image_tools()); + } + Scenario::SingleImage | Scenario::MultiImage => unreachable!("single-turn scenario"), + } + } + } +} + +fn assert_tool_output_carries_image(output: &Value) { + let parts = output + .as_array() + .expect("a tool returning an image submits a content array, not a string"); + assert_eq!( + parts.iter().map(|part| &part["type"]).collect::>(), + vec!["input_text", "input_image"] + ); + assert_eq!(parts[1]["image_url"], data_url(RED_BLUE_PNG)); + assert_eq!(parts[1]["detail"], "low"); +} + +fn assert_terminal_contract( + turn: &support::Turn, + expected_item: &str, + provider: Provider, + previous_response_id: Option<&str>, +) -> Value { + let response = terminal_response(turn); + assert_eq!(response["object"], "response"); + assert_eq!(response["status"], "completed"); + assert!(response["id"].as_str().is_some_and(|id| !id.is_empty())); + assert!(response["created_at"].as_u64().is_some_and(|created| created > 0)); + assert!(response["error"].is_null(), "a completed response carries no error"); + assert!(response["incomplete_details"].is_null()); + let model = response["model"].as_str().expect("response should name its model"); + match provider { + // OpenAI answers with the dated snapshot of the requested alias. + Provider::OpenAi => assert!( + model.starts_with(provider.model()), + "{model} should be a {} snapshot", + provider.model() + ), + Provider::Gateway => assert_eq!(model, provider.model()), + } + assert_eq!( + response["previous_response_id"].as_str(), + previous_response_id, + "the response must echo the continuation it was asked for" + ); + assert_eq!(output_types(&response), vec![expected_item]); + match expected_item { + "message" => { + let message = &response["output"][0]; + assert_eq!(message["role"], "assistant"); + assert_eq!(message["status"], "completed"); + assert!( + message["content"] + .as_array() + .is_some_and(|parts| parts.iter().any(|part| part["type"] == "output_text")), + "message should carry an output_text part" + ); + assert!(!message_text(&response).trim().is_empty(), "message should carry text"); + } + "function_call" => { + let call = function_call(&response); + assert_eq!(call["name"], "view_image"); + assert_eq!(call["status"], "completed"); + assert!(call["call_id"].as_str().is_some_and(|id| !id.is_empty())); + let arguments: Value = serde_json::from_str(call["arguments"].as_str().expect("arguments string")) + .expect("function call arguments should be a JSON object"); + assert_eq!(arguments["path"], "diagram.png"); + } + other => panic!("unexpected output item {other}"), + } + assert!( + response["usage"]["input_tokens"] + .as_u64() + .is_some_and(|tokens| tokens > 0) + ); + assert!( + response["usage"]["output_tokens"] + .as_u64() + .is_some_and(|tokens| tokens > 0) + ); + response +} + +fn lifecycle_event_name(event: &Value) -> String { + let event_type = event["type"].as_str().expect("stream event should contain a type"); + if matches!(event_type, "response.output_item.added" | "response.output_item.done") { + let item_type = event["item"]["type"] + .as_str() + .expect("output-item lifecycle event should contain an item type"); + format!("{event_type}:{item_type}") + } else { + event_type.to_owned() + } +} + +/// Collapse runs of deltas so the lifecycle does not depend on how a model +/// happened to chunk its output. +fn normalized_streaming_lifecycle(events: &[Value]) -> Vec { + let mut lifecycle: Vec = Vec::new(); + for event in events { + let event_name = lifecycle_event_name(event); + let is_delta = matches!( + event_name.as_str(), + "response.output_text.delta" | "response.function_call_arguments.delta" + ); + if is_delta && lifecycle.last() == Some(&event_name) { + continue; + } + lifecycle.push(event_name); + } + lifecycle +} + +fn expected_lifecycle(expected_item: &str) -> &'static [&'static str] { + match expected_item { + "message" => MESSAGE_LIFECYCLE, + "function_call" => FUNCTION_CALL_LIFECYCLE, + other => panic!("unexpected output item {other}"), + } +} + +fn assert_stream_events_contract(events: &[Value], expected_item: &str) { + let sequence_numbers = events + .iter() + .map(|event| { + event["sequence_number"] + .as_u64() + .expect("every stream event should contain a sequence number") + }) + .collect::>(); + assert_eq!( + sequence_numbers, + (0..u64::try_from(events.len()).expect("stream length should fit in u64")).collect::>(), + "stream sequence numbers should be unique and contiguous" + ); + assert_eq!( + normalized_streaming_lifecycle(events), + expected_lifecycle(expected_item) + ); + + // One response, one output item, one content part: every event that names + // an identifier or index must agree with the first one that introduced it. + let response_id = events[0]["response"]["id"] + .as_str() + .expect("response.created should carry the response id"); + let added = events + .iter() + .find(|event| event["type"] == "response.output_item.added") + .expect("stream should add the output item"); + let item_id = added["item"]["id"].as_str().expect("output item should carry an id"); + assert!(!item_id.is_empty()); + for event in events { + if let Some(response) = event.get("response") { + assert_eq!( + response["id"], response_id, + "{}: response id must not change", + event["type"] + ); + } + if let Some(item) = event.get("item") { + assert_eq!(item["id"], item_id, "{}: output item id must not change", event["type"]); + assert_eq!( + item["type"], expected_item, + "{}: output item type must not change", + event["type"] + ); + } + if let Some(event_item_id) = event.get("item_id") { + assert_eq!( + event_item_id, item_id, + "{}: item_id must name the added item", + event["type"] + ); + } + if let Some(output_index) = event.get("output_index") { + assert_eq!(output_index, 0, "{}: the only item keeps output index 0", event["type"]); + } + if let Some(content_index) = event.get("content_index") { + assert_eq!( + content_index, 0, + "{}: the only part keeps content index 0", + event["type"] + ); + } + } + let terminal = terminal_event_response(events); + assert_eq!( + terminal["output"][0]["id"], item_id, + "the completed output must be the streamed item" + ); + + let (delta_type, done_type, done_field) = match expected_item { + "message" => ("response.output_text.delta", "response.output_text.done", "text"), + "function_call" => ( + "response.function_call_arguments.delta", + "response.function_call_arguments.done", + "arguments", + ), + other => panic!("unexpected output item {other}"), + }; + let delta_text = events + .iter() + .filter(|event| event["type"] == delta_type) + .map(|event| event["delta"].as_str().expect("delta event should contain delta text")) + .collect::(); + assert!(!delta_text.is_empty(), "{delta_type} events should contain text"); + let done_events = events + .iter() + .filter(|event| event["type"] == done_type) + .collect::>(); + assert_eq!(done_events.len(), 1); + assert_eq!(done_events[0][done_field].as_str(), Some(delta_text.as_str())); + + let terminal_text = match expected_item { + "message" => message_text(&terminal), + _ => function_call(&terminal)["arguments"] + .as_str() + .expect("arguments string") + .to_owned(), + }; + assert_eq!( + terminal_text, delta_text, + "the completed response should carry the streamed text" + ); +} + +fn request_payload(turn: &support::Turn, previous_response_id: Option<&str>) -> RequestPayload { + let body = &turn.request.body; + serde_json::from_value(json!({ + "model": body.model, + "input": body.input, + "store": body.store, + "stream": body.stream, + "max_output_tokens": body.max_output_tokens, + "previous_response_id": previous_response_id, + "tools": (!body.tools.is_empty()).then_some(&body.tools), + "tool_choice": body.tool_choice, + })) + .expect("recorded request should satisfy the gateway request schema") +} + +struct ReplayedTurn { + response: ResponsePayload, + /// Every SSE event the gateway emitted; empty for a non-streaming replay. + events: Vec, +} + +async fn execute(payload: RequestPayload, fixture: &support::TestFixture) -> ReplayedTurn { + let streaming = payload.stream; + let result = ExecuteRequest::new(payload, fixture.exec_ctx.clone()) + .run() + .await + .expect("recorded response should replay through the gateway"); + if !streaming { + return ReplayedTurn { + response: support::unwrap_blocking(result), + events: Vec::new(), + }; + } + let Either::Right(stream) = result else { + panic!("streaming request should return a stream"); + }; + let chunks: Vec = stream.collect().await; + let events = support::streamed_sse_events(&chunks); + let response = + serde_json::from_value(terminal_event_response(&events)).expect("completed event should carry a response"); + ReplayedTurn { response, events } +} + +/// Replay every recorded turn through the gateway against a mock upstream that +/// serves the recorded responses, continuing the second turn from the id the +/// gateway itself assigned to the first. +async fn replay(cassette: &support::Cassette) -> (Vec, Vec) { + let turns = cassette.turns.iter().collect::>(); + let fixture = support::TestFixture::new(&turns).await; + let mut replayed = Vec::with_capacity(turns.len()); + let mut previous_response_id: Option = None; + for turn in &turns { + let replayed_turn = execute(request_payload(turn, previous_response_id.as_deref()), &fixture).await; + previous_response_id = Some(replayed_turn.response.id.clone()); + replayed.push(replayed_turn); + } + let requests = fixture.request_bodies().await; + assert_eq!( + requests.len(), + turns.len(), + "each turn should reach the upstream exactly once" + ); + (replayed, requests) +} + +fn replayed_output_types(response: &ResponsePayload) -> Vec<&'static str> { + response + .output + .iter() + .map(|item| match item { + OutputItem::Message(_) => "message", + OutputItem::FunctionCall(_) => "function_call", + _ => "other", + }) + .collect() +} + +fn assert_replayed_turns(scenario: Scenario, replayed: &[ReplayedTurn], streaming: bool) { + for (turn, expected_item) in replayed.iter().zip(scenario.expected_items()) { + assert_eq!(turn.response.status, "completed"); + assert_eq!(replayed_output_types(&turn.response), vec![*expected_item]); + if streaming { + assert_stream_events_contract(&turn.events, expected_item); + } else { + assert!(turn.events.is_empty()); + } + } +} + +/// What the gateway forwarded upstream: the first turn exactly as the client +/// sent it, and the second turn as rehydrated history with every image part +/// intact and in order. +fn assert_upstream_requests(scenario: Scenario, requests: &[Value], replayed: &[ReplayedTurn], streaming: bool) { + let first = &requests[0]; + assert_eq!(first["stream"], streaming); + if scenario == Scenario::ToolImage { + // A tool turn runs through the typed executor, which lifts a string + // input into the equivalent user message item before forwarding. + assert_eq!( + first["input"], + json!([{"type": "message", "role": "user", "content": TOOL_PROMPT}]) + ); + assert_eq!(first["tools"], Value::Array(view_image_tools())); + } else { + assert_eq!( + first["input"], + first_turn_input(scenario), + "the image turn reaches the upstream exactly as the client sent it" + ); + } + if scenario.turns() == 1 { + return; + } + + let second = &requests[1]; + assert_eq!(second["stream"], streaming); + assert!( + second.get("previous_response_id").is_none_or(Value::is_null), + "the gateway rehydrates history itself instead of forwarding its own response id" + ); + let history = second["input"] + .as_array() + .expect("continuation should forward rehydrated item history"); + assert_eq!( + history.len(), + 3, + "history should be: first turn, model output, client turn" + ); + match scenario { + Scenario::Continuation => { + assert_eq!(history[0]["role"], "user"); + assert_eq!( + history[0]["content"], + first_turn_input(scenario)[0]["content"], + "the stored image turn must rehydrate with the image part intact" + ); + assert_eq!(history[1]["type"], "message"); + assert_eq!(history[1]["role"], "assistant"); + let assistant_text = history[1]["content"] + .as_array() + .expect("assistant history should carry content parts") + .iter() + .filter(|part| part["type"] == "output_text") + .filter_map(|part| part["text"].as_str()) + .collect::(); + assert_eq!(assistant_text, support::output_text(&replayed[0].response)); + assert_eq!(history[2]["role"], "user"); + assert_eq!( + support::request_input_texts(second).last().map(String::as_str), + Some(FOLLOW_UP_PROMPT) + ); + } + Scenario::ToolImage => { + assert_eq!(history[0]["role"], "user"); + assert_eq!(history[0]["content"], TOOL_PROMPT); + assert_eq!(history[1]["type"], "function_call"); + assert_eq!(history[1]["name"], "view_image"); + let call_id = history[1]["call_id"] + .as_str() + .expect("rehydrated call keeps its call_id"); + assert_eq!(history[2]["type"], "function_call_output"); + assert_eq!( + history[2]["call_id"], call_id, + "the tool output must answer the rehydrated call" + ); + assert_tool_output_carries_image(&history[2]["output"]); + assert_eq!(second["tools"], Value::Array(view_image_tools())); + } + Scenario::SingleImage | Scenario::MultiImage => unreachable!("single-turn scenario"), + } +} + +fn check_scenario_recordings(pair: &Pair) { + assert_request_contract(pair); + for (provider, cassette) in [(Provider::OpenAi, &pair.openai), (Provider::Gateway, &pair.gateway)] { + let mut previous_response_id: Option = None; + for (turn, expected_item) in cassette.turns.iter().zip(pair.scenario.expected_items()) { + let response = assert_terminal_contract(turn, expected_item, provider, previous_response_id.as_deref()); + if pair.streaming { + assert_stream_events_contract(&support::recorded_named_sse_events(turn), expected_item); + } + previous_response_id = response["id"].as_str().map(str::to_owned); + } + } +} + +async fn check_scenario_replay(pair: &Pair) { + for cassette in [&pair.openai, &pair.gateway] { + let (replayed, requests) = replay(cassette).await; + assert_replayed_turns(pair.scenario, &replayed, pair.streaming); + assert_upstream_requests(pair.scenario, &requests, &replayed, pair.streaming); + } +} + +#[test] +fn image_fixtures_embed_the_committed_pngs() { + let single = fixture_json(SINGLE_IMAGE_TURN); + assert_eq!(image_urls_of(&single[0]["content"]), vec![data_url(RED_BLUE_PNG)]); + + let multi = fixture_json(MULTI_IMAGE_TURN); + let parts = multi[0]["content"].as_array().expect("content parts"); + assert_eq!( + parts.iter().map(|part| &part["type"]).collect::>(), + vec!["input_text", "input_image", "input_text", "input_image", "input_text"], + "text and images must interleave so ordering is observable" + ); + assert_eq!( + image_urls_of(&multi[0]["content"]), + vec![data_url(RED_BLUE_PNG), data_url(GREEN_YELLOW_PNG)] + ); + assert_ne!(RED_BLUE_PNG, GREEN_YELLOW_PNG, "the two images must be distinguishable"); + + let tools = view_image_tools(); + assert_eq!(tools.len(), 1); + assert_eq!(tools[0]["type"], "function"); + assert_eq!(tools[0]["name"], "view_image"); +} + +#[tokio::test] +async fn recorded_nonstreaming_image_scenarios_match_openai_contract() { + for scenario in Scenario::ALL { + let pair = load_pair(scenario, false); + check_scenario_recordings(&pair); + check_scenario_replay(&pair).await; + } +} + +#[tokio::test] +async fn recorded_streaming_image_scenarios_match_openai_contract() { + for scenario in Scenario::ALL { + let pair = load_pair(scenario, true); + check_scenario_recordings(&pair); + check_scenario_replay(&pair).await; + } +} diff --git a/crates/agentic-server/src/openapi.rs b/crates/agentic-server/src/openapi.rs index fe86bfb0..bb469657 100644 --- a/crates/agentic-server/src/openapi.rs +++ b/crates/agentic-server/src/openapi.rs @@ -37,6 +37,7 @@ use utoipa::OpenApi; agentic_core::types::io::InputTextContent, agentic_core::types::io::InputImageContent, agentic_core::types::io::InputFileContent, + agentic_core::types::io::RefusalContent, agentic_core::types::io::InputContent, agentic_core::types::io::InputFunctionToolCall, agentic_core::types::io::FunctionToolResultMessage, diff --git a/crates/agentic-server/tests/compaction_test.rs b/crates/agentic-server/tests/compaction_test.rs index 2c02dcef..145c8b31 100644 --- a/crates/agentic-server/tests/compaction_test.rs +++ b/crates/agentic-server/tests/compaction_test.rs @@ -445,3 +445,108 @@ async fn compact_endpoint_rejects_missing_context() { assert_eq!(response.status(), reqwest::StatusCode::BAD_REQUEST); assert!(requests.lock().await.is_empty()); } + +// --- Image preservation across compaction (issue #253) --- +// +// Token estimation for image-bearing messages is issue #255; these tests only +// assert that compaction keeps a retained image-bearing user message intact. +const RED_PIXEL_PNG: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR42mP4z8AAAAMBAQD3A0FDAAAAAElFTkSuQmCC"; + +fn image_message_content() -> serde_json::Value { + serde_json::json!([ + {"type": "input_text", "text": "retained"}, + {"type": "input_image", "image_url": RED_PIXEL_PNG, "detail": "low"} + ]) +} + +#[tokio::test] +async fn compaction_window_retains_image_bearing_user_message() { + let (model_url, model_requests, _model) = spawn_compaction_model().await; + let (gateway_url, _gateway) = spawn_gateway(test_state(&test_config(&model_url))).await; + let content = image_message_content(); + + let response = reqwest::Client::new() + .post(format!("{gateway_url}/v1/responses")) + .json(&serde_json::json!({ + "model": "test-model", + "store": false, + "stream": false, + "input": [ + {"type": "message", "role": "user", "content": "superseded by the checkpoint"}, + { + "type": "message", + "id": "msg_keep", + "role": "user", + "status": "completed", + "content": content + }, + {"type": "compaction", "encrypted_content": "summary so far"}, + {"type": "message", "role": "user", "content": "after the checkpoint"} + ] + })) + .send() + .await + .expect("compacted continuation request"); + + assert_eq!(response.status(), reqwest::StatusCode::OK); + let requests = model_requests.lock().await; + assert_eq!(requests.len(), 1); + let model_input = requests[0]["input"].as_array().expect("model input"); + assert_eq!(model_input.len(), 3, "only the retained window reaches the model"); + assert_eq!( + model_input[0]["content"], content, + "a retained user message must keep its image parts" + ); + assert_eq!(model_input[1]["role"], "assistant"); + assert_eq!(model_input[1]["content"][0]["text"], "summary so far"); + assert_eq!(model_input[2]["content"], "after the checkpoint"); +} + +#[tokio::test] +async fn compact_endpoint_preserves_retained_image_message() { + let (model_url, model_requests, _model) = spawn_compaction_model().await; + let (gateway_url, _gateway) = spawn_gateway(test_state(&test_config(&model_url))).await; + let content = image_message_content(); + + let response = reqwest::Client::new() + .post(format!("{gateway_url}/v1/responses/compact")) + .json(&serde_json::json!({ + "model": "test-model", + "input": [ + {"type": "message", "role": "user", "content": content}, + {"type": "function_call_output", "call_id": "call_1", "output": "large result"} + ], + "tools": [] + })) + .send() + .await + .expect("compact request"); + + assert_eq!(response.status(), reqwest::StatusCode::OK); + let body: serde_json::Value = response.json().await.expect("compact response JSON"); + assert_eq!(body["output"][0]["type"], "message"); + assert_eq!( + body["output"][0]["content"], content, + "the compacted window must carry the image-bearing user message forward" + ); + assert_eq!(body["output"][1]["type"], "compaction"); + + // Reusing the compacted window must still send the image to the model. + let reused = reqwest::Client::new() + .post(format!("{gateway_url}/v1/responses")) + .json(&serde_json::json!({ + "model": "test-model", + "input": body["output"].clone(), + "store": false, + "stream": false + })) + .send() + .await + .expect("reuse compacted output"); + assert_eq!(reused.status(), reqwest::StatusCode::OK); + + let requests = model_requests.lock().await; + assert_eq!(requests.len(), 2); + assert_eq!(requests[1]["input"][0]["content"], content); + assert_eq!(requests[1]["input"][1]["role"], "assistant"); +} diff --git a/crates/agentic-server/tests/responses_test.rs b/crates/agentic-server/tests/responses_test.rs index fd1f1326..d42f0176 100644 --- a/crates/agentic-server/tests/responses_test.rs +++ b/crates/agentic-server/tests/responses_test.rs @@ -1634,3 +1634,597 @@ async fn test_raised_request_size_limit_admits_larger_body() { assert_eq!(requests.len(), 1); assert_eq!(requests[0]["input"][0]["content"].as_str().unwrap().len(), 4 * 1024); } + +// --- Image preservation through the Responses HTTP transport (issue #253) --- +// +// A 1x1 red PNG and a 1x1 blue PNG, inline as data URLs. They are real, valid +// PNGs with distinguishable pixels, so an ordering assertion cannot pass by +// accident, and no binary fixture has to ship with the tests. The gateway never +// decodes them: decoding and preprocessing stay in vLLM. +const RED_PIXEL_PNG: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR42mP4z8AAAAMBAQD3A0FDAAAAAElFTkSuQmCC"; +const BLUE_PIXEL_PNG: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR42mNgYPgPAAEDAQA2dBFAAAAAAElFTkSuQmCC"; + +fn image_part(image_url: &str, detail: Option<&str>) -> serde_json::Value { + match detail { + Some(detail) => serde_json::json!({"type": "input_image", "image_url": image_url, "detail": detail}), + None => serde_json::json!({"type": "input_image", "image_url": image_url}), + } +} + +/// Spawn a mock vLLM that captures every request body and answers each one with a +/// distinct completed assistant message, so a stored turn rehydrates real history. +async fn spawn_mock_vllm_json_capture_answers() +-> (String, Arc>>, tokio::task::JoinHandle<()>) { + let requests = Arc::new(Mutex::new(Vec::new())); + let route_requests = Arc::clone(&requests); + let turn = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let app = Router::new().route( + "/v1/responses", + post(move |body: Bytes| { + let route_requests = Arc::clone(&route_requests); + let turn = Arc::clone(&turn); + async move { + let body = serde_json::from_slice::(&body).unwrap(); + route_requests.lock().await.push(body); + let index = turn.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + 1; + let payload = serde_json::json!({ + "id": format!("resp_upstream_{index}"), + "object": "response", + "status": "completed", + "model": "test", + "created_at": 0, + "output": [{ + "id": format!("msg_upstream_{index}"), + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": format!("ANSWER {index}")}] + }] + }); + axum::response::Response::builder() + .status(200) + .header("Content-Type", "application/json") + .body(axum::body::Body::from(payload.to_string())) + .unwrap() + .into_response() + } + }), + ); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let handle = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + (format!("http://{addr}"), requests, handle) +} + +async fn post_response(gateway_url: &str, body: &serde_json::Value) -> serde_json::Value { + let response = reqwest::Client::new() + .post(format!("{gateway_url}/v1/responses")) + .json(body) + .send() + .await + .expect("response request"); + assert_eq!(response.status(), StatusCode::OK); + response.json().await.expect("response JSON") +} + +#[tokio::test] +async fn test_http_preserves_mixed_text_and_image_ordering() { + let (llm_url, requests, _llm) = spawn_mock_vllm_json_capture().await; + let fixture = storage_backed_state(&llm_url).await; + let (gateway_url, _gateway) = spawn_gateway(fixture.state.clone()).await; + let content = serde_json::json!([ + {"type": "input_text", "text": "first"}, + image_part(RED_PIXEL_PNG, Some("low")), + {"type": "input_text", "text": "between"}, + image_part(BLUE_PIXEL_PNG, Some("high")), + {"type": "input_text", "text": "last"} + ]); + + post_response( + &gateway_url, + &serde_json::json!({ + "model": "test", + "input": [{"type": "message", "role": "user", "content": content}], + "store": true, + "stream": false + }), + ) + .await; + + let requests = requests.lock().await; + assert_eq!(requests.len(), 1); + assert_eq!( + requests[0]["input"][0]["content"], content, + "mixed text and image parts must reach vLLM in the order the client sent them" + ); +} + +#[tokio::test] +async fn test_http_preserves_multiple_images_across_messages() { + let (llm_url, requests, _llm) = spawn_mock_vllm_json_capture().await; + let fixture = storage_backed_state(&llm_url).await; + let (gateway_url, _gateway) = spawn_gateway(fixture.state.clone()).await; + let input = serde_json::json!([ + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "look at this"}, image_part(RED_PIXEL_PNG, None)] + }, + {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "I see red."}]}, + { + "type": "message", + "role": "user", + "content": [image_part(BLUE_PIXEL_PNG, None), {"type": "input_text", "text": "and this?"}] + } + ]); + + post_response( + &gateway_url, + &serde_json::json!({"model": "test", "input": input, "store": true, "stream": false}), + ) + .await; + + let requests = requests.lock().await; + assert_eq!(requests.len(), 1); + assert_eq!(requests[0]["input"], input); + let images = requests[0]["input"] + .as_array() + .expect("input items") + .iter() + .filter_map(|item| item["content"].as_array()) + .flatten() + .filter(|part| part["type"] == "input_image") + .map(|part| part["image_url"].as_str().expect("image URL")) + .collect::>(); + assert_eq!( + images, + vec![RED_PIXEL_PNG, BLUE_PIXEL_PNG], + "each turn's image must survive, in turn order" + ); +} + +#[tokio::test] +async fn test_http_client_view_image_tool_output_reaches_next_round() { + let (llm_url, requests, _llm) = spawn_mock_vllm_json_capture().await; + let fixture = storage_backed_state(&llm_url).await; + let (gateway_url, _gateway) = spawn_gateway(fixture.state.clone()).await; + let tool_output = serde_json::json!([ + {"type": "input_text", "text": "attached local image path: diagram.png"}, + image_part(RED_PIXEL_PNG, None) + ]); + + post_response( + &gateway_url, + &serde_json::json!({ + "model": "test", + "tools": [{ + "type": "function", + "name": "view_image", + "description": "Attach a local image to the conversation.", + "parameters": {"type": "object", "properties": {"path": {"type": "string"}}} + }], + "input": [ + {"type": "message", "role": "user", "content": "look at diagram.png"}, + { + "type": "function_call", + "call_id": "call_view_image_1", + "name": "view_image", + "arguments": "{\"path\":\"diagram.png\"}" + }, + {"type": "function_call_output", "call_id": "call_view_image_1", "output": tool_output} + ], + "store": true, + "stream": false + }), + ) + .await; + + let requests = requests.lock().await; + let output = requests[0]["input"] + .as_array() + .expect("input items") + .iter() + .find(|item| item["type"] == "function_call_output") + .map(|item| &item["output"]) + .expect("client tool output should reach the next inference round"); + assert!( + output.is_array(), + "structured tool output must stay an array, not an escaped JSON string: {output}" + ); + assert_eq!(output, &tool_output); +} + +#[tokio::test] +async fn test_http_custom_tool_image_output_normalizes_without_stringifying() { + let (llm_url, requests, _llm) = spawn_mock_vllm_json_capture().await; + let fixture = storage_backed_state(&llm_url).await; + let (gateway_url, _gateway) = spawn_gateway(fixture.state.clone()).await; + let tool_output = serde_json::json!([ + {"type": "input_text", "text": "screenshot"}, + image_part(BLUE_PIXEL_PNG, Some("auto")) + ]); + + post_response( + &gateway_url, + &serde_json::json!({ + "model": "test", + "tools": [{"type": "custom", "name": "grab_screenshot", "description": "Capture the screen."}], + "input": [ + { + "type": "custom_tool_call", + "id": "ctc_1", + "call_id": "call_custom_1", + "name": "grab_screenshot", + "input": "screen", + "status": "completed" + }, + {"type": "custom_tool_call_output", "call_id": "call_custom_1", "output": tool_output} + ], + "store": true, + "stream": false + }), + ) + .await; + + let requests = requests.lock().await; + let output = requests[0]["input"] + .as_array() + .expect("input items") + .iter() + .find(|item| item["type"] == "function_call_output") + .map(|item| &item["output"]) + .expect("a custom-tool output must normalize to a function-tool output"); + assert!( + output.is_array(), + "normalization must not stringify the array: {output}" + ); + assert_eq!(output, &tool_output); +} + +#[tokio::test] +async fn test_http_previous_response_id_continuation_preserves_images() { + let (llm_url, requests, _llm) = spawn_mock_vllm_json_capture_answers().await; + let fixture = storage_backed_state(&llm_url).await; + let (gateway_url, _gateway) = spawn_gateway(fixture.state.clone()).await; + let content = serde_json::json!([ + {"type": "input_text", "text": "describe this"}, + image_part(RED_PIXEL_PNG, Some("low")) + ]); + + let first = post_response( + &gateway_url, + &serde_json::json!({ + "model": "test", + "input": [{"type": "message", "role": "user", "content": content}], + "store": true, + "stream": false + }), + ) + .await; + let previous_response_id = first["id"].as_str().expect("stored response ID"); + + post_response( + &gateway_url, + &serde_json::json!({ + "model": "test", + "previous_response_id": previous_response_id, + "input": [{"type": "message", "role": "user", "content": "and now?"}], + "store": true, + "stream": false + }), + ) + .await; + + let requests = requests.lock().await; + assert_eq!(requests.len(), 2); + let history = requests[1]["input"].as_array().expect("rehydrated history"); + assert_eq!( + history[0]["content"], content, + "the stored image must survive the round trip through the response store" + ); + assert_eq!(history[1]["role"], "assistant"); + assert_eq!(history[2]["content"], "and now?"); +} + +#[tokio::test] +async fn test_http_typed_path_preserves_extension_fields_on_image_parts() { + // The typed store=true path must forward a known part exactly as sent, + // unmodeled fields included, and persist them for continuation — the same + // guarantee the raw store=false path gives by forwarding bytes verbatim. + // Tool-output parts share the same content structs; their round trip is + // covered by the core unit tests. + let (llm_url, requests, _llm) = spawn_mock_vllm_json_capture_answers().await; + let fixture = storage_backed_state(&llm_url).await; + let (gateway_url, _gateway) = spawn_gateway(fixture.state.clone()).await; + let content = serde_json::json!([ + {"type": "input_text", "text": "describe this", "x_future_text": 1}, + {"type": "input_image", "image_url": RED_PIXEL_PNG, "detail": "low", "x_future_field": "kept"} + ]); + let first = post_response( + &gateway_url, + &serde_json::json!({ + "model": "test", + "input": [{"type": "message", "role": "user", "content": content}], + "store": true, + "stream": false + }), + ) + .await; + let previous_response_id = first["id"].as_str().expect("stored response ID"); + + post_response( + &gateway_url, + &serde_json::json!({ + "model": "test", + "previous_response_id": previous_response_id, + "input": [{"type": "message", "role": "user", "content": "and now?"}], + "store": true, + "stream": false + }), + ) + .await; + + let requests = requests.lock().await; + assert_eq!(requests.len(), 2); + assert_eq!( + requests[0]["input"][0]["content"], content, + "extension fields must be forwarded on the first turn" + ); + let history = requests[1]["input"].as_array().expect("rehydrated history"); + assert_eq!( + history[0]["content"], content, + "extension fields must survive persistence and rehydration" + ); + assert_eq!(history[2]["content"], "and now?"); +} + +#[tokio::test] +async fn test_http_conversation_rehydration_preserves_images() { + let (llm_url, requests, _llm) = spawn_mock_vllm_json_capture_answers().await; + let fixture = storage_backed_state(&llm_url).await; + let (gateway_url, _gateway) = spawn_gateway(fixture.state.clone()).await; + let client = reqwest::Client::new(); + let conversation_id = create_conversation(&client, &gateway_url).await; + let content = serde_json::json!([ + {"type": "input_text", "text": "conversation image"}, + image_part(BLUE_PIXEL_PNG, None) + ]); + + post_response( + &gateway_url, + &serde_json::json!({ + "model": "test", + "conversation_id": conversation_id, + "input": [{"type": "message", "role": "user", "content": content}], + "store": true, + "stream": false + }), + ) + .await; + post_response( + &gateway_url, + &serde_json::json!({ + "model": "test", + "conversation_id": conversation_id, + "input": [{"type": "message", "role": "user", "content": "follow up"}], + "store": true, + "stream": false + }), + ) + .await; + + let requests = requests.lock().await; + assert_eq!(requests.len(), 2); + let history = requests[1]["input"].as_array().expect("rehydrated history"); + assert_eq!(history[0]["content"], content); + assert_eq!(history.last().expect("newest turn")["content"], "follow up"); +} + +#[tokio::test] +async fn test_http_stored_tool_image_output_survives_continuation() { + let (llm_url, requests, _llm) = spawn_mock_vllm_json_capture_answers().await; + let fixture = storage_backed_state(&llm_url).await; + let (gateway_url, _gateway) = spawn_gateway(fixture.state.clone()).await; + let tool_output = serde_json::json!([ + {"type": "input_text", "text": "attached local image path: diagram.png"}, + image_part(RED_PIXEL_PNG, None) + ]); + + let first = post_response( + &gateway_url, + &serde_json::json!({ + "model": "test", + "tools": [{ + "type": "function", + "name": "view_image", + "description": "Attach a local image to the conversation.", + "parameters": {"type": "object", "properties": {"path": {"type": "string"}}} + }], + "input": [ + {"type": "message", "role": "user", "content": "look at diagram.png"}, + { + "type": "function_call", + "call_id": "call_view_image_1", + "name": "view_image", + "arguments": "{\"path\":\"diagram.png\"}" + }, + {"type": "function_call_output", "call_id": "call_view_image_1", "output": tool_output} + ], + "store": true, + "stream": false + }), + ) + .await; + + post_response( + &gateway_url, + &serde_json::json!({ + "model": "test", + "previous_response_id": first["id"].as_str().expect("stored response ID"), + "input": [{"type": "message", "role": "user", "content": "describe it"}], + "store": true, + "stream": false + }), + ) + .await; + + let requests = requests.lock().await; + let stored_output = requests[1]["input"] + .as_array() + .expect("rehydrated history") + .iter() + .find(|item| item["type"] == "function_call_output") + .map(|item| &item["output"]) + .expect("the stored tool output must rehydrate"); + assert!(stored_output.is_array(), "persistence must not stringify the array"); + assert_eq!(stored_output, &tool_output); +} + +#[tokio::test] +async fn test_store_false_proxies_image_content_verbatim() { + let (llm_url, requests, _llm) = spawn_mock_vllm_json_capture_bytes().await; + let (gateway_url, _gateway) = spawn_gateway(test_state(&test_config(&llm_url))).await; + // An unmodeled field on the image part proves the raw proxy forwards bytes + // rather than reserializing through the gateway's typed input model. + let request_body = format!( + r#"{{"model":"test","store":false,"stream":false,"input":[{{"type":"message","role":"user","content":[{{"type":"input_image","image_url":"{RED_PIXEL_PNG}","detail":"low","x_future_field":"kept"}},{{"type":"input_text","text":"raw proxy"}}]}}]}}"# + ); + + let response = reqwest::Client::new() + .post(format!("{gateway_url}/v1/responses")) + .header("Content-Type", "application/json") + .body(request_body.clone()) + .send() + .await + .expect("raw proxy request"); + + assert_eq!(response.status(), StatusCode::OK); + let requests = requests.lock().await; + assert_eq!(requests.len(), 1); + assert_eq!( + std::str::from_utf8(&requests[0]).expect("upstream body should be UTF-8"), + request_body, + "a stateless request must reach vLLM byte for byte" + ); +} + +#[tokio::test] +async fn test_http_text_only_model_still_forwards_images_unchanged() { + // The gateway never strips image content on the model's behalf: whether a + // model accepts images is the upstream's decision, so a request naming a + // text-only model must still reach it with the image part intact. That is + // what lets a missing image be attributed to the client rather than to the + // gateway. + let (llm_url, requests, _llm) = spawn_mock_vllm_json_capture().await; + let fixture = storage_backed_state(&llm_url).await; + let (gateway_url, _gateway) = spawn_gateway(fixture.state.clone()).await; + let content = serde_json::json!([ + {"type": "input_text", "text": "text-only model"}, + image_part(RED_PIXEL_PNG, None) + ]); + + post_response( + &gateway_url, + &serde_json::json!({ + "model": "text-only-model", + "input": [{"type": "message", "role": "user", "content": content}], + "store": true, + "stream": false + }), + ) + .await; + + let requests = requests.lock().await; + assert_eq!(requests[0]["model"], "text-only-model"); + assert_eq!(requests[0]["input"][0]["content"], content); +} + +async fn stored_row_counts(pool: &Arc) -> (i64, i64) { + let responses = sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM responses") + .fetch_one(pool.as_ref()) + .await + .expect("response count"); + let items = sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM items") + .fetch_one(pool.as_ref()) + .await + .expect("item count"); + (responses, items) +} + +/// Post a typed request that must be rejected before inference, returning the +/// error message once the upstream and storage are proven untouched. +async fn post_rejected_message_content(input: serde_json::Value, expected_path: &str, expected_type: &str) -> String { + let (llm_url, requests, _llm) = spawn_mock_vllm_json_capture().await; + let fixture = storage_backed_state(&llm_url).await; + let (gateway_url, _gateway) = spawn_gateway(fixture.state.clone()).await; + + let response = reqwest::Client::new() + .post(format!("{gateway_url}/v1/responses")) + .json(&serde_json::json!({"model": "test", "input": input, "store": true, "stream": false})) + .send() + .await + .expect("response request"); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let body: serde_json::Value = response.json().await.expect("error JSON"); + assert_eq!(body["error"]["type"], "invalid_request_error", "{body}"); + let message = body["error"]["message"].as_str().expect("error message").to_owned(); + assert!(message.contains(expected_path), "{message}"); + assert!(message.contains(expected_type), "{message}"); + + assert!( + requests.lock().await.is_empty(), + "a rejected message must never reach the upstream" + ); + assert_eq!( + stored_row_counts(&fixture.pool).await, + (0, 0), + "a rejected message must not be persisted, not even as a synthetic part" + ); + message +} + +#[tokio::test] +async fn test_http_unmodeled_content_part_is_rejected_not_dropped() { + // A part the gateway cannot model is rejected instead of dropped: dropping + // it would forward a message the client did not send, and would forward a + // different message than the raw store=false path does for the same bytes. + let message = post_rejected_message_content( + serde_json::json!([{"type": "message", "role": "user", "content": [ + {"type": "input_text", "text": "before"}, + {"type": "input_audio", "audio_url": "https://example.com/clip.wav"}, + image_part(RED_PIXEL_PNG, None) + ]}]), + "input[0].content[1]", + "`input_audio`", + ) + .await; + assert!( + !message.contains("clip.wav"), + "do not reflect part contents in the error: {message}" + ); +} + +#[tokio::test] +async fn test_http_message_with_only_unmodeled_content_is_rejected() { + post_rejected_message_content( + serde_json::json!([{"type": "message", "role": "user", "content": [ + {"type": "input_audio", "audio_url": "https://example.com/clip.wav"} + ]}]), + "input[0].content[0]", + "`input_audio`", + ) + .await; +} + +#[tokio::test] +async fn test_http_message_with_empty_content_is_rejected() { + post_rejected_message_content( + serde_json::json!([ + {"type": "message", "role": "user", "content": [image_part(RED_PIXEL_PNG, None)]}, + {"type": "message", "role": "user", "content": []} + ]), + "input[1].content", + "at least one content part", + ) + .await; +} diff --git a/crates/agentic-server/tests/responses_websocket_test.rs b/crates/agentic-server/tests/responses_websocket_test.rs index b7ba9e2d..81a72c2c 100644 --- a/crates/agentic-server/tests/responses_websocket_test.rs +++ b/crates/agentic-server/tests/responses_websocket_test.rs @@ -3474,3 +3474,348 @@ async fn websocket_unstored_fork_keeps_pinned_parent_after_source_failure() { assert!(!requests[0]["input"].to_string().contains("source fails")); assert_no_response_state(&fixture.pool).await; } + +// --- Image preservation through the Responses WebSocket transport (issue #253) --- +// +// The same 1x1 red and blue PNGs used by the HTTP tests: real, valid, and +// distinguishable, so ordering assertions cannot pass by accident. +const RED_PIXEL_PNG: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR42mP4z8AAAAMBAQD3A0FDAAAAAElFTkSuQmCC"; +const BLUE_PIXEL_PNG: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR42mNgYPgPAAEDAQA2dBFAAAAAAElFTkSuQmCC"; + +fn image_part(image_url: &str, detail: Option<&str>) -> Value { + match detail { + Some(detail) => json!({"type": "input_image", "image_url": image_url, "detail": detail}), + None => json!({"type": "input_image", "image_url": image_url}), + } +} + +#[tokio::test] +async fn test_websocket_preserves_mixed_text_and_image_ordering() { + let mock = MockResponsesServer::start(vec![sse_response("resp_upstream_1", "msg_upstream_1", "I see red.")]).await; + let fixture = storage_backed_state(&mock.url).await; + let (gateway_url, _gateway) = spawn_gateway(fixture.state.clone()).await; + let mut ws = connect_responses_ws(&gateway_url).await; + let content = json!([ + {"type": "input_text", "text": "first"}, + image_part(RED_PIXEL_PNG, Some("low")), + {"type": "input_text", "text": "between"}, + image_part(BLUE_PIXEL_PNG, Some("high")), + {"type": "input_text", "text": "last"} + ]); + + send_json( + &mut ws, + json!({ + "type": "response.create", + "model": "test-model", + "input": [{"type": "message", "role": "user", "content": content}], + "store": true, + "stream": true + }), + ) + .await; + + let events = recv_until_completed(&mut ws).await; + assert_eq!( + events.last().unwrap()["response"]["output"][0]["content"][0]["text"], + "I see red." + ); + + let requests = mock.request_bodies().await; + assert_eq!(requests.len(), 1); + assert_eq!( + requests[0]["input"][0]["content"], content, + "streaming must not reorder or drop image parts" + ); +} + +#[tokio::test] +async fn test_websocket_multiple_images_across_messages_keep_order() { + let mock = MockResponsesServer::start(vec![sse_response("resp_upstream_1", "msg_upstream_1", "Both seen.")]).await; + let fixture = storage_backed_state(&mock.url).await; + let (gateway_url, _gateway) = spawn_gateway(fixture.state.clone()).await; + let mut ws = connect_responses_ws(&gateway_url).await; + let input = json!([ + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "look at this"}, image_part(RED_PIXEL_PNG, None)] + }, + {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "I see red."}]}, + { + "type": "message", + "role": "user", + "content": [image_part(BLUE_PIXEL_PNG, None), {"type": "input_text", "text": "and this?"}] + } + ]); + + send_json( + &mut ws, + json!({ + "type": "response.create", + "model": "test-model", + "input": input, + "store": true, + "stream": true + }), + ) + .await; + recv_until_completed(&mut ws).await; + + let requests = mock.request_bodies().await; + let images = requests[0]["input"] + .as_array() + .expect("input items") + .iter() + .filter_map(|item| item["content"].as_array()) + .flatten() + .filter(|part| part["type"] == "input_image") + .map(|part| part["image_url"].as_str().expect("image URL")) + .collect::>(); + assert_eq!(images, vec![RED_PIXEL_PNG, BLUE_PIXEL_PNG]); + assert_eq!(requests[0]["input"], input); +} + +#[tokio::test] +async fn test_websocket_view_image_tool_output_reaches_next_round() { + let mock = MockResponsesServer::start(vec![ + sse_function_call_response("resp_upstream_1", "view_image"), + sse_response("resp_after_image", "msg_after_image", "A red pixel."), + ]) + .await; + let fixture = storage_backed_state(&mock.url).await; + let (gateway_url, _gateway) = spawn_gateway(fixture.state.clone()).await; + let mut ws = connect_responses_ws(&gateway_url).await; + let view_image_tool = json!({ + "type": "function", + "name": "view_image", + "description": "Attach a local image to the conversation.", + "parameters": {"type": "object", "properties": {"path": {"type": "string"}}} + }); + + send_json( + &mut ws, + json!({ + "type": "response.create", + "model": "test-model", + "input": [{"type": "message", "role": "user", "content": "look at diagram.png"}], + "tools": [view_image_tool], + "store": true, + "stream": true + }), + ) + .await; + + let first = recv_until_completed(&mut ws).await; + let first_completed = first.last().unwrap(); + assert_eq!(first_completed["response"]["output"][0]["type"], "function_call"); + assert_eq!(first_completed["response"]["output"][0]["name"], "view_image"); + let previous_response_id = first_completed["response"]["id"].as_str().unwrap(); + + // The client executes `view_image` locally and returns structured content. + let tool_output = json!([ + {"type": "input_text", "text": "attached local image path: diagram.png"}, + image_part(RED_PIXEL_PNG, None) + ]); + send_json( + &mut ws, + json!({ + "type": "response.create", + "model": "test-model", + "previous_response_id": previous_response_id, + "input": [{"type": "function_call_output", "call_id": "call_1", "output": tool_output}], + "tools": [view_image_tool], + "store": true, + "stream": true + }), + ) + .await; + + let second = recv_until_completed(&mut ws).await; + assert_eq!( + second.last().unwrap()["response"]["output"][0]["content"][0]["text"], + "A red pixel." + ); + + let requests = mock.request_bodies().await; + assert_eq!(requests.len(), 2); + let continuation = requests[1]["input"].as_array().expect("continuation input"); + let output = continuation + .iter() + .find(|item| item["type"] == "function_call_output") + .map(|item| &item["output"]) + .expect("the client tool output must reach the next inference round"); + assert!( + output.is_array(), + "structured tool output must stay an array, not an escaped JSON string: {output}" + ); + assert_eq!(output, &tool_output); + assert!( + continuation + .iter() + .any(|item| item["type"] == "function_call" && item["name"] == "view_image"), + "the call the output resolves must be replayed alongside it" + ); +} + +#[tokio::test] +async fn test_websocket_custom_tool_image_output_round_trip() { + let mock = MockResponsesServer::start(vec![ + sse_custom_tool_call_response(), + sse_response("resp_after_custom", "msg_after_custom", "Screenshot received."), + ]) + .await; + let fixture = storage_backed_state(&mock.url).await; + let (gateway_url, _gateway) = spawn_gateway(fixture.state.clone()).await; + let mut ws = connect_responses_ws(&gateway_url).await; + + send_json( + &mut ws, + json!({ + "type": "response.create", + "model": "test-model", + "input": [{"type": "message", "role": "user", "content": "grab a screenshot"}], + "tools": [{"type": "custom", "name": "apply_patch", "description": "Apply a patch."}], + "store": true, + "stream": true + }), + ) + .await; + let first = recv_until_completed(&mut ws).await; + let previous_response_id = first.last().unwrap()["response"]["id"].as_str().unwrap(); + + let tool_output = json!([ + {"type": "input_text", "text": "screenshot"}, + image_part(BLUE_PIXEL_PNG, Some("auto")) + ]); + send_json( + &mut ws, + json!({ + "type": "response.create", + "model": "test-model", + "previous_response_id": previous_response_id, + "input": [{"type": "custom_tool_call_output", "call_id": "call_custom_1", "output": tool_output}], + "store": true, + "stream": true + }), + ) + .await; + let second = recv_until_completed(&mut ws).await; + assert_eq!( + second.last().unwrap()["response"]["output"][0]["content"][0]["text"], + "Screenshot received." + ); + + let requests = mock.request_bodies().await; + let output = requests[1]["input"] + .as_array() + .expect("continuation input") + .iter() + .find(|item| item["type"] == "function_call_output") + .map(|item| &item["output"]) + .expect("a custom-tool output must normalize to a function-tool output"); + assert!( + output.is_array(), + "normalization must not stringify the array: {output}" + ); + assert_eq!(output, &tool_output); +} + +#[tokio::test] +async fn test_websocket_continuation_rehydrates_images() { + let mock = MockResponsesServer::start(vec![ + sse_response("resp_upstream_1", "msg_upstream_1", "I see red."), + sse_response("resp_upstream_2", "msg_upstream_2", "Still red."), + ]) + .await; + let fixture = storage_backed_state(&mock.url).await; + let (gateway_url, _gateway) = spawn_gateway(fixture.state.clone()).await; + let mut ws = connect_responses_ws(&gateway_url).await; + let content = json!([ + {"type": "input_text", "text": "describe this"}, + image_part(RED_PIXEL_PNG, Some("low")) + ]); + + send_json( + &mut ws, + json!({ + "type": "response.create", + "model": "test-model", + "input": [{"type": "message", "role": "user", "content": content}], + "store": true, + "stream": true + }), + ) + .await; + let first = recv_until_completed(&mut ws).await; + let previous_response_id = first.last().unwrap()["response"]["id"].as_str().unwrap(); + + send_json( + &mut ws, + json!({ + "type": "response.create", + "model": "test-model", + "previous_response_id": previous_response_id, + "input": [{"type": "message", "role": "user", "content": "and now?"}], + "store": true, + "stream": true + }), + ) + .await; + recv_until_completed(&mut ws).await; + + let requests = mock.request_bodies().await; + assert_eq!(requests.len(), 2); + let history = requests[1]["input"].as_array().expect("rehydrated history"); + assert_eq!( + history[0]["content"], content, + "the stored image must survive rehydration over the WebSocket transport" + ); + assert_eq!(history[1]["role"], "assistant"); + assert_eq!(history[2]["content"], "and now?"); +} + +#[tokio::test] +async fn test_websocket_unmodeled_content_part_is_rejected_not_dropped() { + let mock = MockResponsesServer::start(vec![sse_response("resp_unused", "msg_unused", "unreachable")]).await; + let fixture = storage_backed_state(&mock.url).await; + let (gateway_url, _gateway) = spawn_gateway(fixture.state.clone()).await; + let mut ws = connect_responses_ws(&gateway_url).await; + + send_json( + &mut ws, + json!({ + "type": "response.create", + "model": "test-model", + "input": [{"type": "message", "role": "user", "content": [ + {"type": "input_text", "text": "before"}, + {"type": "input_audio", "audio_url": "https://example.com/clip.wav"}, + image_part(RED_PIXEL_PNG, None) + ]}], + "store": true, + "stream": true + }), + ) + .await; + + let error = recv_json(&mut ws).await; + assert_eq!(error["type"], "error"); + assert_eq!(error["status"], StatusCode::BAD_REQUEST.as_u16()); + assert_eq!(error["error"]["type"], "invalid_request_error"); + let message = error["error"]["message"].as_str().expect("error message"); + assert!(message.contains("input[0].content[1]"), "{message}"); + assert!(message.contains("`input_audio`"), "{message}"); + assert!(!message.contains("clip.wav"), "do not reflect part contents: {message}"); + + assert!( + mock.request_bodies().await.is_empty(), + "a rejected message must never reach the upstream" + ); + for table in ["responses", "items"] { + let count: i64 = sqlx::query_scalar(&format!("SELECT COUNT(*) FROM {table}")) + .fetch_one(fixture.pool.as_ref()) + .await + .unwrap(); + assert_eq!(count, 0, "a rejected message must not be persisted in {table}"); + } +}