diff --git a/Package.swift b/Package.swift index 431f6339..554c8173 100644 --- a/Package.swift +++ b/Package.swift @@ -49,6 +49,11 @@ let package = Package( package: "mlx-swift-examples", condition: .when(traits: ["MLX"]) ), + .product( + name: "MLXVLM", + package: "mlx-swift-examples", + condition: .when(traits: ["MLX"]) + ), .product( name: "MLXLMCommon", package: "mlx-swift-examples", diff --git a/README.md b/README.md index dc426b36..efdaf933 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,9 @@ let response = try await session.respond { } ``` +> [!NOTE] +> Image inputs are not yet supported by Apple Foundation Models. + ### Core ML Run [Core ML](https://developer.apple.com/documentation/coreml) models @@ -136,6 +139,9 @@ Enable the trait in Package.swift: ) ``` +> [!NOTE] +> Image inputs are not currently supported with `CoreMLLanguageModel`. + ### MLX Run [MLX](https://github.com/ml-explore/mlx-swift) models on Apple Silicon @@ -150,6 +156,22 @@ let response = try await session.respond { } ``` +Vision support depends on the specific MLX model you load. +Use a vision‑capable model for multimodal prompts +(for example, a VLM variant). +The following shows extracting text from an image: + +```swift +let ocr = try await session.respond( + to: "Extract the total amount from this receipt", + images: [ + .init(url: URL(fileURLWithPath: "/path/to/receipt_page1.png")), + .init(url: URL(fileURLWithPath: "/path/to/receipt_page2.png")) + ] +) +print(ocr.content) +``` + Enable the trait in Package.swift: ```swift @@ -184,6 +206,9 @@ Enable the trait in Package.swift: ) ``` +> [!NOTE] +> Image inputs are not currently supported with `LlamaLanguageModel`. + ### OpenAI Supports both @@ -197,9 +222,17 @@ let model = OpenAILanguageModel( ) let session = LanguageModelSession(model: model) -let response = try await session.respond { - Prompt("Write a haiku about Swift") -} +let response = try await session.respond( + to: "List the objects you see", + images: [ + .init(url: URL(string: "https://example.com/desk.jpg")!), + .init( + data: try Data(contentsOf: URL(fileURLWithPath: "/path/to/closeup.png")), + mimeType: "image/png" + ) + ] +) +print(response.content) ``` For OpenAI-compatible endpoints that use older Chat Completions API: @@ -229,6 +262,20 @@ let response = try await session.respond { } ``` +You can include images with your prompt. +You can point to remote URLs or construct from image data: + +```swift +let response = try await session.respond( + to: "Explain the key parts of this diagram", + image: .init( + data: try Data(contentsOf: URL(fileURLWithPath: "/path/to/diagram.png")), + mimeType: "image/png" + ) +) +print(response.content) +``` + ### Google Gemini Uses the [Gemini API](https://ai.google.dev/api/generate-content) with Gemini models: @@ -245,6 +292,16 @@ let response = try await session.respond { } ``` +Send images with your prompt using remote or local sources: + +```swift +let response = try await session.respond( + to: "Identify the plants in this photo", + image: .init(url: URL(string: "https://example.com/garden.jpg")!) +) +print(response.content) +``` + Gemini models use an internal ["thinking process"](https://ai.google.dev/gemini-api/docs/thinking) that improves reasoning and multi-step planning. You can configure how much Gemini should "think" using the `thinking` parameter: @@ -293,11 +350,12 @@ let model = GeminiLanguageModel( ### Ollama -Run models locally via Ollama's [HTTP API](https://github.com/ollama/ollama/blob/main/docs/api.md): +Run models locally via Ollama's +[HTTP API](https://github.com/ollama/ollama/blob/main/docs/api.md): ```swift // Default: connects to http://localhost:11434 -let model = OllamaLanguageModel(model: "qwen3") +let model = OllamaLanguageModel(model: "qwen3") // `ollama pull qwen3:8b` // Custom endpoint let model = OllamaLanguageModel( @@ -311,7 +369,22 @@ let response = try await session.respond { } ``` -First, pull the model: `ollama pull qwen3:0.6b` +For local models, make sure you’re using a vision‑capable model +(for example, a `-vl` variant). +You can combine multiple images: + +```swift +let model = OllamaLanguageModel(model: "qwen3-vl") // `ollama pull qwen3-vl:8b` +let session = LanguageModelSession(model: model) +let response = try await session.respond( + to: "Compare these posters and summarize their differences", + images: [ + .init(url: URL(string: "https://example.com/poster1.jpg")!), + .init(url: URL(fileURLWithPath: "/path/to/poster2.jpg")) + ] +) +print(response.content) +``` ## Testing diff --git a/Sources/AnyLanguageModel/LanguageModelSession.swift b/Sources/AnyLanguageModel/LanguageModelSession.swift index 3818050a..50ea2ec3 100644 --- a/Sources/AnyLanguageModel/LanguageModelSession.swift +++ b/Sources/AnyLanguageModel/LanguageModelSession.swift @@ -166,6 +166,74 @@ public final class LanguageModelSession: @unchecked Sendable { public let transcriptEntries: ArraySlice } + @discardableResult + nonisolated public func respond( + to prompt: Prompt, + generating type: Content.Type = Content.self, + includeSchemaInPrompt: Bool = true, + options: GenerationOptions = GenerationOptions() + ) async throws -> Response where Content: Generable { + try await wrapRespond { + // Add prompt to transcript + let promptEntry = Transcript.Entry.prompt( + Transcript.Prompt( + segments: [.text(.init(content: prompt.description))], + options: options, + responseFormat: nil + ) + ) + await MainActor.run { + self.transcript.append(promptEntry) + } + + let response = try await model.respond( + within: self, + to: prompt, + generating: type, + includeSchemaInPrompt: includeSchemaInPrompt, + options: options + ) + + // Add response entries to transcript + await MainActor.run { + self.transcript.append(contentsOf: response.transcriptEntries) + } + + return response + } + } + + nonisolated public func streamResponse( + to prompt: Prompt, + generating type: Content.Type = Content.self, + includeSchemaInPrompt: Bool = true, + options: GenerationOptions = GenerationOptions() + ) -> sending ResponseStream where Content: Generable { + // Create prompt entry that will be added when stream starts + let promptEntry = Transcript.Entry.prompt( + Transcript.Prompt( + segments: [.text(.init(content: prompt.description))], + options: options, + responseFormat: nil + ) + ) + + return wrapStream( + model.streamResponse( + within: self, + to: prompt, + generating: type, + includeSchemaInPrompt: includeSchemaInPrompt, + options: options + ), + promptEntry: promptEntry + ) + } +} + +// MARK: - String Response Convenience Methods + +extension LanguageModelSession { @discardableResult nonisolated public func respond( to prompt: Prompt, @@ -195,6 +263,36 @@ public final class LanguageModelSession: @unchecked Sendable { try await respond(to: try prompt(), options: options) } + public func streamResponse( + to prompt: Prompt, + options: GenerationOptions = GenerationOptions() + ) -> sending ResponseStream { + streamResponse( + to: prompt, + generating: String.self, + includeSchemaInPrompt: true, + options: options + ) + } + + public func streamResponse( + to prompt: String, + options: GenerationOptions = GenerationOptions() + ) -> sending ResponseStream { + streamResponse(to: Prompt(prompt), options: options) + } + + public func streamResponse( + options: GenerationOptions = GenerationOptions(), + @PromptBuilder prompt: () throws -> Prompt + ) rethrows -> sending ResponseStream { + streamResponse(to: try prompt(), options: options) + } +} + +// MARK: - GeneratedContent with Schema Convenience Methods + +extension LanguageModelSession { @discardableResult nonisolated public func respond( to prompt: Prompt, @@ -240,43 +338,47 @@ public final class LanguageModelSession: @unchecked Sendable { ) } - @discardableResult - nonisolated public func respond( + nonisolated public func streamResponse( to prompt: Prompt, - generating type: Content.Type = Content.self, + schema: GenerationSchema, includeSchemaInPrompt: Bool = true, options: GenerationOptions = GenerationOptions() - ) async throws -> Response where Content: Generable { - try await wrapRespond { - // Add prompt to transcript - let promptEntry = Transcript.Entry.prompt( - Transcript.Prompt( - segments: [.text(.init(content: prompt.description))], - options: options, - responseFormat: nil - ) - ) - await MainActor.run { - self.transcript.append(promptEntry) - } - - let response = try await model.respond( - within: self, - to: prompt, - generating: type, - includeSchemaInPrompt: includeSchemaInPrompt, - options: options - ) + ) -> sending ResponseStream { + streamResponse( + to: prompt, + generating: GeneratedContent.self, + includeSchemaInPrompt: includeSchemaInPrompt, + options: options + ) + } - // Add response entries to transcript - await MainActor.run { - self.transcript.append(contentsOf: response.transcriptEntries) - } + nonisolated public func streamResponse( + to prompt: String, + schema: GenerationSchema, + includeSchemaInPrompt: Bool = true, + options: GenerationOptions = GenerationOptions() + ) -> sending ResponseStream { + streamResponse( + to: Prompt(prompt), + schema: schema, + includeSchemaInPrompt: includeSchemaInPrompt, + options: options + ) + } - return response - } + nonisolated public func streamResponse( + schema: GenerationSchema, + includeSchemaInPrompt: Bool = true, + options: GenerationOptions = GenerationOptions(), + @PromptBuilder prompt: () throws -> Prompt + ) rethrows -> sending ResponseStream { + streamResponse(to: try prompt(), schema: schema, includeSchemaInPrompt: includeSchemaInPrompt, options: options) } +} + +// MARK: - Generic Content Convenience Methods +extension LanguageModelSession { @discardableResult nonisolated public func respond( to prompt: String, @@ -307,124 +409,215 @@ public final class LanguageModelSession: @unchecked Sendable { ) } - nonisolated public func streamResponse( - to prompt: Prompt, - schema: GenerationSchema, + nonisolated public func streamResponse( + to prompt: String, + generating type: Content.Type = Content.self, includeSchemaInPrompt: Bool = true, options: GenerationOptions = GenerationOptions() - ) -> sending ResponseStream { + ) -> sending ResponseStream where Content: Generable { streamResponse( - to: prompt, - generating: GeneratedContent.self, + to: Prompt(prompt), + generating: type, includeSchemaInPrompt: includeSchemaInPrompt, options: options ) } - nonisolated public func streamResponse( - to prompt: String, - schema: GenerationSchema, + public func streamResponse( + generating type: Content.Type = Content.self, includeSchemaInPrompt: Bool = true, - options: GenerationOptions = GenerationOptions() - ) -> sending ResponseStream { + options: GenerationOptions = GenerationOptions(), + @PromptBuilder prompt: () throws -> Prompt + ) rethrows -> sending ResponseStream where Content: Generable { streamResponse( - to: Prompt(prompt), - schema: schema, + to: try prompt(), + generating: type, includeSchemaInPrompt: includeSchemaInPrompt, options: options ) } +} - nonisolated public func streamResponse( - schema: GenerationSchema, - includeSchemaInPrompt: Bool = true, - options: GenerationOptions = GenerationOptions(), - @PromptBuilder prompt: () throws -> Prompt - ) rethrows -> sending ResponseStream { - streamResponse(to: try prompt(), schema: schema, includeSchemaInPrompt: includeSchemaInPrompt, options: options) - } +// MARK: - Image Convenience Methods - nonisolated public func streamResponse( - to prompt: Prompt, - generating type: Content.Type = Content.self, - includeSchemaInPrompt: Bool = true, +extension LanguageModelSession { + @discardableResult + nonisolated public func respond( + to prompt: String, + image: Transcript.ImageSegment, options: GenerationOptions = GenerationOptions() - ) -> sending ResponseStream where Content: Generable { - // Create prompt entry that will be added when stream starts - let promptEntry = Transcript.Entry.prompt( - Transcript.Prompt( - segments: [.text(.init(content: prompt.description))], - options: options, - responseFormat: nil - ) + ) async throws -> Response { + try await respond( + to: prompt, + images: [image], + options: options ) + } - return wrapStream( - model.streamResponse( - within: self, - to: prompt, - generating: type, - includeSchemaInPrompt: includeSchemaInPrompt, - options: options - ), - promptEntry: promptEntry + @discardableResult + nonisolated public func respond( + to prompt: String, + images: [Transcript.ImageSegment], + options: GenerationOptions = GenerationOptions() + ) async throws -> Response { + try await respond( + to: prompt, + images: images, + generating: String.self, + includeSchemaInPrompt: true, + options: options ) } - nonisolated public func streamResponse( + @discardableResult + nonisolated public func respond( to prompt: String, + image: Transcript.ImageSegment, generating type: Content.Type = Content.self, includeSchemaInPrompt: Bool = true, options: GenerationOptions = GenerationOptions() - ) -> sending ResponseStream where Content: Generable { - streamResponse( - to: Prompt(prompt), + ) async throws -> Response where Content: Generable { + try await respond( + to: prompt, + images: [image], generating: type, includeSchemaInPrompt: includeSchemaInPrompt, options: options ) } - public func streamResponse( + @discardableResult + nonisolated public func respond( + to prompt: String, + images: [Transcript.ImageSegment], generating type: Content.Type = Content.self, includeSchemaInPrompt: Bool = true, - options: GenerationOptions = GenerationOptions(), - @PromptBuilder prompt: () throws -> Prompt - ) rethrows -> sending ResponseStream where Content: Generable { + options: GenerationOptions = GenerationOptions() + ) async throws -> Response where Content: Generable { + try await wrapRespond { + // Build segments from text and images + var segments: [Transcript.Segment] = [] + if !prompt.isEmpty { + segments.append(.text(.init(content: prompt))) + } + segments.append(contentsOf: images.map { .image($0) }) + + // Add prompt to transcript + let promptEntry = Transcript.Entry.prompt( + Transcript.Prompt( + segments: segments, + options: options, + responseFormat: nil + ) + ) + await MainActor.run { + self.transcript.append(promptEntry) + } + + // Extract text content for the Prompt parameter + let textPrompt = Prompt(prompt) + + let response = try await model.respond( + within: self, + to: textPrompt, + generating: type, + includeSchemaInPrompt: includeSchemaInPrompt, + options: options + ) + + // Add response entries to transcript + await MainActor.run { + self.transcript.append(contentsOf: response.transcriptEntries) + } + + return response + } + } + + public func streamResponse( + to prompt: String, + image: Transcript.ImageSegment, + options: GenerationOptions = GenerationOptions() + ) -> sending ResponseStream { streamResponse( - to: try prompt(), - generating: type, - includeSchemaInPrompt: includeSchemaInPrompt, + to: prompt, + images: [image], options: options ) } public func streamResponse( - to prompt: Prompt, + to prompt: String, + images: [Transcript.ImageSegment], options: GenerationOptions = GenerationOptions() ) -> sending ResponseStream { streamResponse( to: prompt, + images: images, generating: String.self, includeSchemaInPrompt: true, options: options ) } - public func streamResponse( + nonisolated public func streamResponse( to prompt: String, + image: Transcript.ImageSegment, + generating type: Content.Type = Content.self, + includeSchemaInPrompt: Bool = true, options: GenerationOptions = GenerationOptions() - ) -> sending ResponseStream { - streamResponse(to: Prompt(prompt), options: options) + ) -> sending ResponseStream where Content: Generable { + streamResponse( + to: prompt, + images: [image], + generating: type, + includeSchemaInPrompt: includeSchemaInPrompt, + options: options + ) } - public func streamResponse( - options: GenerationOptions = GenerationOptions(), - @PromptBuilder prompt: () throws -> Prompt - ) rethrows -> sending ResponseStream { - streamResponse(to: try prompt(), options: options) + nonisolated public func streamResponse( + to prompt: String, + images: [Transcript.ImageSegment], + generating type: Content.Type = Content.self, + includeSchemaInPrompt: Bool = true, + options: GenerationOptions = GenerationOptions() + ) -> sending ResponseStream where Content: Generable { + // Build segments from text and images + var segments: [Transcript.Segment] = [] + if !prompt.isEmpty { + segments.append(.text(.init(content: prompt))) + } + segments.append(contentsOf: images.map { .image($0) }) + + // Create prompt entry that will be added when stream starts + let promptEntry = Transcript.Entry.prompt( + Transcript.Prompt( + segments: segments, + options: options, + responseFormat: nil + ) + ) + + // Extract text content for the Prompt parameter + let textPrompt = Prompt(prompt) + + return wrapStream( + model.streamResponse( + within: self, + to: textPrompt, + generating: type, + includeSchemaInPrompt: includeSchemaInPrompt, + options: options + ), + promptEntry: promptEntry + ) } +} + +// MARK: - +extension LanguageModelSession { @discardableResult public func logFeedbackAttachment( sentiment: LanguageModelFeedback.Sentiment?, @@ -440,19 +633,7 @@ public final class LanguageModelSession: @unchecked Sendable { } } -private actor RespondingState { - private var count = 0 - - func increment() -> Int { - count += 1 - return count - } - - func decrement() -> Int { - count = max(0, count - 1) - return count - } -} +// MARK: - extension LanguageModelSession { public enum GenerationError: Error, LocalizedError { @@ -649,3 +830,19 @@ extension LanguageModelSession.ResponseStream: AsyncSequence { ) } } + +// MARK: - + +private actor RespondingState { + private var count = 0 + + func increment() -> Int { + count += 1 + return count + } + + func decrement() -> Int { + count = max(0, count - 1) + return count + } +} diff --git a/Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift b/Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift index 97795f1a..3df7e396 100644 --- a/Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift @@ -100,8 +100,9 @@ public struct AnthropicLanguageModel: LanguageModel { let url = baseURL.appendingPathComponent("v1/messages") let headers = buildHeaders() + let userSegments = extractPromptSegments(from: session, fallbackText: prompt.description) let messages = [ - AnthropicMessage(role: .user, content: [.text(.init(text: prompt.description))]) + AnthropicMessage(role: .user, content: convertSegmentsToAnthropicContent(userSegments)) ] // Convert available tools to Anthropic format @@ -170,8 +171,9 @@ public struct AnthropicLanguageModel: LanguageModel { fatalError("AnthropicLanguageModel only supports generating String content") } + let userSegments = extractPromptSegments(from: session, fallbackText: prompt.description) let messages = [ - AnthropicMessage(role: .user, content: [.text(.init(text: prompt.description))]) + AnthropicMessage(role: .user, content: convertSegmentsToAnthropicContent(userSegments)) ] let url = baseURL.appendingPathComponent("v1/messages") @@ -383,11 +385,12 @@ private struct AnthropicMessage: Codable, Sendable { private enum AnthropicContent: Codable, Sendable { case text(AnthropicText) + case image(AnthropicImage) case toolUse(AnthropicToolUse) enum CodingKeys: String, CodingKey { case type } - enum ContentType: String, Codable { case text = "text", toolUse = "tool_use" } + enum ContentType: String, Codable { case text = "text", image = "image", toolUse = "tool_use" } init(from decoder: any Decoder) throws { let container = try decoder.container(keyedBy: CodingKeys.self) @@ -395,6 +398,8 @@ private enum AnthropicContent: Codable, Sendable { switch type { case .text: self = .text(try AnthropicText(from: decoder)) + case .image: + self = .image(try AnthropicImage(from: decoder)) case .toolUse: self = .toolUse(try AnthropicToolUse(from: decoder)) } @@ -403,6 +408,7 @@ private enum AnthropicContent: Codable, Sendable { func encode(to encoder: any Encoder) throws { switch self { case .text(let t): try t.encode(to: encoder) + case .image(let i): try i.encode(to: encoder) case .toolUse(let u): try u.encode(to: encoder) } } @@ -418,6 +424,77 @@ private struct AnthropicText: Codable, Sendable { } } +private struct AnthropicImage: Codable, Sendable { + struct Source: Codable, Sendable { + let type: String + let mediaType: String? + let data: String? + let url: String? + + enum CodingKeys: String, CodingKey { + case type + case mediaType = "media_type" + case data + case url + } + } + + let type: String + let source: Source + + init(base64Data: String, mimeType: String) { + self.type = "image" + self.source = Source(type: "base64", mediaType: mimeType, data: base64Data, url: nil) + } + + init(url: String) { + self.type = "image" + self.source = Source(type: "url", mediaType: nil, data: nil, url: url) + } +} + +private func convertSegmentsToAnthropicContent(_ segments: [Transcript.Segment]) -> [AnthropicContent] { + var blocks: [AnthropicContent] = [] + blocks.reserveCapacity(segments.count) + for segment in segments { + switch segment { + case .text(let t): + blocks.append(.text(AnthropicText(text: t.content))) + case .structure(let s): + blocks.append(.text(AnthropicText(text: s.content.jsonString))) + case .image(let img): + switch img.source { + case .url(let url): + blocks.append(.image(AnthropicImage(url: url.absoluteString))) + case .data(let data, let mimeType): + blocks.append(.image(AnthropicImage(base64Data: data.base64EncodedString(), mimeType: mimeType))) + } + } + } + return blocks +} + +private func extractPromptSegments(from session: LanguageModelSession, fallbackText: String) -> [Transcript.Segment] { + for entry in session.transcript.reversed() { + if case .prompt(let p) = entry { + // Skip prompts that are effectively empty (single empty text block) + let hasMeaningfulContent = p.segments.contains { segment in + switch segment { + case .text(let t): + return !t.content.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + case .structure: + return true + case .image: + return true + } + } + if hasMeaningfulContent { return p.segments } + // Otherwise continue searching older entries + } + } + return [.text(.init(content: fallbackText))] +} + private struct AnthropicToolUse: Codable, Sendable { let type: String let id: String diff --git a/Sources/AnyLanguageModel/Models/CoreMLLanguageModel.swift b/Sources/AnyLanguageModel/Models/CoreMLLanguageModel.swift index 68587e9a..bc8b1e21 100644 --- a/Sources/AnyLanguageModel/Models/CoreMLLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/CoreMLLanguageModel.swift @@ -80,6 +80,8 @@ fatalError("CoreMLLanguageModel only supports generating String content") } + try validateNoImageSegments(in: session) + // Convert AnyLanguageModel GenerationOptions to swift-transformers GenerationConfig let generationConfig = toGenerationConfig(options) @@ -122,6 +124,17 @@ fatalError("CoreMLLanguageModel only supports generating String content") } + // Validate that no image segments are present + do { + try validateNoImageSegments(in: session) + } catch { + return LanguageModelSession.ResponseStream( + stream: AsyncThrowingStream { continuation in + continuation.finish(throwing: error) + } + ) + } + // Convert AnyLanguageModel GenerationOptions to swift-transformers GenerationConfig let generationConfig = toGenerationConfig(options) @@ -210,6 +223,8 @@ /// The model file was found but is corrupted, incompatible, or otherwise invalid. case modelInvalid(URL, underlyingError: Error) + /// Image segments are not supported in CoreMLLanguageModel + case unsupportedFeature public var errorDescription: String? { switch self { @@ -221,6 +236,33 @@ case .modelInvalid(let url, let underlyingError): return "Core ML model at \(url.path) is invalid or corrupted: \(underlyingError.localizedDescription). Please verify the model file is valid and compatible with the current Core ML version." + case .unsupportedFeature: + return "This CoreMLLanguageModel does not support image segments" + } + } + } + + // MARK: - Image Validation + + private func validateNoImageSegments(in session: LanguageModelSession) throws { + // Check for image segments in instructions + if let instructions = session.instructions { + for segment in instructions.segments { + if case .image = segment { + throw CoreMLLanguageModelError.unsupportedFeature + } + } + } + + // Check for image segments in the most recent prompt + for entry in session.transcript.reversed() { + if case .prompt(let p) = entry { + for segment in p.segments { + if case .image = segment { + throw CoreMLLanguageModelError.unsupportedFeature + } + } + break } } } @@ -245,7 +287,7 @@ config.topK = k case .nucleus(let p, _): config.doSample = true - config.topP = p + config.topP = Float(p) } } diff --git a/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift b/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift index 88429e03..c1f9318c 100644 --- a/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift @@ -97,8 +97,9 @@ public struct GeminiLanguageModel: LanguageModel { .appendingPathComponent("models/\(model):generateContent") let headers = buildHeaders() + let userSegments = extractPromptSegments(from: session, fallbackText: prompt.description) var contents = [ - GeminiContent(role: .user, parts: [.text(GeminiTextPart(text: prompt.description))]) + GeminiContent(role: .user, parts: convertSegmentsToGeminiParts(userSegments)) ] let geminiTools = try buildTools(from: session.tools) @@ -195,8 +196,9 @@ public struct GeminiLanguageModel: LanguageModel { fatalError("GeminiLanguageModel only supports generating String content") } + let userSegments = extractPromptSegments(from: session, fallbackText: prompt.description) let contents = [ - GeminiContent(role: .user, parts: [.text(GeminiTextPart(text: prompt.description))]) + GeminiContent(role: .user, parts: convertSegmentsToGeminiParts(userSegments)) ] var streamURL = @@ -442,6 +444,9 @@ private func toJSONValue(_ toolOutput: Transcript.ToolOutput) throws -> [String: if let jsonString = String(data: data, encoding: .utf8) { result["result"] = .string(jsonString) } + case .image: + // Ignore images in tool outputs for Gemini conversion + break } } @@ -511,12 +516,16 @@ private enum GeminiPart: Codable, Sendable { case text(GeminiTextPart) case functionCall(GeminiFunctionCall) case functionResponse(GeminiFunctionResponse) + case inlineData(GeminiInlineData) + case fileData(GeminiFileData) enum CodingKeys: String, CodingKey { case text case functionCall case functionResponse case thoughtSignature + case inlineData + case fileData } init(from decoder: any Decoder) throws { @@ -530,6 +539,10 @@ private enum GeminiPart: Codable, Sendable { self = .functionCall(try container.decode(GeminiFunctionCall.self, forKey: .functionCall)) } else if container.contains(.functionResponse) { self = .functionResponse(try container.decode(GeminiFunctionResponse.self, forKey: .functionResponse)) + } else if container.contains(.inlineData) { + self = .inlineData(try container.decode(GeminiInlineData.self, forKey: .inlineData)) + } else if container.contains(.fileData) { + self = .fileData(try container.decode(GeminiFileData.self, forKey: .fileData)) } else { throw DecodingError.dataCorrupted( DecodingError.Context( @@ -549,6 +562,10 @@ private enum GeminiPart: Codable, Sendable { try container.encode(call, forKey: .functionCall) case .functionResponse(let response): try container.encode(response, forKey: .functionResponse) + case .inlineData(let data): + try container.encode(data, forKey: .inlineData) + case .fileData(let data): + try container.encode(data, forKey: .fileData) } } } @@ -557,6 +574,54 @@ private struct GeminiTextPart: Codable, Sendable { let text: String } +private struct GeminiInlineData: Codable, Sendable { + let mimeType: String + let data: String + + enum CodingKeys: String, CodingKey { + case mimeType = "mime_type" + case data + } +} + +private struct GeminiFileData: Codable, Sendable { + let fileURI: String + + enum CodingKeys: String, CodingKey { + case fileURI = "file_uri" + } +} + +private func convertSegmentsToGeminiParts(_ segments: [Transcript.Segment]) -> [GeminiPart] { + var parts: [GeminiPart] = [] + parts.reserveCapacity(segments.count) + for segment in segments { + switch segment { + case .text(let t): + parts.append(.text(GeminiTextPart(text: t.content))) + case .structure(let s): + parts.append(.text(GeminiTextPart(text: s.content.jsonString))) + case .image(let img): + switch img.source { + case .data(let data, let mime): + parts.append(.inlineData(GeminiInlineData(mimeType: mime, data: data.base64EncodedString()))) + case .url(let url): + parts.append(.fileData(GeminiFileData(fileURI: url.absoluteString))) + } + } + } + return parts +} + +private func extractPromptSegments(from session: LanguageModelSession, fallbackText: String) -> [Transcript.Segment] { + for entry in session.transcript.reversed() { + if case .prompt(let p) = entry { + return p.segments + } + } + return [.text(.init(content: fallbackText))] +} + private struct GeminiFunctionCall: Codable, Sendable { let name: String let args: [String: JSONValue]? diff --git a/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift b/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift index 71cf860b..2b5b0855 100644 --- a/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift @@ -111,6 +111,9 @@ import Foundation fatalError("LlamaLanguageModel only supports generating String content") } + // Validate that no image segments are present + try validateNoImageSegments(in: session) + try await ensureModelLoaded() let contextParams = createContextParams(from: options) @@ -154,6 +157,17 @@ import Foundation fatalError("LlamaLanguageModel only supports generating String content") } + // Validate that no image segments are present + do { + try validateNoImageSegments(in: session) + } catch { + return LanguageModelSession.ResponseStream( + stream: AsyncThrowingStream { continuation in + continuation.finish(throwing: error) + } + ) + } + let maxTokens = options.maximumResponseTokens ?? 100 let stream: AsyncThrowingStream.Snapshot, any Error> = @@ -526,6 +540,31 @@ import Foundation } } + // MARK: - Image Validation + + private func validateNoImageSegments(in session: LanguageModelSession) throws { + // Check for image segments in instructions + if let instructions = session.instructions { + for segment in instructions.segments { + if case .image = segment { + throw LlamaLanguageModelError.unsupportedFeature + } + } + } + + // Check for image segments in the most recent prompt + for entry in session.transcript.reversed() { + if case .prompt(let p) = entry { + for segment in p.segments { + if case .image = segment { + throw LlamaLanguageModelError.unsupportedFeature + } + } + break + } + } + } + // MARK: - Helper Methods private func tokenizeText(vocab: OpaquePointer, text: String) throws -> [llama_token] { @@ -601,6 +640,7 @@ import Foundation case decodingFailed case invalidModelPath case insufficientMemory + case unsupportedFeature public var errorDescription: String? { switch self { @@ -618,6 +658,8 @@ import Foundation return "Invalid model file path" case .insufficientMemory: return "Insufficient memory for operation" + case .unsupportedFeature: + return "This LlamaLanguageModel does not support image segments" } } } diff --git a/Sources/AnyLanguageModel/Models/MLXLanguageModel.swift b/Sources/AnyLanguageModel/Models/MLXLanguageModel.swift index 2a5e1b3a..dda5ead7 100644 --- a/Sources/AnyLanguageModel/Models/MLXLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/MLXLanguageModel.swift @@ -1,6 +1,19 @@ import Foundation + +#if canImport(UIKit) + import UIKit + import CoreImage +#endif + +#if canImport(AppKit) + import AppKit + import CoreImage +#endif + #if MLX import MLXLMCommon + import MLX + import MLXVLM import Tokenizers /// A language model that runs locally using MLX. @@ -38,8 +51,7 @@ import Foundation fatalError("MLXLanguageModel only supports generating String content") } - // Load model context via MLXLMCommon - let context = try await MLXLMCommon.loadModel(id: modelId) + let context = try await loadModel(id: modelId) // Convert session tools to MLX ToolSpec format let toolSpecs: [ToolSpec]? = @@ -53,7 +65,9 @@ import Foundation let generateParameters = toGenerateParameters(options) // Start with user prompt - var chat: [MLXLMCommon.Chat.Message] = [.user(prompt.description)] + let userSegments = extractPromptSegments(from: session, fallbackText: prompt.description) + let userMessage = convertSegmentsToMLXMessage(userSegments) + var chat: [MLXLMCommon.Chat.Message] = [userMessage] var allTextChunks: [String] = [] var allEntries: [Transcript.Entry] = [] @@ -62,7 +76,8 @@ import Foundation // Build user input with current chat history and tools let userInput = MLXLMCommon.UserInput( chat: chat, - tools: toolSpecs + processing: .init(resize: .init(width: 512, height: 512)), + tools: toolSpecs, ) let lmInput = try await context.processor.prepare(input: userInput) @@ -164,6 +179,56 @@ import Foundation ) } + // MARK: - Segment Extraction + + private func extractPromptSegments(from session: LanguageModelSession, fallbackText: String) -> [Transcript.Segment] + { + // Prefer the most recent Transcript.Prompt entry if present + for entry in session.transcript.reversed() { + if case .prompt(let p) = entry { + return p.segments + } + } + return [.text(.init(content: fallbackText))] + } + + private func convertSegmentsToMLXMessage(_ segments: [Transcript.Segment]) -> MLXLMCommon.Chat.Message { + var textParts: [String] = [] + var images: [MLXLMCommon.UserInput.Image] = [] + + for segment in segments { + switch segment { + case .text(let text): + textParts.append(text.content) + case .structure(let structured): + textParts.append(structured.content.jsonString) + case .image(let imageSegment): + switch imageSegment.source { + case .url(let url): + images.append(.url(url)) + case .data(let data, _): + #if canImport(UIKit) + if let uiImage = UIKit.UIImage(data: data), + let ciImage = CIImage(image: uiImage) + { + images.append(.ciImage(ciImage)) + } + #elseif canImport(AppKit) + if let nsImage = AppKit.NSImage(data: data), + let cgImage = nsImage.cgImage(forProposedRect: nil, context: nil, hints: nil) + { + let ciImage = CIImage(cgImage: cgImage) + images.append(.ciImage(ciImage)) + } + #endif + } + } + } + + let content = textParts.joined(separator: "\n") + return MLXLMCommon.Chat.Message(role: .user, content: content, images: images) + } + // MARK: - Tool Conversion private func convertToolToMLXSpec(_ tool: any Tool) -> ToolSpec { @@ -267,6 +332,9 @@ import Foundation case .structure(let structuredSegment): // structured content already has jsonString property textParts.append(structuredSegment.content.jsonString) + case .image: + // Image segments are not supported in MLX tool output + break } } return textParts.joined(separator: "\n") diff --git a/Sources/AnyLanguageModel/Models/OllamaLanguageModel.swift b/Sources/AnyLanguageModel/Models/OllamaLanguageModel.swift index f45d1e31..a5c5c115 100644 --- a/Sources/AnyLanguageModel/Models/OllamaLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/OllamaLanguageModel.swift @@ -62,8 +62,10 @@ public struct OllamaLanguageModel: LanguageModel { fatalError("OllamaLanguageModel only supports generating String content") } + let userSegments = extractPromptSegments(from: session, fallbackText: prompt.description) + let (ollamaText, ollamaImages) = convertSegmentsToOllama(userSegments) let messages = [ - OllamaMessage(role: .user, content: prompt.description) + OllamaMessage(role: .user, content: ollamaText) ] let ollamaOptions = convertOptions(options) let ollamaTools = try session.tools.map { tool in @@ -75,7 +77,8 @@ public struct OllamaLanguageModel: LanguageModel { messages: messages, tools: ollamaTools.isEmpty ? nil : ollamaTools, options: ollamaOptions, - stream: false + stream: false, + images: ollamaImages.isEmpty ? nil : ollamaImages ) let url = baseURL.appendingPathComponent("api/chat") @@ -119,8 +122,10 @@ public struct OllamaLanguageModel: LanguageModel { fatalError("OllamaLanguageModel only supports generating String content") } + let userSegments = extractPromptSegments(from: session, fallbackText: prompt.description) + let (ollamaText, ollamaImages) = convertSegmentsToOllama(userSegments) let messages = [ - OllamaMessage(role: .user, content: prompt.description) + OllamaMessage(role: .user, content: ollamaText) ] let ollamaOptions = convertOptions(options) let ollamaTools = try? session.tools.map { tool in @@ -132,7 +137,8 @@ public struct OllamaLanguageModel: LanguageModel { messages: messages, tools: (ollamaTools?.isEmpty == false) ? ollamaTools : nil, options: ollamaOptions, - stream: true + stream: true, + images: (ollamaImages.isEmpty ? nil : ollamaImages) ) let url = baseURL.appendingPathComponent("api/chat") @@ -312,7 +318,8 @@ private func createChatParams( messages: [OllamaMessage], tools: [[String: JSONValue]]?, options: [String: JSONValue]?, - stream: Bool + stream: Bool, + images: [String]? ) throws -> [String: JSONValue] { var params: [String: JSONValue] = [ "model": .string(model), @@ -328,6 +335,10 @@ private func createChatParams( params["options"] = .object(options) } + if let images, !images.isEmpty { + params["images"] = .array(images.map { .string($0) }) + } + return params } @@ -345,6 +356,37 @@ private struct OllamaMessage: Hashable, Codable, Sendable { let content: String } +private func convertSegmentsToOllama(_ segments: [Transcript.Segment]) -> (String, [String]) { + var textParts: [String] = [] + var images: [String] = [] + for segment in segments { + switch segment { + case .text(let t): + textParts.append(t.content) + case .structure(let s): + textParts.append(s.content.jsonString) + case .image(let img): + switch img.source { + case .data(let data, _): + images.append(data.base64EncodedString()) + case .url(let url): + // Ollama supports base64 images; include URL as text if provided + textParts.append(url.absoluteString) + } + } + } + return (textParts.joined(separator: "\n"), images) +} + +private func extractPromptSegments(from session: LanguageModelSession, fallbackText: String) -> [Transcript.Segment] { + for entry in session.transcript.reversed() { + if case .prompt(let p) = entry { + return p.segments + } + } + return [.text(.init(content: fallbackText))] +} + private struct ChatResponse: Decodable, Sendable { let model: String let createdAt: Date diff --git a/Sources/AnyLanguageModel/Models/OpenAILanguageModel.swift b/Sources/AnyLanguageModel/Models/OpenAILanguageModel.swift index e09afef3..c8d2d714 100644 --- a/Sources/AnyLanguageModel/Models/OpenAILanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/OpenAILanguageModel.swift @@ -88,10 +88,13 @@ public struct OpenAILanguageModel: LanguageModel { } var messages: [OpenAIMessage] = [] - if let instructions = session.instructions { - messages.append(OpenAIMessage(role: .system, content: .text(instructions.description))) + if let systemSegments = extractInstructionSegments(from: session) { + messages.append( + OpenAIMessage(role: .system, content: .blocks(convertSegmentsToOpenAIBlocks(systemSegments))) + ) } - messages.append(OpenAIMessage(role: .user, content: .text(prompt.description))) + let userSegments = extractPromptSegments(from: session, fallbackText: prompt.description) + messages.append(OpenAIMessage(role: .user, content: .blocks(convertSegmentsToOpenAIBlocks(userSegments)))) // Convert tools if any are available in the session let openAITools: [OpenAITool]? = { @@ -234,10 +237,13 @@ public struct OpenAILanguageModel: LanguageModel { } var messages: [OpenAIMessage] = [] - if let instructions = session.instructions { - messages.append(OpenAIMessage(role: .system, content: .text(instructions.description))) + if let systemSegments = extractInstructionSegments(from: session) { + messages.append( + OpenAIMessage(role: .system, content: .blocks(convertSegmentsToOpenAIBlocks(systemSegments))) + ) } - messages.append(OpenAIMessage(role: .user, content: .text(prompt.description))) + let userSegments = extractPromptSegments(from: session, fallbackText: prompt.description) + messages.append(OpenAIMessage(role: .user, content: .blocks(convertSegmentsToOpenAIBlocks(userSegments)))) // Convert tools if any are available in the session let openAITools: [OpenAITool]? = { @@ -439,19 +445,65 @@ private enum Responses { options: GenerationOptions, stream: Bool ) -> JSONValue { - // Extract the system and user message content for the request + // Build input blocks from the user message content let systemMessage = messages.first { $0.role == .system } let userMessage = messages.first { $0.role == .user } - let inputText = if case .text(let text) = userMessage?.content { text } else { "" } var body: [String: JSONValue] = [ "model": .string(model), - "input": .string(inputText), "stream": .bool(stream), ] - if case .text(let instructions) = systemMessage?.content { - body["instructions"] = .string(instructions) + if let userMessage { + // Wrap user content into a single top-level message as required by Responses API + let contentBlocks: [JSONValue] + switch userMessage.content { + case .text(let text): + contentBlocks = [ + .object(["type": .string("input_text"), "text": .string(text)]) + ] + case .blocks(let blocks): + contentBlocks = blocks.map { block in + switch block { + case .text(let text): + return .object(["type": .string("input_text"), "text": .string(text)]) + case .imageURL(let url): + return .object([ + "type": .string("input_image"), + "image_url": .object(["url": .string(url)]), + ]) + } + } + } + + body["input"] = .array([ + .object([ + "type": .string("message"), + "role": .string("user"), + "content": .array(contentBlocks), + ]) + ]) + } else { + body["input"] = .array([ + .object([ + "type": .string("message"), + "role": .string("user"), + "content": .array([]), + ]) + ]) + } + + if let systemMessage { + switch systemMessage.content { + case .text(let text): + body["instructions"] = .string(text) + case .blocks(let blocks): + // Concatenate text blocks for instructions; ignore images + let text = blocks.compactMap { if case .text(let t) = $0 { return t } else { return nil } }.joined( + separator: "\n" + ) + if !text.isEmpty { body["instructions"] = .string(text) } + } } if let tools { @@ -490,6 +542,7 @@ private struct OpenAIMessage: Hashable, Codable, Sendable { enum Content: Hashable, Codable, Sendable { case text(String) + case blocks([Block]) } let role: Role @@ -512,10 +565,101 @@ private struct OpenAIMessage: Hashable, Codable, Sendable { "content": .array([.object(["type": .string("text"), "text": .string(text)])]), ]) } + case .blocks(let blocks): + switch apiVariant { + case .chatCompletions: + // Chat Completions accepts array of content parts + return .object([ + "role": .string(role.rawValue), + "content": .array(blocks.map { $0.jsonValueForChatCompletions }), + ]) + case .responses: + // Responses expects message content blocks + return .object([ + "role": .string(role.rawValue), + "content": .array(blocks.map { $0.jsonValueForResponses }), + ]) + } } } } +private enum Block: Hashable, Codable, Sendable { + case text(String) + case imageURL(String) + + var jsonValueForChatCompletions: JSONValue { + switch self { + case .text(let text): + return .object(["type": .string("text"), "text": .string(text)]) + case .imageURL(let url): + return .object([ + "type": .string("image_url"), + "image_url": .object(["url": .string(url)]), + ]) + } + } + + var jsonValueForResponses: JSONValue { + switch self { + case .text(let text): + return .object(["type": .string("text"), "text": .string(text)]) + case .imageURL(let url): + // Responses API uses input_image at top-level input, but inside messages we mirror block + return .object([ + "type": .string("input_image"), + "image_url": .object(["url": .string(url)]), + ]) + } + } +} + +private func convertSegmentsToOpenAIBlocks(_ segments: [Transcript.Segment]) -> [Block] { + var blocks: [Block] = [] + blocks.reserveCapacity(segments.count) + for segment in segments { + switch segment { + case .text(let text): + blocks.append(.text(text.content)) + case .structure(let structured): + blocks.append(.text(structured.content.jsonString)) + case .image(let image): + switch image.source { + case .url(let url): + blocks.append(.imageURL(url.absoluteString)) + case .data(let data, let mimeType): + let b64 = data.base64EncodedString() + let dataURL = "data:\(mimeType);base64,\(b64)" + blocks.append(.imageURL(dataURL)) + } + } + } + return blocks +} + +private func extractPromptSegments(from session: LanguageModelSession, fallbackText: String) -> [Transcript.Segment] { + // Prefer the most recent Transcript.Prompt entry if present + for entry in session.transcript.reversed() { + if case .prompt(let p) = entry { + return p.segments + } + } + return [.text(.init(content: fallbackText))] +} + +private func extractInstructionSegments(from session: LanguageModelSession) -> [Transcript.Segment]? { + // Prefer the first Transcript.Instructions entry if present + for entry in session.transcript { + if case .instructions(let i) = entry { + return i.segments + } + } + if let instructions = session.instructions?.description, !instructions.isEmpty { + return [.text(.init(content: instructions))] + } + return nil +} + private struct OpenAITool: Hashable, Codable, Sendable { let type: String let function: OpenAIFunction diff --git a/Sources/AnyLanguageModel/Transcript.swift b/Sources/AnyLanguageModel/Transcript.swift index 69b054b6..3dae191a 100644 --- a/Sources/AnyLanguageModel/Transcript.swift +++ b/Sources/AnyLanguageModel/Transcript.swift @@ -1,4 +1,4 @@ -import struct Foundation.UUID +import Foundation /// A type that represents a conversation history between a user and a language model. public struct Transcript: Sendable, Equatable, Codable { @@ -61,9 +61,12 @@ public struct Transcript: Sendable, Equatable, Codable { /// A segment containing text. case text(TextSegment) - /// A segment containing structured content + /// A segment containing structured content. case structure(StructuredSegment) + /// A segment containing an image. + case image(ImageSegment) + /// The stable identity of the entity associated with this instance. public var id: String { switch self { @@ -71,6 +74,8 @@ public struct Transcript: Sendable, Equatable, Codable { return textSegment.id case .structure(let structuredSegment): return structuredSegment.id + case .image(let imageSegment): + return imageSegment.id } } } @@ -106,6 +111,97 @@ public struct Transcript: Sendable, Equatable, Codable { } } + /// A segment that represents an image for multi‑modal prompts and outputs. + /// + /// Use this type to include images alongside text and structured content when + /// constructing `Transcript` entries. Images can be provided as raw data with a + /// MIME type or by URL. + public struct ImageSegment: Sendable, Identifiable, Equatable, Codable { + /// The stable identity of the entity associated with this instance. + public var id: String + + /// The source of the image data. + public let source: Source + + /// The origin of an image's content. + public enum Source: Sendable, Equatable, Codable { + /// Image bytes and their MIME type (for example, `image/jpeg`). + case data(Data, mimeType: String) + /// A URL that references an image. + case url(URL) + + private enum CodingKeys: String, CodingKey { case kind, data, mimeType, url } + + public init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + let kind = try container.decode(String.self, forKey: .kind) + switch kind { + case "data": + let data = try container.decode(Data.self, forKey: .data) + let mimeType = try container.decode(String.self, forKey: .mimeType) + self = .data(data, mimeType: mimeType) + case "url": + let url = try container.decode(URL.self, forKey: .url) + self = .url(url) + default: + throw DecodingError.dataCorrupted( + .init(codingPath: [CodingKeys.kind], debugDescription: "Unknown image source kind: \(kind)") + ) + } + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + switch self { + case .data(let data, let mimeType): + try container.encode("data", forKey: .kind) + try container.encode(data, forKey: .data) + try container.encode(mimeType, forKey: .mimeType) + case .url(let url): + try container.encode("url", forKey: .kind) + try container.encode(url, forKey: .url) + } + } + } + + /// Creates an image segment from a source. + /// + /// - Parameters: + /// - id: A unique identifier for this segment. Defaults to a generated UUID. + /// - source: The image source. + public init(id: String = UUID().uuidString, source: Source) { + self.id = id + self.source = source + } + + /// Creates an image segment from raw bytes. + /// + /// - Parameters: + /// - id: A unique identifier for this segment. Defaults to a generated UUID. + /// - data: The encoded image bytes. + /// - mimeType: The MIME type corresponding to the image data (for example, `image/png`). + public init(id: String = UUID().uuidString, data: Data, mimeType: String) { + self.id = id + self.source = .data(data, mimeType: mimeType) + } + + /// Creates an image segment from a URL. + /// + /// - Parameters: + /// - id: A unique identifier for this segment. Defaults to a generated UUID. + /// - url: A URL that references an image. + public init(id: String = UUID().uuidString, url: URL) { + self.id = id + self.source = .url(url) + } + } + + /// Errors that can occur when converting platform images to encoded data. + public enum ImageEncodingError: Error { + /// The image couldn't be converted to the requested format. + case imageConversionFailed + } + /// Instructions you provide to the model that define its behavior. /// /// Instructions are typically provided to define the role and behavior of the model. Apple trains the model @@ -333,6 +429,8 @@ extension Transcript.Segment: CustomStringConvertible { return textSegment.description case .structure(let structuredSegment): return structuredSegment.description + case .image: + return "" } } } @@ -347,6 +445,152 @@ extension Transcript.StructuredSegment: CustomStringConvertible { } } +extension Transcript.ImageSegment { + /// Preferred image encodings for image conversion. + public enum Format: Sendable { + /// JPEG encoding with the specified compression quality. + case jpeg(compressionQuality: Double = 0.9) + /// PNG encoding. + case png + } +} + +#if canImport(UIKit) + import UIKit + + extension Transcript.ImageSegment { + fileprivate static func encode(_ image: UIImage, format: Format) throws -> (Data, String) { + switch format { + case .jpeg(let quality): + guard let data = image.jpegData(compressionQuality: quality) else { + throw Transcript.ImageEncodingError.imageConversionFailed + } + return (data, "image/jpeg") + case .png: + guard let data = image.pngData() else { + throw Transcript.ImageEncodingError.imageConversionFailed + } + return (data, "image/png") + } + } + + /// Creates an image segment by encoding a UIKit image. + /// + /// - Parameters: + /// - image: The source image to encode. + /// - format: The target encoding. Defaults to JPEG with 0.9 quality. + /// - Throws: ``Transcript/ImageEncodingError-swift.enum/imageConversionFailed`` if encoding fails. + public init(image: UIImage, format: Format = .jpeg()) throws { + let (data, mimeType) = try Self.encode(image, format: format) + self.init(data: data, mimeType: mimeType) + } + } +#endif + +#if canImport(AppKit) + import AppKit + + extension Transcript.ImageSegment { + fileprivate static func encode(_ image: NSImage, format: Format) throws -> (Data, String) { + guard let tiffData = image.tiffRepresentation, + let bitmapImage = NSBitmapImageRep(data: tiffData) + else { + throw Transcript.ImageEncodingError.imageConversionFailed + } + + switch format { + case .jpeg(let quality): + guard + let data = bitmapImage.representation( + using: .jpeg, + properties: [.compressionFactor: quality] + ) + else { + throw Transcript.ImageEncodingError.imageConversionFailed + } + return (data, "image/jpeg") + case .png: + guard + let data = bitmapImage.representation( + using: .png, + properties: [:] + ) + else { + throw Transcript.ImageEncodingError.imageConversionFailed + } + return (data, "image/png") + } + } + + /// Creates an image segment by encoding an AppKit image. + /// + /// - Parameters: + /// - image: The source image to encode. + /// - format: The target encoding. Defaults to JPEG with 0.9 quality. + /// - Throws: ``Transcript/ImageEncodingError-swift.enum/imageConversionFailed`` if encoding fails. + public init(image: NSImage, format: Format = .jpeg()) throws { + let (data, mimeType) = try Self.encode(image, format: format) + self.init(data: data, mimeType: mimeType) + } + } +#endif + +#if canImport(CoreGraphics) + import CoreGraphics + import ImageIO + import UniformTypeIdentifiers + + extension Transcript.ImageSegment { + fileprivate static func encode(_ image: CGImage, format: Format) throws -> (Data, String) { + let data = NSMutableData() + let utType: UTType + + switch format { + case .jpeg: + utType = .jpeg + case .png: + utType = .png + } + + guard + let destination = CGImageDestinationCreateWithData( + data, + utType.identifier as CFString, + 1, + nil + ) + else { + throw Transcript.ImageEncodingError.imageConversionFailed + } + + var properties: [CFString: Any] = [:] + if case .jpeg(let quality) = format { + properties[kCGImageDestinationLossyCompressionQuality] = quality + } + + CGImageDestinationAddImage(destination, image, properties as CFDictionary) + + guard CGImageDestinationFinalize(destination) else { + throw Transcript.ImageEncodingError.imageConversionFailed + } + + let mimeType = (utType == .jpeg) ? "image/jpeg" : "image/png" + return (data as Data, mimeType) + } + + /// Creates an image segment by encoding a CoreGraphics image. + /// + /// - Parameters: + /// - image: The source image to encode. + /// - format: The target encoding. Defaults to JPEG with 0.9 quality. + /// - Throws: ``Transcript/ImageEncodingError-swift.enum/imageConversionFailed`` if encoding fails. + public init(image: CGImage, format: Format = .jpeg()) throws { + let (data, mimeType) = try Self.encode(image, format: format) + self.init(data: data, mimeType: mimeType) + } + } +#endif + extension Transcript.Instructions: CustomStringConvertible { public var description: String { "Instructions(segments: \(segments.count), tools: \(toolDefinitions.count))" diff --git a/Tests/AnyLanguageModelTests/AnthropicLanguageModelTests.swift b/Tests/AnyLanguageModelTests/AnthropicLanguageModelTests.swift index 8fccc314..2a03db30 100644 --- a/Tests/AnyLanguageModelTests/AnthropicLanguageModelTests.swift +++ b/Tests/AnyLanguageModelTests/AnthropicLanguageModelTests.swift @@ -99,4 +99,32 @@ struct AnthropicLanguageModelTests { } #expect(foundToolOutput) } + + @Test func multimodalWithImageURL() async throws { + let transcript = Transcript(entries: [ + .prompt( + Transcript.Prompt(segments: [ + .text(.init(content: "Describe this image")), + .image(.init(url: testImageURL)), + ]) + ) + ]) + let session = LanguageModelSession(model: model, transcript: transcript) + let response = try await session.respond(to: "") + #expect(!response.content.isEmpty) + } + + @Test func multimodalWithImageData() async throws { + let transcript = Transcript(entries: [ + .prompt( + Transcript.Prompt(segments: [ + .text(.init(content: "Describe this image")), + .image(.init(data: testImageData, mimeType: "image/png")), + ]) + ) + ]) + let session = LanguageModelSession(model: model, transcript: transcript) + let response = try await session.respond(to: "") + #expect(!response.content.isEmpty) + } } diff --git a/Tests/AnyLanguageModelTests/CoreMLLanguageModelTests.swift b/Tests/AnyLanguageModelTests/CoreMLLanguageModelTests.swift index be568b97..f725c82a 100644 --- a/Tests/AnyLanguageModelTests/CoreMLLanguageModelTests.swift +++ b/Tests/AnyLanguageModelTests/CoreMLLanguageModelTests.swift @@ -133,5 +133,39 @@ import Testing // Note: We can't easily test token count without access to the tokenizer // but we can verify the response is not empty } + + @Test @available(macOS 15.0, iOS 18.0, tvOS 18.0, visionOS 2.0, watchOS 11.0, *) + func multimodal_rejectsImageURL() async throws { + let model = try await getModel() + let session = LanguageModelSession(model: model) + let prompt = Transcript.Prompt(segments: [ + .text(.init(content: "Describe this image")), + .image(.init(url: testImageURL)), + ]) + do { + _ = try await session.respond(to: prompt) + Issue.record("Expected error when image segments are present") + } catch { + // CoreMLUnsupportedFeatureError is a private struct, so we just check that an error is thrown + #expect(true) + } + } + + @Test @available(macOS 15.0, iOS 18.0, tvOS 18.0, visionOS 2.0, watchOS 11.0, *) + func multimodal_rejectsImageData() async throws { + let model = try await getModel() + let session = LanguageModelSession(model: model) + let prompt = Transcript.Prompt(segments: [ + .text(.init(content: "Describe this image")), + .image(.init(data: testImageData, mimeType: "image/jpeg")), + ]) + do { + _ = try await session.respond(to: prompt) + Issue.record("Expected error when image segments are present") + } catch { + // CoreMLUnsupportedFeatureError is a private struct, so we just check that an error is thrown + #expect(true) + } + } } #endif // CoreML diff --git a/Tests/AnyLanguageModelTests/GeminiLanguageModelTests.swift b/Tests/AnyLanguageModelTests/GeminiLanguageModelTests.swift index 420d36da..447fdd1b 100644 --- a/Tests/AnyLanguageModelTests/GeminiLanguageModelTests.swift +++ b/Tests/AnyLanguageModelTests/GeminiLanguageModelTests.swift @@ -119,4 +119,32 @@ struct GeminiLanguageModelTests { let response = try await session.respond(to: "What coffee shops are nearby?") #expect(!response.content.isEmpty) } + + @Test func multimodalWithImageURL() async throws { + let transcript = Transcript(entries: [ + .prompt( + Transcript.Prompt(segments: [ + .text(.init(content: "Describe this image")), + .image(.init(url: testImageURL)), + ]) + ) + ]) + let session = LanguageModelSession(model: model, transcript: transcript) + let response = try await session.respond(to: "") + #expect(!response.content.isEmpty) + } + + @Test func multimodalWithImageData() async throws { + let transcript = Transcript(entries: [ + .prompt( + Transcript.Prompt(segments: [ + .text(.init(content: "Describe this image")), + .image(.init(data: testImageData, mimeType: "image/png")), + ]) + ) + ]) + let session = LanguageModelSession(model: model, transcript: transcript) + let response = try await session.respond(to: "") + #expect(!response.content.isEmpty) + } } diff --git a/Tests/AnyLanguageModelTests/LlamaLanguageModelTests.swift b/Tests/AnyLanguageModelTests/LlamaLanguageModelTests.swift index 8c3746c0..162a23d8 100644 --- a/Tests/AnyLanguageModelTests/LlamaLanguageModelTests.swift +++ b/Tests/AnyLanguageModelTests/LlamaLanguageModelTests.swift @@ -139,5 +139,35 @@ import Testing ) #expect(!response.content.isEmpty) } + + @Test func multimodal_rejectsImageURL() async throws { + let session = LanguageModelSession(model: model) + let prompt = Transcript.Prompt(segments: [ + .text(.init(content: "Describe this image")), + .image(.init(url: testImageURL)), + ]) + do { + _ = try await session.respond(to: prompt) + Issue.record("Expected error when image segments are present") + } catch let error as LlamaLanguageModelError { + #expect(error == .unsupportedFeature) + } + } + + @Test func multimodal_rejectsImageData() async throws { + let session = LanguageModelSession(model: model) + + let data = Data(png1x1) + let prompt = Transcript.Prompt(segments: [ + .text(.init(content: "Describe this image")), + .image(.init(data: data, mimeType: "image/png")), + ]) + do { + _ = try await session.respond(to: prompt) + Issue.record("Expected error when image segments are present") + } catch let error as LlamaLanguageModelError { + #expect(error == .unsupportedFeature) + } + } } #endif // Llama diff --git a/Tests/AnyLanguageModelTests/MLXLanguageModelTests.swift b/Tests/AnyLanguageModelTests/MLXLanguageModelTests.swift index 32aafdd9..9f72715d 100644 --- a/Tests/AnyLanguageModelTests/MLXLanguageModelTests.swift +++ b/Tests/AnyLanguageModelTests/MLXLanguageModelTests.swift @@ -33,6 +33,7 @@ import Testing struct MLXLanguageModelTests { // Qwen3-0.6B is a small model that supports tool calling let model = MLXLanguageModel(modelId: "mlx-community/Qwen3-0.6B-4bit") + let visionModel = MLXLanguageModel(modelId: "mlx-community/Qwen2-VL-2B-Instruct-4bit") @Test func basicResponse() async throws { let session = LanguageModelSession(model: model) @@ -79,5 +80,33 @@ import Testing #expect(first.arguments.city.contains("San Francisco")) } } + + @Test func multimodalWithImageURL() async throws { + let transcript = Transcript(entries: [ + .prompt( + Transcript.Prompt(segments: [ + .text(.init(content: "Describe this image")), + .image(.init(url: testImageURL)), + ]) + ) + ]) + let session = LanguageModelSession(model: visionModel, transcript: transcript) + let response = try await session.respond(to: "") + #expect(!response.content.isEmpty) + } + + @Test func multimodalWithImageData() async throws { + let transcript = Transcript(entries: [ + .prompt( + Transcript.Prompt(segments: [ + .text(.init(content: "Describe this image")), + .image(.init(data: testImageData, mimeType: "image/png")), + ]) + ) + ]) + let session = LanguageModelSession(model: visionModel, transcript: transcript) + let response = try await session.respond(to: "") + #expect(!response.content.isEmpty) + } } #endif // MLX diff --git a/Tests/AnyLanguageModelTests/MockLanguageModelTests.swift b/Tests/AnyLanguageModelTests/MockLanguageModelTests.swift index 5608d046..5ec4ac82 100644 --- a/Tests/AnyLanguageModelTests/MockLanguageModelTests.swift +++ b/Tests/AnyLanguageModelTests/MockLanguageModelTests.swift @@ -146,4 +146,120 @@ struct MockLanguageModelTests { #expect(!entry.id.isEmpty) } } + + @Test func respondWithSingleImage() async throws { + let model = MockLanguageModel.echo + let session = LanguageModelSession(model: model) + + let image = Transcript.ImageSegment(url: testImageURL) + let response = try await session.respond(to: "Describe this image", image: image) + + #expect(response.content.contains("Describe this image")) + + // Verify transcript has prompt with text + image and a response + #expect(session.transcript.count == 2) + if case .prompt(let promptEntry) = session.transcript[0] { + #expect(promptEntry.segments.count == 2) + // Expect one text and one image segment + let kinds = promptEntry.segments.map { segment -> String in + switch segment { + case .text: return "text"; + case .image: return "image"; + case .structure: return "structure" + } + } + #expect(kinds.contains("text")) + #expect(kinds.contains("image")) + } else { + Issue.record("First entry should be prompt with image") + } + } + + @Test func respondWithMultipleImages() async throws { + let model = MockLanguageModel.echo + let session = LanguageModelSession(model: model) + + let images: [Transcript.ImageSegment] = [ + .init(url: testImageURL), + .init(data: testImageData, mimeType: "image/png"), + ] + let response = try await session.respond(to: "Classify these", images: images) + + #expect(response.content.contains("Classify these")) + #expect(session.transcript.count == 2) + if case .prompt(let promptEntry) = session.transcript[0] { + #expect(promptEntry.segments.count == 3) + let imageCount = promptEntry.segments.reduce(into: 0) { count, seg in if case .image = seg { count += 1 } } + #expect(imageCount == 2) + } else { + Issue.record("First entry should be prompt with images") + } + } + + @Test func respondGeneratingStringWithImages() async throws { + let model = MockLanguageModel.echo + let session = LanguageModelSession(model: model) + + let images: [Transcript.ImageSegment] = [ + .init(url: testImageURL) + ] + let response: LanguageModelSession.Response = try await session.respond( + to: "What do you see?", + images: images, + generating: String.self, + includeSchemaInPrompt: true + ) + + #expect(response.content.contains("What do you see?")) + #expect(session.transcript.count == 2) + if case .prompt(let promptEntry) = session.transcript[0] { + #expect(promptEntry.segments.count == 2) + } else { + Issue.record("First entry should be prompt with image") + } + } + + @Test func streamResponseWithSingleImage() async throws { + let model = MockLanguageModel.streamingMock() + let session = LanguageModelSession(model: model) + + let image = Transcript.ImageSegment(url: testImageURL) + let stream = session.streamResponse(to: "Stream about image", image: image) + + var snapshots = 0 + for try await _ in stream { snapshots += 1 } + #expect(snapshots >= 1) + + // Prompt added at start, response added at end (append may occur after stream finishes) + try await Task.sleep(for: .milliseconds(10)) + #expect(session.transcript.count == 2) + if case .prompt(let promptEntry) = session.transcript[0] { + #expect(promptEntry.segments.count == 2) + } else { + Issue.record("First entry should be prompt with image") + } + } + + @Test func streamResponseWithMultipleImages() async throws { + let model = MockLanguageModel.streamingMock() + let session = LanguageModelSession(model: model) + + let images: [Transcript.ImageSegment] = [ + .init(url: testImageURL), + .init(data: testImageData, mimeType: "image/png"), + ] + let stream = session.streamResponse(to: "Stream about images", images: images) + + for try await _ in stream { /* drain */ } + // Response append occurs after stream finishes; wait briefly + try await Task.sleep(for: .milliseconds(10)) + #expect(session.transcript.count == 2) + if case .prompt(let promptEntry) = session.transcript[0] { + #expect(promptEntry.segments.count == 3) + let imageCount = promptEntry.segments.reduce(into: 0) { c, seg in if case .image = seg { c += 1 } } + #expect(imageCount == 2) + } else { + Issue.record("First entry should be prompt with images") + } + } } diff --git a/Tests/AnyLanguageModelTests/OllamaLanguageModelTests.swift b/Tests/AnyLanguageModelTests/OllamaLanguageModelTests.swift index c4d4f77d..50c4c437 100644 --- a/Tests/AnyLanguageModelTests/OllamaLanguageModelTests.swift +++ b/Tests/AnyLanguageModelTests/OllamaLanguageModelTests.swift @@ -108,4 +108,32 @@ struct OllamaLanguageModelTests { Issue.record("Expected successful tool call") } } + + @Test func multimodalWithImageURL() async throws { + let transcript = Transcript(entries: [ + .prompt( + Transcript.Prompt(segments: [ + .text(.init(content: "Describe this image")), + .image(.init(url: testImageURL)), + ]) + ) + ]) + let session = LanguageModelSession(model: model, transcript: transcript) + let response = try await session.respond(to: "") + #expect(!response.content.isEmpty) + } + + @Test func multimodalWithImageData() async throws { + let transcript = Transcript(entries: [ + .prompt( + Transcript.Prompt(segments: [ + .text(.init(content: "Describe this image")), + .image(.init(data: testImageData, mimeType: "image/png")), + ]) + ) + ]) + let session = LanguageModelSession(model: model, transcript: transcript) + let response = try await session.respond(to: "") + #expect(!response.content.isEmpty) + } } diff --git a/Tests/AnyLanguageModelTests/OpenAILanguageModelTests.swift b/Tests/AnyLanguageModelTests/OpenAILanguageModelTests.swift index d76c47af..11028eab 100644 --- a/Tests/AnyLanguageModelTests/OpenAILanguageModelTests.swift +++ b/Tests/AnyLanguageModelTests/OpenAILanguageModelTests.swift @@ -89,6 +89,34 @@ struct OpenAILanguageModelTests { #expect(!response.content.isEmpty) } + @Test func multimodalWithImageURL() async throws { + let transcript = Transcript(entries: [ + .prompt( + Transcript.Prompt(segments: [ + .text(.init(content: "Describe this image")), + .image(.init(url: testImageURL)), + ]) + ) + ]) + let session = LanguageModelSession(model: model, transcript: transcript) + let response = try await session.respond(to: "") + #expect(!response.content.isEmpty) + } + + @Test func multimodalWithImageData() async throws { + let transcript = Transcript(entries: [ + .prompt( + Transcript.Prompt(segments: [ + .text(.init(content: "Describe this image")), + .image(.init(data: testImageData, mimeType: "image/png")), + ]) + ) + ]) + let session = LanguageModelSession(model: model, transcript: transcript) + let response = try await session.respond(to: "") + #expect(!response.content.isEmpty) + } + @Test func conversationContext() async throws { let session = LanguageModelSession(model: model) @@ -181,6 +209,34 @@ struct OpenAILanguageModelTests { #expect(!response.content.isEmpty) } + @Test func multimodalWithImageURL() async throws { + let transcript = Transcript(entries: [ + .prompt( + Transcript.Prompt(segments: [ + .text(.init(content: "Describe this image")), + .image(.init(url: testImageURL)), + ]) + ) + ]) + let session = LanguageModelSession(model: model, transcript: transcript) + let response = try await session.respond(to: "") + #expect(!response.content.isEmpty) + } + + @Test func multimodalWithImageData() async throws { + let transcript = Transcript(entries: [ + .prompt( + Transcript.Prompt(segments: [ + .text(.init(content: "Describe this image")), + .image(.init(data: testImageData, mimeType: "image/png")), + ]) + ) + ]) + let session = LanguageModelSession(model: model, transcript: transcript) + let response = try await session.respond(to: "") + #expect(!response.content.isEmpty) + } + @Test func conversationContext() async throws { let session = LanguageModelSession(model: model) diff --git a/Tests/AnyLanguageModelTests/Shared/Images.swift b/Tests/AnyLanguageModelTests/Shared/Images.swift new file mode 100644 index 00000000..ba380224 --- /dev/null +++ b/Tests/AnyLanguageModelTests/Shared/Images.swift @@ -0,0 +1,33 @@ +import Foundation + +let testImageURL = URL(string: "https://github.com/huggingface.png")! + +// Minimal 1x1 PNG +let testImageData = Data([ + // PNG signature + 0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, + + // IHDR chunk + 0x00, 0x00, 0x00, 0x0D, // IHDR length (13) + 0x49, 0x48, 0x44, 0x52, // "IHDR" + 0x00, 0x00, 0x00, 0x01, // width = 1 + 0x00, 0x00, 0x00, 0x01, // height = 1 + 0x01, // bit depth = 1 (1-bit grayscale) + 0x00, // color type = 0 (grayscale) + 0x00, // compression method = 0 (deflate) + 0x00, // filter method = 0 (none) + 0x00, // interlace method = 0 (no interlace) + 0x37, 0x6E, 0xF9, 0x24, // IHDR CRC32 + + // IDAT chunk + 0x00, 0x00, 0x00, 0x0A, // IDAT length (10) + 0x49, 0x44, 0x41, 0x54, // "IDAT" + 0x78, 0x01, // zlib header (deflate, no compression) + 0x63, 0x60, 0x00, 0x00, 0x00, 0x02, 0x00, 0x01, // compressed data (scanline 00 00) + 0x73, 0x75, 0x01, 0x18, // IDAT CRC32 + + // IEND chunk + 0x00, 0x00, 0x00, 0x00, // IEND length (0) + 0x49, 0x45, 0x4E, 0x44, // "IEND" + 0xAE, 0x42, 0x60, 0x82, // IEND CRC32 +])