From 777b1ab59805c3b8e99fde7c3513886b15cbc217 Mon Sep 17 00:00:00 2001 From: Mattt Zmuda Date: Wed, 5 Nov 2025 05:16:07 -0600 Subject: [PATCH 01/16] Add APIs for image segments in transcript --- Sources/AnyLanguageModel/Transcript.swift | 142 +++++++++++++++++++++- 1 file changed, 140 insertions(+), 2 deletions(-) diff --git a/Sources/AnyLanguageModel/Transcript.swift b/Sources/AnyLanguageModel/Transcript.swift index 69b054b6..d0c6d8ce 100644 --- a/Sources/AnyLanguageModel/Transcript.swift +++ b/Sources/AnyLanguageModel/Transcript.swift @@ -1,4 +1,4 @@ -import struct Foundation.UUID +import Foundation /// A type that represents a conversation history between a user and a language model. public struct Transcript: Sendable, Equatable, Codable { @@ -61,9 +61,12 @@ public struct Transcript: Sendable, Equatable, Codable { /// A segment containing text. case text(TextSegment) - /// A segment containing structured content + /// A segment containing structured content. case structure(StructuredSegment) + /// A segment containing an image. + case image(ImageSegment) + /// The stable identity of the entity associated with this instance. public var id: String { switch self { @@ -71,6 +74,8 @@ public struct Transcript: Sendable, Equatable, Codable { return textSegment.id case .structure(let structuredSegment): return structuredSegment.id + case .image(let imageSegment): + return imageSegment.id } } } @@ -106,6 +111,91 @@ public struct Transcript: Sendable, Equatable, Codable { } } + /// A segment that represents an image for multi‑modal prompts and outputs. + /// + /// Use this type to include images alongside text and structured content when + /// constructing `Transcript` entries. Images can be provided as raw data with a + /// MIME type or by URL. + public struct ImageSegment: Sendable, Identifiable, Equatable, Codable { + /// The stable identity of the entity associated with this instance. + public var id: String + + /// The source of the image data. + public let source: Source + + /// The origin of an image's content. + public enum Source: Sendable, Equatable, Codable { + /// Image bytes and their MIME type (for example, `image/jpeg`). + case data(Data, mimeType: String) + /// A URL that references an image. + case url(URL) + + private enum CodingKeys: String, CodingKey { case kind, data, mimeType, url } + + public init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + let kind = try container.decode(String.self, forKey: .kind) + switch kind { + case "data": + let data = try container.decode(Data.self, forKey: .data) + let mimeType = try container.decode(String.self, forKey: .mimeType) + self = .data(data, mimeType: mimeType) + case "url": + let url = try container.decode(URL.self, forKey: .url) + self = .url(url) + default: + throw DecodingError.dataCorrupted( + .init(codingPath: [CodingKeys.kind], debugDescription: "Unknown image source kind: \(kind)") + ) + } + } + + public func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + switch self { + case .data(let data, let mimeType): + try container.encode("data", forKey: .kind) + try container.encode(data, forKey: .data) + try container.encode(mimeType, forKey: .mimeType) + case .url(let url): + try container.encode("url", forKey: .kind) + try container.encode(url, forKey: .url) + } + } + } + + /// Creates an image segment from a source. + /// + /// - Parameters: + /// - id: A unique identifier for this segment. Defaults to a generated UUID. + /// - source: The image source. + public init(id: String = UUID().uuidString, source: Source) { + self.id = id + self.source = source + } + + /// Creates an image segment from raw bytes. + /// + /// - Parameters: + /// - id: A unique identifier for this segment. Defaults to a generated UUID. + /// - data: The encoded image bytes. + /// - mimeType: The MIME type corresponding to the image data (for example, `image/png`). + public init(id: String = UUID().uuidString, data: Data, mimeType: String) { + self.id = id + self.source = .data(data, mimeType: mimeType) + } + + /// Creates an image segment from a URL. + /// + /// - Parameters: + /// - id: A unique identifier for this segment. Defaults to a generated UUID. + /// - url: A URL that references an image. + public init(id: String = UUID().uuidString, url: URL) { + self.id = id + self.source = .url(url) + } + } + /// Instructions you provide to the model that define its behavior. /// /// Instructions are typically provided to define the role and behavior of the model. Apple trains the model @@ -333,6 +423,8 @@ extension Transcript.Segment: CustomStringConvertible { return textSegment.description case .structure(let structuredSegment): return structuredSegment.description + case .image: + return "" } } } @@ -347,6 +439,52 @@ extension Transcript.StructuredSegment: CustomStringConvertible { } } +#if canImport(UIKit) + import UIKit + + extension Transcript.ImageSegment { + /// Preferred image encodings for UIKit image conversion. + public enum ImageFormat { + /// JPEG encoding with the specified compression quality. + case jpeg(compressionQuality: Double = 0.9) + /// PNG encoding. + case png + + fileprivate func encode(_ image: UIImage) throws -> (Data, String) { + switch self { + case .jpeg(let quality): + guard let data = image.jpegData(compressionQuality: quality) else { + throw ImageEncodingError.imageConversionFailed + } + return (data, "image/jpeg") + case .png: + guard let data = image.pngData() else { + throw ImageEncodingError.imageConversionFailed + } + return (data, "image/png") + } + } + } + + /// Errors that can occur when converting a `UIImage` to encoded data. + public enum ImageEncodingError: Error { + /// The image couldn't be converted to the requested format. + case imageConversionFailed + } + + /// Creates an image segment by encoding a UIKit image. + /// + /// - Parameters: + /// - image: The source image to encode. + /// - format: The target encoding. Defaults to JPEG with 0.9 quality. + /// - Throws: ``Transcript/ImageSegment/ImageEncodingError-swift.enum/imageConversionFailed`` if encoding fails. + public init(image: UIImage, format: ImageFormat = .jpeg()) throws { + let (data, mimeType) = try format.encode(image) + self.init(data: data, mimeType: mimeType) + } + } +#endif + extension Transcript.Instructions: CustomStringConvertible { public var description: String { "Instructions(segments: \(segments.count), tools: \(toolDefinitions.count))" From 25c08266d95a08a614ee823c95e6e1bc04015b24 Mon Sep 17 00:00:00 2001 From: Mattt Zmuda Date: Wed, 5 Nov 2025 05:27:18 -0600 Subject: [PATCH 02/16] Add convenience methods for AppKit / NSImage --- Sources/AnyLanguageModel/Transcript.swift | 74 ++++++++++++++++++++--- 1 file changed, 65 insertions(+), 9 deletions(-) diff --git a/Sources/AnyLanguageModel/Transcript.swift b/Sources/AnyLanguageModel/Transcript.swift index d0c6d8ce..04f059f8 100644 --- a/Sources/AnyLanguageModel/Transcript.swift +++ b/Sources/AnyLanguageModel/Transcript.swift @@ -196,6 +196,12 @@ public struct Transcript: Sendable, Equatable, Codable { } } + /// Errors that can occur when converting platform images to encoded data. + public enum ImageEncodingError: Error { + /// The image couldn't be converted to the requested format. + case imageConversionFailed + } + /// Instructions you provide to the model that define its behavior. /// /// Instructions are typically provided to define the role and behavior of the model. Apple trains the model @@ -454,30 +460,24 @@ extension Transcript.StructuredSegment: CustomStringConvertible { switch self { case .jpeg(let quality): guard let data = image.jpegData(compressionQuality: quality) else { - throw ImageEncodingError.imageConversionFailed + throw Transcript.ImageEncodingError.imageConversionFailed } return (data, "image/jpeg") case .png: guard let data = image.pngData() else { - throw ImageEncodingError.imageConversionFailed + throw Transcript.ImageEncodingError.imageConversionFailed } return (data, "image/png") } } } - /// Errors that can occur when converting a `UIImage` to encoded data. - public enum ImageEncodingError: Error { - /// The image couldn't be converted to the requested format. - case imageConversionFailed - } - /// Creates an image segment by encoding a UIKit image. /// /// - Parameters: /// - image: The source image to encode. /// - format: The target encoding. Defaults to JPEG with 0.9 quality. - /// - Throws: ``Transcript/ImageSegment/ImageEncodingError-swift.enum/imageConversionFailed`` if encoding fails. + /// - Throws: ``Transcript/ImageEncodingError-swift.enum/imageConversionFailed`` if encoding fails. public init(image: UIImage, format: ImageFormat = .jpeg()) throws { let (data, mimeType) = try format.encode(image) self.init(data: data, mimeType: mimeType) @@ -485,6 +485,62 @@ extension Transcript.StructuredSegment: CustomStringConvertible { } #endif +#if canImport(AppKit) + import AppKit + + extension Transcript.ImageSegment { + /// Preferred image encodings for AppKit image conversion. + public enum ImageFormat { + /// JPEG encoding with the specified compression quality. + case jpeg(compressionQuality: Double = 0.9) + /// PNG encoding. + case png + + fileprivate func encode(_ image: NSImage) throws -> (Data, String) { + guard let tiffData = image.tiffRepresentation, + let bitmapImage = NSBitmapImageRep(data: tiffData) + else { + throw Transcript.ImageEncodingError.imageConversionFailed + } + + switch self { + case .jpeg(let quality): + guard + let data = bitmapImage.representation( + using: .jpeg, + properties: [.compressionFactor: quality] + ) + else { + throw Transcript.ImageEncodingError.imageConversionFailed + } + return (data, "image/jpeg") + case .png: + guard + let data = bitmapImage.representation( + using: .png, + properties: [:] + ) + else { + throw Transcript.ImageEncodingError.imageConversionFailed + } + return (data, "image/png") + } + } + } + + /// Creates an image segment by encoding an AppKit image. + /// + /// - Parameters: + /// - image: The source image to encode. + /// - format: The target encoding. Defaults to JPEG with 0.9 quality. + /// - Throws: ``Transcript/ImageEncodingError-swift.enum/imageConversionFailed`` if encoding fails. + public init(image: NSImage, format: ImageFormat = .jpeg()) throws { + let (data, mimeType) = try format.encode(image) + self.init(data: data, mimeType: mimeType) + } + } +#endif + extension Transcript.Instructions: CustomStringConvertible { public var description: String { "Instructions(segments: \(segments.count), tools: \(toolDefinitions.count))" From 18f79784b59380ca34a90357f4a63e9284b6c533 Mon Sep 17 00:00:00 2001 From: Mattt Zmuda Date: Wed, 5 Nov 2025 05:40:09 -0600 Subject: [PATCH 03/16] Add convenience methods for CoreGraphics / CGImage --- Sources/AnyLanguageModel/Transcript.swift | 160 ++++++++++++++-------- 1 file changed, 105 insertions(+), 55 deletions(-) diff --git a/Sources/AnyLanguageModel/Transcript.swift b/Sources/AnyLanguageModel/Transcript.swift index 04f059f8..3dae191a 100644 --- a/Sources/AnyLanguageModel/Transcript.swift +++ b/Sources/AnyLanguageModel/Transcript.swift @@ -445,30 +445,32 @@ extension Transcript.StructuredSegment: CustomStringConvertible { } } +extension Transcript.ImageSegment { + /// Preferred image encodings for image conversion. + public enum Format: Sendable { + /// JPEG encoding with the specified compression quality. + case jpeg(compressionQuality: Double = 0.9) + /// PNG encoding. + case png + } +} + #if canImport(UIKit) import UIKit extension Transcript.ImageSegment { - /// Preferred image encodings for UIKit image conversion. - public enum ImageFormat { - /// JPEG encoding with the specified compression quality. - case jpeg(compressionQuality: Double = 0.9) - /// PNG encoding. - case png - - fileprivate func encode(_ image: UIImage) throws -> (Data, String) { - switch self { - case .jpeg(let quality): - guard let data = image.jpegData(compressionQuality: quality) else { - throw Transcript.ImageEncodingError.imageConversionFailed - } - return (data, "image/jpeg") - case .png: - guard let data = image.pngData() else { - throw Transcript.ImageEncodingError.imageConversionFailed - } - return (data, "image/png") + fileprivate static func encode(_ image: UIImage, format: Format) throws -> (Data, String) { + switch format { + case .jpeg(let quality): + guard let data = image.jpegData(compressionQuality: quality) else { + throw Transcript.ImageEncodingError.imageConversionFailed } + return (data, "image/jpeg") + case .png: + guard let data = image.pngData() else { + throw Transcript.ImageEncodingError.imageConversionFailed + } + return (data, "image/png") } } @@ -478,8 +480,8 @@ extension Transcript.StructuredSegment: CustomStringConvertible { /// - image: The source image to encode. /// - format: The target encoding. Defaults to JPEG with 0.9 quality. /// - Throws: ``Transcript/ImageEncodingError-swift.enum/imageConversionFailed`` if encoding fails. - public init(image: UIImage, format: ImageFormat = .jpeg()) throws { - let (data, mimeType) = try format.encode(image) + public init(image: UIImage, format: Format = .jpeg()) throws { + let (data, mimeType) = try Self.encode(image, format: format) self.init(data: data, mimeType: mimeType) } } @@ -489,42 +491,34 @@ extension Transcript.StructuredSegment: CustomStringConvertible { import AppKit extension Transcript.ImageSegment { - /// Preferred image encodings for AppKit image conversion. - public enum ImageFormat { - /// JPEG encoding with the specified compression quality. - case jpeg(compressionQuality: Double = 0.9) - /// PNG encoding. - case png - - fileprivate func encode(_ image: NSImage) throws -> (Data, String) { - guard let tiffData = image.tiffRepresentation, - let bitmapImage = NSBitmapImageRep(data: tiffData) + fileprivate static func encode(_ image: NSImage, format: Format) throws -> (Data, String) { + guard let tiffData = image.tiffRepresentation, + let bitmapImage = NSBitmapImageRep(data: tiffData) + else { + throw Transcript.ImageEncodingError.imageConversionFailed + } + + switch format { + case .jpeg(let quality): + guard + let data = bitmapImage.representation( + using: .jpeg, + properties: [.compressionFactor: quality] + ) else { throw Transcript.ImageEncodingError.imageConversionFailed } - - switch self { - case .jpeg(let quality): - guard - let data = bitmapImage.representation( - using: .jpeg, - properties: [.compressionFactor: quality] - ) - else { - throw Transcript.ImageEncodingError.imageConversionFailed - } - return (data, "image/jpeg") - case .png: - guard - let data = bitmapImage.representation( - using: .png, - properties: [:] - ) - else { - throw Transcript.ImageEncodingError.imageConversionFailed - } - return (data, "image/png") + return (data, "image/jpeg") + case .png: + guard + let data = bitmapImage.representation( + using: .png, + properties: [:] + ) + else { + throw Transcript.ImageEncodingError.imageConversionFailed } + return (data, "image/png") } } @@ -534,8 +528,64 @@ extension Transcript.StructuredSegment: CustomStringConvertible { /// - image: The source image to encode. /// - format: The target encoding. Defaults to JPEG with 0.9 quality. /// - Throws: ``Transcript/ImageEncodingError-swift.enum/imageConversionFailed`` if encoding fails. - public init(image: NSImage, format: ImageFormat = .jpeg()) throws { - let (data, mimeType) = try format.encode(image) + public init(image: NSImage, format: Format = .jpeg()) throws { + let (data, mimeType) = try Self.encode(image, format: format) + self.init(data: data, mimeType: mimeType) + } + } +#endif + +#if canImport(CoreGraphics) + import CoreGraphics + import ImageIO + import UniformTypeIdentifiers + + extension Transcript.ImageSegment { + fileprivate static func encode(_ image: CGImage, format: Format) throws -> (Data, String) { + let data = NSMutableData() + let utType: UTType + + switch format { + case .jpeg: + utType = .jpeg + case .png: + utType = .png + } + + guard + let destination = CGImageDestinationCreateWithData( + data, + utType.identifier as CFString, + 1, + nil + ) + else { + throw Transcript.ImageEncodingError.imageConversionFailed + } + + var properties: [CFString: Any] = [:] + if case .jpeg(let quality) = format { + properties[kCGImageDestinationLossyCompressionQuality] = quality + } + + CGImageDestinationAddImage(destination, image, properties as CFDictionary) + + guard CGImageDestinationFinalize(destination) else { + throw Transcript.ImageEncodingError.imageConversionFailed + } + + let mimeType = (utType == .jpeg) ? "image/jpeg" : "image/png" + return (data as Data, mimeType) + } + + /// Creates an image segment by encoding a CoreGraphics image. + /// + /// - Parameters: + /// - image: The source image to encode. + /// - format: The target encoding. Defaults to JPEG with 0.9 quality. + /// - Throws: ``Transcript/ImageEncodingError-swift.enum/imageConversionFailed`` if encoding fails. + public init(image: CGImage, format: Format = .jpeg()) throws { + let (data, mimeType) = try Self.encode(image, format: format) self.init(data: data, mimeType: mimeType) } } From 9b132167fa01bd07600164d34de60b22be487394 Mon Sep 17 00:00:00 2001 From: Mattt Zmuda Date: Wed, 5 Nov 2025 07:06:14 -0600 Subject: [PATCH 04/16] Add convenience methods for responding to image or images --- .../LanguageModelSession.swift | 407 +++++++++++++----- 1 file changed, 302 insertions(+), 105 deletions(-) diff --git a/Sources/AnyLanguageModel/LanguageModelSession.swift b/Sources/AnyLanguageModel/LanguageModelSession.swift index 3818050a..50ea2ec3 100644 --- a/Sources/AnyLanguageModel/LanguageModelSession.swift +++ b/Sources/AnyLanguageModel/LanguageModelSession.swift @@ -166,6 +166,74 @@ public final class LanguageModelSession: @unchecked Sendable { public let transcriptEntries: ArraySlice } + @discardableResult + nonisolated public func respond( + to prompt: Prompt, + generating type: Content.Type = Content.self, + includeSchemaInPrompt: Bool = true, + options: GenerationOptions = GenerationOptions() + ) async throws -> Response where Content: Generable { + try await wrapRespond { + // Add prompt to transcript + let promptEntry = Transcript.Entry.prompt( + Transcript.Prompt( + segments: [.text(.init(content: prompt.description))], + options: options, + responseFormat: nil + ) + ) + await MainActor.run { + self.transcript.append(promptEntry) + } + + let response = try await model.respond( + within: self, + to: prompt, + generating: type, + includeSchemaInPrompt: includeSchemaInPrompt, + options: options + ) + + // Add response entries to transcript + await MainActor.run { + self.transcript.append(contentsOf: response.transcriptEntries) + } + + return response + } + } + + nonisolated public func streamResponse( + to prompt: Prompt, + generating type: Content.Type = Content.self, + includeSchemaInPrompt: Bool = true, + options: GenerationOptions = GenerationOptions() + ) -> sending ResponseStream where Content: Generable { + // Create prompt entry that will be added when stream starts + let promptEntry = Transcript.Entry.prompt( + Transcript.Prompt( + segments: [.text(.init(content: prompt.description))], + options: options, + responseFormat: nil + ) + ) + + return wrapStream( + model.streamResponse( + within: self, + to: prompt, + generating: type, + includeSchemaInPrompt: includeSchemaInPrompt, + options: options + ), + promptEntry: promptEntry + ) + } +} + +// MARK: - String Response Convenience Methods + +extension LanguageModelSession { @discardableResult nonisolated public func respond( to prompt: Prompt, @@ -195,6 +263,36 @@ public final class LanguageModelSession: @unchecked Sendable { try await respond(to: try prompt(), options: options) } + public func streamResponse( + to prompt: Prompt, + options: GenerationOptions = GenerationOptions() + ) -> sending ResponseStream { + streamResponse( + to: prompt, + generating: String.self, + includeSchemaInPrompt: true, + options: options + ) + } + + public func streamResponse( + to prompt: String, + options: GenerationOptions = GenerationOptions() + ) -> sending ResponseStream { + streamResponse(to: Prompt(prompt), options: options) + } + + public func streamResponse( + options: GenerationOptions = GenerationOptions(), + @PromptBuilder prompt: () throws -> Prompt + ) rethrows -> sending ResponseStream { + streamResponse(to: try prompt(), options: options) + } +} + +// MARK: - GeneratedContent with Schema Convenience Methods + +extension LanguageModelSession { @discardableResult nonisolated public func respond( to prompt: Prompt, @@ -240,43 +338,47 @@ public final class LanguageModelSession: @unchecked Sendable { ) } - @discardableResult - nonisolated public func respond( + nonisolated public func streamResponse( to prompt: Prompt, - generating type: Content.Type = Content.self, + schema: GenerationSchema, includeSchemaInPrompt: Bool = true, options: GenerationOptions = GenerationOptions() - ) async throws -> Response where Content: Generable { - try await wrapRespond { - // Add prompt to transcript - let promptEntry = Transcript.Entry.prompt( - Transcript.Prompt( - segments: [.text(.init(content: prompt.description))], - options: options, - responseFormat: nil - ) - ) - await MainActor.run { - self.transcript.append(promptEntry) - } - - let response = try await model.respond( - within: self, - to: prompt, - generating: type, - includeSchemaInPrompt: includeSchemaInPrompt, - options: options - ) + ) -> sending ResponseStream { + streamResponse( + to: prompt, + generating: GeneratedContent.self, + includeSchemaInPrompt: includeSchemaInPrompt, + options: options + ) + } - // Add response entries to transcript - await MainActor.run { - self.transcript.append(contentsOf: response.transcriptEntries) - } + nonisolated public func streamResponse( + to prompt: String, + schema: GenerationSchema, + includeSchemaInPrompt: Bool = true, + options: GenerationOptions = GenerationOptions() + ) -> sending ResponseStream { + streamResponse( + to: Prompt(prompt), + schema: schema, + includeSchemaInPrompt: includeSchemaInPrompt, + options: options + ) + } - return response - } + nonisolated public func streamResponse( + schema: GenerationSchema, + includeSchemaInPrompt: Bool = true, + options: GenerationOptions = GenerationOptions(), + @PromptBuilder prompt: () throws -> Prompt + ) rethrows -> sending ResponseStream { + streamResponse(to: try prompt(), schema: schema, includeSchemaInPrompt: includeSchemaInPrompt, options: options) } +} + +// MARK: - Generic Content Convenience Methods +extension LanguageModelSession { @discardableResult nonisolated public func respond( to prompt: String, @@ -307,124 +409,215 @@ public final class LanguageModelSession: @unchecked Sendable { ) } - nonisolated public func streamResponse( - to prompt: Prompt, - schema: GenerationSchema, + nonisolated public func streamResponse( + to prompt: String, + generating type: Content.Type = Content.self, includeSchemaInPrompt: Bool = true, options: GenerationOptions = GenerationOptions() - ) -> sending ResponseStream { + ) -> sending ResponseStream where Content: Generable { streamResponse( - to: prompt, - generating: GeneratedContent.self, + to: Prompt(prompt), + generating: type, includeSchemaInPrompt: includeSchemaInPrompt, options: options ) } - nonisolated public func streamResponse( - to prompt: String, - schema: GenerationSchema, + public func streamResponse( + generating type: Content.Type = Content.self, includeSchemaInPrompt: Bool = true, - options: GenerationOptions = GenerationOptions() - ) -> sending ResponseStream { + options: GenerationOptions = GenerationOptions(), + @PromptBuilder prompt: () throws -> Prompt + ) rethrows -> sending ResponseStream where Content: Generable { streamResponse( - to: Prompt(prompt), - schema: schema, + to: try prompt(), + generating: type, includeSchemaInPrompt: includeSchemaInPrompt, options: options ) } +} - nonisolated public func streamResponse( - schema: GenerationSchema, - includeSchemaInPrompt: Bool = true, - options: GenerationOptions = GenerationOptions(), - @PromptBuilder prompt: () throws -> Prompt - ) rethrows -> sending ResponseStream { - streamResponse(to: try prompt(), schema: schema, includeSchemaInPrompt: includeSchemaInPrompt, options: options) - } +// MARK: - Image Convenience Methods - nonisolated public func streamResponse( - to prompt: Prompt, - generating type: Content.Type = Content.self, - includeSchemaInPrompt: Bool = true, +extension LanguageModelSession { + @discardableResult + nonisolated public func respond( + to prompt: String, + image: Transcript.ImageSegment, options: GenerationOptions = GenerationOptions() - ) -> sending ResponseStream where Content: Generable { - // Create prompt entry that will be added when stream starts - let promptEntry = Transcript.Entry.prompt( - Transcript.Prompt( - segments: [.text(.init(content: prompt.description))], - options: options, - responseFormat: nil - ) + ) async throws -> Response { + try await respond( + to: prompt, + images: [image], + options: options ) + } - return wrapStream( - model.streamResponse( - within: self, - to: prompt, - generating: type, - includeSchemaInPrompt: includeSchemaInPrompt, - options: options - ), - promptEntry: promptEntry + @discardableResult + nonisolated public func respond( + to prompt: String, + images: [Transcript.ImageSegment], + options: GenerationOptions = GenerationOptions() + ) async throws -> Response { + try await respond( + to: prompt, + images: images, + generating: String.self, + includeSchemaInPrompt: true, + options: options ) } - nonisolated public func streamResponse( + @discardableResult + nonisolated public func respond( to prompt: String, + image: Transcript.ImageSegment, generating type: Content.Type = Content.self, includeSchemaInPrompt: Bool = true, options: GenerationOptions = GenerationOptions() - ) -> sending ResponseStream where Content: Generable { - streamResponse( - to: Prompt(prompt), + ) async throws -> Response where Content: Generable { + try await respond( + to: prompt, + images: [image], generating: type, includeSchemaInPrompt: includeSchemaInPrompt, options: options ) } - public func streamResponse( + @discardableResult + nonisolated public func respond( + to prompt: String, + images: [Transcript.ImageSegment], generating type: Content.Type = Content.self, includeSchemaInPrompt: Bool = true, - options: GenerationOptions = GenerationOptions(), - @PromptBuilder prompt: () throws -> Prompt - ) rethrows -> sending ResponseStream where Content: Generable { + options: GenerationOptions = GenerationOptions() + ) async throws -> Response where Content: Generable { + try await wrapRespond { + // Build segments from text and images + var segments: [Transcript.Segment] = [] + if !prompt.isEmpty { + segments.append(.text(.init(content: prompt))) + } + segments.append(contentsOf: images.map { .image($0) }) + + // Add prompt to transcript + let promptEntry = Transcript.Entry.prompt( + Transcript.Prompt( + segments: segments, + options: options, + responseFormat: nil + ) + ) + await MainActor.run { + self.transcript.append(promptEntry) + } + + // Extract text content for the Prompt parameter + let textPrompt = Prompt(prompt) + + let response = try await model.respond( + within: self, + to: textPrompt, + generating: type, + includeSchemaInPrompt: includeSchemaInPrompt, + options: options + ) + + // Add response entries to transcript + await MainActor.run { + self.transcript.append(contentsOf: response.transcriptEntries) + } + + return response + } + } + + public func streamResponse( + to prompt: String, + image: Transcript.ImageSegment, + options: GenerationOptions = GenerationOptions() + ) -> sending ResponseStream { streamResponse( - to: try prompt(), - generating: type, - includeSchemaInPrompt: includeSchemaInPrompt, + to: prompt, + images: [image], options: options ) } public func streamResponse( - to prompt: Prompt, + to prompt: String, + images: [Transcript.ImageSegment], options: GenerationOptions = GenerationOptions() ) -> sending ResponseStream { streamResponse( to: prompt, + images: images, generating: String.self, includeSchemaInPrompt: true, options: options ) } - public func streamResponse( + nonisolated public func streamResponse( to prompt: String, + image: Transcript.ImageSegment, + generating type: Content.Type = Content.self, + includeSchemaInPrompt: Bool = true, options: GenerationOptions = GenerationOptions() - ) -> sending ResponseStream { - streamResponse(to: Prompt(prompt), options: options) + ) -> sending ResponseStream where Content: Generable { + streamResponse( + to: prompt, + images: [image], + generating: type, + includeSchemaInPrompt: includeSchemaInPrompt, + options: options + ) } - public func streamResponse( - options: GenerationOptions = GenerationOptions(), - @PromptBuilder prompt: () throws -> Prompt - ) rethrows -> sending ResponseStream { - streamResponse(to: try prompt(), options: options) + nonisolated public func streamResponse( + to prompt: String, + images: [Transcript.ImageSegment], + generating type: Content.Type = Content.self, + includeSchemaInPrompt: Bool = true, + options: GenerationOptions = GenerationOptions() + ) -> sending ResponseStream where Content: Generable { + // Build segments from text and images + var segments: [Transcript.Segment] = [] + if !prompt.isEmpty { + segments.append(.text(.init(content: prompt))) + } + segments.append(contentsOf: images.map { .image($0) }) + + // Create prompt entry that will be added when stream starts + let promptEntry = Transcript.Entry.prompt( + Transcript.Prompt( + segments: segments, + options: options, + responseFormat: nil + ) + ) + + // Extract text content for the Prompt parameter + let textPrompt = Prompt(prompt) + + return wrapStream( + model.streamResponse( + within: self, + to: textPrompt, + generating: type, + includeSchemaInPrompt: includeSchemaInPrompt, + options: options + ), + promptEntry: promptEntry + ) } +} + +// MARK: - +extension LanguageModelSession { @discardableResult public func logFeedbackAttachment( sentiment: LanguageModelFeedback.Sentiment?, @@ -440,19 +633,7 @@ public final class LanguageModelSession: @unchecked Sendable { } } -private actor RespondingState { - private var count = 0 - - func increment() -> Int { - count += 1 - return count - } - - func decrement() -> Int { - count = max(0, count - 1) - return count - } -} +// MARK: - extension LanguageModelSession { public enum GenerationError: Error, LocalizedError { @@ -649,3 +830,19 @@ extension LanguageModelSession.ResponseStream: AsyncSequence { ) } } + +// MARK: - + +private actor RespondingState { + private var count = 0 + + func increment() -> Int { + count += 1 + return count + } + + func decrement() -> Int { + count = max(0, count - 1) + return count + } +} From 3c3609752c33bdad8fd921eb557d25a8a87898bd Mon Sep 17 00:00:00 2001 From: Mattt Zmuda Date: Wed, 5 Nov 2025 07:06:44 -0600 Subject: [PATCH 05/16] Add test coverage for respond overloads --- .../MockLanguageModelTests.swift | 116 ++++++++++++++++++ .../AnyLanguageModelTests/Shared/Images.swift | 33 +++++ 2 files changed, 149 insertions(+) create mode 100644 Tests/AnyLanguageModelTests/Shared/Images.swift diff --git a/Tests/AnyLanguageModelTests/MockLanguageModelTests.swift b/Tests/AnyLanguageModelTests/MockLanguageModelTests.swift index 5608d046..5ec4ac82 100644 --- a/Tests/AnyLanguageModelTests/MockLanguageModelTests.swift +++ b/Tests/AnyLanguageModelTests/MockLanguageModelTests.swift @@ -146,4 +146,120 @@ struct MockLanguageModelTests { #expect(!entry.id.isEmpty) } } + + @Test func respondWithSingleImage() async throws { + let model = MockLanguageModel.echo + let session = LanguageModelSession(model: model) + + let image = Transcript.ImageSegment(url: testImageURL) + let response = try await session.respond(to: "Describe this image", image: image) + + #expect(response.content.contains("Describe this image")) + + // Verify transcript has prompt with text + image and a response + #expect(session.transcript.count == 2) + if case .prompt(let promptEntry) = session.transcript[0] { + #expect(promptEntry.segments.count == 2) + // Expect one text and one image segment + let kinds = promptEntry.segments.map { segment -> String in + switch segment { + case .text: return "text"; + case .image: return "image"; + case .structure: return "structure" + } + } + #expect(kinds.contains("text")) + #expect(kinds.contains("image")) + } else { + Issue.record("First entry should be prompt with image") + } + } + + @Test func respondWithMultipleImages() async throws { + let model = MockLanguageModel.echo + let session = LanguageModelSession(model: model) + + let images: [Transcript.ImageSegment] = [ + .init(url: testImageURL), + .init(data: testImageData, mimeType: "image/png"), + ] + let response = try await session.respond(to: "Classify these", images: images) + + #expect(response.content.contains("Classify these")) + #expect(session.transcript.count == 2) + if case .prompt(let promptEntry) = session.transcript[0] { + #expect(promptEntry.segments.count == 3) + let imageCount = promptEntry.segments.reduce(into: 0) { count, seg in if case .image = seg { count += 1 } } + #expect(imageCount == 2) + } else { + Issue.record("First entry should be prompt with images") + } + } + + @Test func respondGeneratingStringWithImages() async throws { + let model = MockLanguageModel.echo + let session = LanguageModelSession(model: model) + + let images: [Transcript.ImageSegment] = [ + .init(url: testImageURL) + ] + let response: LanguageModelSession.Response = try await session.respond( + to: "What do you see?", + images: images, + generating: String.self, + includeSchemaInPrompt: true + ) + + #expect(response.content.contains("What do you see?")) + #expect(session.transcript.count == 2) + if case .prompt(let promptEntry) = session.transcript[0] { + #expect(promptEntry.segments.count == 2) + } else { + Issue.record("First entry should be prompt with image") + } + } + + @Test func streamResponseWithSingleImage() async throws { + let model = MockLanguageModel.streamingMock() + let session = LanguageModelSession(model: model) + + let image = Transcript.ImageSegment(url: testImageURL) + let stream = session.streamResponse(to: "Stream about image", image: image) + + var snapshots = 0 + for try await _ in stream { snapshots += 1 } + #expect(snapshots >= 1) + + // Prompt added at start, response added at end (append may occur after stream finishes) + try await Task.sleep(for: .milliseconds(10)) + #expect(session.transcript.count == 2) + if case .prompt(let promptEntry) = session.transcript[0] { + #expect(promptEntry.segments.count == 2) + } else { + Issue.record("First entry should be prompt with image") + } + } + + @Test func streamResponseWithMultipleImages() async throws { + let model = MockLanguageModel.streamingMock() + let session = LanguageModelSession(model: model) + + let images: [Transcript.ImageSegment] = [ + .init(url: testImageURL), + .init(data: testImageData, mimeType: "image/png"), + ] + let stream = session.streamResponse(to: "Stream about images", images: images) + + for try await _ in stream { /* drain */ } + // Response append occurs after stream finishes; wait briefly + try await Task.sleep(for: .milliseconds(10)) + #expect(session.transcript.count == 2) + if case .prompt(let promptEntry) = session.transcript[0] { + #expect(promptEntry.segments.count == 3) + let imageCount = promptEntry.segments.reduce(into: 0) { c, seg in if case .image = seg { c += 1 } } + #expect(imageCount == 2) + } else { + Issue.record("First entry should be prompt with images") + } + } } diff --git a/Tests/AnyLanguageModelTests/Shared/Images.swift b/Tests/AnyLanguageModelTests/Shared/Images.swift new file mode 100644 index 00000000..ba380224 --- /dev/null +++ b/Tests/AnyLanguageModelTests/Shared/Images.swift @@ -0,0 +1,33 @@ +import Foundation + +let testImageURL = URL(string: "https://github.com/huggingface.png")! + +// Minimal 1x1 PNG +let testImageData = Data([ + // PNG signature + 0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, + + // IHDR chunk + 0x00, 0x00, 0x00, 0x0D, // IHDR length (13) + 0x49, 0x48, 0x44, 0x52, // "IHDR" + 0x00, 0x00, 0x00, 0x01, // width = 1 + 0x00, 0x00, 0x00, 0x01, // height = 1 + 0x01, // bit depth = 1 (1-bit grayscale) + 0x00, // color type = 0 (grayscale) + 0x00, // compression method = 0 (deflate) + 0x00, // filter method = 0 (none) + 0x00, // interlace method = 0 (no interlace) + 0x37, 0x6E, 0xF9, 0x24, // IHDR CRC32 + + // IDAT chunk + 0x00, 0x00, 0x00, 0x0A, // IDAT length (10) + 0x49, 0x44, 0x41, 0x54, // "IDAT" + 0x78, 0x01, // zlib header (deflate, no compression) + 0x63, 0x60, 0x00, 0x00, 0x00, 0x02, 0x00, 0x01, // compressed data (scanline 00 00) + 0x73, 0x75, 0x01, 0x18, // IDAT CRC32 + + // IEND chunk + 0x00, 0x00, 0x00, 0x00, // IEND length (0) + 0x49, 0x45, 0x4E, 0x44, // "IEND" + 0xAE, 0x42, 0x60, 0x82, // IEND CRC32 +]) From 21aab475be90fe1b5e3ddd084f199ae72cedd543 Mon Sep 17 00:00:00 2001 From: Mattt Zmuda Date: Wed, 5 Nov 2025 07:07:05 -0600 Subject: [PATCH 06/16] Implement image support in model providers --- .../Models/AnthropicLanguageModel.swift | 83 ++++++++- .../Models/CoreMLLanguageModel.swift | 6 +- .../Models/GeminiLanguageModel.swift | 69 +++++++- .../Models/LlamaLanguageModel.swift | 3 + .../Models/MLXLanguageModel.swift | 16 ++ .../Models/OllamaLanguageModel.swift | 52 +++++- .../Models/OpenAILanguageModel.swift | 166 ++++++++++++++++-- .../AnthropicLanguageModelTests.swift | 28 +++ .../CoreMLLanguageModelTests.swift | 47 +++++ .../GeminiLanguageModelTests.swift | 28 +++ .../LlamaLanguageModelTests.swift | 30 ++++ .../MLXLanguageModelTests.swift | 31 ++++ .../OllamaLanguageModelTests.swift | 28 +++ .../OpenAILanguageModelTests.swift | 56 ++++++ 14 files changed, 621 insertions(+), 22 deletions(-) diff --git a/Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift b/Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift index 97795f1a..3df7e396 100644 --- a/Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/AnthropicLanguageModel.swift @@ -100,8 +100,9 @@ public struct AnthropicLanguageModel: LanguageModel { let url = baseURL.appendingPathComponent("v1/messages") let headers = buildHeaders() + let userSegments = extractPromptSegments(from: session, fallbackText: prompt.description) let messages = [ - AnthropicMessage(role: .user, content: [.text(.init(text: prompt.description))]) + AnthropicMessage(role: .user, content: convertSegmentsToAnthropicContent(userSegments)) ] // Convert available tools to Anthropic format @@ -170,8 +171,9 @@ public struct AnthropicLanguageModel: LanguageModel { fatalError("AnthropicLanguageModel only supports generating String content") } + let userSegments = extractPromptSegments(from: session, fallbackText: prompt.description) let messages = [ - AnthropicMessage(role: .user, content: [.text(.init(text: prompt.description))]) + AnthropicMessage(role: .user, content: convertSegmentsToAnthropicContent(userSegments)) ] let url = baseURL.appendingPathComponent("v1/messages") @@ -383,11 +385,12 @@ private struct AnthropicMessage: Codable, Sendable { private enum AnthropicContent: Codable, Sendable { case text(AnthropicText) + case image(AnthropicImage) case toolUse(AnthropicToolUse) enum CodingKeys: String, CodingKey { case type } - enum ContentType: String, Codable { case text = "text", toolUse = "tool_use" } + enum ContentType: String, Codable { case text = "text", image = "image", toolUse = "tool_use" } init(from decoder: any Decoder) throws { let container = try decoder.container(keyedBy: CodingKeys.self) @@ -395,6 +398,8 @@ private enum AnthropicContent: Codable, Sendable { switch type { case .text: self = .text(try AnthropicText(from: decoder)) + case .image: + self = .image(try AnthropicImage(from: decoder)) case .toolUse: self = .toolUse(try AnthropicToolUse(from: decoder)) } @@ -403,6 +408,7 @@ private enum AnthropicContent: Codable, Sendable { func encode(to encoder: any Encoder) throws { switch self { case .text(let t): try t.encode(to: encoder) + case .image(let i): try i.encode(to: encoder) case .toolUse(let u): try u.encode(to: encoder) } } @@ -418,6 +424,77 @@ private struct AnthropicText: Codable, Sendable { } } +private struct AnthropicImage: Codable, Sendable { + struct Source: Codable, Sendable { + let type: String + let mediaType: String? + let data: String? + let url: String? + + enum CodingKeys: String, CodingKey { + case type + case mediaType = "media_type" + case data + case url + } + } + + let type: String + let source: Source + + init(base64Data: String, mimeType: String) { + self.type = "image" + self.source = Source(type: "base64", mediaType: mimeType, data: base64Data, url: nil) + } + + init(url: String) { + self.type = "image" + self.source = Source(type: "url", mediaType: nil, data: nil, url: url) + } +} + +private func convertSegmentsToAnthropicContent(_ segments: [Transcript.Segment]) -> [AnthropicContent] { + var blocks: [AnthropicContent] = [] + blocks.reserveCapacity(segments.count) + for segment in segments { + switch segment { + case .text(let t): + blocks.append(.text(AnthropicText(text: t.content))) + case .structure(let s): + blocks.append(.text(AnthropicText(text: s.content.jsonString))) + case .image(let img): + switch img.source { + case .url(let url): + blocks.append(.image(AnthropicImage(url: url.absoluteString))) + case .data(let data, let mimeType): + blocks.append(.image(AnthropicImage(base64Data: data.base64EncodedString(), mimeType: mimeType))) + } + } + } + return blocks +} + +private func extractPromptSegments(from session: LanguageModelSession, fallbackText: String) -> [Transcript.Segment] { + for entry in session.transcript.reversed() { + if case .prompt(let p) = entry { + // Skip prompts that are effectively empty (single empty text block) + let hasMeaningfulContent = p.segments.contains { segment in + switch segment { + case .text(let t): + return !t.content.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + case .structure: + return true + case .image: + return true + } + } + if hasMeaningfulContent { return p.segments } + // Otherwise continue searching older entries + } + } + return [.text(.init(content: fallbackText))] +} + private struct AnthropicToolUse: Codable, Sendable { let type: String let id: String diff --git a/Sources/AnyLanguageModel/Models/CoreMLLanguageModel.swift b/Sources/AnyLanguageModel/Models/CoreMLLanguageModel.swift index 68587e9a..1cd88a2b 100644 --- a/Sources/AnyLanguageModel/Models/CoreMLLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/CoreMLLanguageModel.swift @@ -210,6 +210,8 @@ /// The model file was found but is corrupted, incompatible, or otherwise invalid. case modelInvalid(URL, underlyingError: Error) + /// Image segments are not supported in CoreMLLanguageModel + case unsupportedFeature public var errorDescription: String? { switch self { @@ -221,6 +223,8 @@ case .modelInvalid(let url, let underlyingError): return "Core ML model at \(url.path) is invalid or corrupted: \(underlyingError.localizedDescription). Please verify the model file is valid and compatible with the current Core ML version." + case .unsupportedFeature: + return "This CoreMLLanguageModel does not support image segments" } } } @@ -245,7 +249,7 @@ config.topK = k case .nucleus(let p, _): config.doSample = true - config.topP = p + config.topP = Float(p) } } diff --git a/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift b/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift index 88429e03..c1f9318c 100644 --- a/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift @@ -97,8 +97,9 @@ public struct GeminiLanguageModel: LanguageModel { .appendingPathComponent("models/\(model):generateContent") let headers = buildHeaders() + let userSegments = extractPromptSegments(from: session, fallbackText: prompt.description) var contents = [ - GeminiContent(role: .user, parts: [.text(GeminiTextPart(text: prompt.description))]) + GeminiContent(role: .user, parts: convertSegmentsToGeminiParts(userSegments)) ] let geminiTools = try buildTools(from: session.tools) @@ -195,8 +196,9 @@ public struct GeminiLanguageModel: LanguageModel { fatalError("GeminiLanguageModel only supports generating String content") } + let userSegments = extractPromptSegments(from: session, fallbackText: prompt.description) let contents = [ - GeminiContent(role: .user, parts: [.text(GeminiTextPart(text: prompt.description))]) + GeminiContent(role: .user, parts: convertSegmentsToGeminiParts(userSegments)) ] var streamURL = @@ -442,6 +444,9 @@ private func toJSONValue(_ toolOutput: Transcript.ToolOutput) throws -> [String: if let jsonString = String(data: data, encoding: .utf8) { result["result"] = .string(jsonString) } + case .image: + // Ignore images in tool outputs for Gemini conversion + break } } @@ -511,12 +516,16 @@ private enum GeminiPart: Codable, Sendable { case text(GeminiTextPart) case functionCall(GeminiFunctionCall) case functionResponse(GeminiFunctionResponse) + case inlineData(GeminiInlineData) + case fileData(GeminiFileData) enum CodingKeys: String, CodingKey { case text case functionCall case functionResponse case thoughtSignature + case inlineData + case fileData } init(from decoder: any Decoder) throws { @@ -530,6 +539,10 @@ private enum GeminiPart: Codable, Sendable { self = .functionCall(try container.decode(GeminiFunctionCall.self, forKey: .functionCall)) } else if container.contains(.functionResponse) { self = .functionResponse(try container.decode(GeminiFunctionResponse.self, forKey: .functionResponse)) + } else if container.contains(.inlineData) { + self = .inlineData(try container.decode(GeminiInlineData.self, forKey: .inlineData)) + } else if container.contains(.fileData) { + self = .fileData(try container.decode(GeminiFileData.self, forKey: .fileData)) } else { throw DecodingError.dataCorrupted( DecodingError.Context( @@ -549,6 +562,10 @@ private enum GeminiPart: Codable, Sendable { try container.encode(call, forKey: .functionCall) case .functionResponse(let response): try container.encode(response, forKey: .functionResponse) + case .inlineData(let data): + try container.encode(data, forKey: .inlineData) + case .fileData(let data): + try container.encode(data, forKey: .fileData) } } } @@ -557,6 +574,54 @@ private struct GeminiTextPart: Codable, Sendable { let text: String } +private struct GeminiInlineData: Codable, Sendable { + let mimeType: String + let data: String + + enum CodingKeys: String, CodingKey { + case mimeType = "mime_type" + case data + } +} + +private struct GeminiFileData: Codable, Sendable { + let fileURI: String + + enum CodingKeys: String, CodingKey { + case fileURI = "file_uri" + } +} + +private func convertSegmentsToGeminiParts(_ segments: [Transcript.Segment]) -> [GeminiPart] { + var parts: [GeminiPart] = [] + parts.reserveCapacity(segments.count) + for segment in segments { + switch segment { + case .text(let t): + parts.append(.text(GeminiTextPart(text: t.content))) + case .structure(let s): + parts.append(.text(GeminiTextPart(text: s.content.jsonString))) + case .image(let img): + switch img.source { + case .data(let data, let mime): + parts.append(.inlineData(GeminiInlineData(mimeType: mime, data: data.base64EncodedString()))) + case .url(let url): + parts.append(.fileData(GeminiFileData(fileURI: url.absoluteString))) + } + } + } + return parts +} + +private func extractPromptSegments(from session: LanguageModelSession, fallbackText: String) -> [Transcript.Segment] { + for entry in session.transcript.reversed() { + if case .prompt(let p) = entry { + return p.segments + } + } + return [.text(.init(content: fallbackText))] +} + private struct GeminiFunctionCall: Codable, Sendable { let name: String let args: [String: JSONValue]? diff --git a/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift b/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift index 71cf860b..b77640f2 100644 --- a/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift @@ -601,6 +601,7 @@ import Foundation case decodingFailed case invalidModelPath case insufficientMemory + case unsupportedFeature public var errorDescription: String? { switch self { @@ -618,6 +619,8 @@ import Foundation return "Invalid model file path" case .insufficientMemory: return "Insufficient memory for operation" + case .unsupportedFeature: + return "This LlamaLanguageModel does not support image segments" } } } diff --git a/Sources/AnyLanguageModel/Models/MLXLanguageModel.swift b/Sources/AnyLanguageModel/Models/MLXLanguageModel.swift index 2a5e1b3a..69a3f7c9 100644 --- a/Sources/AnyLanguageModel/Models/MLXLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/MLXLanguageModel.swift @@ -148,6 +148,19 @@ import Foundation } } + /// Errors that can occur when working with MLX language models. + public enum MLXLanguageModelError: LocalizedError { + /// Image segments are not supported in MLXLanguageModel + case unsupportedFeature + + public var errorDescription: String? { + switch self { + case .unsupportedFeature: + return "This MLXLanguageModel does not support image segments" + } + } + } + // MARK: - Options Mapping private func toGenerateParameters(_ options: GenerationOptions) -> MLXLMCommon.GenerateParameters { @@ -267,6 +280,9 @@ import Foundation case .structure(let structuredSegment): // structured content already has jsonString property textParts.append(structuredSegment.content.jsonString) + case .image: + // Image segments are not supported in MLX tool output + break } } return textParts.joined(separator: "\n") diff --git a/Sources/AnyLanguageModel/Models/OllamaLanguageModel.swift b/Sources/AnyLanguageModel/Models/OllamaLanguageModel.swift index f45d1e31..a5c5c115 100644 --- a/Sources/AnyLanguageModel/Models/OllamaLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/OllamaLanguageModel.swift @@ -62,8 +62,10 @@ public struct OllamaLanguageModel: LanguageModel { fatalError("OllamaLanguageModel only supports generating String content") } + let userSegments = extractPromptSegments(from: session, fallbackText: prompt.description) + let (ollamaText, ollamaImages) = convertSegmentsToOllama(userSegments) let messages = [ - OllamaMessage(role: .user, content: prompt.description) + OllamaMessage(role: .user, content: ollamaText) ] let ollamaOptions = convertOptions(options) let ollamaTools = try session.tools.map { tool in @@ -75,7 +77,8 @@ public struct OllamaLanguageModel: LanguageModel { messages: messages, tools: ollamaTools.isEmpty ? nil : ollamaTools, options: ollamaOptions, - stream: false + stream: false, + images: ollamaImages.isEmpty ? nil : ollamaImages ) let url = baseURL.appendingPathComponent("api/chat") @@ -119,8 +122,10 @@ public struct OllamaLanguageModel: LanguageModel { fatalError("OllamaLanguageModel only supports generating String content") } + let userSegments = extractPromptSegments(from: session, fallbackText: prompt.description) + let (ollamaText, ollamaImages) = convertSegmentsToOllama(userSegments) let messages = [ - OllamaMessage(role: .user, content: prompt.description) + OllamaMessage(role: .user, content: ollamaText) ] let ollamaOptions = convertOptions(options) let ollamaTools = try? session.tools.map { tool in @@ -132,7 +137,8 @@ public struct OllamaLanguageModel: LanguageModel { messages: messages, tools: (ollamaTools?.isEmpty == false) ? ollamaTools : nil, options: ollamaOptions, - stream: true + stream: true, + images: (ollamaImages.isEmpty ? nil : ollamaImages) ) let url = baseURL.appendingPathComponent("api/chat") @@ -312,7 +318,8 @@ private func createChatParams( messages: [OllamaMessage], tools: [[String: JSONValue]]?, options: [String: JSONValue]?, - stream: Bool + stream: Bool, + images: [String]? ) throws -> [String: JSONValue] { var params: [String: JSONValue] = [ "model": .string(model), @@ -328,6 +335,10 @@ private func createChatParams( params["options"] = .object(options) } + if let images, !images.isEmpty { + params["images"] = .array(images.map { .string($0) }) + } + return params } @@ -345,6 +356,37 @@ private struct OllamaMessage: Hashable, Codable, Sendable { let content: String } +private func convertSegmentsToOllama(_ segments: [Transcript.Segment]) -> (String, [String]) { + var textParts: [String] = [] + var images: [String] = [] + for segment in segments { + switch segment { + case .text(let t): + textParts.append(t.content) + case .structure(let s): + textParts.append(s.content.jsonString) + case .image(let img): + switch img.source { + case .data(let data, _): + images.append(data.base64EncodedString()) + case .url(let url): + // Ollama supports base64 images; include URL as text if provided + textParts.append(url.absoluteString) + } + } + } + return (textParts.joined(separator: "\n"), images) +} + +private func extractPromptSegments(from session: LanguageModelSession, fallbackText: String) -> [Transcript.Segment] { + for entry in session.transcript.reversed() { + if case .prompt(let p) = entry { + return p.segments + } + } + return [.text(.init(content: fallbackText))] +} + private struct ChatResponse: Decodable, Sendable { let model: String let createdAt: Date diff --git a/Sources/AnyLanguageModel/Models/OpenAILanguageModel.swift b/Sources/AnyLanguageModel/Models/OpenAILanguageModel.swift index e09afef3..c8d2d714 100644 --- a/Sources/AnyLanguageModel/Models/OpenAILanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/OpenAILanguageModel.swift @@ -88,10 +88,13 @@ public struct OpenAILanguageModel: LanguageModel { } var messages: [OpenAIMessage] = [] - if let instructions = session.instructions { - messages.append(OpenAIMessage(role: .system, content: .text(instructions.description))) + if let systemSegments = extractInstructionSegments(from: session) { + messages.append( + OpenAIMessage(role: .system, content: .blocks(convertSegmentsToOpenAIBlocks(systemSegments))) + ) } - messages.append(OpenAIMessage(role: .user, content: .text(prompt.description))) + let userSegments = extractPromptSegments(from: session, fallbackText: prompt.description) + messages.append(OpenAIMessage(role: .user, content: .blocks(convertSegmentsToOpenAIBlocks(userSegments)))) // Convert tools if any are available in the session let openAITools: [OpenAITool]? = { @@ -234,10 +237,13 @@ public struct OpenAILanguageModel: LanguageModel { } var messages: [OpenAIMessage] = [] - if let instructions = session.instructions { - messages.append(OpenAIMessage(role: .system, content: .text(instructions.description))) + if let systemSegments = extractInstructionSegments(from: session) { + messages.append( + OpenAIMessage(role: .system, content: .blocks(convertSegmentsToOpenAIBlocks(systemSegments))) + ) } - messages.append(OpenAIMessage(role: .user, content: .text(prompt.description))) + let userSegments = extractPromptSegments(from: session, fallbackText: prompt.description) + messages.append(OpenAIMessage(role: .user, content: .blocks(convertSegmentsToOpenAIBlocks(userSegments)))) // Convert tools if any are available in the session let openAITools: [OpenAITool]? = { @@ -439,19 +445,65 @@ private enum Responses { options: GenerationOptions, stream: Bool ) -> JSONValue { - // Extract the system and user message content for the request + // Build input blocks from the user message content let systemMessage = messages.first { $0.role == .system } let userMessage = messages.first { $0.role == .user } - let inputText = if case .text(let text) = userMessage?.content { text } else { "" } var body: [String: JSONValue] = [ "model": .string(model), - "input": .string(inputText), "stream": .bool(stream), ] - if case .text(let instructions) = systemMessage?.content { - body["instructions"] = .string(instructions) + if let userMessage { + // Wrap user content into a single top-level message as required by Responses API + let contentBlocks: [JSONValue] + switch userMessage.content { + case .text(let text): + contentBlocks = [ + .object(["type": .string("input_text"), "text": .string(text)]) + ] + case .blocks(let blocks): + contentBlocks = blocks.map { block in + switch block { + case .text(let text): + return .object(["type": .string("input_text"), "text": .string(text)]) + case .imageURL(let url): + return .object([ + "type": .string("input_image"), + "image_url": .object(["url": .string(url)]), + ]) + } + } + } + + body["input"] = .array([ + .object([ + "type": .string("message"), + "role": .string("user"), + "content": .array(contentBlocks), + ]) + ]) + } else { + body["input"] = .array([ + .object([ + "type": .string("message"), + "role": .string("user"), + "content": .array([]), + ]) + ]) + } + + if let systemMessage { + switch systemMessage.content { + case .text(let text): + body["instructions"] = .string(text) + case .blocks(let blocks): + // Concatenate text blocks for instructions; ignore images + let text = blocks.compactMap { if case .text(let t) = $0 { return t } else { return nil } }.joined( + separator: "\n" + ) + if !text.isEmpty { body["instructions"] = .string(text) } + } } if let tools { @@ -490,6 +542,7 @@ private struct OpenAIMessage: Hashable, Codable, Sendable { enum Content: Hashable, Codable, Sendable { case text(String) + case blocks([Block]) } let role: Role @@ -512,10 +565,101 @@ private struct OpenAIMessage: Hashable, Codable, Sendable { "content": .array([.object(["type": .string("text"), "text": .string(text)])]), ]) } + case .blocks(let blocks): + switch apiVariant { + case .chatCompletions: + // Chat Completions accepts array of content parts + return .object([ + "role": .string(role.rawValue), + "content": .array(blocks.map { $0.jsonValueForChatCompletions }), + ]) + case .responses: + // Responses expects message content blocks + return .object([ + "role": .string(role.rawValue), + "content": .array(blocks.map { $0.jsonValueForResponses }), + ]) + } } } } +private enum Block: Hashable, Codable, Sendable { + case text(String) + case imageURL(String) + + var jsonValueForChatCompletions: JSONValue { + switch self { + case .text(let text): + return .object(["type": .string("text"), "text": .string(text)]) + case .imageURL(let url): + return .object([ + "type": .string("image_url"), + "image_url": .object(["url": .string(url)]), + ]) + } + } + + var jsonValueForResponses: JSONValue { + switch self { + case .text(let text): + return .object(["type": .string("text"), "text": .string(text)]) + case .imageURL(let url): + // Responses API uses input_image at top-level input, but inside messages we mirror block + return .object([ + "type": .string("input_image"), + "image_url": .object(["url": .string(url)]), + ]) + } + } +} + +private func convertSegmentsToOpenAIBlocks(_ segments: [Transcript.Segment]) -> [Block] { + var blocks: [Block] = [] + blocks.reserveCapacity(segments.count) + for segment in segments { + switch segment { + case .text(let text): + blocks.append(.text(text.content)) + case .structure(let structured): + blocks.append(.text(structured.content.jsonString)) + case .image(let image): + switch image.source { + case .url(let url): + blocks.append(.imageURL(url.absoluteString)) + case .data(let data, let mimeType): + let b64 = data.base64EncodedString() + let dataURL = "data:\(mimeType);base64,\(b64)" + blocks.append(.imageURL(dataURL)) + } + } + } + return blocks +} + +private func extractPromptSegments(from session: LanguageModelSession, fallbackText: String) -> [Transcript.Segment] { + // Prefer the most recent Transcript.Prompt entry if present + for entry in session.transcript.reversed() { + if case .prompt(let p) = entry { + return p.segments + } + } + return [.text(.init(content: fallbackText))] +} + +private func extractInstructionSegments(from session: LanguageModelSession) -> [Transcript.Segment]? { + // Prefer the first Transcript.Instructions entry if present + for entry in session.transcript { + if case .instructions(let i) = entry { + return i.segments + } + } + if let instructions = session.instructions?.description, !instructions.isEmpty { + return [.text(.init(content: instructions))] + } + return nil +} + private struct OpenAITool: Hashable, Codable, Sendable { let type: String let function: OpenAIFunction diff --git a/Tests/AnyLanguageModelTests/AnthropicLanguageModelTests.swift b/Tests/AnyLanguageModelTests/AnthropicLanguageModelTests.swift index 8fccc314..2a03db30 100644 --- a/Tests/AnyLanguageModelTests/AnthropicLanguageModelTests.swift +++ b/Tests/AnyLanguageModelTests/AnthropicLanguageModelTests.swift @@ -99,4 +99,32 @@ struct AnthropicLanguageModelTests { } #expect(foundToolOutput) } + + @Test func multimodalWithImageURL() async throws { + let transcript = Transcript(entries: [ + .prompt( + Transcript.Prompt(segments: [ + .text(.init(content: "Describe this image")), + .image(.init(url: testImageURL)), + ]) + ) + ]) + let session = LanguageModelSession(model: model, transcript: transcript) + let response = try await session.respond(to: "") + #expect(!response.content.isEmpty) + } + + @Test func multimodalWithImageData() async throws { + let transcript = Transcript(entries: [ + .prompt( + Transcript.Prompt(segments: [ + .text(.init(content: "Describe this image")), + .image(.init(data: testImageData, mimeType: "image/png")), + ]) + ) + ]) + let session = LanguageModelSession(model: model, transcript: transcript) + let response = try await session.respond(to: "") + #expect(!response.content.isEmpty) + } } diff --git a/Tests/AnyLanguageModelTests/CoreMLLanguageModelTests.swift b/Tests/AnyLanguageModelTests/CoreMLLanguageModelTests.swift index be568b97..d53c71f1 100644 --- a/Tests/AnyLanguageModelTests/CoreMLLanguageModelTests.swift +++ b/Tests/AnyLanguageModelTests/CoreMLLanguageModelTests.swift @@ -133,5 +133,52 @@ import Testing // Note: We can't easily test token count without access to the tokenizer // but we can verify the response is not empty } + + @Test @available(macOS 15.0, iOS 18.0, tvOS 18.0, visionOS 2.0, watchOS 11.0, *) + func multimodal_rejectsImageURL() async throws { + let model = try await getModel() + let session = LanguageModelSession(model: model) + let prompt = Transcript.Prompt(segments: [ + .text(.init(content: "Describe this image")), + .image(.init(url: testImageURL)), + ]) + do { + _ = try await session.respond(to: prompt) + Issue.record("Expected error when image segments are present") + } catch { + // CoreMLUnsupportedFeatureError is a private struct, so we just check that an error is thrown + #expect(true) + } + } + + @Test @available(macOS 15.0, iOS 18.0, tvOS 18.0, visionOS 2.0, watchOS 11.0, *) + func multimodal_rejectsImageData() async throws { + let model = try await getModel() + let session = LanguageModelSession(model: model) + // Minimal 1x1 PNG + let png1x1: [UInt8] = [ + 0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, + 0x00, 0x00, 0x00, 0x0D, 0x49, 0x48, 0x44, 0x52, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, + 0x08, 0x02, 0x00, 0x00, 0x00, 0x90, 0x77, 0x53, + 0xDE, 0x00, 0x00, 0x00, 0x0A, 0x49, 0x44, 0x41, + 0x54, 0x78, 0x9C, 0x63, 0xF8, 0x0F, 0x00, 0x01, + 0x01, 0x01, 0x00, 0x18, 0xDD, 0x8D, 0xF7, 0x00, + 0x00, 0x00, 0x00, 0x49, 0x45, 0x4E, 0x44, 0xAE, + 0x42, 0x60, 0x82, + ] + let data = Data(png1x1) + let prompt = Transcript.Prompt(segments: [ + .text(.init(content: "Describe this image")), + .image(.init(data: data, mimeType: "image/png")), + ]) + do { + _ = try await session.respond(to: prompt) + Issue.record("Expected error when image segments are present") + } catch { + // CoreMLUnsupportedFeatureError is a private struct, so we just check that an error is thrown + #expect(true) + } + } } #endif // CoreML diff --git a/Tests/AnyLanguageModelTests/GeminiLanguageModelTests.swift b/Tests/AnyLanguageModelTests/GeminiLanguageModelTests.swift index 420d36da..447fdd1b 100644 --- a/Tests/AnyLanguageModelTests/GeminiLanguageModelTests.swift +++ b/Tests/AnyLanguageModelTests/GeminiLanguageModelTests.swift @@ -119,4 +119,32 @@ struct GeminiLanguageModelTests { let response = try await session.respond(to: "What coffee shops are nearby?") #expect(!response.content.isEmpty) } + + @Test func multimodalWithImageURL() async throws { + let transcript = Transcript(entries: [ + .prompt( + Transcript.Prompt(segments: [ + .text(.init(content: "Describe this image")), + .image(.init(url: testImageURL)), + ]) + ) + ]) + let session = LanguageModelSession(model: model, transcript: transcript) + let response = try await session.respond(to: "") + #expect(!response.content.isEmpty) + } + + @Test func multimodalWithImageData() async throws { + let transcript = Transcript(entries: [ + .prompt( + Transcript.Prompt(segments: [ + .text(.init(content: "Describe this image")), + .image(.init(data: testImageData, mimeType: "image/png")), + ]) + ) + ]) + let session = LanguageModelSession(model: model, transcript: transcript) + let response = try await session.respond(to: "") + #expect(!response.content.isEmpty) + } } diff --git a/Tests/AnyLanguageModelTests/LlamaLanguageModelTests.swift b/Tests/AnyLanguageModelTests/LlamaLanguageModelTests.swift index 8c3746c0..162a23d8 100644 --- a/Tests/AnyLanguageModelTests/LlamaLanguageModelTests.swift +++ b/Tests/AnyLanguageModelTests/LlamaLanguageModelTests.swift @@ -139,5 +139,35 @@ import Testing ) #expect(!response.content.isEmpty) } + + @Test func multimodal_rejectsImageURL() async throws { + let session = LanguageModelSession(model: model) + let prompt = Transcript.Prompt(segments: [ + .text(.init(content: "Describe this image")), + .image(.init(url: testImageURL)), + ]) + do { + _ = try await session.respond(to: prompt) + Issue.record("Expected error when image segments are present") + } catch let error as LlamaLanguageModelError { + #expect(error == .unsupportedFeature) + } + } + + @Test func multimodal_rejectsImageData() async throws { + let session = LanguageModelSession(model: model) + + let data = Data(png1x1) + let prompt = Transcript.Prompt(segments: [ + .text(.init(content: "Describe this image")), + .image(.init(data: data, mimeType: "image/png")), + ]) + do { + _ = try await session.respond(to: prompt) + Issue.record("Expected error when image segments are present") + } catch let error as LlamaLanguageModelError { + #expect(error == .unsupportedFeature) + } + } } #endif // Llama diff --git a/Tests/AnyLanguageModelTests/MLXLanguageModelTests.swift b/Tests/AnyLanguageModelTests/MLXLanguageModelTests.swift index 32aafdd9..dbb9b8df 100644 --- a/Tests/AnyLanguageModelTests/MLXLanguageModelTests.swift +++ b/Tests/AnyLanguageModelTests/MLXLanguageModelTests.swift @@ -79,5 +79,36 @@ import Testing #expect(first.arguments.city.contains("San Francisco")) } } + + @Test func multimodal_rejectsImageURL() async throws { + let session = LanguageModelSession(model: model) + let prompt = Transcript.Prompt(segments: [ + .text(.init(content: "Describe this image")), + .image(.init(url: testImageURL)), + ]) + do { + _ = try await session.respond(to: prompt) + Issue.record("Expected error when image segments are present") + } catch { + // MLXUnsupportedFeatureError is a private struct, so we just check that an error is thrown + #expect(true) + } + } + + @Test func multimodal_rejectsImageData() async throws { + let session = LanguageModelSession(model: model) + let data = Data(png1x1) + let prompt = Transcript.Prompt(segments: [ + .text(.init(content: "Describe this image")), + .image(.init(data: data, mimeType: "image/png")), + ]) + do { + _ = try await session.respond(to: prompt) + Issue.record("Expected error when image segments are present") + } catch { + // MLXUnsupportedFeatureError is a private struct, so we just check that an error is thrown + #expect(true) + } + } } #endif // MLX diff --git a/Tests/AnyLanguageModelTests/OllamaLanguageModelTests.swift b/Tests/AnyLanguageModelTests/OllamaLanguageModelTests.swift index c4d4f77d..50c4c437 100644 --- a/Tests/AnyLanguageModelTests/OllamaLanguageModelTests.swift +++ b/Tests/AnyLanguageModelTests/OllamaLanguageModelTests.swift @@ -108,4 +108,32 @@ struct OllamaLanguageModelTests { Issue.record("Expected successful tool call") } } + + @Test func multimodalWithImageURL() async throws { + let transcript = Transcript(entries: [ + .prompt( + Transcript.Prompt(segments: [ + .text(.init(content: "Describe this image")), + .image(.init(url: testImageURL)), + ]) + ) + ]) + let session = LanguageModelSession(model: model, transcript: transcript) + let response = try await session.respond(to: "") + #expect(!response.content.isEmpty) + } + + @Test func multimodalWithImageData() async throws { + let transcript = Transcript(entries: [ + .prompt( + Transcript.Prompt(segments: [ + .text(.init(content: "Describe this image")), + .image(.init(data: testImageData, mimeType: "image/png")), + ]) + ) + ]) + let session = LanguageModelSession(model: model, transcript: transcript) + let response = try await session.respond(to: "") + #expect(!response.content.isEmpty) + } } diff --git a/Tests/AnyLanguageModelTests/OpenAILanguageModelTests.swift b/Tests/AnyLanguageModelTests/OpenAILanguageModelTests.swift index d76c47af..11028eab 100644 --- a/Tests/AnyLanguageModelTests/OpenAILanguageModelTests.swift +++ b/Tests/AnyLanguageModelTests/OpenAILanguageModelTests.swift @@ -89,6 +89,34 @@ struct OpenAILanguageModelTests { #expect(!response.content.isEmpty) } + @Test func multimodalWithImageURL() async throws { + let transcript = Transcript(entries: [ + .prompt( + Transcript.Prompt(segments: [ + .text(.init(content: "Describe this image")), + .image(.init(url: testImageURL)), + ]) + ) + ]) + let session = LanguageModelSession(model: model, transcript: transcript) + let response = try await session.respond(to: "") + #expect(!response.content.isEmpty) + } + + @Test func multimodalWithImageData() async throws { + let transcript = Transcript(entries: [ + .prompt( + Transcript.Prompt(segments: [ + .text(.init(content: "Describe this image")), + .image(.init(data: testImageData, mimeType: "image/png")), + ]) + ) + ]) + let session = LanguageModelSession(model: model, transcript: transcript) + let response = try await session.respond(to: "") + #expect(!response.content.isEmpty) + } + @Test func conversationContext() async throws { let session = LanguageModelSession(model: model) @@ -181,6 +209,34 @@ struct OpenAILanguageModelTests { #expect(!response.content.isEmpty) } + @Test func multimodalWithImageURL() async throws { + let transcript = Transcript(entries: [ + .prompt( + Transcript.Prompt(segments: [ + .text(.init(content: "Describe this image")), + .image(.init(url: testImageURL)), + ]) + ) + ]) + let session = LanguageModelSession(model: model, transcript: transcript) + let response = try await session.respond(to: "") + #expect(!response.content.isEmpty) + } + + @Test func multimodalWithImageData() async throws { + let transcript = Transcript(entries: [ + .prompt( + Transcript.Prompt(segments: [ + .text(.init(content: "Describe this image")), + .image(.init(data: testImageData, mimeType: "image/png")), + ]) + ) + ]) + let session = LanguageModelSession(model: model, transcript: transcript) + let response = try await session.respond(to: "") + #expect(!response.content.isEmpty) + } + @Test func conversationContext() async throws { let session = LanguageModelSession(model: model) From cebd9c5e553ad490cb932e98ef57afed36dc82ae Mon Sep 17 00:00:00 2001 From: noorbhatia Date: Fri, 7 Nov 2025 00:55:38 +0530 Subject: [PATCH 07/16] Add image input support for MLX models --- Package.swift | 5 ++ .../Models/MLXLanguageModel.swift | 69 +++++++++++++++++-- 2 files changed, 70 insertions(+), 4 deletions(-) diff --git a/Package.swift b/Package.swift index 431f6339..554c8173 100644 --- a/Package.swift +++ b/Package.swift @@ -49,6 +49,11 @@ let package = Package( package: "mlx-swift-examples", condition: .when(traits: ["MLX"]) ), + .product( + name: "MLXVLM", + package: "mlx-swift-examples", + condition: .when(traits: ["MLX"]) + ), .product( name: "MLXLMCommon", package: "mlx-swift-examples", diff --git a/Sources/AnyLanguageModel/Models/MLXLanguageModel.swift b/Sources/AnyLanguageModel/Models/MLXLanguageModel.swift index 69a3f7c9..77228fd6 100644 --- a/Sources/AnyLanguageModel/Models/MLXLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/MLXLanguageModel.swift @@ -1,6 +1,16 @@ import Foundation +import CoreImage +#if canImport(UIKit) + import UIKit +#endif +#if canImport(AppKit) + import AppKit +#endif + #if MLX import MLXLMCommon + import MLX + import MLXVLM import Tokenizers /// A language model that runs locally using MLX. @@ -38,8 +48,9 @@ import Foundation fatalError("MLXLanguageModel only supports generating String content") } - // Load model context via MLXLMCommon - let context = try await MLXLMCommon.loadModel(id: modelId) + // Set GPU memory limit to prevent out of memory issues + MLX.GPU.set(cacheLimit: 20 * 1024 * 1024) + let context = try await loadModel(id: modelId) // Convert session tools to MLX ToolSpec format let toolSpecs: [ToolSpec]? = @@ -53,7 +64,9 @@ import Foundation let generateParameters = toGenerateParameters(options) // Start with user prompt - var chat: [MLXLMCommon.Chat.Message] = [.user(prompt.description)] + let userSegments = extractPromptSegments(from: session, fallbackText: prompt.description) + let userMessage = convertSegmentsToMLXMessage(userSegments) + var chat: [MLXLMCommon.Chat.Message] = [userMessage] var allTextChunks: [String] = [] var allEntries: [Transcript.Entry] = [] @@ -62,7 +75,8 @@ import Foundation // Build user input with current chat history and tools let userInput = MLXLMCommon.UserInput( chat: chat, - tools: toolSpecs + processing: .init(resize: .init(width: 512, height: 512)), + tools: toolSpecs, ) let lmInput = try await context.processor.prepare(input: userInput) @@ -177,6 +191,53 @@ import Foundation ) } + // MARK: - Segment Extraction + + private func extractPromptSegments(from session: LanguageModelSession, fallbackText: String) -> [Transcript.Segment] { + // Prefer the most recent Transcript.Prompt entry if present + for entry in session.transcript.reversed() { + if case .prompt(let p) = entry { + return p.segments + } + } + return [.text(.init(content: fallbackText))] + } + + private func convertSegmentsToMLXMessage(_ segments: [Transcript.Segment]) -> MLXLMCommon.Chat.Message { + var textParts: [String] = [] + var images: [MLXLMCommon.UserInput.Image] = [] + + for segment in segments { + switch segment { + case .text(let text): + textParts.append(text.content) + case .structure(let structured): + textParts.append(structured.content.jsonString) + case .image(let imageSegment): + switch imageSegment.source { + case .url(let url): + images.append(.url(url)) + case .data(let data, _): + #if canImport(UIKit) + if let uiImage = UIKit.UIImage(data: data), + let ciImage = CIImage(image: uiImage) { + images.append(.ciImage(ciImage)) + } + #elseif canImport(AppKit) + if let nsImage = AppKit.NSImage(data: data), + let cgImage = nsImage.cgImage(forProposedRect: nil, context: nil, hints: nil) { + let ciImage = CIImage(cgImage: cgImage) + images.append(.ciImage(ciImage)) + } + #endif + } + } + } + + let content = textParts.joined(separator: "\n") + return MLXLMCommon.Chat.Message(role: .user, content: content, images: images) + } + // MARK: - Tool Conversion private func convertToolToMLXSpec(_ tool: any Tool) -> ToolSpec { From 44b21bd301c17d11b0281124b7a3f0606b2e004a Mon Sep 17 00:00:00 2001 From: noorbhatia Date: Fri, 7 Nov 2025 17:03:48 +0530 Subject: [PATCH 08/16] Update MLX tests to support image inputs --- .../MLXLanguageModelTests.swift | 48 +++++++++---------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/Tests/AnyLanguageModelTests/MLXLanguageModelTests.swift b/Tests/AnyLanguageModelTests/MLXLanguageModelTests.swift index dbb9b8df..9f72715d 100644 --- a/Tests/AnyLanguageModelTests/MLXLanguageModelTests.swift +++ b/Tests/AnyLanguageModelTests/MLXLanguageModelTests.swift @@ -33,6 +33,7 @@ import Testing struct MLXLanguageModelTests { // Qwen3-0.6B is a small model that supports tool calling let model = MLXLanguageModel(modelId: "mlx-community/Qwen3-0.6B-4bit") + let visionModel = MLXLanguageModel(modelId: "mlx-community/Qwen2-VL-2B-Instruct-4bit") @Test func basicResponse() async throws { let session = LanguageModelSession(model: model) @@ -80,35 +81,32 @@ import Testing } } - @Test func multimodal_rejectsImageURL() async throws { - let session = LanguageModelSession(model: model) - let prompt = Transcript.Prompt(segments: [ - .text(.init(content: "Describe this image")), - .image(.init(url: testImageURL)), + @Test func multimodalWithImageURL() async throws { + let transcript = Transcript(entries: [ + .prompt( + Transcript.Prompt(segments: [ + .text(.init(content: "Describe this image")), + .image(.init(url: testImageURL)), + ]) + ) ]) - do { - _ = try await session.respond(to: prompt) - Issue.record("Expected error when image segments are present") - } catch { - // MLXUnsupportedFeatureError is a private struct, so we just check that an error is thrown - #expect(true) - } + let session = LanguageModelSession(model: visionModel, transcript: transcript) + let response = try await session.respond(to: "") + #expect(!response.content.isEmpty) } - @Test func multimodal_rejectsImageData() async throws { - let session = LanguageModelSession(model: model) - let data = Data(png1x1) - let prompt = Transcript.Prompt(segments: [ - .text(.init(content: "Describe this image")), - .image(.init(data: data, mimeType: "image/png")), + @Test func multimodalWithImageData() async throws { + let transcript = Transcript(entries: [ + .prompt( + Transcript.Prompt(segments: [ + .text(.init(content: "Describe this image")), + .image(.init(data: testImageData, mimeType: "image/png")), + ]) + ) ]) - do { - _ = try await session.respond(to: prompt) - Issue.record("Expected error when image segments are present") - } catch { - // MLXUnsupportedFeatureError is a private struct, so we just check that an error is thrown - #expect(true) - } + let session = LanguageModelSession(model: visionModel, transcript: transcript) + let response = try await session.respond(to: "") + #expect(!response.content.isEmpty) } } #endif // MLX From ac4de228b866c7a3f5392692235edb835ebecf96 Mon Sep 17 00:00:00 2001 From: Mattt Zmuda Date: Sun, 9 Nov 2025 06:49:19 -0500 Subject: [PATCH 09/16] Conditionalize import of CoreImage --- .../AnyLanguageModel/Models/MLXLanguageModel.swift | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/Sources/AnyLanguageModel/Models/MLXLanguageModel.swift b/Sources/AnyLanguageModel/Models/MLXLanguageModel.swift index 77228fd6..be66e908 100644 --- a/Sources/AnyLanguageModel/Models/MLXLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/MLXLanguageModel.swift @@ -1,10 +1,13 @@ import Foundation -import CoreImage + #if canImport(UIKit) import UIKit + import CoreImage #endif + #if canImport(AppKit) import AppKit + import CoreImage #endif #if MLX @@ -193,7 +196,8 @@ import CoreImage // MARK: - Segment Extraction - private func extractPromptSegments(from session: LanguageModelSession, fallbackText: String) -> [Transcript.Segment] { + private func extractPromptSegments(from session: LanguageModelSession, fallbackText: String) -> [Transcript.Segment] + { // Prefer the most recent Transcript.Prompt entry if present for entry in session.transcript.reversed() { if case .prompt(let p) = entry { @@ -220,12 +224,14 @@ import CoreImage case .data(let data, _): #if canImport(UIKit) if let uiImage = UIKit.UIImage(data: data), - let ciImage = CIImage(image: uiImage) { + let ciImage = CIImage(image: uiImage) + { images.append(.ciImage(ciImage)) } #elseif canImport(AppKit) if let nsImage = AppKit.NSImage(data: data), - let cgImage = nsImage.cgImage(forProposedRect: nil, context: nil, hints: nil) { + let cgImage = nsImage.cgImage(forProposedRect: nil, context: nil, hints: nil) + { let ciImage = CIImage(cgImage: cgImage) images.append(.ciImage(ciImage)) } From 4f256a9f1af0d2848587dc649978704a7c123b2d Mon Sep 17 00:00:00 2001 From: Mattt Zmuda Date: Sun, 9 Nov 2025 06:56:25 -0500 Subject: [PATCH 10/16] Update CoreMLLanguageModel to throw unsupportedFeature if image segments are in session --- .../Models/CoreMLLanguageModel.swift | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/Sources/AnyLanguageModel/Models/CoreMLLanguageModel.swift b/Sources/AnyLanguageModel/Models/CoreMLLanguageModel.swift index 1cd88a2b..bc8b1e21 100644 --- a/Sources/AnyLanguageModel/Models/CoreMLLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/CoreMLLanguageModel.swift @@ -80,6 +80,8 @@ fatalError("CoreMLLanguageModel only supports generating String content") } + try validateNoImageSegments(in: session) + // Convert AnyLanguageModel GenerationOptions to swift-transformers GenerationConfig let generationConfig = toGenerationConfig(options) @@ -122,6 +124,17 @@ fatalError("CoreMLLanguageModel only supports generating String content") } + // Validate that no image segments are present + do { + try validateNoImageSegments(in: session) + } catch { + return LanguageModelSession.ResponseStream( + stream: AsyncThrowingStream { continuation in + continuation.finish(throwing: error) + } + ) + } + // Convert AnyLanguageModel GenerationOptions to swift-transformers GenerationConfig let generationConfig = toGenerationConfig(options) @@ -229,6 +242,31 @@ } } + // MARK: - Image Validation + + private func validateNoImageSegments(in session: LanguageModelSession) throws { + // Check for image segments in instructions + if let instructions = session.instructions { + for segment in instructions.segments { + if case .image = segment { + throw CoreMLLanguageModelError.unsupportedFeature + } + } + } + + // Check for image segments in the most recent prompt + for entry in session.transcript.reversed() { + if case .prompt(let p) = entry { + for segment in p.segments { + if case .image = segment { + throw CoreMLLanguageModelError.unsupportedFeature + } + } + break + } + } + } + // MARK: - private func toGenerationConfig(_ options: GenerationOptions) -> GenerationConfig { From f1f70070a10fadc09acb8bb8f8809b025fbe6362 Mon Sep 17 00:00:00 2001 From: Mattt Zmuda Date: Sun, 9 Nov 2025 07:00:53 -0500 Subject: [PATCH 11/16] Update LlamaLanguageModel to throw unsupportedFeature if image segments are in session --- .../Models/LlamaLanguageModel.swift | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift b/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift index b77640f2..2b5b0855 100644 --- a/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift @@ -111,6 +111,9 @@ import Foundation fatalError("LlamaLanguageModel only supports generating String content") } + // Validate that no image segments are present + try validateNoImageSegments(in: session) + try await ensureModelLoaded() let contextParams = createContextParams(from: options) @@ -154,6 +157,17 @@ import Foundation fatalError("LlamaLanguageModel only supports generating String content") } + // Validate that no image segments are present + do { + try validateNoImageSegments(in: session) + } catch { + return LanguageModelSession.ResponseStream( + stream: AsyncThrowingStream { continuation in + continuation.finish(throwing: error) + } + ) + } + let maxTokens = options.maximumResponseTokens ?? 100 let stream: AsyncThrowingStream.Snapshot, any Error> = @@ -526,6 +540,31 @@ import Foundation } } + // MARK: - Image Validation + + private func validateNoImageSegments(in session: LanguageModelSession) throws { + // Check for image segments in instructions + if let instructions = session.instructions { + for segment in instructions.segments { + if case .image = segment { + throw LlamaLanguageModelError.unsupportedFeature + } + } + } + + // Check for image segments in the most recent prompt + for entry in session.transcript.reversed() { + if case .prompt(let p) = entry { + for segment in p.segments { + if case .image = segment { + throw LlamaLanguageModelError.unsupportedFeature + } + } + break + } + } + } + // MARK: - Helper Methods private func tokenizeText(vocab: OpaquePointer, text: String) throws -> [llama_token] { From b7bb9ec4125c53af585e52c7ce6ed3a697aaca3a Mon Sep 17 00:00:00 2001 From: Mattt Zmuda Date: Sun, 9 Nov 2025 07:27:01 -0500 Subject: [PATCH 12/16] Update README --- README.md | 83 +++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 77 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index dc426b36..dd75877d 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,9 @@ let response = try await session.respond { } ``` +> [!NOTE] +> Image inputs are not yet supported by Apple Foundation Models. + ### Core ML Run [Core ML](https://developer.apple.com/documentation/coreml) models @@ -136,6 +139,9 @@ Enable the trait in Package.swift: ) ``` +> [!NOTE] +> Image inputs are not currently supported with `CoreMLLanguageModel`. + ### MLX Run [MLX](https://github.com/ml-explore/mlx-swift) models on Apple Silicon @@ -150,6 +156,22 @@ let response = try await session.respond { } ``` +Vision support depends on the specific MLX model you load. +Use a vision‑capable model for multimodal prompts +(for example, a VLM variant). +The following shows extracting text from an image: + +```swift +let ocr = try await session.respond( + to: "Extract the total amount from this receipt", + images: [ + .init(url: URL(fileURLWithPath: "/path/to/receipt_page1.png")), + .init(url: URL(fileURLWithPath: "/path/to/receipt_page2.png")) + ] +) +print(ocr.content) +``` + Enable the trait in Package.swift: ```swift @@ -184,6 +206,9 @@ Enable the trait in Package.swift: ) ``` +> [!NOTE] +> Image inputs are not currently supported with `LlamaLanguageModel`. + ### OpenAI Supports both @@ -197,9 +222,17 @@ let model = OpenAILanguageModel( ) let session = LanguageModelSession(model: model) -let response = try await session.respond { - Prompt("Write a haiku about Swift") -} +let response = try await session.respond( + to: "List the objects you see", + images: [ + .init(url: URL(string: "https://example.com/desk.jpg")!), + .init( + data: try Data(contentsOf: URL(fileURLWithPath: "/path/to/closeup.png")), + mimeType: "image/png" + ) + ] +) +print(response.content) ``` For OpenAI-compatible endpoints that use older Chat Completions API: @@ -229,6 +262,18 @@ let response = try await session.respond { } ``` +You can include images with your question. +You can point to remote URLs or construct from bytes if you already have image data: + +```swift +let data = try Data(contentsOf: URL(fileURLWithPath: "/path/to/diagram.png")) +let response = try await session.respond( + to: "Explain the key parts of this diagram", + image: .init(data: data, mimeType: "image/png") +) +print(response.content) +``` + ### Google Gemini Uses the [Gemini API](https://ai.google.dev/api/generate-content) with Gemini models: @@ -245,6 +290,16 @@ let response = try await session.respond { } ``` +Send images with your prompt using remote or local sources: + +```swift +let response = try await session.respond( + to: "Identify the plants in this photo", + image: .init(url: URL(string: "https://example.com/garden.jpg")!) +) +print(response.content) +``` + Gemini models use an internal ["thinking process"](https://ai.google.dev/gemini-api/docs/thinking) that improves reasoning and multi-step planning. You can configure how much Gemini should "think" using the `thinking` parameter: @@ -293,11 +348,12 @@ let model = GeminiLanguageModel( ### Ollama -Run models locally via Ollama's [HTTP API](https://github.com/ollama/ollama/blob/main/docs/api.md): +Run models locally via Ollama's +[HTTP API](https://github.com/ollama/ollama/blob/main/docs/api.md): ```swift // Default: connects to http://localhost:11434 -let model = OllamaLanguageModel(model: "qwen3") +let model = OllamaLanguageModel(model: "qwen3") // `ollama pull qwen3:8b` // Custom endpoint let model = OllamaLanguageModel( @@ -311,7 +367,22 @@ let response = try await session.respond { } ``` -First, pull the model: `ollama pull qwen3:0.6b` +For local models, make sure you’re using a vision‑capable model +(for example, a “‑VL” variant). +You can combine multiple images: + +```swift +let model = OllamaLanguageModel(model: "qwen3-vl") // `ollama pull qwen3-vl:8b` +let session = LanguageModelSession(model: model) +let response = try await session.respond( + to: "Compare these posters and summarize their differences", + images: [ + .init(url: URL(string: "https://example.com/poster1.jpg")!), + .init(url: URL(fileURLWithPath: "/path/to/poster2.jpg")) + ] +) +print(response.content) +``` ## Testing From 25b78766ac9902125c578f4ed1357a6e147e41bb Mon Sep 17 00:00:00 2001 From: Mattt Zmuda Date: Sun, 9 Nov 2025 07:32:12 -0500 Subject: [PATCH 13/16] Remove MLX.GPU.set, as callers can set this themselves --- Sources/AnyLanguageModel/Models/MLXLanguageModel.swift | 2 -- 1 file changed, 2 deletions(-) diff --git a/Sources/AnyLanguageModel/Models/MLXLanguageModel.swift b/Sources/AnyLanguageModel/Models/MLXLanguageModel.swift index be66e908..58a4e094 100644 --- a/Sources/AnyLanguageModel/Models/MLXLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/MLXLanguageModel.swift @@ -51,8 +51,6 @@ import Foundation fatalError("MLXLanguageModel only supports generating String content") } - // Set GPU memory limit to prevent out of memory issues - MLX.GPU.set(cacheLimit: 20 * 1024 * 1024) let context = try await loadModel(id: modelId) // Convert session tools to MLX ToolSpec format From f6275cf1f9bd5e1c3135c0651fb1d44f7f9b5d1e Mon Sep 17 00:00:00 2001 From: Mattt Zmuda Date: Sun, 9 Nov 2025 07:33:28 -0500 Subject: [PATCH 14/16] Remove unused MLXLanguageModelError enum --- .../AnyLanguageModel/Models/MLXLanguageModel.swift | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/Sources/AnyLanguageModel/Models/MLXLanguageModel.swift b/Sources/AnyLanguageModel/Models/MLXLanguageModel.swift index 58a4e094..dda5ead7 100644 --- a/Sources/AnyLanguageModel/Models/MLXLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/MLXLanguageModel.swift @@ -163,19 +163,6 @@ import Foundation } } - /// Errors that can occur when working with MLX language models. - public enum MLXLanguageModelError: LocalizedError { - /// Image segments are not supported in MLXLanguageModel - case unsupportedFeature - - public var errorDescription: String? { - switch self { - case .unsupportedFeature: - return "This MLXLanguageModel does not support image segments" - } - } - } - // MARK: - Options Mapping private func toGenerateParameters(_ options: GenerationOptions) -> MLXLMCommon.GenerateParameters { From e702206f5dbe033428396fc0e3b3c6a805ef9eb2 Mon Sep 17 00:00:00 2001 From: Mattt Zmuda Date: Sun, 9 Nov 2025 07:45:32 -0500 Subject: [PATCH 15/16] Replace stray png1x1 with shared testImageData --- .../CoreMLLanguageModelTests.swift | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/Tests/AnyLanguageModelTests/CoreMLLanguageModelTests.swift b/Tests/AnyLanguageModelTests/CoreMLLanguageModelTests.swift index d53c71f1..f725c82a 100644 --- a/Tests/AnyLanguageModelTests/CoreMLLanguageModelTests.swift +++ b/Tests/AnyLanguageModelTests/CoreMLLanguageModelTests.swift @@ -155,22 +155,9 @@ import Testing func multimodal_rejectsImageData() async throws { let model = try await getModel() let session = LanguageModelSession(model: model) - // Minimal 1x1 PNG - let png1x1: [UInt8] = [ - 0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, - 0x00, 0x00, 0x00, 0x0D, 0x49, 0x48, 0x44, 0x52, - 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, - 0x08, 0x02, 0x00, 0x00, 0x00, 0x90, 0x77, 0x53, - 0xDE, 0x00, 0x00, 0x00, 0x0A, 0x49, 0x44, 0x41, - 0x54, 0x78, 0x9C, 0x63, 0xF8, 0x0F, 0x00, 0x01, - 0x01, 0x01, 0x00, 0x18, 0xDD, 0x8D, 0xF7, 0x00, - 0x00, 0x00, 0x00, 0x49, 0x45, 0x4E, 0x44, 0xAE, - 0x42, 0x60, 0x82, - ] - let data = Data(png1x1) let prompt = Transcript.Prompt(segments: [ .text(.init(content: "Describe this image")), - .image(.init(data: data, mimeType: "image/png")), + .image(.init(data: testImageData, mimeType: "image/jpeg")), ]) do { _ = try await session.respond(to: prompt) From e89eae08f2bd6c4d3a2288e79c5a67d8bf88e3d3 Mon Sep 17 00:00:00 2001 From: Mattt Zmuda Date: Sun, 9 Nov 2025 07:49:46 -0500 Subject: [PATCH 16/16] Update README --- README.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index dd75877d..efdaf933 100644 --- a/README.md +++ b/README.md @@ -262,14 +262,16 @@ let response = try await session.respond { } ``` -You can include images with your question. -You can point to remote URLs or construct from bytes if you already have image data: +You can include images with your prompt. +You can point to remote URLs or construct from image data: ```swift -let data = try Data(contentsOf: URL(fileURLWithPath: "/path/to/diagram.png")) let response = try await session.respond( to: "Explain the key parts of this diagram", - image: .init(data: data, mimeType: "image/png") + image: .init( + data: try Data(contentsOf: URL(fileURLWithPath: "/path/to/diagram.png")), + mimeType: "image/png" + ) ) print(response.content) ``` @@ -368,7 +370,7 @@ let response = try await session.respond { ``` For local models, make sure you’re using a vision‑capable model -(for example, a “‑VL” variant). +(for example, a `-vl` variant). You can combine multiple images: ```swift