From ace671e22b127cb495204aa5fb112e333660828f Mon Sep 17 00:00:00 2001 From: dijix009 Date: Tue, 23 Jun 2026 16:01:06 +0700 Subject: [PATCH 1/6] =?UTF-8?q?feat(cloud):=20CloudInference=20module=20?= =?UTF-8?q?=E2=80=94=20Anthropic=20Messages=20adapter=20(phase=201)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First phase of optional hosted-model support: a self-contained, MLX-free CloudInference module that adapts Anthropic's Messages API to Interless's backend-agnostic TokenChunk stream. No wiring yet — local MLX remains the only active backend until phase 2 routes roles to it. - CloudModelClient protocol; CloudProvider + CloudModelResolver (the `provider/model` id convention — `anthropic/claude-opus-4-8` is cloud, bare / Hugging Face ids stay local); CloudKeyProvider with Keychain (SecretStore, account `anthropic.apiKey`) + ANTHROPIC_API_KEY env fallback. - HTTPTransport seam + URLSessionHTTPTransport (the app's first outbound HTTP), injectable so tests run with no network. - AnthropicModelClient: maps GenerationRequest → Messages (system hoisted, text- mode messages with same-role merge, tools→input_schema, max_tokens, temperature); parses streamed SSE → TokenChunk (text deltas, tool_use → ModelToolCall, terminal usage/stop_reason); maps non-2xx → InferenceError with a clear message; clear missing-key error. Module depends only on Shared + InterlessSecurity. Tests drive a fake transport with canned SSE (request mapping, text+tool_use+terminal stream, 401 error, missing key, resolver). Full suite green (370 tests). --- CloudInference/AnthropicModelClient.swift | 276 ++++++++++++++++++ CloudInference/CloudModelClient.swift | 95 ++++++ CloudInference/HTTPTransport.swift | 63 ++++ Package.swift | 16 + Security/KeychainSecretStore.swift | 1 + .../AnthropicModelClientTests.swift | 175 +++++++++++ 6 files changed, 626 insertions(+) create mode 100644 CloudInference/AnthropicModelClient.swift create mode 100644 CloudInference/CloudModelClient.swift create mode 100644 CloudInference/HTTPTransport.swift create mode 100644 Tests/CloudInferenceTests/AnthropicModelClientTests.swift diff --git a/CloudInference/AnthropicModelClient.swift b/CloudInference/AnthropicModelClient.swift new file mode 100644 index 0000000..e286a0d --- /dev/null +++ b/CloudInference/AnthropicModelClient.swift @@ -0,0 +1,276 @@ +import Foundation +import Shared + +/// Streams from Anthropic's Messages API and adapts it to Interless's +/// backend-agnostic `TokenChunk` stream. MLX-free; transport + key provider are +/// injected so it is fully unit-testable without network. +/// +/// Tool calling is text-mode-consistent with the local path: prior tool results +/// are sent as plain user turns (no structured tool_use/tool_result id threading), +/// while `tools` are still advertised so the model can emit a fresh `tool_use`, +/// which is surfaced as a `ModelToolCall` chunk. +public struct AnthropicModelClient: CloudModelClient { + private let transport: any HTTPTransport + private let keyProvider: any CloudKeyProvider + private let baseURL: URL + private let apiVersion: String + private let defaultMaxTokens: Int + + public init( + transport: any HTTPTransport = URLSessionHTTPTransport(), + keyProvider: any CloudKeyProvider = KeychainCloudKeyProvider(), + baseURL: URL = URL(string: "https://api.anthropic.com")!, + apiVersion: String = "2023-06-01", + defaultMaxTokens: Int = 4096 + ) { + self.transport = transport + self.keyProvider = keyProvider + self.baseURL = baseURL + self.apiVersion = apiVersion + self.defaultMaxTokens = defaultMaxTokens + } + + public func validate() async throws { + guard await keyProvider.apiKey(for: .anthropic) != nil else { + throw InferenceError.generationFailed(Self.missingKeyMessage) + } + } + + public func stream(model: String, request: GenerationRequest) -> AsyncThrowingStream { + AsyncThrowingStream { continuation in + let task = Task { + do { + guard let key = await keyProvider.apiKey(for: .anthropic) else { + throw InferenceError.generationFailed(Self.missingKeyMessage) + } + let body = try Self.makeRequestBody( + model: model, request: request, defaultMaxTokens: defaultMaxTokens) + let spec = HTTPRequestSpec( + url: baseURL.appendingPathComponent("v1/messages"), + method: "POST", + headers: [ + "x-api-key": key, + "anthropic-version": apiVersion, + "content-type": "application/json", + "accept": "text/event-stream", + ], + body: body) + let (head, lines) = try await transport.stream(spec) + guard head.statusCode == 200 else { + var raw = "" + for try await line in lines { raw += line } + throw InferenceError.generationFailed(Self.errorMessage(status: head.statusCode, body: raw)) + } + + var index = 0 + var promptTokens = 0 + var outputTokens = 0 + var stopReason = "" + var pendingToolName: String? + var pendingToolJSON = "" + + for try await line in lines { + try Task.checkCancellation() + guard line.hasPrefix("data:") else { continue } + let payload = line.dropFirst("data:".count).trimmingCharacters(in: .whitespaces) + guard !payload.isEmpty, payload != "[DONE]", + let data = payload.data(using: .utf8), + let event = try? JSONDecoder().decode(AnthropicStreamEvent.self, from: data) else { + continue + } + switch event.type { + case "message_start": + promptTokens = event.message?.usage?.input_tokens ?? 0 + case "content_block_start": + if event.content_block?.type == "tool_use" { + pendingToolName = event.content_block?.name + pendingToolJSON = "" + } + case "content_block_delta": + if event.delta?.type == "text_delta", let text = event.delta?.text, !text.isEmpty { + continuation.yield(TokenChunk(text: text, index: index, isFinal: false)) + index += 1 + } else if event.delta?.type == "input_json_delta", let partial = event.delta?.partial_json { + pendingToolJSON += partial + } + case "content_block_stop": + if let name = pendingToolName { + continuation.yield(TokenChunk( + text: "", + index: index, + isFinal: false, + toolCall: ModelToolCall(name: name, arguments: Self.parseToolArguments(pendingToolJSON)))) + index += 1 + pendingToolName = nil + pendingToolJSON = "" + } + case "message_delta": + if let stop = event.delta?.stop_reason { stopReason = stop } + if let out = event.usage?.output_tokens { outputTokens = out } + case "error": + throw InferenceError.generationFailed(event.error?.message ?? "Anthropic stream error") + default: + break + } + } + + continuation.yield(TokenChunk( + text: "", + index: index, + isFinal: true, + info: TokenChunk.CompletionInfo( + promptTokenCount: promptTokens, + generationTokenCount: outputTokens, + stopReason: stopReason.isEmpty ? "stop" : stopReason))) + continuation.finish() + } catch is CancellationError { + continuation.finish(throwing: InferenceError.cancelled) + } catch { + continuation.finish(throwing: error) + } + } + continuation.onTermination = { _ in task.cancel() } + } + } + + // MARK: - Request mapping + + static let missingKeyMessage = + "Anthropic API key not set. Add it in Settings or set ANTHROPIC_API_KEY." + + static func makeRequestBody(model: String, request: GenerationRequest, defaultMaxTokens: Int) throws -> Data { + let (system, messages) = mapMessages(request) + var object: [String: JSONValue] = [ + "model": .string(model), + "max_tokens": .int(max(1, request.maxTokens ?? defaultMaxTokens)), + "temperature": .double(Double(request.temperature)), + "stream": .bool(true), + "messages": .array(messages), + ] + if let system, !system.isEmpty { + object["system"] = .string(system) + } + if !request.tools.isEmpty { + object["tools"] = .array(request.tools.map { tool in + .object([ + "name": .string(tool.name), + "description": .string(tool.description), + "input_schema": normalizedSchema(tool.parameters), + ]) + }) + } + return try JSONEncoder().encode(JSONValue.object(object)) + } + + /// Maps the request into a top-level `system` string + alternating user/assistant + /// messages. System turns are hoisted out (Anthropic takes system separately); + /// `tool` turns become user text; consecutive same-role turns are merged so the + /// transcript alternates as Anthropic expects. + static func mapMessages(_ request: GenerationRequest) -> (system: String?, messages: [JSONValue]) { + let chat: [GenerationRequest.ChatMessage] + switch request.input { + case let .prompt(text): + chat = [.init(role: .user, content: text)] + case let .messages(messages): + chat = messages + } + + var systemParts: [String] = [] + var turns: [(role: String, content: String)] = [] + for message in chat { + switch message.role { + case .system: + systemParts.append(message.content) + case .user: + turns.append((role: "user", content: message.content)) + case .assistant: + turns.append((role: "assistant", content: message.content)) + case .tool: + turns.append((role: "user", content: "[tool result]\n" + message.content)) + } + } + + var merged: [(role: String, content: String)] = [] + for turn in turns { + if var last = merged.last, last.role == turn.role { + last.content += "\n\n" + turn.content + merged[merged.count - 1] = last + } else { + merged.append(turn) + } + } + + let messages = merged.map { turn in + JSONValue.object(["role": .string(turn.role), "content": .string(turn.content)]) + } + let system = systemParts.isEmpty ? nil : systemParts.joined(separator: "\n\n") + return (system, messages) + } + + /// Anthropic's `input_schema` must be a JSON Schema object. The registry already + /// produces object schemas; wrap defensively if not. + static func normalizedSchema(_ schema: JSONValue) -> JSONValue { + if case .object = schema { return schema } + return .object(["type": .string("object")]) + } + + static func parseToolArguments(_ json: String) -> [String: JSONValue] { + let trimmed = json.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty, + let data = trimmed.data(using: .utf8), + let value = try? JSONDecoder().decode(JSONValue.self, from: data), + case let .object(object) = value else { + return [:] + } + return object + } + + static func errorMessage(status: Int, body: String) -> String { + if let data = body.data(using: .utf8), + let parsed = try? JSONDecoder().decode(AnthropicErrorEnvelope.self, from: data), + let message = parsed.error?.message { + return "Anthropic request failed (\(status)): \(message)" + } + let trimmed = body.trimmingCharacters(in: .whitespacesAndNewlines) + return trimmed.isEmpty + ? "Anthropic request failed (\(status))." + : "Anthropic request failed (\(status)): \(trimmed.prefix(500))" + } +} + +// MARK: - Wire decoding + +struct AnthropicStreamEvent: Decodable { + let type: String + let delta: Delta? + let content_block: ContentBlock? + let message: MessageStart? + let usage: Usage? + let error: ErrorBody? + + struct Delta: Decodable { + let type: String? + let text: String? + let partial_json: String? + let stop_reason: String? + } + struct ContentBlock: Decodable { + let type: String? + let name: String? + } + struct MessageStart: Decodable { + let usage: Usage? + } + struct Usage: Decodable { + let input_tokens: Int? + let output_tokens: Int? + } + struct ErrorBody: Decodable { + let type: String? + let message: String? + } +} + +struct AnthropicErrorEnvelope: Decodable { + let error: AnthropicStreamEvent.ErrorBody? +} diff --git a/CloudInference/CloudModelClient.swift b/CloudInference/CloudModelClient.swift new file mode 100644 index 0000000..8e382b9 --- /dev/null +++ b/CloudInference/CloudModelClient.swift @@ -0,0 +1,95 @@ +import Foundation +import Shared +import InterlessSecurity + +/// Hosted model providers Interless can route a role to. Local MLX is the +/// default; these are opt-in. (OpenAI is a planned sibling.) +public enum CloudProvider: String, Sendable, Equatable, CaseIterable { + case anthropic + + /// Keychain account holding this provider's API key. + public var keychainAccount: String { + switch self { + case .anthropic: return InterlessSecrets.anthropicAPIKeyAccount + } + } + + /// Environment variable consulted as a fallback (for the CLI / headless). + public var environmentVariable: String { + switch self { + case .anthropic: return "ANTHROPIC_API_KEY" + } + } +} + +/// A model id resolved to a hosted provider, e.g. `anthropic/claude-opus-4-8`. +public struct CloudModelID: Sendable, Equatable { + public var provider: CloudProvider + public var model: String + + public init(provider: CloudProvider, model: String) { + self.provider = provider + self.model = model + } +} + +/// Resolves a role's model-id string to a hosted provider, or `nil` for a local +/// MLX model. The convention is `provider/model`; bare ids (incl. Hugging Face +/// repo ids like `mlx-community/…`) stay local because only known provider +/// prefixes match. +public enum CloudModelResolver { + public static func resolve(_ id: String) -> CloudModelID? { + let parts = id.split(separator: "/", maxSplits: 1, omittingEmptySubsequences: false).map(String.init) + guard parts.count == 2, + let provider = CloudProvider(rawValue: parts[0].lowercased()), + !parts[1].isEmpty else { + return nil + } + return CloudModelID(provider: provider, model: parts[1]) + } + + public static func isCloud(_ id: String) -> Bool { + resolve(id) != nil + } +} + +/// Streams generation from a hosted model. The model is passed explicitly (the +/// id is resolved from the role's loaded handle by the caller). A `nil`-key or +/// transport failure surfaces as a thrown `InferenceError` on the stream. +public protocol CloudModelClient: Sendable { + func stream(model: String, request: GenerationRequest) -> AsyncThrowingStream + /// Cheap precondition check (e.g. an API key is present). Throws a clear + /// `InferenceError` when the client cannot be used. + func validate() async throws +} + +/// Supplies provider API keys. Default reads the Keychain `SecretStore`, then +/// falls back to the provider's environment variable. +public protocol CloudKeyProvider: Sendable { + func apiKey(for provider: CloudProvider) async -> String? +} + +public struct KeychainCloudKeyProvider: CloudKeyProvider { + private let secretStore: any SecretStore + private let environment: [String: String] + + public init( + secretStore: any SecretStore = KeychainSecretStore(), + environment: [String: String] = ProcessInfo.processInfo.environment + ) { + self.secretStore = secretStore + self.environment = environment + } + + public func apiKey(for provider: CloudProvider) async -> String? { + if let stored = (try? await secretStore.read( + service: InterlessSecrets.service, account: provider.keychainAccount)) ?? nil, + !stored.isEmpty { + return stored + } + if let env = environment[provider.environmentVariable], !env.isEmpty { + return env + } + return nil + } +} diff --git a/CloudInference/HTTPTransport.swift b/CloudInference/HTTPTransport.swift new file mode 100644 index 0000000..b287959 --- /dev/null +++ b/CloudInference/HTTPTransport.swift @@ -0,0 +1,63 @@ +import Foundation + +/// A single outbound HTTP request. Kept minimal and value-typed so the transport +/// can be faked in tests (this is the app's only outbound network path). +public struct HTTPRequestSpec: Sendable, Equatable { + public var url: URL + public var method: String + public var headers: [String: String] + public var body: Data + + public init(url: URL, method: String = "POST", headers: [String: String] = [:], body: Data = Data()) { + self.url = url + self.method = method + self.headers = headers + self.body = body + } +} + +public struct HTTPResponseHead: Sendable, Equatable { + public var statusCode: Int + public init(statusCode: Int) { self.statusCode = statusCode } +} + +/// Sends a request and streams the response body back as UTF-8 lines +/// (newline-delimited — sufficient for SSE, whose framing is line based). +/// Injectable so tests drive canned responses with no real network. +public protocol HTTPTransport: Sendable { + func stream(_ request: HTTPRequestSpec) async throws -> (head: HTTPResponseHead, lines: AsyncThrowingStream) +} + +public struct URLSessionHTTPTransport: HTTPTransport { + private let session: URLSession + + public init(session: URLSession = .shared) { + self.session = session + } + + public func stream(_ request: HTTPRequestSpec) async throws -> (head: HTTPResponseHead, lines: AsyncThrowingStream) { + var urlRequest = URLRequest(url: request.url) + urlRequest.httpMethod = request.method + urlRequest.httpBody = request.body + for (key, value) in request.headers { + urlRequest.setValue(value, forHTTPHeaderField: key) + } + + let (bytes, response) = try await session.bytes(for: urlRequest) + let status = (response as? HTTPURLResponse)?.statusCode ?? 0 + let lines = AsyncThrowingStream { continuation in + let task = Task { + do { + for try await line in bytes.lines { + continuation.yield(line) + } + continuation.finish() + } catch { + continuation.finish(throwing: error) + } + } + continuation.onTermination = { _ in task.cancel() } + } + return (HTTPResponseHead(statusCode: status), lines) + } +} diff --git a/Package.swift b/Package.swift index 2da068d..e17533b 100644 --- a/Package.swift +++ b/Package.swift @@ -18,6 +18,7 @@ let package = Package( .library(name: "Workspace", targets: ["Workspace"]), .library(name: "Tooling", targets: ["Tooling"]), .library(name: "InterlessSecurity", targets: ["InterlessSecurity"]), + .library(name: "CloudInference", targets: ["CloudInference"]), .library(name: "Agents", targets: ["Agents"]), .library(name: "AgentCLI", targets: ["AgentCLI"]), .library(name: "UI", targets: ["UI"]), @@ -114,6 +115,14 @@ let package = Package( exclude: ["README.md"], swiftSettings: [.swiftLanguageMode(.v6)] ), + // MARK: - CloudInference (optional hosted-model backends, e.g. Anthropic; + // MLX-free — depends only on Shared value types + Keychain secrets). + .target( + name: "CloudInference", + dependencies: ["Shared", "InterlessSecurity"], + path: "CloudInference", + swiftSettings: [.swiftLanguageMode(.v6)] + ), // MARK: - Agents (Phase 3; orchestration/runtime; no UI/Persistence imports) .target( name: "Agents", @@ -225,6 +234,13 @@ let package = Package( path: "Tests/SecurityTests", swiftSettings: [.swiftLanguageMode(.v6)] ), + // Fast CloudInference tests: fake HTTP transport + canned SSE, no network. + .testTarget( + name: "CloudInferenceTests", + dependencies: ["CloudInference", "Shared"], + path: "Tests/CloudInferenceTests", + swiftSettings: [.swiftLanguageMode(.v6)] + ), // Fast Agent tests: fake model/search + real restricted tooling. .testTarget( name: "AgentsTests", diff --git a/Security/KeychainSecretStore.swift b/Security/KeychainSecretStore.swift index c161fc4..fec417a 100644 --- a/Security/KeychainSecretStore.swift +++ b/Security/KeychainSecretStore.swift @@ -84,4 +84,5 @@ public struct KeychainSecretStore: SecretStore { public enum InterlessSecrets { public static let service = "dev.interless.secrets" public static let huggingFaceTokenAccount = "huggingface.token" + public static let anthropicAPIKeyAccount = "anthropic.apiKey" } diff --git a/Tests/CloudInferenceTests/AnthropicModelClientTests.swift b/Tests/CloudInferenceTests/AnthropicModelClientTests.swift new file mode 100644 index 0000000..61ea3a5 --- /dev/null +++ b/Tests/CloudInferenceTests/AnthropicModelClientTests.swift @@ -0,0 +1,175 @@ +import Foundation +import Testing +import Shared +@testable import CloudInference + +struct AnthropicModelClientTests { + + @Test func resolverDistinguishesCloudFromLocal() { + #expect(CloudModelResolver.resolve("anthropic/claude-opus-4-8") + == CloudModelID(provider: .anthropic, model: "claude-opus-4-8")) + // Hugging Face repo ids and bare ids stay local. + #expect(CloudModelResolver.resolve("mlx-community/gemma-2-2b-it-4bit") == nil) + #expect(CloudModelResolver.resolve("claude") == nil) + #expect(CloudModelResolver.isCloud("anthropic/claude-haiku-4-5")) + #expect(!CloudModelResolver.isCloud("mlx-community/x")) + } + + @Test func streamsTextDeltasThenToolUseThenFinal() async throws { + let transport = FakeHTTPTransport(status: 200, lines: [ + "event: message_start", + #"data: {"type":"message_start","message":{"usage":{"input_tokens":10}}}"#, + "", + "event: content_block_delta", + #"data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hello"}}"#, + "", + "event: content_block_start", + #"data: {"type":"content_block_start","index":1,"content_block":{"type":"tool_use","id":"tu_1","name":"read_file"}}"#, + "event: content_block_delta", + #"data: {"type":"content_block_delta","index":1,"delta":{"type":"input_json_delta","partial_json":"{\"path\":\"a.txt\"}"}}"#, + "event: content_block_stop", + #"data: {"type":"content_block_stop","index":1}"#, + "event: message_delta", + #"data: {"type":"message_delta","delta":{"stop_reason":"tool_use"},"usage":{"output_tokens":5}}"#, + "event: message_stop", + #"data: {"type":"message_stop"}"#, + ]) + let client = AnthropicModelClient(transport: transport, keyProvider: StubKeyProvider(key: "k")) + let chunks = try await collect(client.stream(model: "claude-opus-4-8", request: .prompt("hi"))) + + // Ascending index, exactly one terminal chunk. + #expect(chunks.map(\.index) == Array(0..) async throws -> [TokenChunk] { + var result: [TokenChunk] = [] + for try await chunk in stream { result.append(chunk) } + return result +} + +private struct StubKeyProvider: CloudKeyProvider { + let key: String? + func apiKey(for provider: CloudProvider) async -> String? { key } +} + +private actor FakeHTTPTransport: HTTPTransport { + let status: Int + let scriptedLines: [String] + private var captured: HTTPRequestSpec? + + init(status: Int, lines: [String]) { + self.status = status + self.scriptedLines = lines + } + + func stream(_ request: HTTPRequestSpec) async throws -> (head: HTTPResponseHead, lines: AsyncThrowingStream) { + captured = request + let scripted = scriptedLines + let lines = AsyncThrowingStream { continuation in + for line in scripted { continuation.yield(line) } + continuation.finish() + } + return (HTTPResponseHead(statusCode: status), lines) + } + + func capturedRequest() -> HTTPRequestSpec? { captured } +} + +private extension JSONValue { + /// Array accessor for tests (the Shared type exposes object/string/stringArray + /// but not a plain array getter). + var arrayValueForTest: [JSONValue]? { + if case let .array(values) = self { return values } + return nil + } +} From 504d61f3cde4eb8a4a6c9edec98bf6b076a3ea13 Mon Sep 17 00:00:00 2001 From: dijix009 Date: Tue, 23 Jun 2026 16:08:24 +0700 Subject: [PATCH 2/6] feat(cloud): per-role local/cloud routing backend (phase 2a) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire the Anthropic adapter into the inference seam without changing the controller or agents. - RemoteInferenceBackend (MLXEngine): an InferenceBackend over a CloudModelClient. Lifecycle is hollow — no local weights/KV/GPU/RAM, so load/unload/clearKVCache/ draft are no-ops and memory readings are zero; load validates the API key so a missing key surfaces at load time. generate delegates to the client using the model id on the handle (provider prefix stripped). embed throws (embeddings stay local). - RoutingInferenceBackend (MLXEngine): dispatches by model id — a `provider/model` id (anthropic/…) routes to the remote backend, everything else (bare / Hugging Face ids) to local MLX. No mutable routing table: load/generate/embed route by id, role-keyed calls forward to both sub-backends (no-op where absent), and countTokens/gpuMemory/footprint defer to local (correct for cloud roles too). - EngineBootstrap composes RoutingInferenceBackend(local: MLXBackend, remote: RemoteInferenceBackend(AnthropicModelClient())). Local-only behavior is unchanged; the remote backend is inert until a cloud id is loaded. MLXEngine now depends on CloudInference. Tests: remote backend delegates + strips the prefix, embed throws; routing dispatches local vs cloud by id. Full suite green (373 tests). Note: consent (allowCloudModels) + the loadModels/RuntimeConfigMapper wiring that lets a user actually select a cloud model land in phase 2b; UI in phase 3. --- MLXEngine/EngineBootstrap.swift | 10 ++- MLXEngine/RemoteInferenceBackend.swift | 52 ++++++++++++ MLXEngine/RoutingInferenceBackend.swift | 81 +++++++++++++++++++ Package.swift | 3 +- Tests/MLXEngineTests/RemoteBackendTests.swift | 77 ++++++++++++++++++ 5 files changed, 221 insertions(+), 2 deletions(-) create mode 100644 MLXEngine/RemoteInferenceBackend.swift create mode 100644 MLXEngine/RoutingInferenceBackend.swift create mode 100644 Tests/MLXEngineTests/RemoteBackendTests.swift diff --git a/MLXEngine/EngineBootstrap.swift b/MLXEngine/EngineBootstrap.swift index bdeac10..d78d2de 100644 --- a/MLXEngine/EngineBootstrap.swift +++ b/MLXEngine/EngineBootstrap.swift @@ -1,5 +1,6 @@ import Core import Shared +import CloudInference /// Phase 1 composition root for the inference stack. /// @@ -21,8 +22,15 @@ public enum EngineBootstrap { let budget = ResourceBudget.resolved(for: resourceProfile) var engineTuning = budget.engineTuning if let gpuCacheLimitBytes { engineTuning.gpuCacheLimitBytes = gpuCacheLimitBytes } + // Local MLX is the default; the routing backend sends only `provider/model` + // ids (e.g. `anthropic/…`) to the hosted backend. Local ids are unaffected, + // and the remote backend is inert until a cloud id is loaded (gated by the + // consent + key checks in the app layer). + let backend = RoutingInferenceBackend( + local: MLXBackend(engineTuning: engineTuning), + remote: RemoteInferenceBackend(client: AnthropicModelClient())) let controller = InferenceController( - backend: MLXBackend(engineTuning: engineTuning), + backend: backend, memoryMonitor: MemoryPressureMonitor(thresholds: thresholds), memoryCoordinator: memoryCoordinator, resourceProfile: resourceProfile diff --git a/MLXEngine/RemoteInferenceBackend.swift b/MLXEngine/RemoteInferenceBackend.swift new file mode 100644 index 0000000..352af74 --- /dev/null +++ b/MLXEngine/RemoteInferenceBackend.swift @@ -0,0 +1,52 @@ +import Foundation +import Shared +import CloudInference + +/// `InferenceBackend` over a hosted `CloudModelClient` (e.g. Anthropic). The +/// lifecycle is intentionally hollow: there are no local weights, KV cache, or +/// GPU/RAM to manage, so `load`/`unload`/`clearKVCache`/draft ops are no-ops and +/// memory readings are zero. `generate` delegates to the client using the model +/// id carried on the handle (provider prefix stripped). Embeddings stay local. +public struct RemoteInferenceBackend: InferenceBackend { + private let client: any CloudModelClient + + public init(client: any CloudModelClient) { + self.client = client + } + + public func load( + id: String, + role: ModelRole, + quantization: QuantizationLevel, + toolCallFormat: ModelToolCallFormat?, + progressHandler: (@Sendable (Double) -> Void)? + ) async throws -> LoadedModelHandle { + // Nothing to download; surface a missing-key error here so it appears at + // "load" time rather than mid-generation. + try await client.validate() + progressHandler?(1.0) + return LoadedModelHandle(role: role, id: id, quantization: quantization) + } + + public func generate(request: GenerationRequest, handle: LoadedModelHandle) -> AsyncThrowingStream { + client.stream(model: Self.modelName(from: handle.id), request: request) + } + + public func embed(texts: [String], handle: LoadedModelHandle) async throws -> [EmbeddingVector] { + throw InferenceError.generationFailed("Cloud embeddings are not supported; embeddings run locally.") + } + + public func unload(role: ModelRole) async {} + public func clearKVCache(role: ModelRole) async {} + public func unloadDraftModel(forRole role: ModelRole) async {} + + public func gpuMemory() async -> GPUMemory { GPUMemory() } + public func footprint() async -> MemoryFootprint { + MemoryFootprint(processFootprintBytes: 0, totalUnifiedBytes: 0) + } + + /// `anthropic/claude-opus-4-8` → `claude-opus-4-8`; leaves bare ids unchanged. + static func modelName(from id: String) -> String { + CloudModelResolver.resolve(id)?.model ?? id + } +} diff --git a/MLXEngine/RoutingInferenceBackend.swift b/MLXEngine/RoutingInferenceBackend.swift new file mode 100644 index 0000000..c72f987 --- /dev/null +++ b/MLXEngine/RoutingInferenceBackend.swift @@ -0,0 +1,81 @@ +import Foundation +import Shared +import CloudInference + +/// Dispatches each backend call to the local MLX backend or a hosted (cloud) +/// backend, decided by the model id: a `provider/model` id (e.g. +/// `anthropic/claude-opus-4-8`) routes to `remote`; everything else (bare ids, +/// Hugging Face repo ids) routes to `local`. This is how a role becomes local or +/// cloud — `InferenceController` and the agents are unchanged. +/// +/// No mutable routing table is needed: `load`/`generate`/`embed` route by id (the +/// handle carries it), and role-keyed calls forward to both sub-backends (the one +/// that never loaded the role is a harmless no-op). +public struct RoutingInferenceBackend: InferenceBackend { + private let local: any InferenceBackend + private let remote: any InferenceBackend + + public init(local: any InferenceBackend, remote: any InferenceBackend) { + self.local = local + self.remote = remote + } + + private func backend(for id: String) -> any InferenceBackend { + CloudModelResolver.isCloud(id) ? remote : local + } + + public func load( + id: String, + role: ModelRole, + quantization: QuantizationLevel, + toolCallFormat: ModelToolCallFormat?, + progressHandler: (@Sendable (Double) -> Void)? + ) async throws -> LoadedModelHandle { + try await backend(for: id).load( + id: id, role: role, quantization: quantization, + toolCallFormat: toolCallFormat, progressHandler: progressHandler) + } + + public func generate(request: GenerationRequest, handle: LoadedModelHandle) -> AsyncThrowingStream { + backend(for: handle.id).generate(request: request, handle: handle) + } + + public func embed(texts: [String], handle: LoadedModelHandle) async throws -> [EmbeddingVector] { + try await backend(for: handle.id).embed(texts: texts, handle: handle) + } + + public func unload(role: ModelRole) async { + await local.unload(role: role) + await remote.unload(role: role) + } + + public func clearKVCache(role: ModelRole) async { + await local.clearKVCache(role: role) + await remote.clearKVCache(role: role) + } + + public func loadDraftModel( + id: String, + forRole role: ModelRole, + quantization: QuantizationLevel, + progressHandler: (@Sendable (Double) -> Void)? + ) async throws -> LoadedModelHandle { + // Speculative drafting is an MLX-only feature; always local. + try await local.loadDraftModel( + id: id, forRole: role, quantization: quantization, progressHandler: progressHandler) + } + + public func unloadDraftModel(forRole role: ModelRole) async { + await local.unloadDraftModel(forRole: role) + } + + public func countTokens(_ text: String, role: ModelRole) async -> Int { + // Local gives a tokenizer-true count for local roles and a deterministic + // estimate when the role has no local model (i.e. a cloud role) — exactly + // what context fitting needs in both cases. + await local.countTokens(text, role: role) + } + + public func gpuMemory() async -> GPUMemory { await local.gpuMemory() } + public func footprint() async -> MemoryFootprint { await local.footprint() } +} diff --git a/Package.swift b/Package.swift index e17533b..a953096 100644 --- a/Package.swift +++ b/Package.swift @@ -63,6 +63,7 @@ let package = Package( dependencies: [ "Shared", "Core", + "CloudInference", .product(name: "MLXLLM", package: "mlx-swift-lm"), .product(name: "MLXLMCommon", package: "mlx-swift-lm"), .product(name: "MLXEmbedders", package: "mlx-swift-lm"), @@ -183,7 +184,7 @@ let package = Package( // Fast, model-free unit tests (default `swift test`). .testTarget( name: "MLXEngineTests", - dependencies: ["MLXEngine", "Core", "Shared"], + dependencies: ["MLXEngine", "Core", "Shared", "CloudInference"], path: "Tests/MLXEngineTests", swiftSettings: [.swiftLanguageMode(.v6)] ), diff --git a/Tests/MLXEngineTests/RemoteBackendTests.swift b/Tests/MLXEngineTests/RemoteBackendTests.swift new file mode 100644 index 0000000..3a83861 --- /dev/null +++ b/Tests/MLXEngineTests/RemoteBackendTests.swift @@ -0,0 +1,77 @@ +import Foundation +import Testing +import Shared +import CloudInference +@testable import MLXEngine + +struct RemoteBackendTests { + + @Test func remoteBackendDelegatesAndStripsProviderPrefix() async throws { + let client = FakeCloudModelClient(texts: ["A", "B"]) + let backend = RemoteInferenceBackend(client: client) + + let handle = try await backend.load( + id: "anthropic/claude-opus-4-8", role: .orchestrator, + quantization: .q8, toolCallFormat: nil, progressHandler: nil) + #expect(handle.id == "anthropic/claude-opus-4-8") + + let chunks = try await collect(backend.generate( + request: .prompt("hi", role: .orchestrator), handle: handle)) + #expect(client.capturedModel == "claude-opus-4-8") + #expect(chunks.filter { !$0.text.isEmpty }.map(\.text) == ["A", "B"]) + #expect(chunks.last?.isFinal == true) + } + + @Test func remoteBackendEmbedThrows() async throws { + let backend = RemoteInferenceBackend(client: FakeCloudModelClient(texts: [])) + let handle = LoadedModelHandle(role: .embeddings, id: "anthropic/x", quantization: .q8) + await #expect(throws: InferenceError.self) { + _ = try await backend.embed(texts: ["x"], handle: handle) + } + } + + @Test func routingDispatchesByModelId() async throws { + let local = FakeBackend() + await local.setScriptedTokens(["LOCAL"]) + let remote = RemoteInferenceBackend(client: FakeCloudModelClient(texts: ["REMOTE"])) + let routing = RoutingInferenceBackend(local: local, remote: remote) + + let localHandle = try await routing.load( + id: "mlx-community/model", role: .utility, + quantization: .q4, toolCallFormat: nil, progressHandler: nil) + let localChunks = try await collect(routing.generate( + request: .prompt("x", role: .utility), handle: localHandle)) + #expect(localChunks.filter { !$0.text.isEmpty }.map(\.text) == ["LOCAL"]) + + let cloudHandle = try await routing.load( + id: "anthropic/claude-haiku-4-5", role: .orchestrator, + quantization: .q8, toolCallFormat: nil, progressHandler: nil) + let cloudChunks = try await collect(routing.generate( + request: .prompt("y", role: .orchestrator), handle: cloudHandle)) + #expect(cloudChunks.filter { !$0.text.isEmpty }.map(\.text) == ["REMOTE"]) + } +} + +private final class FakeCloudModelClient: CloudModelClient, @unchecked Sendable { + private let texts: [String] + private(set) var capturedModel: String? + + init(texts: [String]) { self.texts = texts } + + func validate() async throws {} + + func stream(model: String, request: GenerationRequest) -> AsyncThrowingStream { + capturedModel = model + let texts = self.texts + return AsyncThrowingStream { continuation in + var index = 0 + for text in texts { + continuation.yield(TokenChunk(text: text, index: index, isFinal: false)) + index += 1 + } + continuation.yield(TokenChunk(text: "", index: index, isFinal: true, + info: TokenChunk.CompletionInfo(stopReason: "stop"))) + continuation.finish() + } + } +} From e61b2a23a754ee03115167cf5f660cc5f8361ec8 Mon Sep 17 00:00:00 2001 From: dijix009 Date: Tue, 23 Jun 2026 16:18:08 +0700 Subject: [PATCH 3/6] feat(cloud): consent gate + load wiring for cloud roles (phase 2b) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make hosted models actually selectable per role, gated by explicit consent. - ModelSettingsViewState gains allowCloudModels (default false) — distinct from allowNetworkTools because it sends prompts + workspace context to the provider. A migration-tolerant init(from:) is added so older persisted settings (missing the new key) still decode. - AppDependencyFactory.loadModels validates cloud usage before loading: a cloud orchestrator/utility role is refused unless allowCloudModels is on, and cloud embedding models are rejected (embeddings stay local). Errors surface through the existing invalidModelSettings path. Cloud roles otherwise load through the routing backend with no download/quantization (RAM-free). AppCore now depends on CloudInference (for the provider/model resolver). Tests: local roles never need consent; cloud roles require it; cloud embeddings are unsupported even with consent. Full suite green (377 tests). UI (key field, consent toggle, cloud status) lands in phase 3. --- AppCore/AppDependencyFactory.swift | 28 +++++++++++++++ AppCore/WorkspaceSessionModel.swift | 33 ++++++++++++++++++ Package.swift | 2 +- Tests/AppCoreTests/CloudUsageTests.swift | 43 ++++++++++++++++++++++++ UI/ModelSettingsView.swift | 35 +++++++++++++++++++ UI/PresentationModels.swift | 43 +++++++++++++++++++++++- 6 files changed, 182 insertions(+), 2 deletions(-) create mode 100644 Tests/AppCoreTests/CloudUsageTests.swift diff --git a/AppCore/AppDependencyFactory.swift b/AppCore/AppDependencyFactory.swift index 300a37a..8cc24b5 100644 --- a/AppCore/AppDependencyFactory.swift +++ b/AppCore/AppDependencyFactory.swift @@ -1,5 +1,6 @@ import Foundation import Agents +import CloudInference import Core import MLXEngine import Persistence @@ -293,6 +294,11 @@ public struct LiveAppDependencyFactory: AppDependencyFactory { let orchestratorModelID = Self.agentModelID(agentCatalog: agentCatalog, agentIDs: ["build", "plan"], fallback: runtimeSettings.orchestratorModelID) let utilityModelID = Self.agentModelID(agentCatalog: agentCatalog, agentIDs: ["general"], fallback: runtimeSettings.utilityModelID) let singleModelID = Self.agentModelID(agentCatalog: agentCatalog, agentIDs: ["general", "build"], fallback: runtimeSettings.orchestratorModelID) + try Self.validateCloudUsage( + orchestrator: singleAgentMode ? singleModelID : orchestratorModelID, + utility: singleAgentMode ? "" : utilityModelID, + embeddings: runtimeSettings.embeddingsModelID, + allowCloudModels: runtimeSettings.allowCloudModels) await resolvedController.unload(role: .orchestrator) await resolvedController.unload(role: .utility) await resolvedController.unload(role: .embeddings) @@ -353,6 +359,28 @@ public struct LiveAppDependencyFactory: AppDependencyFactory { }) } + /// Gates hosted (cloud) model usage: cloud orchestrator/utility roles require + /// explicit consent, and cloud embedding models are unsupported. Reuses the + /// `invalidModelSettings` surface so the message reaches the UI like any other + /// settings problem. + static func validateCloudUsage( + orchestrator: String, + utility: String, + embeddings: String, + allowCloudModels: Bool + ) throws { + var errors: [String] = [] + if CloudModelResolver.isCloud(embeddings) { + errors.append("Cloud embedding models are not supported; use a local embeddings model.") + } + if !allowCloudModels { + for id in [orchestrator, utility] where CloudModelResolver.isCloud(id) { + errors.append("\"\(id)\" is a cloud model. Enable \"Allow cloud models\" in Settings to use it.") + } + } + guard errors.isEmpty else { throw AppRuntimeError.invalidModelSettings(errors) } + } + private static func makeAgent( root: URL, store: any WorkspaceIndexStore, diff --git a/AppCore/WorkspaceSessionModel.swift b/AppCore/WorkspaceSessionModel.swift index 3d1c7a8..c3c8809 100644 --- a/AppCore/WorkspaceSessionModel.swift +++ b/AppCore/WorkspaceSessionModel.swift @@ -346,6 +346,39 @@ public final class WorkspaceSessionModel { } } + public func saveAnthropicAPIKey(_ key: String) { + let trimmed = key.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return } + Task { + do { + try await secretStore.save( + trimmed, + service: InterlessSecrets.service, + account: InterlessSecrets.anthropicAPIKeyAccount) + appendNotice(severity: .info, title: "Key saved", message: "Anthropic API key was saved in Keychain.") + await publish(.init(kind: .model, message: "Saved cloud provider key", metadata: ["provider": "anthropic", "store": "keychain"])) + } catch { + appendNotice(severity: .error, title: "Key save failed", message: String(describing: error)) + await recordFailure(kind: .model, message: "Failed to save Anthropic API key.") + } + } + } + + public func deleteAnthropicAPIKey() { + Task { + do { + try await secretStore.delete( + service: InterlessSecrets.service, + account: InterlessSecrets.anthropicAPIKeyAccount) + appendNotice(severity: .info, title: "Key deleted", message: "Anthropic API key was removed from Keychain.") + await publish(.init(kind: .model, message: "Deleted cloud provider key", metadata: ["provider": "anthropic", "store": "keychain"])) + } catch { + appendNotice(severity: .error, title: "Key delete failed", message: String(describing: error)) + await recordFailure(kind: .model, message: "Failed to delete Anthropic API key.") + } + } + } + public func clearPersistedHistory() { Task { do { diff --git a/Package.swift b/Package.swift index a953096..5357146 100644 --- a/Package.swift +++ b/Package.swift @@ -149,7 +149,7 @@ let package = Package( name: "AppCore", dependencies: [ "UI", "Shared", "Core", "Agents", "MLXEngine", "InterlessSecurity", - "Persistence", "Workspace", "Tooling", + "Persistence", "Workspace", "Tooling", "CloudInference", ], path: "AppCore", swiftSettings: [.swiftLanguageMode(.v6)] diff --git a/Tests/AppCoreTests/CloudUsageTests.swift b/Tests/AppCoreTests/CloudUsageTests.swift new file mode 100644 index 0000000..65481c9 --- /dev/null +++ b/Tests/AppCoreTests/CloudUsageTests.swift @@ -0,0 +1,43 @@ +import Testing +@testable import AppCore + +struct CloudUsageTests { + + @Test func localRolesNeverRequireConsent() throws { + // Bare / Hugging Face ids are local — allowed regardless of consent. + try LiveAppDependencyFactory.validateCloudUsage( + orchestrator: "mlx-community/orchestrator", + utility: "mlx-community/utility", + embeddings: "nomic-embed", + allowCloudModels: false) + } + + @Test func cloudRolesAllowedOnlyWithConsent() throws { + // With consent, cloud orchestrator + cloud sub-agent are fine. + try LiveAppDependencyFactory.validateCloudUsage( + orchestrator: "anthropic/claude-opus-4-8", + utility: "anthropic/claude-haiku-4-5", + embeddings: "nomic-embed", + allowCloudModels: true) + } + + @Test func cloudRoleWithoutConsentThrows() { + #expect(throws: (any Error).self) { + try LiveAppDependencyFactory.validateCloudUsage( + orchestrator: "anthropic/claude-opus-4-8", + utility: "mlx-community/utility", + embeddings: "nomic-embed", + allowCloudModels: false) + } + } + + @Test func cloudEmbeddingsUnsupportedEvenWithConsent() { + #expect(throws: (any Error).self) { + try LiveAppDependencyFactory.validateCloudUsage( + orchestrator: "mlx-community/orchestrator", + utility: "mlx-community/utility", + embeddings: "anthropic/embed", + allowCloudModels: true) + } + } +} diff --git a/UI/ModelSettingsView.swift b/UI/ModelSettingsView.swift index b29b798..241afc5 100644 --- a/UI/ModelSettingsView.swift +++ b/UI/ModelSettingsView.swift @@ -15,6 +15,8 @@ public struct ModelSettingsView: View { public var onApplyRecommendations: @MainActor () -> Void public var onSaveHuggingFaceToken: @MainActor (String) -> Void public var onDeleteHuggingFaceToken: @MainActor () -> Void + public var onSaveAnthropicAPIKey: @MainActor (String) -> Void + public var onDeleteAnthropicAPIKey: @MainActor () -> Void private let headerTitle: String? private let headerSubtitle: String private let embedsInParentScroll: Bool @@ -22,6 +24,7 @@ public struct ModelSettingsView: View { private let showsResourceProfileControl: Bool private let showsDangerZone: Bool @State private var huggingFaceToken = "" + @State private var anthropicAPIKey = "" @State private var showAdvanced = false public init( @@ -38,6 +41,8 @@ public struct ModelSettingsView: View { onApplyRecommendations: @escaping @MainActor () -> Void = {}, onSaveHuggingFaceToken: @escaping @MainActor (String) -> Void = { _ in }, onDeleteHuggingFaceToken: @escaping @MainActor () -> Void = {}, + onSaveAnthropicAPIKey: @escaping @MainActor (String) -> Void = { _ in }, + onDeleteAnthropicAPIKey: @escaping @MainActor () -> Void = {}, headerTitle: String? = "Providers", headerSubtitle: String = "Local MLX model setup and native tool-call compatibility.", embedsInParentScroll: Bool = false, @@ -58,6 +63,8 @@ public struct ModelSettingsView: View { self.onApplyRecommendations = onApplyRecommendations self.onSaveHuggingFaceToken = onSaveHuggingFaceToken self.onDeleteHuggingFaceToken = onDeleteHuggingFaceToken + self.onSaveAnthropicAPIKey = onSaveAnthropicAPIKey + self.onDeleteAnthropicAPIKey = onDeleteAnthropicAPIKey self.headerTitle = headerTitle self.headerSubtitle = headerSubtitle self.embedsInParentScroll = embedsInParentScroll @@ -268,6 +275,25 @@ public struct ModelSettingsView: View { Text("Tokens are stored only in Keychain.") .font(.metaMono) .foregroundStyle(Theme.C.textSecondary) + settingsDivider() + controlRow("Anthropic API key") { + SecureField("sk-ant-…", text: $anthropicAPIKey) + .textContentType(.password) + .textFieldStyle(.roundedBorder) + .frame(maxWidth: 360) + } + HStack(spacing: .space2) { + Button("Save Key") { + onSaveAnthropicAPIKey(anthropicAPIKey) + anthropicAPIKey = "" + } + .disabled(anthropicAPIKey.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty) + Button("Delete Key", action: onDeleteAnthropicAPIKey) + } + .font(.bodyS.weight(.semibold)) + Text("Stored only in Keychain. Used when a role's model id is a cloud id (e.g. anthropic/claude-opus-4-8) and cloud models are allowed below.") + .font(.metaMono) + .foregroundStyle(Theme.C.textSecondary) if showsResourceProfileControl { settingsDivider() controlRow("Resource profile") { @@ -325,6 +351,15 @@ public struct ModelSettingsView: View { .foregroundStyle(Theme.C.danger) .fixedSize(horizontal: false, vertical: true) } + Toggle("Allow cloud models", isOn: $settings.allowCloudModels) + .font(.bodyS) + .accessibilityHint("Sends prompts and workspace context to the hosted model provider") + if let warning = settings.cloudModelsWarning { + Label(warning, systemImage: "cloud") + .font(.metaMono) + .foregroundStyle(Theme.C.danger) + .fixedSize(horizontal: false, vertical: true) + } } .padding(.space3) .frame(maxWidth: .infinity, alignment: .leading) diff --git a/UI/PresentationModels.swift b/UI/PresentationModels.swift index 5c2a603..13f9472 100644 --- a/UI/PresentationModels.swift +++ b/UI/PresentationModels.swift @@ -1851,6 +1851,11 @@ public struct ModelSettingsViewState: Sendable, Equatable, Codable { /// tokens the orchestrator verifies. Both models must share a tokenizer. public var enableSpeculativeDecoding: Bool public var speculativeDraftModelID: String + /// Explicit consent to use hosted (cloud) models for orchestrator/sub-agent + /// roles. Off by default; when off, a role configured with a `provider/model` + /// id (e.g. `anthropic/…`) is refused. Distinct from `allowNetworkTools` + /// (shell/network) because it sends prompts + workspace context to the provider. + public var allowCloudModels: Bool public init( orchestratorModelID: String = "", @@ -1866,7 +1871,8 @@ public struct ModelSettingsViewState: Sendable, Equatable, Codable { maxToolIterations: Int = 4, resourceProfile: ResourceProfile = .automatic, enableSpeculativeDecoding: Bool = false, - speculativeDraftModelID: String = "" + speculativeDraftModelID: String = "", + allowCloudModels: Bool = false ) { self.orchestratorModelID = orchestratorModelID self.utilityModelID = utilityModelID @@ -1882,6 +1888,37 @@ public struct ModelSettingsViewState: Sendable, Equatable, Codable { self.resourceProfile = resourceProfile self.enableSpeculativeDecoding = enableSpeculativeDecoding self.speculativeDraftModelID = speculativeDraftModelID + self.allowCloudModels = allowCloudModels + } + + private enum CodingKeys: String, CodingKey { + case orchestratorModelID, utilityModelID, embeddingsModelID + case orchestratorQuantization, utilityQuantization, embeddingsQuantization + case toolCallFormat, allowWrites, allowNetworkTools, persistPromptHistory + case maxToolIterations, resourceProfile, enableSpeculativeDecoding + case speculativeDraftModelID, allowCloudModels + } + + /// Migration-tolerant decode: every field falls back to its default so older + /// persisted settings (missing newer keys like `allowCloudModels`) still load. + public init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + self.init( + orchestratorModelID: try c.decodeIfPresent(String.self, forKey: .orchestratorModelID) ?? "", + utilityModelID: try c.decodeIfPresent(String.self, forKey: .utilityModelID) ?? "", + embeddingsModelID: try c.decodeIfPresent(String.self, forKey: .embeddingsModelID) ?? "", + orchestratorQuantization: try c.decodeIfPresent(QuantizationLevel.self, forKey: .orchestratorQuantization) ?? .defaultFor(.orchestrator), + utilityQuantization: try c.decodeIfPresent(QuantizationLevel.self, forKey: .utilityQuantization) ?? .defaultFor(.utility), + embeddingsQuantization: try c.decodeIfPresent(QuantizationLevel.self, forKey: .embeddingsQuantization) ?? .defaultFor(.embeddings), + toolCallFormat: try c.decodeIfPresent(ModelToolCallFormat.self, forKey: .toolCallFormat), + allowWrites: try c.decodeIfPresent(Bool.self, forKey: .allowWrites) ?? false, + allowNetworkTools: try c.decodeIfPresent(Bool.self, forKey: .allowNetworkTools) ?? false, + persistPromptHistory: try c.decodeIfPresent(Bool.self, forKey: .persistPromptHistory) ?? true, + maxToolIterations: try c.decodeIfPresent(Int.self, forKey: .maxToolIterations) ?? 4, + resourceProfile: try c.decodeIfPresent(ResourceProfile.self, forKey: .resourceProfile) ?? .automatic, + enableSpeculativeDecoding: try c.decodeIfPresent(Bool.self, forKey: .enableSpeculativeDecoding) ?? false, + speculativeDraftModelID: try c.decodeIfPresent(String.self, forKey: .speculativeDraftModelID) ?? "", + allowCloudModels: try c.decodeIfPresent(Bool.self, forKey: .allowCloudModels) ?? false) } public func usesSingleAgentMode( @@ -1930,6 +1967,10 @@ public struct ModelSettingsViewState: Sendable, Equatable, Codable { public var networkToolWarning: String? { allowNetworkTools ? "Trusted process/network tools can execute workspace scripts and commands." : nil } + + public var cloudModelsWarning: String? { + allowCloudModels ? "Cloud models send your prompts and workspace context to the hosted provider (e.g. Anthropic)." : nil + } } public struct RecommendedModel: Identifiable, Sendable, Equatable, Codable { From c25732e4e3fd4c385eb21fe3502742df83646d09 Mon Sep 17 00:00:00 2001 From: dijix009 Date: Tue, 23 Jun 2026 16:29:50 +0700 Subject: [PATCH 4/6] feat(cloud): settings UI for cloud models (phase 3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire the cloud-model controls through the view chain so they're user-reachable: WorkspaceActions → WorkspaceView → SettingsHubView → ModelSettingsView, plus the session key handlers in the app composition root. This completes the Settings surface (the field/toggle live in ModelSettingsView and WorkspaceSessionModel from the prior phase): an Anthropic API-key field (Keychain, account anthropic.apiKey), an "Allow cloud models" consent toggle in the Danger Zone with a privacy warning that prompts + workspace context are sent to the provider, and model-id fields that accept anthropic/… ids. Full suite green (377 tests); app builds. --- App/InterlessApp.swift | 2 ++ UI/SettingsHubView.swift | 12 +++++++++++- UI/WorkspaceView.swift | 8 ++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/App/InterlessApp.swift b/App/InterlessApp.swift index 932bb56..9fb1ae5 100644 --- a/App/InterlessApp.swift +++ b/App/InterlessApp.swift @@ -157,6 +157,8 @@ private struct WorkspaceShell: View { cancelModelLoad: { session.cancelModelLoad() }, saveHuggingFaceToken: session.saveHuggingFaceToken, deleteHuggingFaceToken: session.deleteHuggingFaceToken, + saveAnthropicAPIKey: session.saveAnthropicAPIKey, + deleteAnthropicAPIKey: session.deleteAnthropicAPIKey, retryRecoveryAction: session.retryRecoveryAction, dismissRecoveryItem: session.dismissRecoveryItem, clearRecoveryJournal: session.clearRecoveryJournal, diff --git a/UI/SettingsHubView.swift b/UI/SettingsHubView.swift index 9fce6c2..3084b21 100644 --- a/UI/SettingsHubView.swift +++ b/UI/SettingsHubView.swift @@ -27,6 +27,8 @@ public struct SettingsHubView: View { public var onApplyRecommendations: @MainActor () -> Void public var onSaveHuggingFaceToken: @MainActor (String) -> Void public var onDeleteHuggingFaceToken: @MainActor () -> Void + public var onSaveAnthropicAPIKey: @MainActor (String) -> Void + public var onDeleteAnthropicAPIKey: @MainActor () -> Void public var onOpenHealth: @MainActor () -> Void public var onExportDiagnostics: @MainActor () -> Void public var onUpdateModelContextSettings: @MainActor (ModelContextSettingsViewState) -> Void @@ -64,6 +66,8 @@ public struct SettingsHubView: View { onApplyRecommendations: @escaping @MainActor () -> Void, onSaveHuggingFaceToken: @escaping @MainActor (String) -> Void, onDeleteHuggingFaceToken: @escaping @MainActor () -> Void, + onSaveAnthropicAPIKey: @escaping @MainActor (String) -> Void = { _ in }, + onDeleteAnthropicAPIKey: @escaping @MainActor () -> Void = {}, onOpenHealth: @escaping @MainActor () -> Void, onExportDiagnostics: @escaping @MainActor () -> Void, onUpdateModelContextSettings: @escaping @MainActor (ModelContextSettingsViewState) -> Void @@ -77,6 +81,8 @@ public struct SettingsHubView: View { self.onApplyRecommendations = onApplyRecommendations self.onSaveHuggingFaceToken = onSaveHuggingFaceToken self.onDeleteHuggingFaceToken = onDeleteHuggingFaceToken + self.onSaveAnthropicAPIKey = onSaveAnthropicAPIKey + self.onDeleteAnthropicAPIKey = onDeleteAnthropicAPIKey self.onOpenHealth = onOpenHealth self.onExportDiagnostics = onExportDiagnostics self.onUpdateModelContextSettings = onUpdateModelContextSettings @@ -209,7 +215,9 @@ public struct SettingsHubView: View { onDismissOnboarding: onDismissOnboarding, onApplyRecommendations: onApplyRecommendations, onSaveHuggingFaceToken: onSaveHuggingFaceToken, - onDeleteHuggingFaceToken: onDeleteHuggingFaceToken) + onDeleteHuggingFaceToken: onDeleteHuggingFaceToken, + onSaveAnthropicAPIKey: onSaveAnthropicAPIKey, + onDeleteAnthropicAPIKey: onDeleteAnthropicAPIKey) case .usage: settingsScroll { usageSection } case .skills: @@ -373,6 +381,8 @@ public struct SettingsHubView: View { onApplyRecommendations: onApplyRecommendations, onSaveHuggingFaceToken: onSaveHuggingFaceToken, onDeleteHuggingFaceToken: onDeleteHuggingFaceToken, + onSaveAnthropicAPIKey: onSaveAnthropicAPIKey, + onDeleteAnthropicAPIKey: onDeleteAnthropicAPIKey, headerTitle: nil, embedsInParentScroll: true, showsRuntimeControls: false, diff --git a/UI/WorkspaceView.swift b/UI/WorkspaceView.swift index c06d84a..8bf78c6 100644 --- a/UI/WorkspaceView.swift +++ b/UI/WorkspaceView.swift @@ -38,6 +38,8 @@ public struct WorkspaceViewActions { public var cancelModelLoad: @MainActor () -> Void public var saveHuggingFaceToken: @MainActor (String) -> Void public var deleteHuggingFaceToken: @MainActor () -> Void + public var saveAnthropicAPIKey: @MainActor (String) -> Void + public var deleteAnthropicAPIKey: @MainActor () -> Void public var retryRecoveryAction: @MainActor (UUID) -> Void public var dismissRecoveryItem: @MainActor (UUID) -> Void public var clearRecoveryJournal: @MainActor () -> Void @@ -82,6 +84,8 @@ public struct WorkspaceViewActions { cancelModelLoad: @escaping @MainActor () -> Void, saveHuggingFaceToken: @escaping @MainActor (String) -> Void = { _ in }, deleteHuggingFaceToken: @escaping @MainActor () -> Void = {}, + saveAnthropicAPIKey: @escaping @MainActor (String) -> Void = { _ in }, + deleteAnthropicAPIKey: @escaping @MainActor () -> Void = {}, retryRecoveryAction: @escaping @MainActor (UUID) -> Void, dismissRecoveryItem: @escaping @MainActor (UUID) -> Void, clearRecoveryJournal: @escaping @MainActor () -> Void, @@ -125,6 +129,8 @@ public struct WorkspaceViewActions { self.cancelModelLoad = cancelModelLoad self.saveHuggingFaceToken = saveHuggingFaceToken self.deleteHuggingFaceToken = deleteHuggingFaceToken + self.saveAnthropicAPIKey = saveAnthropicAPIKey + self.deleteAnthropicAPIKey = deleteAnthropicAPIKey self.retryRecoveryAction = retryRecoveryAction self.dismissRecoveryItem = dismissRecoveryItem self.clearRecoveryJournal = clearRecoveryJournal @@ -845,6 +851,8 @@ public struct WorkspaceView: View { onApplyRecommendations: actions.applyRecommendedModels, onSaveHuggingFaceToken: actions.saveHuggingFaceToken, onDeleteHuggingFaceToken: actions.deleteHuggingFaceToken, + onSaveAnthropicAPIKey: actions.saveAnthropicAPIKey, + onDeleteAnthropicAPIKey: actions.deleteAnthropicAPIKey, onOpenHealth: actions.openHealth, onExportDiagnostics: actions.exportDiagnostics, onUpdateModelContextSettings: actions.setModelContextSettings) From 138ccdb9cc9d80aa1f0f8c847dd2e2bf511933ca Mon Sep 17 00:00:00 2001 From: dijix009 Date: Tue, 23 Jun 2026 16:58:59 +0700 Subject: [PATCH 5/6] feat(cloud): OpenAI Chat Completions adapter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add OpenAI as a sibling cloud provider behind the existing CloudModelClient abstraction, so a role can use openai/gpt-4o (mix freely with local MLX or Anthropic per role). - CloudProvider gains .openai (Keychain account openai.apiKey, OPENAI_API_KEY env fallback); OpenAIModelClient maps GenerationRequest → Chat Completions (Bearer auth, system kept inline, tool turns as user text + merge, tools via ToolDefinition.schema, max_tokens, stream + usage) and parses streamed chunks → TokenChunk (content deltas; incremental tool_calls accumulated by slot and flushed as ModelToolCall; usage/finish_reason → CompletionInfo). 4xx/5xx → a clear InferenceError. - RemoteInferenceBackend generalized from one client to [CloudProvider: CloudModelClient], resolving the provider from the model id; EngineBootstrap registers anthropic + openai. - Consent + resolution are provider-agnostic, so openai/... is gated by the same allowCloudModels consent (no new gate). - Settings gains an OpenAI API-key field mirroring the Anthropic one, wired through the view chain to WorkspaceSessionModel.save/deleteOpenAIAPIKey. Tests: OpenAI adapter (content+tool_calls stream, request mapping, 401, missing key, resolver) and multi-provider backend dispatch (anthropic vs openai by id). Full suite green (383 tests). Reasoning models (max_completion_tokens / no temperature) are out of scope for this first cut. --- App/InterlessApp.swift | 2 + AppCore/WorkspaceSessionModel.swift | 33 +++ CloudInference/CloudModelClient.swift | 3 + CloudInference/OpenAIModelClient.swift | 246 ++++++++++++++++++ MLXEngine/EngineBootstrap.swift | 5 +- MLXEngine/RemoteInferenceBackend.swift | 41 ++- Security/KeychainSecretStore.swift | 1 + .../OpenAIModelClientTests.swift | 138 ++++++++++ Tests/MLXEngineTests/RemoteBackendTests.swift | 24 +- UI/ModelSettingsView.swift | 26 ++ UI/SettingsHubView.swift | 12 +- UI/WorkspaceView.swift | 8 + 12 files changed, 520 insertions(+), 19 deletions(-) create mode 100644 CloudInference/OpenAIModelClient.swift create mode 100644 Tests/CloudInferenceTests/OpenAIModelClientTests.swift diff --git a/App/InterlessApp.swift b/App/InterlessApp.swift index 9fb1ae5..6924212 100644 --- a/App/InterlessApp.swift +++ b/App/InterlessApp.swift @@ -159,6 +159,8 @@ private struct WorkspaceShell: View { deleteHuggingFaceToken: session.deleteHuggingFaceToken, saveAnthropicAPIKey: session.saveAnthropicAPIKey, deleteAnthropicAPIKey: session.deleteAnthropicAPIKey, + saveOpenAIAPIKey: session.saveOpenAIAPIKey, + deleteOpenAIAPIKey: session.deleteOpenAIAPIKey, retryRecoveryAction: session.retryRecoveryAction, dismissRecoveryItem: session.dismissRecoveryItem, clearRecoveryJournal: session.clearRecoveryJournal, diff --git a/AppCore/WorkspaceSessionModel.swift b/AppCore/WorkspaceSessionModel.swift index c3c8809..b905bc3 100644 --- a/AppCore/WorkspaceSessionModel.swift +++ b/AppCore/WorkspaceSessionModel.swift @@ -379,6 +379,39 @@ public final class WorkspaceSessionModel { } } + public func saveOpenAIAPIKey(_ key: String) { + let trimmed = key.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return } + Task { + do { + try await secretStore.save( + trimmed, + service: InterlessSecrets.service, + account: InterlessSecrets.openAIAPIKeyAccount) + appendNotice(severity: .info, title: "Key saved", message: "OpenAI API key was saved in Keychain.") + await publish(.init(kind: .model, message: "Saved cloud provider key", metadata: ["provider": "openai", "store": "keychain"])) + } catch { + appendNotice(severity: .error, title: "Key save failed", message: String(describing: error)) + await recordFailure(kind: .model, message: "Failed to save OpenAI API key.") + } + } + } + + public func deleteOpenAIAPIKey() { + Task { + do { + try await secretStore.delete( + service: InterlessSecrets.service, + account: InterlessSecrets.openAIAPIKeyAccount) + appendNotice(severity: .info, title: "Key deleted", message: "OpenAI API key was removed from Keychain.") + await publish(.init(kind: .model, message: "Deleted cloud provider key", metadata: ["provider": "openai", "store": "keychain"])) + } catch { + appendNotice(severity: .error, title: "Key delete failed", message: String(describing: error)) + await recordFailure(kind: .model, message: "Failed to delete OpenAI API key.") + } + } + } + public func clearPersistedHistory() { Task { do { diff --git a/CloudInference/CloudModelClient.swift b/CloudInference/CloudModelClient.swift index 8e382b9..2264d0a 100644 --- a/CloudInference/CloudModelClient.swift +++ b/CloudInference/CloudModelClient.swift @@ -6,11 +6,13 @@ import InterlessSecurity /// default; these are opt-in. (OpenAI is a planned sibling.) public enum CloudProvider: String, Sendable, Equatable, CaseIterable { case anthropic + case openai /// Keychain account holding this provider's API key. public var keychainAccount: String { switch self { case .anthropic: return InterlessSecrets.anthropicAPIKeyAccount + case .openai: return InterlessSecrets.openAIAPIKeyAccount } } @@ -18,6 +20,7 @@ public enum CloudProvider: String, Sendable, Equatable, CaseIterable { public var environmentVariable: String { switch self { case .anthropic: return "ANTHROPIC_API_KEY" + case .openai: return "OPENAI_API_KEY" } } } diff --git a/CloudInference/OpenAIModelClient.swift b/CloudInference/OpenAIModelClient.swift new file mode 100644 index 0000000..0b37074 --- /dev/null +++ b/CloudInference/OpenAIModelClient.swift @@ -0,0 +1,246 @@ +import Foundation +import Shared + +/// Streams from OpenAI's Chat Completions API and adapts it to Interless's +/// backend-agnostic `TokenChunk` stream. Sibling of `AnthropicModelClient` behind +/// the same `CloudModelClient` protocol; MLX-free, transport + key injected. +/// +/// Tool calling is text-mode-consistent with the local path: prior tool results +/// are sent as plain user turns (no tool_call_id threading), while `tools` are +/// advertised so the model can emit fresh `tool_calls`, which stream +/// incrementally (name once, arguments across deltas) and are flushed as +/// `ModelToolCall` chunks when the turn finishes. +/// +/// Note: `max_tokens` + `temperature` target the mainstream chat models +/// (gpt-4o / gpt-4.1 class). Reasoning models that require `max_completion_tokens` +/// or reject `temperature` are out of scope for this first cut. +public struct OpenAIModelClient: CloudModelClient { + private let transport: any HTTPTransport + private let keyProvider: any CloudKeyProvider + private let baseURL: URL + private let defaultMaxTokens: Int + + public init( + transport: any HTTPTransport = URLSessionHTTPTransport(), + keyProvider: any CloudKeyProvider = KeychainCloudKeyProvider(), + baseURL: URL = URL(string: "https://api.openai.com")!, + defaultMaxTokens: Int = 4096 + ) { + self.transport = transport + self.keyProvider = keyProvider + self.baseURL = baseURL + self.defaultMaxTokens = defaultMaxTokens + } + + public func validate() async throws { + guard await keyProvider.apiKey(for: .openai) != nil else { + throw InferenceError.generationFailed(Self.missingKeyMessage) + } + } + + public func stream(model: String, request: GenerationRequest) -> AsyncThrowingStream { + AsyncThrowingStream { continuation in + let task = Task { + do { + guard let key = await keyProvider.apiKey(for: .openai) else { + throw InferenceError.generationFailed(Self.missingKeyMessage) + } + let body = try Self.makeRequestBody(model: model, request: request, defaultMaxTokens: defaultMaxTokens) + let spec = HTTPRequestSpec( + url: baseURL.appendingPathComponent("v1/chat/completions"), + method: "POST", + headers: [ + "authorization": "Bearer \(key)", + "content-type": "application/json", + "accept": "text/event-stream", + ], + body: body) + let (head, lines) = try await transport.stream(spec) + guard head.statusCode == 200 else { + var raw = "" + for try await line in lines { raw += line } + throw InferenceError.generationFailed(Self.errorMessage(status: head.statusCode, body: raw)) + } + + var index = 0 + var promptTokens = 0 + var completionTokens = 0 + var stopReason = "" + // Tool calls stream incrementally, keyed by their position. + var toolNames: [Int: String] = [:] + var toolArguments: [Int: String] = [:] + + for try await line in lines { + try Task.checkCancellation() + guard line.hasPrefix("data:") else { continue } + let payload = line.dropFirst("data:".count).trimmingCharacters(in: .whitespaces) + if payload.isEmpty { continue } + if payload == "[DONE]" { break } + guard let data = payload.data(using: .utf8), + let chunk = try? JSONDecoder().decode(OpenAIStreamChunk.self, from: data) else { + continue + } + if let usage = chunk.usage { + promptTokens = usage.prompt_tokens ?? promptTokens + completionTokens = usage.completion_tokens ?? completionTokens + } + guard let choice = chunk.choices?.first else { continue } + if let reason = choice.finish_reason { stopReason = reason } + if let content = choice.delta?.content, !content.isEmpty { + continuation.yield(TokenChunk(text: content, index: index, isFinal: false)) + index += 1 + } + for call in choice.delta?.tool_calls ?? [] { + let slot = call.index ?? 0 + if let name = call.function?.name, !name.isEmpty { + toolNames[slot] = name + } + if let args = call.function?.arguments { + toolArguments[slot, default: ""] += args + } + } + } + + // Flush accumulated tool calls in slot order before completing. + for slot in toolNames.keys.sorted() { + guard let name = toolNames[slot] else { continue } + continuation.yield(TokenChunk( + text: "", + index: index, + isFinal: false, + toolCall: ModelToolCall(name: name, arguments: Self.parseToolArguments(toolArguments[slot] ?? "")))) + index += 1 + } + + continuation.yield(TokenChunk( + text: "", + index: index, + isFinal: true, + info: TokenChunk.CompletionInfo( + promptTokenCount: promptTokens, + generationTokenCount: completionTokens, + stopReason: stopReason.isEmpty ? "stop" : stopReason))) + continuation.finish() + } catch is CancellationError { + continuation.finish(throwing: InferenceError.cancelled) + } catch { + continuation.finish(throwing: error) + } + } + continuation.onTermination = { _ in task.cancel() } + } + } + + // MARK: - Request mapping + + static let missingKeyMessage = + "OpenAI API key not set. Add it in Settings or set OPENAI_API_KEY." + + static func makeRequestBody(model: String, request: GenerationRequest, defaultMaxTokens: Int) throws -> Data { + var object: [String: JSONValue] = [ + "model": .string(model), + "max_tokens": .int(max(1, request.maxTokens ?? defaultMaxTokens)), + "temperature": .double(Double(request.temperature)), + "stream": .bool(true), + "stream_options": .object(["include_usage": .bool(true)]), + "messages": .array(mapMessages(request)), + ] + if !request.tools.isEmpty { + // ToolDefinition.schema is already OpenAI's {type:"function", function:{…}}. + object["tools"] = .array(request.tools.map(\.schema)) + } + return try JSONEncoder().encode(JSONValue.object(object)) + } + + /// Chat messages with `system` kept inline (OpenAI takes it as a message). + /// `tool` turns become user text and consecutive same-role turns are merged. + static func mapMessages(_ request: GenerationRequest) -> [JSONValue] { + let chat: [GenerationRequest.ChatMessage] + switch request.input { + case let .prompt(text): + chat = [.init(role: .user, content: text)] + case let .messages(messages): + chat = messages + } + + var turns: [(role: String, content: String)] = [] + for message in chat { + switch message.role { + case .system: turns.append((role: "system", content: message.content)) + case .user: turns.append((role: "user", content: message.content)) + case .assistant: turns.append((role: "assistant", content: message.content)) + case .tool: turns.append((role: "user", content: "[tool result]\n" + message.content)) + } + } + + var merged: [(role: String, content: String)] = [] + for turn in turns { + if var last = merged.last, last.role == turn.role { + last.content += "\n\n" + turn.content + merged[merged.count - 1] = last + } else { + merged.append(turn) + } + } + + return merged.map { JSONValue.object(["role": .string($0.role), "content": .string($0.content)]) } + } + + static func parseToolArguments(_ json: String) -> [String: JSONValue] { + let trimmed = json.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty, + let data = trimmed.data(using: .utf8), + let value = try? JSONDecoder().decode(JSONValue.self, from: data), + case let .object(object) = value else { + return [:] + } + return object + } + + static func errorMessage(status: Int, body: String) -> String { + if let data = body.data(using: .utf8), + let parsed = try? JSONDecoder().decode(OpenAIErrorEnvelope.self, from: data), + let message = parsed.error?.message { + return "OpenAI request failed (\(status)): \(message)" + } + let trimmed = body.trimmingCharacters(in: .whitespacesAndNewlines) + return trimmed.isEmpty + ? "OpenAI request failed (\(status))." + : "OpenAI request failed (\(status)): \(trimmed.prefix(500))" + } +} + +// MARK: - Wire decoding + +struct OpenAIStreamChunk: Decodable { + let choices: [Choice]? + let usage: Usage? + + struct Choice: Decodable { + let delta: Delta? + let finish_reason: String? + } + struct Delta: Decodable { + let content: String? + let tool_calls: [ToolCallDelta]? + } + struct ToolCallDelta: Decodable { + let index: Int? + let function: FunctionDelta? + } + struct FunctionDelta: Decodable { + let name: String? + let arguments: String? + } + struct Usage: Decodable { + let prompt_tokens: Int? + let completion_tokens: Int? + } +} + +struct OpenAIErrorEnvelope: Decodable { + let error: ErrorBody? + struct ErrorBody: Decodable { + let message: String? + } +} diff --git a/MLXEngine/EngineBootstrap.swift b/MLXEngine/EngineBootstrap.swift index d78d2de..8adb39d 100644 --- a/MLXEngine/EngineBootstrap.swift +++ b/MLXEngine/EngineBootstrap.swift @@ -28,7 +28,10 @@ public enum EngineBootstrap { // consent + key checks in the app layer). let backend = RoutingInferenceBackend( local: MLXBackend(engineTuning: engineTuning), - remote: RemoteInferenceBackend(client: AnthropicModelClient())) + remote: RemoteInferenceBackend(clients: [ + .anthropic: AnthropicModelClient(), + .openai: OpenAIModelClient(), + ])) let controller = InferenceController( backend: backend, memoryMonitor: MemoryPressureMonitor(thresholds: thresholds), diff --git a/MLXEngine/RemoteInferenceBackend.swift b/MLXEngine/RemoteInferenceBackend.swift index 352af74..364eab4 100644 --- a/MLXEngine/RemoteInferenceBackend.swift +++ b/MLXEngine/RemoteInferenceBackend.swift @@ -2,16 +2,25 @@ import Foundation import Shared import CloudInference -/// `InferenceBackend` over a hosted `CloudModelClient` (e.g. Anthropic). The -/// lifecycle is intentionally hollow: there are no local weights, KV cache, or -/// GPU/RAM to manage, so `load`/`unload`/`clearKVCache`/draft ops are no-ops and -/// memory readings are zero. `generate` delegates to the client using the model -/// id carried on the handle (provider prefix stripped). Embeddings stay local. +/// `InferenceBackend` over one or more hosted `CloudModelClient`s (Anthropic, +/// OpenAI, …), selected per call by the model id's provider prefix. The lifecycle +/// is intentionally hollow: there are no local weights, KV cache, or GPU/RAM to +/// manage, so `load`/`unload`/`clearKVCache`/draft ops are no-ops and memory +/// readings are zero. `generate` delegates to the resolved provider's client +/// using the bare model name. Embeddings stay local (throws here). public struct RemoteInferenceBackend: InferenceBackend { - private let client: any CloudModelClient + private let clients: [CloudProvider: any CloudModelClient] - public init(client: any CloudModelClient) { - self.client = client + public init(clients: [CloudProvider: any CloudModelClient]) { + self.clients = clients + } + + private func resolve(_ id: String) -> (model: String, client: any CloudModelClient)? { + guard let resolved = CloudModelResolver.resolve(id), + let client = clients[resolved.provider] else { + return nil + } + return (resolved.model, client) } public func load( @@ -21,6 +30,9 @@ public struct RemoteInferenceBackend: InferenceBackend { toolCallFormat: ModelToolCallFormat?, progressHandler: (@Sendable (Double) -> Void)? ) async throws -> LoadedModelHandle { + guard let (_, client) = resolve(id) else { + throw InferenceError.modelLoadFailed(role: role, underlying: "no configured cloud provider for model id \"\(id)\"") + } // Nothing to download; surface a missing-key error here so it appears at // "load" time rather than mid-generation. try await client.validate() @@ -29,7 +41,13 @@ public struct RemoteInferenceBackend: InferenceBackend { } public func generate(request: GenerationRequest, handle: LoadedModelHandle) -> AsyncThrowingStream { - client.stream(model: Self.modelName(from: handle.id), request: request) + guard let (model, client) = resolve(handle.id) else { + return AsyncThrowingStream { continuation in + continuation.finish(throwing: InferenceError.generationFailed( + "no configured cloud provider for model id \"\(handle.id)\"")) + } + } + return client.stream(model: model, request: request) } public func embed(texts: [String], handle: LoadedModelHandle) async throws -> [EmbeddingVector] { @@ -44,9 +62,4 @@ public struct RemoteInferenceBackend: InferenceBackend { public func footprint() async -> MemoryFootprint { MemoryFootprint(processFootprintBytes: 0, totalUnifiedBytes: 0) } - - /// `anthropic/claude-opus-4-8` → `claude-opus-4-8`; leaves bare ids unchanged. - static func modelName(from id: String) -> String { - CloudModelResolver.resolve(id)?.model ?? id - } } diff --git a/Security/KeychainSecretStore.swift b/Security/KeychainSecretStore.swift index fec417a..db7321a 100644 --- a/Security/KeychainSecretStore.swift +++ b/Security/KeychainSecretStore.swift @@ -85,4 +85,5 @@ public enum InterlessSecrets { public static let service = "dev.interless.secrets" public static let huggingFaceTokenAccount = "huggingface.token" public static let anthropicAPIKeyAccount = "anthropic.apiKey" + public static let openAIAPIKeyAccount = "openai.apiKey" } diff --git a/Tests/CloudInferenceTests/OpenAIModelClientTests.swift b/Tests/CloudInferenceTests/OpenAIModelClientTests.swift new file mode 100644 index 0000000..d0db37d --- /dev/null +++ b/Tests/CloudInferenceTests/OpenAIModelClientTests.swift @@ -0,0 +1,138 @@ +import Foundation +import Testing +import Shared +@testable import CloudInference + +struct OpenAIModelClientTests { + + @Test func resolverHandlesOpenAIIds() { + #expect(CloudModelResolver.resolve("openai/gpt-4o") + == CloudModelID(provider: .openai, model: "gpt-4o")) + #expect(CloudModelResolver.isCloud("openai/gpt-4.1")) + } + + @Test func streamsContentThenToolCallThenFinal() async throws { + let transport = OpenAIFakeTransport(status: 200, lines: [ + #"data: {"choices":[{"delta":{"role":"assistant","content":""},"finish_reason":null}]}"#, + #"data: {"choices":[{"delta":{"content":"Hello"},"finish_reason":null}]}"#, + #"data: {"choices":[{"delta":{"tool_calls":[{"index":0,"id":"call_1","type":"function","function":{"name":"read_file","arguments":""}}]},"finish_reason":null}]}"#, + #"data: {"choices":[{"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"path\":\"a.txt\"}"}}]},"finish_reason":null}]}"#, + #"data: {"choices":[{"delta":{},"finish_reason":"tool_calls"}]}"#, + #"data: {"choices":[],"usage":{"prompt_tokens":10,"completion_tokens":5}}"#, + "data: [DONE]", + ]) + let client = OpenAIModelClient(transport: transport, keyProvider: OpenAIStubKey(key: "k")) + let chunks = try await collectOpenAI(client.stream(model: "gpt-4o", request: .prompt("hi"))) + + #expect(chunks.map(\.index) == Array(0..) async throws -> [TokenChunk] { + var result: [TokenChunk] = [] + for try await chunk in stream { result.append(chunk) } + return result +} + +private struct OpenAIStubKey: CloudKeyProvider { + let key: String? + func apiKey(for provider: CloudProvider) async -> String? { key } +} + +private actor OpenAIFakeTransport: HTTPTransport { + let status: Int + let scriptedLines: [String] + private var captured: HTTPRequestSpec? + + init(status: Int, lines: [String]) { + self.status = status + self.scriptedLines = lines + } + + func stream(_ request: HTTPRequestSpec) async throws -> (head: HTTPResponseHead, lines: AsyncThrowingStream) { + captured = request + let scripted = scriptedLines + let lines = AsyncThrowingStream { continuation in + for line in scripted { continuation.yield(line) } + continuation.finish() + } + return (HTTPResponseHead(statusCode: status), lines) + } + + func capturedRequest() -> HTTPRequestSpec? { captured } +} diff --git a/Tests/MLXEngineTests/RemoteBackendTests.swift b/Tests/MLXEngineTests/RemoteBackendTests.swift index 3a83861..5cd8d77 100644 --- a/Tests/MLXEngineTests/RemoteBackendTests.swift +++ b/Tests/MLXEngineTests/RemoteBackendTests.swift @@ -8,7 +8,7 @@ struct RemoteBackendTests { @Test func remoteBackendDelegatesAndStripsProviderPrefix() async throws { let client = FakeCloudModelClient(texts: ["A", "B"]) - let backend = RemoteInferenceBackend(client: client) + let backend = RemoteInferenceBackend(clients: [.anthropic: client]) let handle = try await backend.load( id: "anthropic/claude-opus-4-8", role: .orchestrator, @@ -23,7 +23,7 @@ struct RemoteBackendTests { } @Test func remoteBackendEmbedThrows() async throws { - let backend = RemoteInferenceBackend(client: FakeCloudModelClient(texts: [])) + let backend = RemoteInferenceBackend(clients: [.anthropic: FakeCloudModelClient(texts: [])]) let handle = LoadedModelHandle(role: .embeddings, id: "anthropic/x", quantization: .q8) await #expect(throws: InferenceError.self) { _ = try await backend.embed(texts: ["x"], handle: handle) @@ -33,7 +33,7 @@ struct RemoteBackendTests { @Test func routingDispatchesByModelId() async throws { let local = FakeBackend() await local.setScriptedTokens(["LOCAL"]) - let remote = RemoteInferenceBackend(client: FakeCloudModelClient(texts: ["REMOTE"])) + let remote = RemoteInferenceBackend(clients: [.anthropic: FakeCloudModelClient(texts: ["REMOTE"])]) let routing = RoutingInferenceBackend(local: local, remote: remote) let localHandle = try await routing.load( @@ -50,6 +50,24 @@ struct RemoteBackendTests { request: .prompt("y", role: .orchestrator), handle: cloudHandle)) #expect(cloudChunks.filter { !$0.text.isEmpty }.map(\.text) == ["REMOTE"]) } + + @Test func remoteBackendRoutesByProviderPrefix() async throws { + let anthropic = FakeCloudModelClient(texts: ["A"]) + let openai = FakeCloudModelClient(texts: ["O"]) + let backend = RemoteInferenceBackend(clients: [.anthropic: anthropic, .openai: openai]) + + let aHandle = try await backend.load( + id: "anthropic/claude-opus-4-8", role: .orchestrator, + quantization: .q8, toolCallFormat: nil, progressHandler: nil) + _ = try await collect(backend.generate(request: .prompt("x", role: .orchestrator), handle: aHandle)) + #expect(anthropic.capturedModel == "claude-opus-4-8") + + let oHandle = try await backend.load( + id: "openai/gpt-4o", role: .utility, + quantization: .q8, toolCallFormat: nil, progressHandler: nil) + _ = try await collect(backend.generate(request: .prompt("y", role: .utility), handle: oHandle)) + #expect(openai.capturedModel == "gpt-4o") + } } private final class FakeCloudModelClient: CloudModelClient, @unchecked Sendable { diff --git a/UI/ModelSettingsView.swift b/UI/ModelSettingsView.swift index 241afc5..60beb71 100644 --- a/UI/ModelSettingsView.swift +++ b/UI/ModelSettingsView.swift @@ -17,6 +17,8 @@ public struct ModelSettingsView: View { public var onDeleteHuggingFaceToken: @MainActor () -> Void public var onSaveAnthropicAPIKey: @MainActor (String) -> Void public var onDeleteAnthropicAPIKey: @MainActor () -> Void + public var onSaveOpenAIAPIKey: @MainActor (String) -> Void + public var onDeleteOpenAIAPIKey: @MainActor () -> Void private let headerTitle: String? private let headerSubtitle: String private let embedsInParentScroll: Bool @@ -25,6 +27,7 @@ public struct ModelSettingsView: View { private let showsDangerZone: Bool @State private var huggingFaceToken = "" @State private var anthropicAPIKey = "" + @State private var openAIAPIKey = "" @State private var showAdvanced = false public init( @@ -43,6 +46,8 @@ public struct ModelSettingsView: View { onDeleteHuggingFaceToken: @escaping @MainActor () -> Void = {}, onSaveAnthropicAPIKey: @escaping @MainActor (String) -> Void = { _ in }, onDeleteAnthropicAPIKey: @escaping @MainActor () -> Void = {}, + onSaveOpenAIAPIKey: @escaping @MainActor (String) -> Void = { _ in }, + onDeleteOpenAIAPIKey: @escaping @MainActor () -> Void = {}, headerTitle: String? = "Providers", headerSubtitle: String = "Local MLX model setup and native tool-call compatibility.", embedsInParentScroll: Bool = false, @@ -65,6 +70,8 @@ public struct ModelSettingsView: View { self.onDeleteHuggingFaceToken = onDeleteHuggingFaceToken self.onSaveAnthropicAPIKey = onSaveAnthropicAPIKey self.onDeleteAnthropicAPIKey = onDeleteAnthropicAPIKey + self.onSaveOpenAIAPIKey = onSaveOpenAIAPIKey + self.onDeleteOpenAIAPIKey = onDeleteOpenAIAPIKey self.headerTitle = headerTitle self.headerSubtitle = headerSubtitle self.embedsInParentScroll = embedsInParentScroll @@ -294,6 +301,25 @@ public struct ModelSettingsView: View { Text("Stored only in Keychain. Used when a role's model id is a cloud id (e.g. anthropic/claude-opus-4-8) and cloud models are allowed below.") .font(.metaMono) .foregroundStyle(Theme.C.textSecondary) + settingsDivider() + controlRow("OpenAI API key") { + SecureField("sk-…", text: $openAIAPIKey) + .textContentType(.password) + .textFieldStyle(.roundedBorder) + .frame(maxWidth: 360) + } + HStack(spacing: .space2) { + Button("Save Key") { + onSaveOpenAIAPIKey(openAIAPIKey) + openAIAPIKey = "" + } + .disabled(openAIAPIKey.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty) + Button("Delete Key", action: onDeleteOpenAIAPIKey) + } + .font(.bodyS.weight(.semibold)) + Text("Stored only in Keychain. Used for cloud ids like openai/gpt-4o when cloud models are allowed below.") + .font(.metaMono) + .foregroundStyle(Theme.C.textSecondary) if showsResourceProfileControl { settingsDivider() controlRow("Resource profile") { diff --git a/UI/SettingsHubView.swift b/UI/SettingsHubView.swift index 3084b21..10c6cdc 100644 --- a/UI/SettingsHubView.swift +++ b/UI/SettingsHubView.swift @@ -29,6 +29,8 @@ public struct SettingsHubView: View { public var onDeleteHuggingFaceToken: @MainActor () -> Void public var onSaveAnthropicAPIKey: @MainActor (String) -> Void public var onDeleteAnthropicAPIKey: @MainActor () -> Void + public var onSaveOpenAIAPIKey: @MainActor (String) -> Void + public var onDeleteOpenAIAPIKey: @MainActor () -> Void public var onOpenHealth: @MainActor () -> Void public var onExportDiagnostics: @MainActor () -> Void public var onUpdateModelContextSettings: @MainActor (ModelContextSettingsViewState) -> Void @@ -68,6 +70,8 @@ public struct SettingsHubView: View { onDeleteHuggingFaceToken: @escaping @MainActor () -> Void, onSaveAnthropicAPIKey: @escaping @MainActor (String) -> Void = { _ in }, onDeleteAnthropicAPIKey: @escaping @MainActor () -> Void = {}, + onSaveOpenAIAPIKey: @escaping @MainActor (String) -> Void = { _ in }, + onDeleteOpenAIAPIKey: @escaping @MainActor () -> Void = {}, onOpenHealth: @escaping @MainActor () -> Void, onExportDiagnostics: @escaping @MainActor () -> Void, onUpdateModelContextSettings: @escaping @MainActor (ModelContextSettingsViewState) -> Void @@ -83,6 +87,8 @@ public struct SettingsHubView: View { self.onDeleteHuggingFaceToken = onDeleteHuggingFaceToken self.onSaveAnthropicAPIKey = onSaveAnthropicAPIKey self.onDeleteAnthropicAPIKey = onDeleteAnthropicAPIKey + self.onSaveOpenAIAPIKey = onSaveOpenAIAPIKey + self.onDeleteOpenAIAPIKey = onDeleteOpenAIAPIKey self.onOpenHealth = onOpenHealth self.onExportDiagnostics = onExportDiagnostics self.onUpdateModelContextSettings = onUpdateModelContextSettings @@ -217,7 +223,9 @@ public struct SettingsHubView: View { onSaveHuggingFaceToken: onSaveHuggingFaceToken, onDeleteHuggingFaceToken: onDeleteHuggingFaceToken, onSaveAnthropicAPIKey: onSaveAnthropicAPIKey, - onDeleteAnthropicAPIKey: onDeleteAnthropicAPIKey) + onDeleteAnthropicAPIKey: onDeleteAnthropicAPIKey, + onSaveOpenAIAPIKey: onSaveOpenAIAPIKey, + onDeleteOpenAIAPIKey: onDeleteOpenAIAPIKey) case .usage: settingsScroll { usageSection } case .skills: @@ -383,6 +391,8 @@ public struct SettingsHubView: View { onDeleteHuggingFaceToken: onDeleteHuggingFaceToken, onSaveAnthropicAPIKey: onSaveAnthropicAPIKey, onDeleteAnthropicAPIKey: onDeleteAnthropicAPIKey, + onSaveOpenAIAPIKey: onSaveOpenAIAPIKey, + onDeleteOpenAIAPIKey: onDeleteOpenAIAPIKey, headerTitle: nil, embedsInParentScroll: true, showsRuntimeControls: false, diff --git a/UI/WorkspaceView.swift b/UI/WorkspaceView.swift index 8bf78c6..c9cb44a 100644 --- a/UI/WorkspaceView.swift +++ b/UI/WorkspaceView.swift @@ -40,6 +40,8 @@ public struct WorkspaceViewActions { public var deleteHuggingFaceToken: @MainActor () -> Void public var saveAnthropicAPIKey: @MainActor (String) -> Void public var deleteAnthropicAPIKey: @MainActor () -> Void + public var saveOpenAIAPIKey: @MainActor (String) -> Void + public var deleteOpenAIAPIKey: @MainActor () -> Void public var retryRecoveryAction: @MainActor (UUID) -> Void public var dismissRecoveryItem: @MainActor (UUID) -> Void public var clearRecoveryJournal: @MainActor () -> Void @@ -86,6 +88,8 @@ public struct WorkspaceViewActions { deleteHuggingFaceToken: @escaping @MainActor () -> Void = {}, saveAnthropicAPIKey: @escaping @MainActor (String) -> Void = { _ in }, deleteAnthropicAPIKey: @escaping @MainActor () -> Void = {}, + saveOpenAIAPIKey: @escaping @MainActor (String) -> Void = { _ in }, + deleteOpenAIAPIKey: @escaping @MainActor () -> Void = {}, retryRecoveryAction: @escaping @MainActor (UUID) -> Void, dismissRecoveryItem: @escaping @MainActor (UUID) -> Void, clearRecoveryJournal: @escaping @MainActor () -> Void, @@ -131,6 +135,8 @@ public struct WorkspaceViewActions { self.deleteHuggingFaceToken = deleteHuggingFaceToken self.saveAnthropicAPIKey = saveAnthropicAPIKey self.deleteAnthropicAPIKey = deleteAnthropicAPIKey + self.saveOpenAIAPIKey = saveOpenAIAPIKey + self.deleteOpenAIAPIKey = deleteOpenAIAPIKey self.retryRecoveryAction = retryRecoveryAction self.dismissRecoveryItem = dismissRecoveryItem self.clearRecoveryJournal = clearRecoveryJournal @@ -853,6 +859,8 @@ public struct WorkspaceView: View { onDeleteHuggingFaceToken: actions.deleteHuggingFaceToken, onSaveAnthropicAPIKey: actions.saveAnthropicAPIKey, onDeleteAnthropicAPIKey: actions.deleteAnthropicAPIKey, + onSaveOpenAIAPIKey: actions.saveOpenAIAPIKey, + onDeleteOpenAIAPIKey: actions.deleteOpenAIAPIKey, onOpenHealth: actions.openHealth, onExportDiagnostics: actions.exportDiagnostics, onUpdateModelContextSettings: actions.setModelContextSettings) From 1b1bd48f5eca2341cdc0b49a945df3e735bb74ee Mon Sep 17 00:00:00 2001 From: dijix009 Date: Thu, 25 Jun 2026 11:43:25 +0700 Subject: [PATCH 6/6] feat(cloud): cloud-aware role mixing (per-role local/cloud) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two role-management decisions, written for the all-local world, blocked mixing models per role (e.g. openai/gpt-5.5 orchestrator + anthropic/claude-haiku-4-5 sub-agent, or a cloud orchestrator + local sub-agent). Fix both: - effectiveSingleAgentMode(settings:orchestratorID:utilityID:): single-agent collapse now only applies when small-RAM AND both roles are local. A cloud role costs zero local RAM, so mixing is allowed even on an 8 GB Mac. Used in loadModels (which model(s) to load) and makeAgent (route/prompt/agent selection) so both agree. - advertisesNativeTools(modelID:toolCallFormat:): cloud models do native tool-calling and no longer require a local toolCallFormat. The main path gates on the orchestrator id, the sub-agent path on the utility id (where read-only sub-agents run); local roles still need a configured format. The sub-agent runs on the Utility role's model; embeddings stay local. Tests: single-agent matrix (small/large RAM × local/cloud) and tool-advertise matrix (cloud/local × format). Full suite green (385 tests). --- AppCore/AppDependencyFactory.swift | 36 +++++++++++++++++++++--- Tests/AppCoreTests/CloudUsageTests.swift | 27 ++++++++++++++++++ 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/AppCore/AppDependencyFactory.swift b/AppCore/AppDependencyFactory.swift index 8cc24b5..5260047 100644 --- a/AppCore/AppDependencyFactory.swift +++ b/AppCore/AppDependencyFactory.swift @@ -215,7 +215,9 @@ public struct LiveAppDependencyFactory: AppDependencyFactory { config: config?.effective, settings: currentSettings, resourceBudget: ResourceBudget.resolved(for: currentSettings.resourceProfile)) - let canAdvertiseNativeTools = runtime.settings.toolCallFormat != nil + let orchestratorModelID = Self.agentModelID(agentCatalog: agentCatalog, agentIDs: ["build", "plan"], fallback: runtime.settings.orchestratorModelID) + let canAdvertiseNativeTools = Self.advertisesNativeTools( + modelID: orchestratorModelID, toolCallFormat: runtime.settings.toolCallFormat) return await Self.makeAgent( root: root, store: store, @@ -257,6 +259,10 @@ public struct LiveAppDependencyFactory: AppDependencyFactory { // Read-only sub-agent in its own context: same workspace tools minus // writes/network and the task tool (no recursion). Synchronous, so it // reuses the orchestrator gate and stays serial / 8GB-safe. + // The sub-agent runs on the utility role's model, so its tool + // advertisement keys off that id (cloud → native tools regardless of + // toolCallFormat). + let subagentModelID = Self.agentModelID(agentCatalog: agentCatalog, agentIDs: ["general"], fallback: runtime.settings.utilityModelID) let subagent = await Self.makeAgent( root: root, store: store, @@ -264,7 +270,7 @@ public struct LiveAppDependencyFactory: AppDependencyFactory { settings: runtime.settings, metricsRecorder: metricsRecorder, includesWorkspaceContext: true, - advertisesTools: runtime.settings.toolCallFormat != nil, + advertisesTools: Self.advertisesNativeTools(modelID: subagentModelID, toolCallFormat: runtime.settings.toolCallFormat), explorationOnly: true, readOnly: true, snapshotStore: snapshotStore, @@ -290,10 +296,11 @@ public struct LiveAppDependencyFactory: AppDependencyFactory { let runtimeSettings = runtime.settings let errors = runtimeSettings.validationErrors() guard errors.isEmpty else { throw AppRuntimeError.invalidModelSettings(errors) } - let singleAgentMode = Self.usesSingleAgentMode(runtimeSettings) let orchestratorModelID = Self.agentModelID(agentCatalog: agentCatalog, agentIDs: ["build", "plan"], fallback: runtimeSettings.orchestratorModelID) let utilityModelID = Self.agentModelID(agentCatalog: agentCatalog, agentIDs: ["general"], fallback: runtimeSettings.utilityModelID) let singleModelID = Self.agentModelID(agentCatalog: agentCatalog, agentIDs: ["general", "build"], fallback: runtimeSettings.orchestratorModelID) + let singleAgentMode = Self.effectiveSingleAgentMode( + settings: runtimeSettings, orchestratorID: orchestratorModelID, utilityID: utilityModelID) try Self.validateCloudUsage( orchestrator: singleAgentMode ? singleModelID : orchestratorModelID, utility: singleAgentMode ? "" : utilityModelID, @@ -402,7 +409,10 @@ public struct LiveAppDependencyFactory: AppDependencyFactory { settings: settings, resourceBudget: budget) let runtimeSettings = runtime.settings - let singleAgentMode = usesSingleAgentMode(runtimeSettings) + let orchestratorModelID = agentModelID(agentCatalog: agentCatalog, agentIDs: ["build", "plan"], fallback: runtimeSettings.orchestratorModelID) + let utilityModelID = agentModelID(agentCatalog: agentCatalog, agentIDs: ["general"], fallback: runtimeSettings.utilityModelID) + let singleAgentMode = effectiveSingleAgentMode( + settings: runtimeSettings, orchestratorID: orchestratorModelID, utilityID: utilityModelID) var policy = runtime.toolPolicy if readOnly { // Sub-agents are read-only regardless of workspace config: deny writes, @@ -507,6 +517,24 @@ public struct LiveAppDependencyFactory: AppDependencyFactory { settings.usesSingleAgentMode() } + /// Single-agent collapse is about LOCAL RAM, not cloud. Cloud roles cost zero + /// local memory, so only collapse to one agent when small-RAM AND both roles + /// are local (two local models won't fit). Otherwise allow per-role mixing + /// (e.g. cloud orchestrator + local/cloud sub-agent) even on an 8 GB Mac. + static func effectiveSingleAgentMode( + settings: ModelSettingsViewState, orchestratorID: String, utilityID: String + ) -> Bool { + guard usesSingleAgentMode(settings) else { return false } + return !CloudModelResolver.isCloud(orchestratorID) && !CloudModelResolver.isCloud(utilityID) + } + + /// Native tool-calling is available for any cloud model (provider-native) and + /// for local models only when a tool-call format is configured. `toolCallFormat` + /// is a local text-grammar concept and must not gate cloud roles. + static func advertisesNativeTools(modelID: String, toolCallFormat: ModelToolCallFormat?) -> Bool { + CloudModelResolver.isCloud(modelID) || toolCallFormat != nil + } + private static func mergeSearchHits( lexical: [SearchHit], semantic: [SearchHit], diff --git a/Tests/AppCoreTests/CloudUsageTests.swift b/Tests/AppCoreTests/CloudUsageTests.swift index 65481c9..fb38576 100644 --- a/Tests/AppCoreTests/CloudUsageTests.swift +++ b/Tests/AppCoreTests/CloudUsageTests.swift @@ -1,4 +1,6 @@ import Testing +import Shared +import UI @testable import AppCore struct CloudUsageTests { @@ -40,4 +42,29 @@ struct CloudUsageTests { allowCloudModels: true) } } + + @Test func singleAgentModeIsCloudAware() { + let small = ModelSettingsViewState(resourceProfile: .smallRAM) + let large = ModelSettingsViewState(resourceProfile: .largeRAM) + // Small RAM + both local → collapse to a single agent (two local models won't fit). + #expect(LiveAppDependencyFactory.effectiveSingleAgentMode( + settings: small, orchestratorID: "mlx-community/a", utilityID: "mlx-community/b")) + // Small RAM but a cloud role costs no local RAM → allow mixing (multi-agent). + #expect(!LiveAppDependencyFactory.effectiveSingleAgentMode( + settings: small, orchestratorID: "openai/gpt-5.5", utilityID: "mlx-community/b")) + #expect(!LiveAppDependencyFactory.effectiveSingleAgentMode( + settings: small, orchestratorID: "mlx-community/a", utilityID: "anthropic/claude-haiku-4-5")) + // Large RAM → always multi-agent. + #expect(!LiveAppDependencyFactory.effectiveSingleAgentMode( + settings: large, orchestratorID: "mlx-community/a", utilityID: "mlx-community/b")) + } + + @Test func nativeToolAdvertisementIsCloudAware() { + // Cloud roles get native tools regardless of toolCallFormat. + #expect(LiveAppDependencyFactory.advertisesNativeTools(modelID: "anthropic/claude-opus-4-8", toolCallFormat: nil)) + #expect(LiveAppDependencyFactory.advertisesNativeTools(modelID: "openai/gpt-4o", toolCallFormat: nil)) + // Local roles still need a configured tool-call format. + #expect(!LiveAppDependencyFactory.advertisesNativeTools(modelID: "mlx-community/x", toolCallFormat: nil)) + #expect(LiveAppDependencyFactory.advertisesNativeTools(modelID: "mlx-community/x", toolCallFormat: .json)) + } }