diff --git a/Sources/TurboFieldfare/Infrastructure/Metal/MetalContext.swift b/Sources/TurboFieldfare/Infrastructure/Metal/MetalContext.swift index 1eea8155..db862311 100644 --- a/Sources/TurboFieldfare/Infrastructure/Metal/MetalContext.swift +++ b/Sources/TurboFieldfare/Infrastructure/Metal/MetalContext.swift @@ -7,6 +7,7 @@ enum MetalError: Error, CustomStringConvertible { case missingShaderResource(String) case missingFunction(String) case libraryCompileFailed(String) + case commandBufferFailed(String) public var description: String { switch self { @@ -15,14 +16,54 @@ enum MetalError: Error, CustomStringConvertible { case .missingShaderResource(let n): return "Shader resource missing: \(n)" case .missingFunction(let n): return "Metal function missing in library: \(n)" case .libraryCompileFailed(let s):return "Metal library compile failed: \(s)" + case .commandBufferFailed(let s): return "Metal command buffer failed: \(s)" } } } -func checkCommandBufferError(_ error: (any Error)?) throws { +func metalCommandBufferStatusName(_ status: MTLCommandBufferStatus) -> String { + switch status { + case .notEnqueued: return "notEnqueued" + case .enqueued: return "enqueued" + case .committed: return "committed" + case .scheduled: return "scheduled" + case .completed: return "completed" + case .error: return "error" + @unknown default: return "unknown(\(status.rawValue))" + } +} + +/// Builds the diagnostic for a command buffer that did not complete, or nil +/// when it did. Status is checked separately because a failed buffer is not +/// guaranteed to carry an error object. +func metalCommandBufferFailureDetail(label: String?, + status: MTLCommandBufferStatus, + error: (any Error)?) -> String? { + if status == .completed && error == nil { return nil } + + var parts = ["label=\(label.map { $0.isEmpty ? "" : $0 } ?? "")"] + parts.append("status=\(metalCommandBufferStatusName(status))") if let error { - throw error + let nsError = error as NSError + parts.append("domain=\(nsError.domain)") + parts.append("code=\(nsError.code)") + parts.append("description=\(nsError.localizedDescription)") + if !nsError.userInfo.isEmpty { + parts.append("userInfoKeys=\(nsError.userInfo.keys.sorted().joined(separator: ","))") + } + } else { + parts.append("error=") + } + return parts.joined(separator: " ") +} + +func checkCommandBufferError(_ commandBuffer: MTLCommandBuffer) throws { + guard let detail = metalCommandBufferFailureDetail(label: commandBuffer.label, + status: commandBuffer.status, + error: commandBuffer.error) else { + return } + throw MetalError.commandBufferFailed(detail) } public struct MetalFunctionConstant: Hashable, Sendable { @@ -62,8 +103,25 @@ public final class MetalContext: @unchecked Sendable { private var pipelineCache: [PipelineCacheKey: MTLComputePipelineState] = [:] private let pipelineCacheLock = NSLock() + private static func relaxInteractivityWatchdog() { + #if os(macOS) + // The AGX driver reads this once at first device creation. Long prefill + // dispatches can otherwise be killed as compositor-impacting on macOS + // 26. Overwrite 0 preserves an operator's explicit stock-behaviour + // override. This relaxes the deadline; it does not guarantee survival. + setenv("AGX_RELAX_CDM_CTXSTORE_TIMEOUT", "1", 0) + #endif + } + + /// Routes every production device creation through the watchdog mitigation + /// before the AGX driver's process-wide one-time environment read. + public static func makeSystemDefaultDevice() -> MTLDevice? { + relaxInteractivityWatchdog() + return MTLCreateSystemDefaultDevice() + } + public init() throws { - guard let dev = MTLCreateSystemDefaultDevice() else { throw MetalError.noDevice } + guard let dev = Self.makeSystemDefaultDevice() else { throw MetalError.noDevice } guard let q = dev.makeCommandQueue() else { throw MetalError.noQueue } self.device = dev self.queue = q diff --git a/Sources/TurboFieldfare/Kernels/Attention/PrefillAttention.swift b/Sources/TurboFieldfare/Kernels/Attention/PrefillAttention.swift index 0f0f4d7d..7b7df5e1 100644 --- a/Sources/TurboFieldfare/Kernels/Attention/PrefillAttention.swift +++ b/Sources/TurboFieldfare/Kernels/Attention/PrefillAttention.swift @@ -57,13 +57,33 @@ final class PrefillAttention { private let psoParamsSmoke: MTLComputePipelineState private let psoFullTensorOps2DValidityV2: MTLComputePipelineState? - init(context: MetalContext) throws { + var tensorOps2DValidityV2Available: Bool { + psoFullTensorOps2DValidityV2 != nil + } + + convenience init(context: MetalContext) throws { + try self.init(context: context, simulatingMissingTensorOps: false) + } + + /// Tests can force the production fallback even on a host where the MSL 4 + /// TensorOps pipeline builds. + init(context: MetalContext, simulatingMissingTensorOps: Bool) throws { self.context = context self.psoCausalTiled = try context.pipeline("attention_prefill_causal_tiled") self.psoParamsSmoke = try context.pipeline("prefill_attention_params_smoke") - self.psoFullTensorOps2DValidityV2 = context.device.supportsFamily(.apple10) - ? try? context.pipeline("attention_prefill_full_tensorops_2d_validity_v2") - : nil + if simulatingMissingTensorOps { + self.psoFullTensorOps2DValidityV2 = nil + } else { + do { + self.psoFullTensorOps2DValidityV2 = try context.pipeline( + "attention_prefill_full_tensorops_2d_validity_v2") + } catch { + self.psoFullTensorOps2DValidityV2 = nil + FileHandle.standardError.write(Data( + ("PrefillAttention: TensorOps 2D pipeline unavailable; " + + "using causal-tiled fallback: \(error)\n").utf8)) + } + } } func encodeCausal(commandBuffer: MTLCommandBuffer, @@ -87,11 +107,14 @@ final class PrefillAttention { let requestsTensorOps = path == .fullTensorOps2DPreferred || path == .fullTensorOps2DValidityV2 - // The pinned model uses 512/16/2 only for full attention; its - // sliding-window layers use 256/16/8. A future model that reuses this - // shape for sliding attention must add a full-visibility check here. + // TensorOps starts its key loop at zero and ignores slidingWindow, so + // these are visibility guards rather than shape optimizations. + let windowNeverClips = effectiveParams.slidingWindow == 0 + || effectiveParams.slidingWindow >= effectiveParams.kvValidCount let tensorOpsShape = requestsTensorOps + && layerKind == .full && kvRingCapacity == 0 + && windowNeverClips && effectiveParams.headDim == 512 && effectiveParams.numQHeads == 16 && effectiveParams.numKVHeads == 2 @@ -103,7 +126,7 @@ final class PrefillAttention { pipeline = tensorOpsPipeline } else if tensorOpsShape && path == .fullTensorOps2DValidityV2 { preconditionFailure( - "TensorOps 2D prefill attention requires Apple10 MPP tensor support") + "TensorOps 2D prefill attention pipeline is unavailable on this Metal stack") } else { // Explicit mode also falls back for incompatible shapes. Benchmark // fixtures must use 512/16/2 to prove that TensorOps ran. diff --git a/Sources/TurboFieldfare/Kernels/Vision/VisionResize.swift b/Sources/TurboFieldfare/Kernels/Vision/VisionResize.swift index 8601a390..0f4c2c61 100644 --- a/Sources/TurboFieldfare/Kernels/Vision/VisionResize.swift +++ b/Sources/TurboFieldfare/Kernels/Vision/VisionResize.swift @@ -171,7 +171,7 @@ public final class VisionResize { commandBuffer.commit() commandBuffer.waitUntilCompleted() - try checkCommandBufferError(commandBuffer.error) + try checkCommandBufferError(commandBuffer) let finished = sourceHeight == destinationHeight ? intermediate : output let rowBytes = sourceHeight == destinationHeight diff --git a/Sources/TurboFieldfare/Runtime/Generation/RawCompletion.swift b/Sources/TurboFieldfare/Runtime/Generation/RawCompletion.swift index af4923f1..11428c44 100644 --- a/Sources/TurboFieldfare/Runtime/Generation/RawCompletion.swift +++ b/Sources/TurboFieldfare/Runtime/Generation/RawCompletion.swift @@ -307,6 +307,6 @@ private func sampleOnce(scratch: RawCompletionScratch, context: MetalContext, history: history, config: config, position: position, outToken: scratch.outToken) cb.commit(); cb.waitUntilCompleted() - try checkCommandBufferError(cb.error) + try checkCommandBufferError(cb) return Int32(bitPattern: scratch.outToken.contents().load(as: UInt32.self)) } diff --git a/Sources/TurboFieldfare/Runtime/Inference/RealForwardRunner.swift b/Sources/TurboFieldfare/Runtime/Inference/RealForwardRunner.swift index d0b12160..a5b13201 100644 --- a/Sources/TurboFieldfare/Runtime/Inference/RealForwardRunner.swift +++ b/Sources/TurboFieldfare/Runtime/Inference/RealForwardRunner.swift @@ -861,6 +861,7 @@ public final class RealForwardRunner: ChunkedPrefillRunner, MultimodalPrefillRun guard var cb = ctx.queue.makeCommandBuffer() else { throw ModelError.residentBufferWrapFailed } + cb.label = "prefill start=\(startPosition) count=\(t) phase=embed" if let embeddingOverride { // The override spans the whole chunk — the planner emits image spans // as standalone chunks — so the INT4 gather it would cover is @@ -1101,6 +1102,8 @@ public final class RealForwardRunner: ChunkedPrefillRunner, MultimodalPrefillRun guard let sharedCB = ctx.queue.makeCommandBuffer() else { throw ModelError.residentBufferWrapFailed } + sharedCB.label = + "prefill start=\(startPosition) count=\(t) layer=\(L) phase=shared_expert" let sharedProj = sharedExpertProjections[L] try prefillSharedExpert.encodeBlock(commandBuffer: sharedCB, x: scratch.denseX, @@ -1241,6 +1244,8 @@ public final class RealForwardRunner: ChunkedPrefillRunner, MultimodalPrefillRun guard let tileCB = ctx.queue.makeCommandBuffer() else { throw ModelError.residentBufferWrapFailed } + tileCB.label = + "prefill start=\(startPosition) count=\(t) layer=\(L) phase=routed_tile" _ = prefillGroupedMoE.encodeStreamedBatched( commandBuffer: tileCB, hidden: scratch.routedX, @@ -1300,6 +1305,8 @@ public final class RealForwardRunner: ChunkedPrefillRunner, MultimodalPrefillRun guard let nextCB = ctx.queue.makeCommandBuffer() else { throw ModelError.residentBufferWrapFailed } + nextCB.label = + "prefill start=\(startPosition) count=\(t) layer=\(L + 1) phase=qkv_attention" cb = nextCB } continue @@ -1311,6 +1318,7 @@ public final class RealForwardRunner: ChunkedPrefillRunner, MultimodalPrefillRun guard let finalCB = ctx.queue.makeCommandBuffer() else { throw ModelError.residentBufferWrapFailed } + finalCB.label = "prefill start=\(startPosition) count=\(t) phase=final_head" if outputMode == .greedyIfAvailable, useFusedGreedyHead { fusionHead.encodeGreedyDecode( commandBuffer: finalCB, @@ -1395,12 +1403,12 @@ public final class RealForwardRunner: ChunkedPrefillRunner, MultimodalPrefillRun waitUntilCompleted(pending.cb) } if let sharedCB = pending.sharedCB { - try checkCommandBufferError(sharedCB.error) + try checkCommandBufferError(sharedCB) } if let phase1HitCB = pending.phase1HitCB { - try checkCommandBufferError(phase1HitCB.error) + try checkCommandBufferError(phase1HitCB) } - try checkCommandBufferError(pending.cb.error) + try checkCommandBufferError(pending.cb) totalCb2Nanos &+= pending.encodeAndCommitNanos } @@ -1595,7 +1603,7 @@ public final class RealForwardRunner: ChunkedPrefillRunner, MultimodalPrefillRun try finishPendingRoutedCommand(pending, waitIfNeeded: false) pendingRoutedCommand = nil } - try checkCommandBufferError(cb.error) + try checkCommandBufferError(cb) totalCb1Nanos &+= clock_gettime_nsec_np(CLOCK_UPTIME_RAW) - tCb1Start - waitNanos // CPU readback to fetch routed-expert blobs from disk. @@ -1873,7 +1881,7 @@ public final class RealForwardRunner: ChunkedPrefillRunner, MultimodalPrefillRun private nonisolated func waitForCompletion(_ cb: MTLCommandBuffer) throws { waitUntilCompleted(cb) - try checkCommandBufferError(cb.error) + try checkCommandBufferError(cb) } private nonisolated func waitUntilCompleted(_ cb: MTLCommandBuffer) { diff --git a/Sources/TurboFieldfare/Runtime/Vision/VisionRuntime.swift b/Sources/TurboFieldfare/Runtime/Vision/VisionRuntime.swift index f286fc4c..a8082e67 100644 --- a/Sources/TurboFieldfare/Runtime/Vision/VisionRuntime.swift +++ b/Sources/TurboFieldfare/Runtime/Vision/VisionRuntime.swift @@ -244,7 +244,7 @@ public final class VisionRuntime { } public static var isSupportedOnDefaultDevice: Bool { - guard let device = MTLCreateSystemDefaultDevice() else { return false } + guard let device = MetalContext.makeSystemDefaultDevice() else { return false } return isSupported(on: device) } @@ -519,8 +519,10 @@ public final class VisionRuntime { let started = clock_gettime_nsec_np(CLOCK_UPTIME_RAW) entry.buffer.waitUntilCompleted() gpuWaitNanoseconds += clock_gettime_nsec_np(CLOCK_UPTIME_RAW) - started - if let error = entry.buffer.error { - throw VisionRuntimeError.commandFailed(String(describing: error)) + if let detail = metalCommandBufferFailureDetail(label: entry.buffer.label, + status: entry.buffer.status, + error: entry.buffer.error) { + throw VisionRuntimeError.commandFailed(detail) } gpuNanoseconds += UInt64( max(0, entry.buffer.gpuEndTime - entry.buffer.gpuStartTime) @@ -844,8 +846,10 @@ public final class VisionRuntime { let commandBuffer = try makeCommandBuffer() commandBuffer.commit() commandBuffer.waitUntilCompleted() - if let error = commandBuffer.error { - throw VisionRuntimeError.commandFailed(String(describing: error)) + if let detail = metalCommandBufferFailureDetail(label: commandBuffer.label, + status: commandBuffer.status, + error: commandBuffer.error) { + throw VisionRuntimeError.commandFailed(detail) } return clock_gettime_nsec_np(CLOCK_UPTIME_RAW) - started } @@ -853,8 +857,10 @@ public final class VisionRuntime { private func commitAndMeasure(_ commandBuffer: MTLCommandBuffer) throws -> UInt64 { commandBuffer.commit() commandBuffer.waitUntilCompleted() - if let error = commandBuffer.error { - throw VisionRuntimeError.commandFailed(String(describing: error)) + if let detail = metalCommandBufferFailureDetail(label: commandBuffer.label, + status: commandBuffer.status, + error: commandBuffer.error) { + throw VisionRuntimeError.commandFailed(detail) } return UInt64(max(0, commandBuffer.gpuEndTime - commandBuffer.gpuStartTime) * 1_000_000_000) diff --git a/Sources/TurboFieldfareCLI/Run.swift b/Sources/TurboFieldfareCLI/Run.swift index 7b30406a..4995d257 100644 --- a/Sources/TurboFieldfareCLI/Run.swift +++ b/Sources/TurboFieldfareCLI/Run.swift @@ -55,7 +55,7 @@ public func run(args: Args, let input = try parseInput(args: args) var selectedDevice: MTLDevice? if input.hasImages { - guard let device = MTLCreateSystemDefaultDevice() else { + guard let device = MetalContext.makeSystemDefaultDevice() else { return errored(stderr, "no Metal device", 1) } try VisionRuntime.requireSupportedDevice(device) @@ -103,7 +103,7 @@ public func run(args: Args, // Hoisted above the `auto` estimate, which needs a device to read image // geometry. Planning never touches the GPU, but building the plan does // need the device the run will use. - guard let device = selectedDevice ?? MTLCreateSystemDefaultDevice() else { + guard let device = selectedDevice ?? MetalContext.makeSystemDefaultDevice() else { return errored(stderr, "no Metal device", 1) } diff --git a/Tests/TurboFieldfare/Core/Infrastructure/InteractivityWatchdogEnvironmentTests.swift b/Tests/TurboFieldfare/Core/Infrastructure/InteractivityWatchdogEnvironmentTests.swift new file mode 100644 index 00000000..e094a420 --- /dev/null +++ b/Tests/TurboFieldfare/Core/Infrastructure/InteractivityWatchdogEnvironmentTests.swift @@ -0,0 +1,52 @@ +import Darwin +import Testing + +@testable import TurboFieldfare + +@Suite(.serialized) struct InteractivityWatchdogEnvironmentTests { + private static let key = "AGX_RELAX_CDM_CTXSTORE_TIMEOUT" + + private static func read() -> String? { + guard let raw = getenv(key) else { return nil } + return String(cString: raw) + } + + private static func withCleanEnvironment(_ body: () throws -> Void) rethrows { + let saved = read() + unsetenv(key) + defer { + if let saved { + setenv(key, saved, 1) + } else { + unsetenv(key) + } + } + try body() + } + + #if os(macOS) + @Test func constructingAMetalContextSetsTheWatchdogRelaxationOnMacOS() throws { + try Self.withCleanEnvironment { + #expect(Self.read() == nil) + _ = try MetalContext() + #expect(Self.read() == "1") + } + } + + @Test func anExplicitOperatorOverrideIsNotOverwritten() throws { + try Self.withCleanEnvironment { + setenv(Self.key, "0", 1) + _ = try MetalContext() + #expect(Self.read() == "0") + } + } + #else + @Test func constructingAMetalContextDoesNotSetTheMacOSWatchdogRelaxation() throws { + try Self.withCleanEnvironment { + #expect(Self.read() == nil) + _ = try MetalContext() + #expect(Self.read() == nil) + } + } + #endif +} diff --git a/Tests/TurboFieldfare/Core/Infrastructure/MetalContextDeviceCreationTests.swift b/Tests/TurboFieldfare/Core/Infrastructure/MetalContextDeviceCreationTests.swift new file mode 100644 index 00000000..6478aace --- /dev/null +++ b/Tests/TurboFieldfare/Core/Infrastructure/MetalContextDeviceCreationTests.swift @@ -0,0 +1,66 @@ +import Foundation +import Testing + +@Suite struct MetalContextDeviceCreationTests { + private static var sourcesDirectory: URL { + URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + .appendingPathComponent("Sources") + } + + private static let sanctionedRelativePath = + "TurboFieldfare/Infrastructure/Metal/MetalContext.swift" + + @Test func onlyMetalContextCreatesTheSystemDevice() throws { + let sources = Self.sourcesDirectory + try #require(FileManager.default.fileExists(atPath: sources.path), + "cannot locate Sources/ from \(#filePath)") + + var enumerationFailures: [String] = [] + var readFailures: [String] = [] + var offenders: [String] = [] + let walker = try #require(FileManager.default.enumerator( + at: sources, + includingPropertiesForKeys: nil, + errorHandler: { url, error in + enumerationFailures.append("\(url.path): \(error)") + return false + })) + let sourcePrefix = sources.standardizedFileURL.path + "/" + + while let url = walker.nextObject() as? URL { + guard url.pathExtension == "swift" else { continue } + let path = url.standardizedFileURL.path + guard path.hasPrefix(sourcePrefix) else { + readFailures.append("cannot make \(path) relative to \(sources.path)") + continue + } + let relativePath = String(path.dropFirst(sourcePrefix.count)) + guard relativePath != Self.sanctionedRelativePath else { continue } + let text: String + do { + text = try String(contentsOf: url, encoding: .utf8) + } catch { + readFailures.append("\(relativePath): \(error)") + continue + } + guard text.contains("MTLCreateSystemDefaultDevice(") else { continue } + + for (index, line) in text.split(separator: "\n", omittingEmptySubsequences: false) + .enumerated() where line.contains("MTLCreateSystemDefaultDevice(") { + offenders.append("\(relativePath):\(index + 1)") + } + } + + #expect(enumerationFailures.isEmpty, + "source enumeration failed: \(enumerationFailures.joined(separator: "; "))") + #expect(readFailures.isEmpty, + "source reads failed: \(readFailures.joined(separator: "; "))") + #expect(offenders.isEmpty, + "Production device creation bypasses MetalContext: \(offenders.joined(separator: ", "))") + } +} diff --git a/Tests/TurboFieldfare/Core/Kernels/Attention/PrefillAttentionTests.swift b/Tests/TurboFieldfare/Core/Kernels/Attention/PrefillAttentionTests.swift index 6e4ed304..c4fb6f39 100644 --- a/Tests/TurboFieldfare/Core/Kernels/Attention/PrefillAttentionTests.swift +++ b/Tests/TurboFieldfare/Core/Kernels/Attention/PrefillAttentionTests.swift @@ -140,9 +140,10 @@ import TurboFieldfareValidationSupport ]) func tensorOps2DFullAttentionMatchesReferenceAtTileBoundaries(_ visibleKeys: Int) throws { let context = try MetalContext() - // Hosted CI has no Apple10 GPU, so it returns without dispatching this - // kernel. Run this suite on Apple10 before changing the TensorOps path. - guard context.device.supportsFamily(.apple10) else { return } + let attention = try PrefillAttention(context: context) + // GPU family is not the capability test: this pipeline compiles and + // dispatches on the project's Apple8 M2 minimum. + guard attention.tensorOps2DValidityV2Available else { return } let fixture = Self.makeFixture(start: visibleKeys - 1, chunk: 1, window: 0, @@ -186,12 +187,61 @@ import TurboFieldfareValidationSupport "preferred TensorOps maxAbs=\(maxAbs) rel=\(rel)") #expect(rel <= 2e-2, "preferred TensorOps rel=\(rel) maxAbs=\(maxAbs)") - if !context.device.supportsFamily(.apple10) { + let attention = try PrefillAttention(context: context) + if attention.tensorOps2DValidityV2Available { + let explicitTensorOps = try Self.runKernel( + fixture, + path: .fullTensorOps2DValidityV2) + #expect(preferred == explicitTensorOps) + } else { let baseline = try Self.runKernel(fixture, path: .causalTiled) #expect(preferred == baseline) } } + @Test func preferredPathUsesTiledWhenTensorOpsIsUnavailable() throws { + let fixture = Self.makeFixture(start: 128, + chunk: 8, + window: 0, + seed: 0xA873, + headDim: 512, + qHeads: 16, + kvHeads: 2) + let preferred = try Self.runKernel( + fixture, + path: .fullTensorOps2DPreferred, + simulatingMissingTensorOps: true) + let tiled = try Self.runKernel(fixture, path: .causalTiled) + + #expect(preferred == tiled) + } + + /// TensorOps starts its key loop at zero, so a production-shaped request + /// with a clipping window must use the tiled path even when the pipeline is + /// available. + @Test func preferredTensorOpsPathRejectsAWindowThatActuallyClips() throws { + let context = try MetalContext() + let attention = try PrefillAttention(context: context) + guard attention.tensorOps2DValidityV2Available else { return } + + let fixture = Self.makeFixture(start: 40, + chunk: 8, + window: 16, + seed: 0xA877, + headDim: 512, + qHeads: 16, + kvHeads: 2) + let preferred = try Self.runKernel( + fixture, + path: .fullTensorOps2DPreferred, + layerKindOverride: .full) + let tiled = try Self.runKernel(fixture, + path: .causalTiled, + layerKindOverride: .full) + + #expect(RelError.maxAbsDiff(preferred, tiled) == 0) + } + private static func makeFixture(start: Int, chunk: Int, window: Int, @@ -315,10 +365,13 @@ import TurboFieldfareValidationSupport private static func runKernel( _ fixture: Fixture, kvRingCapacity: UInt32 = 0, - path: RuntimePrefillAttentionPath = .causalTiled + path: RuntimePrefillAttentionPath = .causalTiled, + simulatingMissingTensorOps: Bool = false, + layerKindOverride: PrefillAttentionLayerKind? = nil ) throws -> [Float] { let ctx = try MetalContext() - let prefill = try PrefillAttention(context: ctx) + let prefill = try PrefillAttention( + context: ctx, simulatingMissingTensorOps: simulatingMissingTensorOps) let qPrefix = 17 let kPrefix = 19 let vPrefix = 23 @@ -363,10 +416,12 @@ import TurboFieldfareValidationSupport outOffset: oPrefix * MemoryLayout.size, params: params, kvRingCapacity: kvRingCapacity, - layerKind: fixture.window == 0 ? .full : .slidingWindow, + layerKind: layerKindOverride + ?? (fixture.window == 0 ? .full : .slidingWindow), path: path) cb.commit() cb.waitUntilCompleted() + try checkCommandBufferError(cb) let out = Fp16Buffer.read(outBuf, count: outCount) var compact = [Float](repeating: 0, count: fixture.chunk * fixture.qHeads * fixture.headDim) diff --git a/Tests/TurboFieldfare/Core/Runtime/Generation/CommandBufferCompletionTests.swift b/Tests/TurboFieldfare/Core/Runtime/Generation/CommandBufferCompletionTests.swift index d010592c..f08a947f 100644 --- a/Tests/TurboFieldfare/Core/Runtime/Generation/CommandBufferCompletionTests.swift +++ b/Tests/TurboFieldfare/Core/Runtime/Generation/CommandBufferCompletionTests.swift @@ -1,22 +1,154 @@ +import Foundation +import Metal import Testing @testable import TurboFieldfare @Suite struct CommandBufferCompletionTests { - private enum SyntheticCommandBufferError: Error, Equatable { - case failed + private static var visionRuntimeSource: URL { + URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + .appendingPathComponent("Sources/TurboFieldfare/Runtime/Vision/VisionRuntime.swift") } - @Test func commandBufferErrorsArePropagated() throws { - try checkCommandBufferError(nil) + private static func nsError(domain: String = "MTLCommandBufferErrorDomain", + code: Int = 1, + description: String, + userInfo extra: [String: Any] = [:]) -> NSError { + var info: [String: Any] = [NSLocalizedDescriptionKey: description] + info.merge(extra) { current, _ in current } + return NSError(domain: domain, code: code, userInfo: info) + } + + @Test func completedBufferWithNoErrorIsNotAFailure() { + #expect(metalCommandBufferFailureDetail(label: "prefill layer=3", + status: .completed, + error: nil) == nil) + } + + @Test func errorStatusWithNoErrorObjectIsStillAFailure() throws { + let detail = try #require( + metalCommandBufferFailureDetail(label: "prefill layer=3", + status: .error, + error: nil)) + #expect(detail.contains("status=error")) + #expect(detail.contains("label=prefill layer=3")) + #expect(detail.contains("error=")) + } + + @Test(arguments: [ + MTLCommandBufferStatus.notEnqueued, + .enqueued, + .committed, + .scheduled, + ]) + func nonCompletedStatusIsAFailure(_ status: MTLCommandBufferStatus) throws { + let detail = try #require( + metalCommandBufferFailureDetail(label: nil, status: status, error: nil)) + #expect(detail.contains("status=\(metalCommandBufferStatusName(status))")) + } + + @Test func interactivityKillDetailPreservesTheIOGPUToken() throws { + let description = + "Impacting Interactivity " + + "(0000000e:kIOGPUCommandBufferCallbackErrorImpactingInteractivity)" + let detail = try #require( + metalCommandBufferFailureDetail( + label: "prefill start=8064 count=128 layer=12 phase=qkv_attention", + status: .error, + error: Self.nsError(description: description))) + + #expect(detail.contains("kIOGPUCommandBufferCallbackErrorImpactingInteractivity")) + #expect(detail.contains("0000000e")) + #expect(detail.contains("domain=MTLCommandBufferErrorDomain")) + #expect(detail.contains("code=1")) + #expect(detail.contains("layer=12")) + } + + @Test func missingLabelIsReportedRatherThanOmitted() throws { + let detail = try #require( + metalCommandBufferFailureDetail(label: nil, + status: .error, + error: Self.nsError(description: "boom"))) + #expect(detail.contains("label=")) + } + + @Test func emptyLabelIsDistinguishedFromAbsentLabel() throws { + let detail = try #require( + metalCommandBufferFailureDetail(label: "", + status: .error, + error: Self.nsError(description: "boom"))) + #expect(detail.contains("label=")) + } + + @Test func userInfoKeysAreNamedButValuesAreNotLeaked() throws { + let detail = try #require( + metalCommandBufferFailureDetail( + label: "x", + status: .error, + error: Self.nsError(description: "boom", + userInfo: ["TFSecretPayload": "do-not-leak-me"]))) + + #expect(detail.contains("userInfoKeys=")) + #expect(detail.contains("TFSecretPayload")) + #expect(!detail.contains("do-not-leak-me")) + } + + @Test func completedStatusCarryingAnErrorIsAFailure() throws { + let detail = try #require( + metalCommandBufferFailureDetail(label: "x", + status: .completed, + error: Self.nsError(description: "boom"))) + #expect(detail.contains("status=completed")) + #expect(detail.contains("description=boom")) + } + + @Test func statusNamesCoverEveryCase() { + #expect(metalCommandBufferStatusName(.notEnqueued) == "notEnqueued") + #expect(metalCommandBufferStatusName(.enqueued) == "enqueued") + #expect(metalCommandBufferStatusName(.committed) == "committed") + #expect(metalCommandBufferStatusName(.scheduled) == "scheduled") + #expect(metalCommandBufferStatusName(.completed) == "completed") + #expect(metalCommandBufferStatusName(.error) == "error") + } + + @Test func realCompletedCommandBufferDoesNotThrow() throws { + let context = try MetalContext() + let commandBuffer = try #require(context.queue.makeCommandBuffer()) + commandBuffer.label = "test empty" + commandBuffer.commit() + commandBuffer.waitUntilCompleted() + try checkCommandBufferError(commandBuffer) + } + + @Test func realNonCompletedCommandBufferThrows() throws { + let context = try MetalContext() + let commandBuffer = try #require(context.queue.makeCommandBuffer()) + commandBuffer.label = "test uncommitted" do { - try checkCommandBufferError(SyntheticCommandBufferError.failed) - Issue.record("expected command-buffer error") - } catch let error as SyntheticCommandBufferError { - #expect(error == .failed) - } catch { - Issue.record("unexpected error: \(error)") + try checkCommandBufferError(commandBuffer) + Issue.record("expected a failure for a buffer that never completed") + } catch let error as MetalError { + #expect("\(error)".contains("test uncommitted")) } } + + /// The shared formatter covers status-only failures only if every vision + /// wait routes through it. A direct `buffer.error` check accepts `.error` + /// with no error object and can consume incomplete GPU output. + @Test func visionRuntimeUsesStatusAwareCommandBufferChecks() throws { + let source = try String(contentsOf: Self.visionRuntimeSource, encoding: .utf8) + #expect(!source.contains("if let error = entry.buffer.error")) + #expect(!source.contains("if let error = commandBuffer.error")) + let checkedWaits = source.components( + separatedBy: "metalCommandBufferFailureDetail(").count - 1 + #expect(checkedWaits == 3, + "expected all three VisionRuntime waits to preserve status diagnostics") + } } diff --git a/docs/OPTIMIZATION_JOURNEY.md b/docs/OPTIMIZATION_JOURNEY.md index 29626a4c..21d95793 100644 --- a/docs/OPTIMIZATION_JOURNEY.md +++ b/docs/OPTIMIZATION_JOURNEY.md @@ -201,16 +201,22 @@ K/V head across eight query heads, but the tiled kernel handled each query head separately. It launched eight threadgroups and read the same K/V data eight times. -TensorOps processes all eight heads at once, making attention 11x faster at -64K. +On Apple10, TensorOps processes all eight heads at once, making attention 11x +faster at 64K. -The full runtime kept much of that gain. On the same 32K input, prefill fell -from 491.09 to 204.29 seconds, a 2.404x speedup without increasing memory use. +The full runtime kept much of that gain. On the same Apple10 32K input, prefill +fell from 491.09 to 204.29 seconds, a 2.404x speedup without increasing memory +use. We nearly threw this result away because the new reduction order changed the final logits slightly. Better checks showed the differences were harmless. -TensorOps is now the Apple10 path. Earlier GPUs keep tiled attention. +Production now selects TensorOps by pipeline capability rather than GPU family. +The pipeline also compiles and dispatches on Apple8 M2, where it measured +9.027-9.294x faster than tiled attention in isolation and reduced a matched +6,784-token prefill from 419.469 to 243.100 seconds without increasing memory +use. Incompatible shapes and stacks that cannot build the pipeline use tiled +attention. ## Sampling removed repeated vocabulary scans diff --git a/docs/RUNTIME_CONTROLS.md b/docs/RUNTIME_CONTROLS.md index 3e45441f..525402ea 100644 --- a/docs/RUNTIME_CONTROLS.md +++ b/docs/RUNTIME_CONTROLS.md @@ -45,6 +45,21 @@ each request and do not require a reload. Each CLI invocation loads a new model process, so its selected runtime settings apply immediately. The server fixes its runtime settings at startup, so changing one means restarting the process. +### macOS interactivity mitigation + +Before the first Metal device is created, TurboFieldfare defaults +`AGX_RELAX_CDM_CTXSTORE_TIMEOUT` to `1`. This relaxes an AGX context-store +deadline that can terminate a long prefill dispatch as +`kIOGPUCommandBufferCallbackErrorImpactingInteractivity` on macOS 26. It is a +mitigation, not a guarantee: failures have also been reported with the setting +enabled. + +Export `AGX_RELAX_CDM_CTXSTORE_TIMEOUT=0` before launching TurboFieldfare to +restore stock driver behavior. An explicit environment value is never +overwritten. If a command buffer still fails, the error includes its prefill +phase label, Metal status, domain, code, and IOGPU diagnostic token without +including prompt or generated content. + ## Image controls Image input needs the companion pack installed beside the text model. Without diff --git a/docs/experiments/EXPERIMENT_INVENTORY.md b/docs/experiments/EXPERIMENT_INVENTORY.md index 634baaf5..de8d741e 100644 --- a/docs/experiments/EXPERIMENT_INVENTORY.md +++ b/docs/experiments/EXPERIMENT_INVENTORY.md @@ -162,7 +162,7 @@ resident set size; and **NLL** is negative log-likelihood. See | [PF-14](summaries/06-prefill.md#pf-14) — QMM TG reuse | Families +3.2-9.7%; current opportunity about 0.41%. | Rejected. | | [PF-15](summaries/06-prefill.md#pf-15) — Batched routed MoE | Isolated +30.91%; balanced end to end about +2%. | Reversed rejection; production. | | [PF-16](summaries/06-prefill.md#pf-16) — Long endpoint gate | Delta-NLL +0.002588; top-1 16/16; RSS 888.3 MiB. | Production; validation result. | -| [PF-17](summaries/06-prefill.md#pf-17) — Apple10 TensorOps full attention | Isolated 11.24x at 16K and 11.63x at 64K; 32K end to end 2.404x. | Production on Apple10; tiled fallback elsewhere. | +| [PF-17](summaries/06-prefill.md#pf-17) — TensorOps full attention | Apple10: isolated 11.24x at 16K and 11.63x at 64K; 32K end to end 2.404x. Apple8 M2: isolated 9.027-9.294x; 6,784-token prefill 1.726x. | Production when the pipeline builds; Apple8 M2 verified; tiled fallback when unavailable or incompatible. | ### Fusions, head, and orchestration diff --git a/docs/experiments/summaries/06-prefill.md b/docs/experiments/summaries/06-prefill.md index 18521883..8f2ecc3c 100644 --- a/docs/experiments/summaries/06-prefill.md +++ b/docs/experiments/summaries/06-prefill.md @@ -13,7 +13,7 @@ failed the M2 long-row gate. | Current result | Disposition | | --- | --- | | Chunk 128, staged affine MPP, and batched routed MoE | Production | -| Apple10 TensorOps full attention | Production on Apple10; tiled fallback elsewhere | +| TensorOps full attention | Production when the pipeline builds; Apple8 M2 verified; tiled fallback when unavailable or incompatible | | Shared/fetch overlap v3 | Rejected and removed | | Shared INT8 QMM, deeper lookahead, and argument-buffer rings | Rejected on M2 | @@ -268,7 +268,7 @@ failed the M2 long-row gate. separate a local tie from a systematic quality change. -### PF-17: Apple10 TensorOps full-prefill attention +### PF-17: TensorOps full-prefill attention - **Hypothesis:** One threadgroup could process all eight Q heads in a full attention GQA group, reuse K/V reads, and map QK/PV onto cooperative matrix @@ -279,13 +279,18 @@ failed the M2 long-row gate. Same-input 32K prefill fell from 491.09 to 204.29 seconds, a 2.404x end-to-end speedup, with identical post-prefill RSS. Direct attention reference checks passed through 64K, and frozen MLX-relative endpoints - matched top-1 at 8K/16K/32K/64K. + matched top-1 at 8K/16K/32K/64K. On Apple8 M2, the same pipeline measured + 9.027-9.294x faster than tiled attention in isolation and reduced a matched + 6,784-token prefill from 419.469 to 243.100 seconds without increasing the + measured footprint. The frozen 8K MLX-relative endpoint selected the + reference top-1. - **What changed the conclusion:** Exact production-logit identity falsely rejected a valid floating-point reduction order. Direct attention error and independent MLX quality gates isolated top-k routing amplification instead of a shader semantic defect. -- **Final disposition:** Production on Apple10; automatic causal-tiled fallback - on earlier GPU families and a named rollback remain. +- **Final disposition:** Production when the MSL 4 TensorOps pipeline builds; + Apple8 M2 is verified. Incompatible shapes and pipeline-build failures use + the automatic causal-tiled fallback. M3 and M4 remain unmeasured. - **Lesson:** Reordered floating-point kernels need a direct numerical oracle plus model-quality gates, not identity with one reduction order.