From 04b9e5308ca2a4ce46c948cd70377917429b3920 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 18 Jul 2026 17:03:34 +0000 Subject: [PATCH] Jetsam-proof stem separation: headroom gates and playback-start hold MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RCA identified the ORT session load (~600 MB transient) plus the first HT-Demucs window's attention arena (up to ~1.7 GB) as the app's largest allocation, landing exactly when playback, UI, and (first run) the model download are also ramping — a per-process-limit kill on devices with a tight cap. Three structural guards: - Hard start gate: a separation may not begin unless os_proc_available_memory headroom exceeds a required budget (1400 MB to start; tune from mem[] breadcrumbs). Deferred work retries on later ensureStems passes. - Per-window floor: abort before the next window if headroom falls under 600 MB, via the existing abort/cleanup path. - Playback-start hold: the session's first stem demand arms a 60 s hold so the peak never stacks on engine/UI ramp-up; one scheduled retry re-drives ensureStems after the hold. --- .../Sources/Ingest/MemoryFootprint.swift | 7 ++-- .../Ingest/PreparationQueue+Stems.swift | 36 +++++++++++++++++++ .../Sources/Ingest/PreparationQueue.swift | 7 ++++ .../Sources/Ingest/StemSeparator.swift | 18 ++++++++++ 4 files changed, 66 insertions(+), 2 deletions(-) diff --git a/Packages/ContinuityKit/Sources/Ingest/MemoryFootprint.swift b/Packages/ContinuityKit/Sources/Ingest/MemoryFootprint.swift index c8c405c..77808a3 100644 --- a/Packages/ContinuityKit/Sources/Ingest/MemoryFootprint.swift +++ b/Packages/ContinuityKit/Sources/Ingest/MemoryFootprint.swift @@ -7,8 +7,11 @@ import os enum MemoryFootprint { private static let logger = Logger(subsystem: "com.continuity.app", category: "mem") + /// Remaining allowance in MB. Used both for breadcrumbs and to GATE heavy work — heavy + /// stages must check this and defer instead of trusting that the budget will stretch. + static var headroomMB: Int { Int(os_proc_available_memory() / 1_048_576) } + static func breadcrumb(_ label: String) { - let remainingMB = Int(os_proc_available_memory() / 1_048_576) - logger.info("mem[\(label, privacy: .public)] \(remainingMB, privacy: .public) MB headroom") + logger.info("mem[\(label, privacy: .public)] \(headroomMB, privacy: .public) MB headroom") } } diff --git a/Packages/ContinuityKit/Sources/Ingest/PreparationQueue+Stems.swift b/Packages/ContinuityKit/Sources/Ingest/PreparationQueue+Stems.swift index 35c1cd8..d174a9a 100644 --- a/Packages/ContinuityKit/Sources/Ingest/PreparationQueue+Stems.swift +++ b/Packages/ContinuityKit/Sources/Ingest/PreparationQueue+Stems.swift @@ -12,12 +12,18 @@ extension PreparationQueue { /// Also refreshes the LRU clock and heals links whose files were evicted. public func ensureStems(for tracks: [Track], in context: ModelContext) { protectedStemKeys = Set(tracks.compactMap(\.youtubeVideoID)) + // First stem demand of the session = playback just started. Hold separation for a + // minute so its peak never stacks on engine/UI/download ramp-up. + if separationAllowedAt == nil { separationAllowedAt = Date().addingTimeInterval(60) } + let holdActive = Date() < (separationAllowedAt ?? .distantPast) for track in tracks { guard !track.isDemo, track.prepState == .ready, let key = track.youtubeVideoID, track.localRelativePath != nil else { continue } reconcileStemLinks(track, in: context) if track.hasStems { StemCache.markUsed(key: key) // actively played material stays cache-resident + } else if holdActive { + scheduleDeferredStemsRetry(for: tracks, in: context) } else { separateStems(track, in: context) } @@ -30,6 +36,20 @@ extension PreparationQueue { } } + /// Re-drives `ensureStems` once after the playback-start hold passes, so stems still arrive + /// mid-song without waiting for the next track change. At most one retry is pending. + private func scheduleDeferredStemsRetry(for tracks: [Track], in context: ModelContext) { + guard !stemsRetryScheduled, let allowedAt = separationAllowedAt else { return } + stemsRetryScheduled = true + Task { @MainActor [weak self] in + try? await Task.sleep(for: .seconds(max(1, allowedAt.timeIntervalSinceNow + 1))) + guard let self else { return } + self.stemsRetryScheduled = false + // Tracks can be deleted during the hold (sync); a dead @Model crashes on access. + self.ensureStems(for: tracks.filter { $0.modelContext != nil }, in: context) + } + } + /// System memory warning: iOS warns before it jetsams. Abort any in-flight stem separation /// (it deletes its partial output; the track plays without stems and retries on a later /// `ensureStems` pass) and drop the cached ORT session — staying alive beats finishing stems. @@ -89,6 +109,22 @@ extension PreparationQueue { if queueDrained { OnnxStemSeparator.releaseSession() } return } + // HARD headroom gate: session load + first window is the app's biggest peak. If + // the process doesn't have the budget, don't start — stems arrive on a later + // ensureStems pass (every track change) once memory frees up. Jetsam-proof by + // construction: no separation may begin that could not complete its peak. + let headroom = MemoryFootprint.headroomMB + guard headroom > StemSeparationBudget.requiredStartMB else { + Logger.stems.notice("deferring separation for \(key, privacy: .public): \(headroom) MB headroom < \(StemSeparationBudget.requiredStartMB) MB required") + await limiter.release() + let queueDrained = await MainActor.run { () -> Bool in + guard let self else { return true } + self.stemsInFlight.remove(key) + return self.stemsInFlight.isEmpty + } + if queueDrained { OnnxStemSeparator.releaseSession() } + return + } do { MemoryFootprint.breadcrumb("separation task begin") let modelURL = try await StemModelStore.ensureModel() diff --git a/Packages/ContinuityKit/Sources/Ingest/PreparationQueue.swift b/Packages/ContinuityKit/Sources/Ingest/PreparationQueue.swift index 662c745..04d7fc0 100644 --- a/Packages/ContinuityKit/Sources/Ingest/PreparationQueue.swift +++ b/Packages/ContinuityKit/Sources/Ingest/PreparationQueue.swift @@ -311,6 +311,13 @@ public final class PreparationQueue { /// cache — by video, not Track row — so the same video in two playlists can't run two /// concurrent separations that rewrite the same output files. var stemsInFlight: Set = [] + /// Separation is held until this moment — armed on the session's FIRST stem demand + /// (i.e. when playback starts). Pressing play spins up the audio engine, UI, artwork, and + /// (first run) the model download all at once; the ORT session-load + first-window peak is + /// the app's largest allocation and must not stack on that ramp. + var separationAllowedAt: Date? + /// At most one pending post-hold retry (ensureStems also re-fires on every track change). + var stemsRetryScheduled = false } diff --git a/Packages/ContinuityKit/Sources/Ingest/StemSeparator.swift b/Packages/ContinuityKit/Sources/Ingest/StemSeparator.swift index a46a9bd..9c52336 100644 --- a/Packages/ContinuityKit/Sources/Ingest/StemSeparator.swift +++ b/Packages/ContinuityKit/Sources/Ingest/StemSeparator.swift @@ -10,6 +10,17 @@ struct StemPaths: Sendable, Equatable { let accompaniment: URL } +/// Memory budgets (MB) gating separation — the ORT session load (~600 MB transient: fp16→fp32 +/// upcast + protobuf parse) plus the first window's attention arena (up to ~1.7 GB for the 7.8 s +/// segment) is the single biggest allocation in the app. Sized from the jetsam RCA estimates; +/// tune against the mem[] breadcrumbs from a real device run. +enum StemSeparationBudget { + /// Headroom required to START a separation (session load + first-window peak + margin). + static let requiredStartMB = 1_400 + /// Mid-track floor: abort before the next window if headroom falls below this. + static let windowFloorMB = 600 +} + enum StemSeparationError: Error, Sendable { case decode(String) case inference(String) @@ -268,6 +279,13 @@ final class OnnxStemSeparator: StemSeparating { if Self.isAbortRequested { throw StemSeparationError.inference("aborted under memory pressure") } + // Don't wait for the system warning: if headroom has fallen below the window + // floor, the next attention spike could be the fatal one — bail now (the error + // path deletes the partial stems; a later ensureStems pass retries). + let headroom = MemoryFootprint.headroomMB + if headroom < StemSeparationBudget.windowFloorMB { + throw StemSeparationError.inference("aborted: \(headroom) MB headroom below window floor") + } let inputValue = try ORTValue(tensorData: inputData, elementType: .float, shape: [1, NSNumber(value: channels), NSNumber(value: segment)]) let outputs = try session.run(withInputs: ["mix": inputValue],