From a755fbba34c6e253ba0f1dc86dee2981da192d15 Mon Sep 17 00:00:00 2001 From: Tomio Ueda Date: Wed, 19 Aug 2026 14:29:42 -0700 Subject: [PATCH] perf(scan): move bundle-size walk off the scan critical path Summing every file inside every plugin bundle during the scan was the dominant cost of a full scan (audio bundles ship large sample/preset trees). BundleMetadataExtractor no longer walks bundle contents; the scan now reads only Info.plist + the Mach-O header. Bundle size is resolved lazily by a new BundleSizeService actor after the plugin list is shown, cached per bundle modification date so a rescan only re-walks bundles that actually changed. Plugin gains a sizeCacheMtime field; PluginReconciler preserves the stored size when the scan produces none (fileSize is now optional on PluginMetadata). Adds BundleSizeService tests (cache hit/miss/stale) and reconciler coverage for size preservation. Full suite: 323 tests pass. Co-Authored-By: Claude Opus 4.8 --- .../Pluginventory/App/AppState.swift | 64 ++++++++ .../Pluginventory/Models/Plugin.swift | 3 + .../Persistence/PluginReconciler.swift | 10 +- .../Scanner/BundleMetadataExtractor.swift | 32 ++-- .../Services/Scanner/BundleSizeService.swift | 124 +++++++++++++++ .../BundleMetadataExtractorTests.swift | 17 ++ .../Services/BundleSizeServiceTests.swift | 146 ++++++++++++++++++ .../Services/PluginReconcilerTests.swift | 50 +++++- 8 files changed, 419 insertions(+), 27 deletions(-) create mode 100644 PluginUpdater/Pluginventory/Services/Scanner/BundleSizeService.swift create mode 100644 PluginUpdater/PluginventoryTests/Services/BundleSizeServiceTests.swift diff --git a/PluginUpdater/Pluginventory/App/AppState.swift b/PluginUpdater/Pluginventory/App/AppState.swift index 7f5fa40..aec6a6d 100644 --- a/PluginUpdater/Pluginventory/App/AppState.swift +++ b/PluginUpdater/Pluginventory/App/AppState.swift @@ -30,7 +30,9 @@ final class AppState { private let vendorURLResolver = VendorURLResolver() private let appUpdateChecker = AppUpdateChecker() private let manifestCacheManager = ManifestCacheManager() + private let bundleSizeService = BundleSizeService() private var prefetchTask: Task? + private var sizeComputationTask: Task? /// Plist fields from most recent scan, keyed by bundleID. /// Used by VendorURLResolver for URL extraction from plist metadata. @@ -268,6 +270,9 @@ final class AppState { NotificationManager.shared.notifyChanges(result.changes) } + // Resolve bundle sizes in the background (off the scan critical path) + startSizeComputation() + // Check for available updates + resolve vendor URLs await checkForUpdates() @@ -307,6 +312,9 @@ final class AppState { applyResult(result, errors: scanResult.errors) + // Refresh sizes for any changed bundles (mtime cache skips the rest) + startSizeComputation() + // Notify for incremental changes NotificationManager.shared.notifyChanges(result.changes) } catch { @@ -466,6 +474,62 @@ final class AppState { } } + // MARK: - Bundle Sizes + + /// Starts (or restarts) the background bundle-size pass. + /// + /// Size is computed off the scan critical path so the plugin list appears + /// immediately after a scan. Results are cached per bundle modification date, + /// so this is near-instant on rescans where nothing changed. + func startSizeComputation() { + sizeComputationTask?.cancel() + sizeComputationTask = Task { [weak self] in + await self?.computePluginSizes() + } + } + + /// Cancels any running background size pass. + func cancelSizeComputation() { + sizeComputationTask?.cancel() + sizeComputationTask = nil + } + + /// Computes bundle sizes for all present plugins and writes them back, + /// skipping bundles whose modification date matches the cached value. + private func computePluginSizes() async { + let context = modelContainer.mainContext + let descriptor = FetchDescriptor( + predicate: #Predicate { !$0.isRemoved } + ) + guard let plugins = try? context.fetch(descriptor), !plugins.isEmpty else { return } + + let requests = plugins.map { plugin in + BundleSizeService.SizeRequest( + path: plugin.path, + cachedMtime: plugin.sizeCacheMtime, + cachedSize: plugin.fileSize + ) + } + + let results = await bundleSizeService.computeSizes(requests) + if Task.isCancelled { return } + + var changed = false + for plugin in plugins { + guard let result = results[plugin.path] else { continue } + if plugin.fileSize != result.size || plugin.sizeCacheMtime != result.mtime { + plugin.fileSize = result.size + plugin.sizeCacheMtime = result.mtime + changed = true + } + } + + if changed { + try? context.save() + AppLogger.shared.info("Bundle sizes updated for \(plugins.count) plugins", category: "scan") + } + } + // MARK: - Project Scanning func performProjectScan() async { diff --git a/PluginUpdater/Pluginventory/Models/Plugin.swift b/PluginUpdater/Pluginventory/Models/Plugin.swift index ee8da02..b1f1792 100644 --- a/PluginUpdater/Pluginventory/Models/Plugin.swift +++ b/PluginUpdater/Pluginventory/Models/Plugin.swift @@ -16,6 +16,9 @@ final class Plugin { /// Comma-separated architecture raw values (e.g. "arm64,x86_64"). var architecturesRaw: String = "" var fileSize: Int64 = 0 + /// Bundle modification date `fileSize` was last computed for. + /// Lets `BundleSizeService` skip re-walking unchanged bundles across scans. + var sizeCacheMtime: Date? var fileCreationDate: Date? @Relationship(deleteRule: .cascade, inverse: \PluginVersion.plugin) diff --git a/PluginUpdater/Pluginventory/Services/Persistence/PluginReconciler.swift b/PluginUpdater/Pluginventory/Services/Persistence/PluginReconciler.swift index 25dfbe7..398bb9f 100644 --- a/PluginUpdater/Pluginventory/Services/Persistence/PluginReconciler.swift +++ b/PluginUpdater/Pluginventory/Services/Persistence/PluginReconciler.swift @@ -143,7 +143,11 @@ actor PluginReconciler { existing.path = metadata.url.path existing.name = metadata.name existing.architectures = metadata.architectures - existing.fileSize = metadata.fileSize + // Preserve the background-computed size when the scan didn't produce one + // (the scan no longer walks bundle contents — see BundleSizeService). + if let size = metadata.fileSize { + existing.fileSize = size + } existing.fileCreationDate = metadata.fileCreationDate // Plugin reappeared after being removed @@ -196,7 +200,9 @@ actor PluginReconciler { ) plugin.architectures = metadata.architectures - plugin.fileSize = metadata.fileSize + if let size = metadata.fileSize { + plugin.fileSize = size + } plugin.fileCreationDate = metadata.fileCreationDate // Initial version record diff --git a/PluginUpdater/Pluginventory/Services/Scanner/BundleMetadataExtractor.swift b/PluginUpdater/Pluginventory/Services/Scanner/BundleMetadataExtractor.swift index 0658961..aee421d 100644 --- a/PluginUpdater/Pluginventory/Services/Scanner/BundleMetadataExtractor.swift +++ b/PluginUpdater/Pluginventory/Services/Scanner/BundleMetadataExtractor.swift @@ -19,7 +19,9 @@ struct PluginMetadata: Sendable { let plistFields: [String: String] let architectures: [CPUArchitecture] - let fileSize: Int64 + /// Total bundle size in bytes. `nil` when not computed during the scan — + /// size is resolved lazily off the critical path by `BundleSizeService`. + let fileSize: Int64? let fileCreationDate: Date? } @@ -72,10 +74,13 @@ enum BundleMetadataExtractor { let executableName = plist["CFBundleExecutable"] as? String let architectures = ArchitectureDetector.detect(bundleURL: bundleURL, executableName: executableName) - // Bundle size: sum all files in the bundle - let fileSize = Self.calculateBundleSize(bundleURL) + // NOTE: Bundle size is deliberately NOT computed here. Recursively summing + // every file inside every bundle was the dominant scan cost (audio plugins + // ship huge sample/preset trees). Size is resolved lazily, cached by bundle + // modification date, via `BundleSizeService` after the scan completes. + let fileSize: Int64? = nil - // File creation date + // File creation date (single stat — cheap) let fileCreationDate = Self.getCreationDate(bundleURL) return PluginMetadata( @@ -152,25 +157,6 @@ enum BundleMetadataExtractor { return String(parts[1]) } - /// Calculates the total size of all files in a bundle directory. - private static func calculateBundleSize(_ bundleURL: URL) -> Int64 { - let fm = FileManager.default - guard let enumerator = fm.enumerator( - at: bundleURL, - includingPropertiesForKeys: [.fileSizeKey], - options: [.skipsHiddenFiles] - ) else { return 0 } - - var total: Int64 = 0 - for case let fileURL as URL in enumerator { - if let values = try? fileURL.resourceValues(forKeys: [.fileSizeKey]), - let size = values.fileSize { - total += Int64(size) - } - } - return total - } - /// Gets the filesystem creation date of a bundle. private static func getCreationDate(_ bundleURL: URL) -> Date? { try? bundleURL.resourceValues(forKeys: [.creationDateKey]).creationDate diff --git a/PluginUpdater/Pluginventory/Services/Scanner/BundleSizeService.swift b/PluginUpdater/Pluginventory/Services/Scanner/BundleSizeService.swift new file mode 100644 index 0000000..e5e7aa7 --- /dev/null +++ b/PluginUpdater/Pluginventory/Services/Scanner/BundleSizeService.swift @@ -0,0 +1,124 @@ +import Foundation + +/// Computes plugin bundle sizes off the scan critical path. +/// +/// Summing every file inside every bundle is expensive — audio plugins ship large +/// sample, preset, and wavetable trees, so a full library can be tens of gigabytes +/// across hundreds of thousands of files. Doing that during the scan was the single +/// biggest source of slow scans. This service moves that work into a background pass +/// and caches each result against the bundle's modification date, so a rescan only +/// re-walks bundles that actually changed. +actor BundleSizeService { + + /// A request to (re)compute the size of a single bundle. + struct SizeRequest: Sendable { + /// Filesystem path of the bundle. Also used as the result key. + let path: String + /// Modification date the cached size was computed for, if any. + let cachedMtime: Date? + /// Previously computed size in bytes (0 if never computed). + let cachedSize: Int64 + } + + /// The resolved size of a bundle plus the modification date it was computed for. + struct SizeResult: Sendable { + let size: Int64 + let mtime: Date? + } + + private let concurrency: Int + + init(concurrency: Int = Constants.Defaults.scanConcurrency) { + self.concurrency = concurrency + } + + // MARK: - Public + + /// Resolves sizes for the given requests, keyed by `path`. + /// + /// For each request the bundle's current modification date is read (one cheap stat). + /// If it matches `cachedMtime` and a cached size exists, the cached size is returned + /// without walking the bundle. Otherwise the bundle is walked to compute a fresh size. + /// Runs with bounded concurrency to avoid saturating the filesystem. + func computeSizes(_ requests: [SizeRequest]) async -> [String: SizeResult] { + guard !requests.isEmpty else { return [:] } + + var results: [String: SizeResult] = [:] + results.reserveCapacity(requests.count) + + await withTaskGroup(of: (String, SizeResult).self) { group in + var inFlight = 0 + + for request in requests { + if inFlight >= concurrency { + if let (path, result) = await group.next() { + results[path] = result + inFlight -= 1 + } + } + + group.addTask { + (request.path, Self.resolve(request)) + } + inFlight += 1 + } + + for await (path, result) in group { + results[path] = result + } + } + + return results + } + + // MARK: - Private + + /// Resolves a single request, honoring the modification-date cache. + private static func resolve(_ request: SizeRequest) -> SizeResult { + let url = URL(fileURLWithPath: request.path) + let currentMtime = modificationDate(of: url) + + // Cache hit: unchanged bundle with a previously computed size. + if let cached = request.cachedMtime, + let current = currentMtime, + cached == current, + request.cachedSize > 0 { + return SizeResult(size: request.cachedSize, mtime: current) + } + + let size = directorySize(at: url) + return SizeResult(size: size, mtime: currentMtime) + } + + /// The bundle's content modification date, or `nil` if unavailable. + static func modificationDate(of url: URL) -> Date? { + try? url.resourceValues(forKeys: [.contentModificationDateKey]).contentModificationDate + } + + /// Recursively sums the size of every regular file in a directory tree. + /// + /// Uses the values prefetched by the enumerator rather than re-stat'ing each file, + /// and prefers total-allocated size (what the file actually occupies on disk), + /// falling back to logical file size. + static func directorySize(at url: URL) -> Int64 { + let keys: [URLResourceKey] = [.isRegularFileKey, .totalFileAllocatedSizeKey, .fileSizeKey] + let fm = FileManager.default + guard let enumerator = fm.enumerator( + at: url, + includingPropertiesForKeys: keys, + options: [.skipsHiddenFiles] + ) else { return 0 } + + var total: Int64 = 0 + for case let fileURL as URL in enumerator { + guard let values = try? fileURL.resourceValues(forKeys: Set(keys)) else { continue } + if values.isRegularFile == false { continue } + if let allocated = values.totalFileAllocatedSize { + total += Int64(allocated) + } else if let logical = values.fileSize { + total += Int64(logical) + } + } + return total + } +} diff --git a/PluginUpdater/PluginventoryTests/Services/BundleMetadataExtractorTests.swift b/PluginUpdater/PluginventoryTests/Services/BundleMetadataExtractorTests.swift index 6f541d5..c122beb 100644 --- a/PluginUpdater/PluginventoryTests/Services/BundleMetadataExtractorTests.swift +++ b/PluginUpdater/PluginventoryTests/Services/BundleMetadataExtractorTests.swift @@ -77,6 +77,23 @@ struct BundleMetadataExtractorTests { #expect(metadata.vendorName == "FabFilter") } + @Test("Does not compute bundle size during extraction") + func doesNotComputeSizeDuringScan() throws { + // Size is resolved lazily by BundleSizeService, not during the scan, + // so extraction must leave it nil (the dominant scan cost is removed). + let plist: [String: Any] = [ + "CFBundleIdentifier": "com.test.plugin", + "CFBundleName": "Test", + "CFBundleShortVersionString": "1.0.0" + ] + + let bundleURL = try createMockBundle(name: "Test", extension: "vst3", plist: plist) + defer { cleanup(bundleURL) } + + let metadata = try BundleMetadataExtractor.extract(from: bundleURL) + #expect(metadata.fileSize == nil) + } + @Test("Extracts metadata from CLAP bundle") func extractsCLAPMetadata() throws { let plist: [String: Any] = [ diff --git a/PluginUpdater/PluginventoryTests/Services/BundleSizeServiceTests.swift b/PluginUpdater/PluginventoryTests/Services/BundleSizeServiceTests.swift new file mode 100644 index 0000000..a7b7e5b --- /dev/null +++ b/PluginUpdater/PluginventoryTests/Services/BundleSizeServiceTests.swift @@ -0,0 +1,146 @@ +import Testing +import Foundation +@testable import Pluginventory + +@Suite("BundleSizeService Tests") +struct BundleSizeServiceTests { + + /// Creates a temporary directory containing files with the given byte lengths. + /// Returns the directory URL; caller is responsible for cleanup. + private func makeTree(fileByteCounts: [Int]) throws -> URL { + let dir = FileManager.default.temporaryDirectory + .appendingPathComponent("SizeServiceTests-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + + for (index, count) in fileByteCounts.enumerated() { + let fileURL = dir.appendingPathComponent("file-\(index).bin") + let data = Data(repeating: 0xAB, count: count) + try data.write(to: fileURL) + } + return dir + } + + private func cleanup(_ url: URL) { + try? FileManager.default.removeItem(at: url) + } + + // MARK: - directorySize + + @Test("directorySize reflects the content written") + func directorySizeSumsFiles() throws { + let dir = try makeTree(fileByteCounts: [5_000, 3_000]) + defer { cleanup(dir) } + + let size = BundleSizeService.directorySize(at: dir) + // Allocated size is >= logical content; never less. + #expect(size >= 8_000) + } + + @Test("directorySize recurses into subdirectories") + func directorySizeRecurses() throws { + let dir = try makeTree(fileByteCounts: [1_000]) + defer { cleanup(dir) } + let sub = dir.appendingPathComponent("nested") + try FileManager.default.createDirectory(at: sub, withIntermediateDirectories: true) + try Data(repeating: 0xCD, count: 4_000).write(to: sub.appendingPathComponent("deep.bin")) + + let size = BundleSizeService.directorySize(at: dir) + #expect(size >= 5_000) + } + + @Test("directorySize is zero for a nonexistent directory") + func directorySizeNonexistent() { + let size = BundleSizeService.directorySize(at: URL(fileURLWithPath: "/nope/does/not/exist")) + #expect(size == 0) + } + + // MARK: - modificationDate + + @Test("modificationDate returns a date for an existing directory") + func modificationDatePresent() throws { + let dir = try makeTree(fileByteCounts: [10]) + defer { cleanup(dir) } + #expect(BundleSizeService.modificationDate(of: dir) != nil) + } + + // MARK: - computeSizes cache behavior + + @Test("Cache miss computes the real size") + func cacheMissComputes() async throws { + let dir = try makeTree(fileByteCounts: [6_000]) + defer { cleanup(dir) } + + let real = BundleSizeService.directorySize(at: dir) + let service = BundleSizeService() + let results = await service.computeSizes([ + .init(path: dir.path, cachedMtime: nil, cachedSize: 0) + ]) + + let result = try #require(results[dir.path]) + #expect(result.size == real) + #expect(result.mtime != nil) + } + + @Test("Cache hit returns the cached size without re-walking") + func cacheHitReturnsCached() async throws { + let dir = try makeTree(fileByteCounts: [6_000]) + defer { cleanup(dir) } + + // Use the bundle's actual current mtime so the request is considered a hit. + let mtime = try #require(BundleSizeService.modificationDate(of: dir)) + let bogusCachedSize: Int64 = 999_999_999 + + let service = BundleSizeService() + let results = await service.computeSizes([ + .init(path: dir.path, cachedMtime: mtime, cachedSize: bogusCachedSize) + ]) + + let result = try #require(results[dir.path]) + // Returns the (bogus) cached size, proving it did not re-walk the tree. + #expect(result.size == bogusCachedSize) + #expect(result.mtime == mtime) + } + + @Test("Stale cache (mtime mismatch) recomputes") + func staleCacheRecomputes() async throws { + let dir = try makeTree(fileByteCounts: [6_000]) + defer { cleanup(dir) } + + let real = BundleSizeService.directorySize(at: dir) + let staleDate = Date(timeIntervalSince1970: 0) + let bogusCachedSize: Int64 = 999_999_999 + + let service = BundleSizeService() + let results = await service.computeSizes([ + .init(path: dir.path, cachedMtime: staleDate, cachedSize: bogusCachedSize) + ]) + + let result = try #require(results[dir.path]) + #expect(result.size == real) + #expect(result.size != bogusCachedSize) + } + + @Test("Empty request list returns empty results") + func emptyRequests() async { + let service = BundleSizeService() + let results = await service.computeSizes([]) + #expect(results.isEmpty) + } + + @Test("Resolves multiple bundles keyed by path") + func multipleBundles() async throws { + let dirA = try makeTree(fileByteCounts: [2_000]) + let dirB = try makeTree(fileByteCounts: [7_000]) + defer { cleanup(dirA); cleanup(dirB) } + + let service = BundleSizeService() + let results = await service.computeSizes([ + .init(path: dirA.path, cachedMtime: nil, cachedSize: 0), + .init(path: dirB.path, cachedMtime: nil, cachedSize: 0) + ]) + + #expect(results.count == 2) + #expect((results[dirA.path]?.size ?? 0) >= 2_000) + #expect((results[dirB.path]?.size ?? 0) >= 7_000) + } +} diff --git a/PluginUpdater/PluginventoryTests/Services/PluginReconcilerTests.swift b/PluginUpdater/PluginventoryTests/Services/PluginReconcilerTests.swift index c4b46ab..83ee3ce 100644 --- a/PluginUpdater/PluginventoryTests/Services/PluginReconcilerTests.swift +++ b/PluginUpdater/PluginventoryTests/Services/PluginReconcilerTests.swift @@ -15,7 +15,8 @@ struct PluginReconcilerTests { bundleID: String = "com.test.plugin", version: String = "1.0.0", format: PluginFormat = .vst3, - vendor: String = "TestVendor" + vendor: String = "TestVendor", + fileSize: Int64? = 1024 ) -> PluginMetadata { PluginMetadata( url: URL(fileURLWithPath: "/Library/Audio/Plug-Ins/VST3/\(name).vst3"), @@ -31,7 +32,7 @@ struct PluginReconcilerTests { parentDirectory: "VST3", plistFields: [:], architectures: [.arm64, .x86_64], - fileSize: 1024, + fileSize: fileSize, fileCreationDate: nil ) } @@ -68,6 +69,51 @@ struct PluginReconcilerTests { #expect(versions.count == 2) } + @Test("Preserves stored size when scan metadata has no size") + func preservesSizeWhenMetadataSizeNil() async throws { + let container = try makeContainer() + + // Pre-populate a plugin with a known, previously computed size. + let context = ModelContext(container) + let existing = Plugin( + name: "Synth1", + bundleIdentifier: "com.test.synth1", + format: .vst3, + currentVersion: "1.0.0", + path: "/Library/Audio/Plug-Ins/VST3/Synth1.vst3", + vendorName: "TestVendor" + ) + existing.fileSize = 5_000_000 + context.insert(existing) + try context.save() + + // Reconcile with metadata that carries no size (the new scan behavior). + let reconciler = PluginReconciler(modelContainer: container) + _ = try await reconciler.reconcile(scannedPlugins: [ + makeMetadata(name: "Synth1", bundleID: "com.test.synth1", version: "1.0.0", fileSize: nil) + ]) + + let verifyContext = ModelContext(container) + let plugins = try verifyContext.fetch(FetchDescriptor()) + let synth = try #require(plugins.first { $0.bundleIdentifier == "com.test.synth1" }) + #expect(synth.fileSize == 5_000_000) + } + + @Test("Applies size when scan metadata provides one") + func appliesSizeWhenMetadataSizePresent() async throws { + let container = try makeContainer() + let reconciler = PluginReconciler(modelContainer: container) + + _ = try await reconciler.reconcile(scannedPlugins: [ + makeMetadata(name: "Synth1", bundleID: "com.test.synth1", fileSize: 2_048) + ]) + + let context = ModelContext(container) + let plugins = try context.fetch(FetchDescriptor()) + let synth = try #require(plugins.first { $0.bundleIdentifier == "com.test.synth1" }) + #expect(synth.fileSize == 2_048) + } + @Test("Detects version updates and records history") func detectsVersionUpdates() async throws { let container = try makeContainer()