diff --git a/BisonNotes AI/BisonNotes AI.xcodeproj/project.pbxproj b/BisonNotes AI/BisonNotes AI.xcodeproj/project.pbxproj index d1f92bc..5c4a473 100644 --- a/BisonNotes AI/BisonNotes AI.xcodeproj/project.pbxproj +++ b/BisonNotes AI/BisonNotes AI.xcodeproj/project.pbxproj @@ -13,8 +13,8 @@ 1433F5462F7C9E44002AB9D6 /* SwiftUI.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 14EA428E2E7F5ECA0059271B /* SwiftUI.framework */; }; 1433F5572F7C9E45002AB9D6 /* BisonNotes Watch WidgetExtension.appex in Embed Foundation Extensions */ = {isa = PBXBuildFile; fileRef = 1433F5442F7C9E44002AB9D6 /* BisonNotes Watch WidgetExtension.appex */; settings = {ATTRIBUTES = (RemoveHeadersOnCopy, ); }; }; 1435711D2F2137E1000BFF00 /* Textual in Frameworks */ = {isa = PBXBuildFile; productRef = 1435711C2F2137E1000BFF00 /* Textual */; }; - 1435735C2F2E92A500E93954 /* BisonNotes Share.appex in Embed Controls Extension */ = {isa = PBXBuildFile; fileRef = 143573522F2E92A500E93954 /* BisonNotes Share.appex */; settings = {ATTRIBUTES = (RemoveHeadersOnCopy, ); }; }; - 1485FEC52E526FB30044121F /* BisonNotes AI Watch App.app in Embed Watch Content */ = {isa = PBXBuildFile; fileRef = 1485FEA52E526FB10044121F /* BisonNotes AI Watch App.app */; settings = {ATTRIBUTES = (RemoveHeadersOnCopy, ); }; }; + 1435735C2F2E92A500E93954 /* BisonNotes Share.appex in Embed Controls Extension */ = {isa = PBXBuildFile; fileRef = 143573522F2E92A500E93954 /* BisonNotes Share.appex */; platformFilter = ios; settings = {ATTRIBUTES = (RemoveHeadersOnCopy, ); }; }; + 1485FEC52E526FB30044121F /* BisonNotes AI Watch App.app in Embed Watch Content */ = {isa = PBXBuildFile; fileRef = 1485FEA52E526FB10044121F /* BisonNotes AI Watch App.app */; platformFilter = ios; settings = {ATTRIBUTES = (RemoveHeadersOnCopy, ); }; }; 1485FED52E5271970044121F /* WatchConnectivity.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 1485FED42E5271970044121F /* WatchConnectivity.framework */; }; 1485FED72E5271AF0044121F /* WatchConnectivity.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 1485FED62E5271AF0044121F /* WatchConnectivity.framework */; }; 1485FEDC2E5272FA0044121F /* WatchRecordingMessage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1485FED92E5272FA0044121F /* WatchRecordingMessage.swift */; }; @@ -23,9 +23,11 @@ 1485FEDF2E5272FA0044121F /* WatchRecordingMessage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1485FED92E5272FA0044121F /* WatchRecordingMessage.swift */; }; 1485FEE02E5272FA0044121F /* WatchAudioChunk.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1485FED82E5272FA0044121F /* WatchAudioChunk.swift */; }; 1485FEE12E5272FA0044121F /* WatchRecordingState.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1485FEDA2E5272FA0044121F /* WatchRecordingState.swift */; }; + 14B0A0012F80000000BBD2FA /* MLXLLM in Frameworks */ = {isa = PBXBuildFile; productRef = 14B0A0042F80000000BBD2FA /* MLXLLM */; }; + 14B0A0022F80000000BBD2FA /* MLXLMCommon in Frameworks */ = {isa = PBXBuildFile; productRef = 14B0A0052F80000000BBD2FA /* MLXLMCommon */; }; 14EA428D2E7F5ECA0059271B /* WidgetKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 14EA428C2E7F5ECA0059271B /* WidgetKit.framework */; }; 14EA428F2E7F5ECA0059271B /* SwiftUI.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 14EA428E2E7F5ECA0059271B /* SwiftUI.framework */; }; - 14EA42A92E7F66370059271B /* BisonNotes AI ControlsExtension.appex in Embed Controls Extension */ = {isa = PBXBuildFile; fileRef = 14EA428B2E7F5ECA0059271B /* BisonNotes AI ControlsExtension.appex */; settings = {ATTRIBUTES = (RemoveHeadersOnCopy, ); }; }; + 14EA42A92E7F66370059271B /* BisonNotes AI ControlsExtension.appex in Embed Controls Extension */ = {isa = PBXBuildFile; fileRef = 14EA428B2E7F5ECA0059271B /* BisonNotes AI ControlsExtension.appex */; platformFilter = ios; settings = {ATTRIBUTES = (RemoveHeadersOnCopy, ); }; }; 14FA00012F30000000BBD2FA /* FluidAudio in Frameworks */ = {isa = PBXBuildFile; productRef = 14FA00022F30000000BBD2FA /* FluidAudio */; }; 14FDF9082E3F8D5600BBD2FA /* AWSBedrock in Frameworks */ = {isa = PBXBuildFile; productRef = 14FDF9072E3F8D5600BBD2FA /* AWSBedrock */; }; 14FDF90A2E3F8D5600BBD2FA /* AWSClientRuntime in Frameworks */ = {isa = PBXBuildFile; productRef = 14FDF9092E3F8D5600BBD2FA /* AWSClientRuntime */; }; @@ -33,8 +35,6 @@ 14FDF90E2E3F8D5600BBD2FA /* AWSTranscribe in Frameworks */ = {isa = PBXBuildFile; productRef = 14FDF90D2E3F8D5600BBD2FA /* AWSTranscribe */; }; 14FDF9102E3F8D5600BBD2FA /* AWSTranscribeStreaming in Frameworks */ = {isa = PBXBuildFile; productRef = 14FDF90F2E3F8D5600BBD2FA /* AWSTranscribeStreaming */; }; 14FDF9122E3F90FB00BBD2FA /* AWSBedrockRuntime in Frameworks */ = {isa = PBXBuildFile; productRef = 14FDF9112E3F90FB00BBD2FA /* AWSBedrockRuntime */; }; - 14B0A0012F80000000BBD2FA /* MLXLLM in Frameworks */ = {isa = PBXBuildFile; productRef = 14B0A0042F80000000BBD2FA /* MLXLLM */; }; - 14B0A0022F80000000BBD2FA /* MLXLMCommon in Frameworks */ = {isa = PBXBuildFile; productRef = 14B0A0052F80000000BBD2FA /* MLXLMCommon */; }; /* End PBXBuildFile section */ /* Begin PBXContainerItemProxy section */ @@ -891,7 +891,7 @@ "@executable_path/Frameworks", "@executable_path/../../Frameworks", ); - MARKETING_VERSION = 1.10; + MARKETING_VERSION = 1.11; PRODUCT_BUNDLE_IDENTIFIER = "Bison-Networking.BisonNotes-AI.watchkitapp.widget"; PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = watchos; @@ -923,7 +923,7 @@ "@executable_path/Frameworks", "@executable_path/../../Frameworks", ); - MARKETING_VERSION = 1.10; + MARKETING_VERSION = 1.11; PRODUCT_BUNDLE_IDENTIFIER = "Bison-Networking.BisonNotes-AI.watchkitapp.widget"; PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = watchos; @@ -955,7 +955,7 @@ "@executable_path/Frameworks", "@executable_path/../../Frameworks", ); - MARKETING_VERSION = 1.10; + MARKETING_VERSION = 1.11; PRODUCT_BUNDLE_IDENTIFIER = "Bison-Networking.BisonNotes-AI.BisonNotes-Share"; PRODUCT_NAME = "$(TARGET_NAME)"; SKIP_INSTALL = YES; @@ -985,7 +985,7 @@ "@executable_path/Frameworks", "@executable_path/../../Frameworks", ); - MARKETING_VERSION = 1.10; + MARKETING_VERSION = 1.11; PRODUCT_BUNDLE_IDENTIFIER = "Bison-Networking.BisonNotes-AI.BisonNotes-Share"; PRODUCT_NAME = "$(TARGET_NAME)"; SKIP_INSTALL = YES; @@ -1020,7 +1020,7 @@ "$(inherited)", "@executable_path/Frameworks", ); - MARKETING_VERSION = 1.10; + MARKETING_VERSION = 1.11; PRODUCT_BUNDLE_IDENTIFIER = "Bison-Networking.BisonNotes-AI.watchkitapp"; PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = watchos; @@ -1054,7 +1054,7 @@ "$(inherited)", "@executable_path/Frameworks", ); - MARKETING_VERSION = 1.10; + MARKETING_VERSION = 1.11; PRODUCT_BUNDLE_IDENTIFIER = "Bison-Networking.BisonNotes-AI.watchkitapp"; PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = watchos; @@ -1074,7 +1074,7 @@ CURRENT_PROJECT_VERSION = 1; DEVELOPMENT_TEAM = 4W55VW7UXX; GENERATE_INFOPLIST_FILE = YES; - MARKETING_VERSION = 1.10; + MARKETING_VERSION = 1.11; PRODUCT_BUNDLE_IDENTIFIER = "Bison-Networking.BisonNotes-AI-Watch-AppTests"; PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = watchos; @@ -1094,7 +1094,7 @@ CURRENT_PROJECT_VERSION = 1; DEVELOPMENT_TEAM = 4W55VW7UXX; GENERATE_INFOPLIST_FILE = YES; - MARKETING_VERSION = 1.10; + MARKETING_VERSION = 1.11; PRODUCT_BUNDLE_IDENTIFIER = "Bison-Networking.BisonNotes-AI-Watch-AppTests"; PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = watchos; @@ -1113,7 +1113,7 @@ CURRENT_PROJECT_VERSION = 1; DEVELOPMENT_TEAM = 4W55VW7UXX; GENERATE_INFOPLIST_FILE = YES; - MARKETING_VERSION = 1.10; + MARKETING_VERSION = 1.11; PRODUCT_BUNDLE_IDENTIFIER = "Bison-Networking.BisonNotes-AI-Watch-AppUITests"; PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = watchos; @@ -1132,7 +1132,7 @@ CURRENT_PROJECT_VERSION = 1; DEVELOPMENT_TEAM = 4W55VW7UXX; GENERATE_INFOPLIST_FILE = YES; - MARKETING_VERSION = 1.10; + MARKETING_VERSION = 1.11; PRODUCT_BUNDLE_IDENTIFIER = "Bison-Networking.BisonNotes-AI-Watch-AppUITests"; PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = watchos; @@ -1270,6 +1270,8 @@ 14DBC8642E34F7B600DAD442 /* Debug */ = { isa = XCBuildConfiguration; buildSettings = { + ALLOW_TARGET_PLATFORM_SPECIALIZATION = YES; + APS_ENVIRONMENT = development; ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; ASSETCATALOG_COMPILER_INCLUDE_ALL_APPICON_ASSETS = YES; @@ -1279,7 +1281,7 @@ DEVELOPMENT_TEAM = 4W55VW7UXX; ENABLE_PREVIEWS = YES; GENERATE_INFOPLIST_FILE = NO; - "INFOPLIST_FILE[sdk=iphone*]" = "BisonNotes AI/Info.plist"; + INFOPLIST_FILE = "BisonNotes AI/Info.plist"; INFOPLIST_KEY_CFBundleDisplayName = "BisonNotes AI"; INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.utilities"; INFOPLIST_KEY_UIApplicationSceneManifest_Generation = YES; @@ -1291,7 +1293,7 @@ "$(inherited)", "@executable_path/Frameworks", ); - MARKETING_VERSION = 1.10; + MARKETING_VERSION = 1.11; PRODUCT_BUNDLE_IDENTIFIER = "Bison-Networking.BisonNotes-AI"; PRODUCT_NAME = "BisonNotes AI"; SUPPORTED_PLATFORMS = "iphoneos iphonesimulator"; @@ -1307,6 +1309,8 @@ 14DBC8652E34F7B600DAD442 /* Release */ = { isa = XCBuildConfiguration; buildSettings = { + ALLOW_TARGET_PLATFORM_SPECIALIZATION = YES; + APS_ENVIRONMENT = production; ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; ASSETCATALOG_COMPILER_INCLUDE_ALL_APPICON_ASSETS = YES; @@ -1316,7 +1320,7 @@ DEVELOPMENT_TEAM = 4W55VW7UXX; ENABLE_PREVIEWS = YES; GENERATE_INFOPLIST_FILE = NO; - "INFOPLIST_FILE[sdk=iphone*]" = "BisonNotes AI/Info.plist"; + INFOPLIST_FILE = "BisonNotes AI/Info.plist"; INFOPLIST_KEY_CFBundleDisplayName = "BisonNotes AI"; INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.utilities"; INFOPLIST_KEY_UIApplicationSceneManifest_Generation = YES; @@ -1328,7 +1332,7 @@ "$(inherited)", "@executable_path/Frameworks", ); - MARKETING_VERSION = 1.10; + MARKETING_VERSION = 1.11; PRODUCT_BUNDLE_IDENTIFIER = "Bison-Networking.BisonNotes-AI"; PRODUCT_NAME = "BisonNotes AI"; SUPPORTED_PLATFORMS = "iphoneos iphonesimulator"; @@ -1350,7 +1354,7 @@ DEVELOPMENT_TEAM = 4W55VW7UXX; GENERATE_INFOPLIST_FILE = YES; IPHONEOS_DEPLOYMENT_TARGET = 18.5; - MARKETING_VERSION = 1.10; + MARKETING_VERSION = 1.11; PRODUCT_BUNDLE_IDENTIFIER = "Bison-Networking.Audio-JournalTests"; PRODUCT_NAME = "$(TARGET_NAME)"; SUPPORTED_PLATFORMS = "iphoneos iphonesimulator"; @@ -1373,7 +1377,7 @@ DEVELOPMENT_TEAM = 4W55VW7UXX; GENERATE_INFOPLIST_FILE = YES; IPHONEOS_DEPLOYMENT_TARGET = 18.5; - MARKETING_VERSION = 1.10; + MARKETING_VERSION = 1.11; PRODUCT_BUNDLE_IDENTIFIER = "Bison-Networking.Audio-JournalTests"; PRODUCT_NAME = "$(TARGET_NAME)"; SUPPORTED_PLATFORMS = "iphoneos iphonesimulator"; @@ -1394,7 +1398,7 @@ CURRENT_PROJECT_VERSION = 1; DEVELOPMENT_TEAM = 4W55VW7UXX; GENERATE_INFOPLIST_FILE = YES; - MARKETING_VERSION = 1.10; + MARKETING_VERSION = 1.11; PRODUCT_BUNDLE_IDENTIFIER = "Bison-Networking.Audio-JournalUITests"; PRODUCT_NAME = "$(TARGET_NAME)"; SUPPORTED_PLATFORMS = "iphoneos iphonesimulator"; @@ -1415,7 +1419,7 @@ CURRENT_PROJECT_VERSION = 1; DEVELOPMENT_TEAM = 4W55VW7UXX; GENERATE_INFOPLIST_FILE = YES; - MARKETING_VERSION = 1.10; + MARKETING_VERSION = 1.11; PRODUCT_BUNDLE_IDENTIFIER = "Bison-Networking.Audio-JournalUITests"; PRODUCT_NAME = "$(TARGET_NAME)"; SUPPORTED_PLATFORMS = "iphoneos iphonesimulator"; @@ -1449,7 +1453,7 @@ "@executable_path/../../Frameworks", "@executable_path/../../../../Frameworks", ); - MARKETING_VERSION = 1.10; + MARKETING_VERSION = 1.11; PRODUCT_BUNDLE_IDENTIFIER = "Bison-Networking.BisonNotes-AI.controls"; PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = iphoneos; @@ -1487,7 +1491,7 @@ "@executable_path/../../Frameworks", "@executable_path/../../../../Frameworks", ); - MARKETING_VERSION = 1.10; + MARKETING_VERSION = 1.11; PRODUCT_BUNDLE_IDENTIFIER = "Bison-Networking.BisonNotes-AI.controls"; PRODUCT_NAME = "$(TARGET_NAME)"; SDKROOT = iphoneos; @@ -1609,12 +1613,20 @@ kind = branch; }; }; + 14B0A0032F80000000BBD2FA /* XCRemoteSwiftPackageReference "mlx-swift-lm" */ = { + isa = XCRemoteSwiftPackageReference; + repositoryURL = "https://github.com/ml-explore/mlx-swift-lm/"; + requirement = { + kind = upToNextMinorVersion; + minimumVersion = 2.31.3; + }; + }; 14FA00032F30000000BBD2FA /* XCRemoteSwiftPackageReference "FluidAudio" */ = { isa = XCRemoteSwiftPackageReference; repositoryURL = "https://github.com/FluidInference/FluidAudio"; requirement = { - kind = upToNextMajorVersion; - minimumVersion = 0.7.8; + kind = exactVersion; + version = 0.14.8; }; }; 14FDF9062E3F8D5600BBD2FA /* XCRemoteSwiftPackageReference "aws-sdk-swift" */ = { @@ -1625,17 +1637,14 @@ minimumVersion = 1.5.12; }; }; - 14B0A0032F80000000BBD2FA /* XCRemoteSwiftPackageReference "mlx-swift-lm" */ = { - isa = XCRemoteSwiftPackageReference; - repositoryURL = "https://github.com/ml-explore/mlx-swift-lm/"; - requirement = { - kind = upToNextMinorVersion; - minimumVersion = 2.31.3; - }; - }; /* End XCRemoteSwiftPackageReference section */ /* Begin XCSwiftPackageProductDependency section */ + 1435711C2F2137E1000BFF00 /* Textual */ = { + isa = XCSwiftPackageProductDependency; + package = 1435711B2F2137E1000BFF00 /* XCRemoteSwiftPackageReference "textual" */; + productName = Textual; + }; 14B0A0042F80000000BBD2FA /* MLXLLM */ = { isa = XCSwiftPackageProductDependency; package = 14B0A0032F80000000BBD2FA /* XCRemoteSwiftPackageReference "mlx-swift-lm" */; @@ -1646,11 +1655,6 @@ package = 14B0A0032F80000000BBD2FA /* XCRemoteSwiftPackageReference "mlx-swift-lm" */; productName = MLXLMCommon; }; - 1435711C2F2137E1000BFF00 /* Textual */ = { - isa = XCSwiftPackageProductDependency; - package = 1435711B2F2137E1000BFF00 /* XCRemoteSwiftPackageReference "textual" */; - productName = Textual; - }; 14FA00022F30000000BBD2FA /* FluidAudio */ = { isa = XCSwiftPackageProductDependency; package = 14FA00032F30000000BBD2FA /* XCRemoteSwiftPackageReference "FluidAudio" */; diff --git a/BisonNotes AI/BisonNotes AI.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/BisonNotes AI/BisonNotes AI.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved index 78dbb70..ab49c55 100644 --- a/BisonNotes AI/BisonNotes AI.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/BisonNotes AI/BisonNotes AI.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -15,8 +15,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/awslabs/aws-crt-swift", "state" : { - "revision" : "d754d289d594adc240f55c90eab212bac818509c", - "version" : "0.58.1" + "revision" : "6780acbc4aaa94ce256ce115c79babba7d9880c2", + "version" : "0.61.1" } }, { @@ -24,8 +24,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/awslabs/aws-sdk-swift", "state" : { - "revision" : "7bd573f504eb7e2744c86c3e31d0a8031752880e", - "version" : "1.6.83" + "revision" : "ac5780c80677dbfc4f976863c5abdac76e710c17", + "version" : "1.7.9" } }, { @@ -42,8 +42,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/FluidInference/FluidAudio", "state" : { - "revision" : "716f1c9648abea9a057f424febe00bf65a77f867", - "version" : "0.13.2" + "revision" : "56607d90b97de7d95a731200563bf49aa5beef20", + "version" : "0.14.8" } }, { @@ -69,8 +69,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/smithy-lang/smithy-swift", "state" : { - "revision" : "470017211ad3bfe78f2649433cf0c9449d5607c5", - "version" : "0.193.0" + "revision" : "7192e229498190e40b1168219809888dd282f778", + "version" : "0.214.0" } }, { @@ -82,13 +82,22 @@ "version" : "1.2.1" } }, + { + "identity" : "swift-argument-parser", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-argument-parser.git", + "state" : { + "revision" : "ca37474853a4b5f59a22c74bfdd449b1f6bc4cc2", + "version" : "1.8.1" + } + }, { "identity" : "swift-asn1", "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-asn1.git", "state" : { - "revision" : "9f542610331815e29cc3821d3b6f488db8715517", - "version" : "1.6.0" + "revision" : "eb50cbd14606a9161cbc5d452f18797c90ef0bab", + "version" : "1.7.0" } }, { @@ -96,8 +105,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-async-algorithms.git", "state" : { - "revision" : "9d349bcc328ac3c31ce40e746b5882742a0d1272", - "version" : "1.1.3" + "revision" : "d0b4a06d0f173a2f3be27d3ea21b3c3aa18db440", + "version" : "1.1.4" } }, { @@ -114,8 +123,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-certificates.git", "state" : { - "revision" : "24ccdeeeed4dfaae7955fcac9dbf5489ed4f1a25", - "version" : "1.18.0" + "revision" : "bde8ca32a096825dfce37467137c903418c1893d", + "version" : "1.19.1" } }, { @@ -123,8 +132,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-collections.git", "state" : { - "revision" : "6675bc0ff86e61436e615df6fc5174e043e57924", - "version" : "1.4.1" + "revision" : "fea17c02d767f46b23070fdfdacc28a03a39232a", + "version" : "1.5.1" } }, { @@ -132,8 +141,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/pointfreeco/swift-concurrency-extras", "state" : { - "revision" : "5a3825302b1a0d744183200915a47b508c828e6f", - "version" : "1.3.2" + "revision" : "a90e2e40a7a840a853dd29e57cbef5dbb72c9d5b", + "version" : "1.4.0" } }, { @@ -150,8 +159,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-crypto.git", "state" : { - "revision" : "fa308c07a6fa04a727212d793e761460e41049c3", - "version" : "4.3.0" + "revision" : "1b6b2e274e85105bfa155183145a1dcfd63331f1", + "version" : "4.5.0" } }, { @@ -168,8 +177,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-http-structured-headers.git", "state" : { - "revision" : "76d7627bd88b47bf5a0f8497dd244885960dde0b", - "version" : "1.6.0" + "revision" : "933538faa42c432d385f02e07df0ace7c5ecfc47", + "version" : "1.7.0" } }, { @@ -195,8 +204,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/huggingface/swift-jinja.git", "state" : { - "revision" : "2039e9760ba1694f9962ccd2c616403983d46895", - "version" : "2.3.3" + "revision" : "0b67ecb79139f6addef8699eff3622808aa6c7dc", + "version" : "2.3.6" } }, { @@ -204,8 +213,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-log.git", "state" : { - "revision" : "bbd81b6725ae874c69e9b8c8804d462356b55523", - "version" : "1.10.1" + "revision" : "2aed77ae5ec9a86d8fe42c12275e4c2653a286ee", + "version" : "1.13.1" } }, { @@ -213,8 +222,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio.git", "state" : { - "revision" : "558f24a4647193b5a0e2104031b71c55d31ff83a", - "version" : "2.97.1" + "revision" : "57c0a08a331aaea9f5d7a932ad94ef43be942a95", + "version" : "2.100.0" } }, { @@ -222,8 +231,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio-extras.git", "state" : { - "revision" : "abcf5312eb8ed2fb11916078aef7c46b06f20813", - "version" : "1.33.0" + "revision" : "d2eeec0339074034f11a040a74aa2a341a2c4506", + "version" : "1.34.1" } }, { @@ -231,8 +240,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio-http2.git", "state" : { - "revision" : "6d8d596f0a9bfebb925733003731fe2d749b7e02", - "version" : "1.42.0" + "revision" : "61d1b44f6e4e118792be1cff88ee2bc0267c6f9a", + "version" : "1.44.0" } }, { @@ -240,8 +249,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio-ssl.git", "state" : { - "revision" : "df9c3406028e3297246e6e7081977a167318b692", - "version" : "2.36.1" + "revision" : "3f337058ccd7243c4cac7911477d8ad4c598d4da", + "version" : "2.37.0" } }, { @@ -249,8 +258,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio-transport-services.git", "state" : { - "revision" : "60c3e187154421171721c1a38e800b390680fb5d", - "version" : "1.26.0" + "revision" : "67787bb645a5e67d2edcdfbe48a216cc549222d5", + "version" : "1.28.0" } }, { @@ -276,8 +285,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/swift-server/swift-service-lifecycle", "state" : { - "revision" : "89888196dd79c61c50bca9a103d8114f32e1e598", - "version" : "2.10.1" + "revision" : "9829955b385e5bb88128b73f1b8389e9b9c3191a", + "version" : "2.11.0" } }, { @@ -313,7 +322,7 @@ "location" : "https://github.com/bisonbet/textual", "state" : { "branch" : "main", - "revision" : "5b06b811c0f5313b6b84bbef98c635a630638c38" + "revision" : "ad589638b23e80557aaf2fa959760feac643a1e1" } }, { diff --git a/BisonNotes AI/BisonNotes AI/AISettingsView.swift b/BisonNotes AI/BisonNotes AI/AISettingsView.swift index dbaffdd..e21a212 100644 --- a/BisonNotes AI/BisonNotes AI/AISettingsView.swift +++ b/BisonNotes AI/BisonNotes AI/AISettingsView.swift @@ -178,20 +178,20 @@ struct AISettingsView: View { private func checkEngineAvailability(_ engineType: AIEngineType) -> Bool { switch engineType { case .openAI: - let apiKey = UserDefaults.standard.string(forKey: "openAIAPIKey") ?? "" + let apiKey = KeychainSecretStore.shared.string(forKey: KeychainSecretStore.openAIAPIKey) ?? "" return !apiKey.isEmpty case .openAICompatible: - let apiKey = UserDefaults.standard.string(forKey: "openAICompatibleAPIKey") ?? "" + let apiKey = KeychainSecretStore.shared.string(forKey: KeychainSecretStore.openAICompatibleAPIKey) ?? "" return !apiKey.isEmpty case .mistralAI: - let apiKey = UserDefaults.standard.string(forKey: "mistralAPIKey") ?? "" + let apiKey = KeychainSecretStore.shared.string(forKey: KeychainSecretStore.mistralAPIKey) ?? "" let isEnabled = UserDefaults.standard.bool(forKey: "enableMistralAI") return !apiKey.isEmpty && isEnabled case .localLLM: let isEnabled = UserDefaults.standard.bool(forKey: AppSettingsKeys.enableOllama) return isEnabled case .googleAIStudio: - let apiKey = UserDefaults.standard.string(forKey: "googleAIStudioAPIKey") ?? "" + let apiKey = KeychainSecretStore.shared.string(forKey: KeychainSecretStore.googleAIStudioAPIKey) ?? "" let isEnabled = UserDefaults.standard.bool(forKey: "enableGoogleAIStudio") return !apiKey.isEmpty && isEnabled case .awsBedrock: @@ -257,7 +257,7 @@ struct AISettingsView: View { } var body: some View { - NavigationView { + NavigationStack { Form { selectedEngineConfigurationSection engineSelectionSection @@ -335,14 +335,40 @@ struct AISettingsView: View { AWSBedrockSettingsView() } .sheet(isPresented: $showingOnDeviceLLMSettings) { + #if targetEnvironment(macCatalyst) + VStack(spacing: 0) { + HStack { + Text("On-Device AI").font(.headline) + Spacer() + Button("Done") { showingOnDeviceLLMSettings = false }.buttonStyle(.bordered) + } + .padding(.horizontal, 16).padding(.vertical, 10) + Divider() + OnDeviceLLMSettingsView() + } + #else NavigationStack { OnDeviceLLMSettingsView() } + #endif } .sheet(isPresented: $showingMLXSwiftSettings) { + #if targetEnvironment(macCatalyst) + VStack(spacing: 0) { + HStack { + Text("MLX Swift").font(.headline) + Spacer() + Button("Done") { showingMLXSwiftSettings = false }.buttonStyle(.bordered) + } + .padding(.horizontal, 16).padding(.vertical, 10) + Divider() + MLXSwiftSettingsView() + } + #else NavigationStack { MLXSwiftSettingsView() } + #endif } .fullScreenCover(isPresented: $showingMistralOnboarding) { MistralOnboardingView(onSetupComplete: { @@ -544,7 +570,7 @@ private extension AISettingsView { case .googleAIStudio: showingGoogleAIStudioSettings = true case .mistralAI: - let mistralKey = UserDefaults.standard.string(forKey: "mistralAPIKey") ?? "" + let mistralKey = KeychainSecretStore.shared.string(forKey: KeychainSecretStore.mistralAPIKey) ?? "" if mistralKey.isEmpty { showingMistralOnboarding = true } else { diff --git a/BisonNotes AI/BisonNotes AI/AWS/AWSBedrockService.swift b/BisonNotes AI/BisonNotes AI/AWS/AWSBedrockService.swift index 23fc0bf..9fbde31 100644 --- a/BisonNotes AI/BisonNotes AI/AWS/AWSBedrockService.swift +++ b/BisonNotes AI/BisonNotes AI/AWS/AWSBedrockService.swift @@ -35,17 +35,26 @@ class AWSBedrockService: ObservableObject { // Use shared AWS credentials for all services let sharedCredentials = AWSCredentialsManager.shared.credentials - - // Ensure environment variables are set from shared credentials - AWSCredentialsManager.shared.initializeEnvironment() + guard config.useProfile || sharedCredentials.isValid else { + throw SummarizationError.aiServiceUnavailable(service: "AWS Bedrock credentials are not configured") + } do { - let clientConfig = try await BedrockRuntimeClient.BedrockRuntimeClientConfig( - region: sharedCredentials.region - ) - - // AWS SDK for Swift will automatically use environment variables - // set by AWSCredentialsManager.initializeEnvironment() + let clientConfig: BedrockRuntimeClient.BedrockRuntimeClientConfig + if config.useProfile { + clientConfig = try await BedrockRuntimeClient.BedrockRuntimeClientConfig( + region: config.region + ) + } else { + clientConfig = try await BedrockRuntimeClient.BedrockRuntimeClientConfig( + awsCredentialIdentityResolver: AWSClientCredentialResolver.staticResolver( + accessKeyId: config.accessKeyId, + secretAccessKey: config.secretAccessKey, + sessionToken: config.sessionToken + ), + region: config.region + ) + } let client = BedrockRuntimeClient(config: clientConfig) self.bedrockClient = client @@ -339,6 +348,12 @@ class AWSBedrockService: ObservableObject { // Convert response body to Data let responseData = Data(responseBody) + let responseLimit = config.model.maxResponseLength + + guard responseData.count <= responseLimit else { + AppLog.shared.networking("AWS Bedrock response rejected: \(responseData.count) bytes exceeds \(responseLimit) byte limit", level: .error) + throw SummarizationError.aiServiceUnavailable(service: "AWS Bedrock response exceeded the maximum allowed size") + } // Log the raw response only when verbose logging is enabled if PerformanceOptimizer.shouldLogEngineInitialization() { @@ -346,7 +361,8 @@ class AWSBedrockService: ObservableObject { } // Parse the model-specific response - let modelResponse = try AWSBedrockModelFactory.parseResponse(for: config.model, data: responseData) + let sanitizedResponseData = sanitizeResponseData(responseData) + let modelResponse = try AWSBedrockModelFactory.parseResponse(for: config.model, data: sanitizedResponseData) AppLog.shared.networking("AWS Bedrock API Success - Model: \(config.model.rawValue), response: \(modelResponse.content.count) chars") @@ -357,6 +373,25 @@ class AWSBedrockService: ObservableObject { throw SummarizationError.aiServiceUnavailable(service: "AWS Bedrock API request failed: \(error.localizedDescription)") } } + + private func sanitizeResponseData(_ data: Data) -> Data { + guard let responseString = String(data: data, encoding: .utf8) else { + return data + } + + let sanitized = String(responseString.unicodeScalars.filter { scalar in + switch scalar.value { + case 0x09, 0x0A, 0x0D: + return true + case 0x00...0x1F, 0x7F: + return false + default: + return true + } + }) + + return sanitized.data(using: .utf8) ?? data + } private func processCompleteStructured(text: String, contentType: ContentType) async throws -> (summary: String, tasks: [TaskItem], reminders: [ReminderItem], titles: [TitleItem], contentType: ContentType) { let systemPrompt = OpenAIPromptGenerator.createSystemPrompt(for: .complete, contentType: contentType) diff --git a/BisonNotes AI/BisonNotes AI/AWS/AWSClientCredentialResolver.swift b/BisonNotes AI/BisonNotes AI/AWS/AWSClientCredentialResolver.swift new file mode 100644 index 0000000..2417d35 --- /dev/null +++ b/BisonNotes AI/BisonNotes AI/AWS/AWSClientCredentialResolver.swift @@ -0,0 +1,30 @@ +// +// AWSClientCredentialResolver.swift +// BisonNotes AI +// + +import AWSSDKIdentity +import SmithyIdentity + +enum AWSClientCredentialResolver { + static func staticResolver( + accessKeyId: String, + secretAccessKey: String, + sessionToken: String? = nil + ) -> StaticAWSCredentialIdentityResolver { + StaticAWSCredentialIdentityResolver( + AWSCredentialIdentity( + accessKey: accessKeyId, + secret: secretAccessKey, + sessionToken: sessionToken + ) + ) + } + + static func staticResolver(credentials: AWSCredentials) -> StaticAWSCredentialIdentityResolver { + staticResolver( + accessKeyId: credentials.accessKeyId, + secretAccessKey: credentials.secretAccessKey + ) + } +} diff --git a/BisonNotes AI/BisonNotes AI/AWS/AWSCredentialsManager.swift b/BisonNotes AI/BisonNotes AI/AWS/AWSCredentialsManager.swift index 5fe9725..a5f8f0a 100644 --- a/BisonNotes AI/BisonNotes AI/AWS/AWSCredentialsManager.swift +++ b/BisonNotes AI/BisonNotes AI/AWS/AWSCredentialsManager.swift @@ -32,22 +32,27 @@ class AWSCredentialsManager: ObservableObject { @Published var credentials: AWSCredentials private let userDefaults = UserDefaults.standard - private let credentialsKey = "AWSCredentials" + private let keychain = KeychainSecretStore.shared + private let credentialsKey = KeychainSecretStore.awsCredentials init() { + self.credentials = .default + migrateLegacyCredentials() + // Load saved credentials or use default - if let data = userDefaults.data(forKey: credentialsKey), + if let data = keychain.data(forKey: credentialsKey), let savedCredentials = try? JSONDecoder().decode(AWSCredentials.self, from: data) { self.credentials = savedCredentials } else { self.credentials = .default } + clearCredentialEnvironment() } func updateCredentials(_ newCredentials: AWSCredentials) { self.credentials = newCredentials saveCredentials() - configureEnvironmentVariables() + clearCredentialEnvironment() } func updateAccessKey(_ accessKey: String) { @@ -79,26 +84,46 @@ class AWSCredentialsManager: ObservableObject { private func saveCredentials() { if let data = try? JSONEncoder().encode(credentials) { - userDefaults.set(data, forKey: credentialsKey) + keychain.setData(data, forKey: credentialsKey) } + userDefaults.removeObject(forKey: credentialsKey) + userDefaults.removeObject(forKey: "awsAccessKey") + userDefaults.removeObject(forKey: "awsSecretKey") + userDefaults.removeObject(forKey: "awsRegion") } - - private func configureEnvironmentVariables() { - // Set environment variables for AWS SDK - if credentials.isValid { - setenv("AWS_ACCESS_KEY_ID", credentials.accessKeyId, 1) - setenv("AWS_SECRET_ACCESS_KEY", credentials.secretAccessKey, 1) - setenv("AWS_DEFAULT_REGION", credentials.region, 1) - AppLog.shared.networking("AWS credentials configured globally") - } else { - AppLog.shared.networking("AWS credentials incomplete - not setting environment variables", level: .error) + + private func migrateLegacyCredentials() { + if keychain.data(forKey: credentialsKey) == nil, + let data = userDefaults.data(forKey: credentialsKey) { + keychain.setData(data, forKey: credentialsKey) + } else if keychain.data(forKey: credentialsKey) == nil { + let accessKey = userDefaults.string(forKey: "awsAccessKey") ?? "" + let secretKey = userDefaults.string(forKey: "awsSecretKey") ?? "" + let region = userDefaults.string(forKey: "awsRegion") ?? AWSCredentials.default.region + + if !accessKey.isEmpty || !secretKey.isEmpty { + let legacyCredentials = AWSCredentials( + accessKeyId: accessKey, + secretAccessKey: secretKey, + region: region.isEmpty ? AWSCredentials.default.region : region + ) + if let data = try? JSONEncoder().encode(legacyCredentials) { + keychain.setData(data, forKey: credentialsKey) + } + } } + + userDefaults.removeObject(forKey: credentialsKey) + userDefaults.removeObject(forKey: "awsAccessKey") + userDefaults.removeObject(forKey: "awsSecretKey") + userDefaults.removeObject(forKey: "awsRegion") } - - // Call this when app starts to ensure environment variables are set - func initializeEnvironment() { - configureEnvironmentVariables() + func clearCredentialEnvironment() { + unsetenv("AWS_ACCESS_KEY_ID") + unsetenv("AWS_SECRET_ACCESS_KEY") + unsetenv("AWS_SESSION_TOKEN") + unsetenv("AWS_DEFAULT_REGION") } } @@ -107,4 +132,4 @@ class AWSCredentialsManager: ObservableObject { extension AWSCredentialsManager { static let shared = AWSCredentialsManager() -} \ No newline at end of file +} diff --git a/BisonNotes AI/BisonNotes AI/AWSBedrockSettingsView.swift b/BisonNotes AI/BisonNotes AI/AWSBedrockSettingsView.swift index 933639e..4abfaae 100644 --- a/BisonNotes AI/BisonNotes AI/AWSBedrockSettingsView.swift +++ b/BisonNotes AI/BisonNotes AI/AWSBedrockSettingsView.swift @@ -9,7 +9,7 @@ import SwiftUI struct AWSBedrockSettingsView: View { @ObservedObject private var credentialsManager = AWSCredentialsManager.shared - @AppStorage("awsBedrockSessionToken") private var sessionToken: String = "" + @SecureStorage(KeychainSecretStore.awsBedrockSessionToken) private var sessionToken: String = "" @AppStorage("awsBedrockModel") private var selectedModel: String = AWSBedrockModel.llama4Maverick.rawValue @AppStorage("awsBedrockTemperature") private var temperature: Double = 0.1 @AppStorage("awsBedrockMaxTokens") private var maxTokens: Int = 4096 @@ -41,7 +41,7 @@ struct AWSBedrockSettingsView: View { } var body: some View { - NavigationView { + NavigationStack { Form { headerSection diff --git a/BisonNotes AI/BisonNotes AI/AWSSettingsView.swift b/BisonNotes AI/BisonNotes AI/AWSSettingsView.swift index b91c824..c13ceae 100644 --- a/BisonNotes AI/BisonNotes AI/AWSSettingsView.swift +++ b/BisonNotes AI/BisonNotes AI/AWSSettingsView.swift @@ -30,7 +30,7 @@ struct AWSSettingsView: View { ] var body: some View { - NavigationView { + NavigationStack { Form { Section(header: Text("AWS Transcribe")) { VStack(alignment: .leading, spacing: 8) { diff --git a/BisonNotes AI/BisonNotes AI/AWSTranscribeService.swift b/BisonNotes AI/BisonNotes AI/AWSTranscribeService.swift index 984f550..c0554d2 100644 --- a/BisonNotes AI/BisonNotes AI/AWSTranscribeService.swift +++ b/BisonNotes AI/BisonNotes AI/AWSTranscribeService.swift @@ -93,14 +93,6 @@ class AWSTranscribeService: NSObject, ObservableObject { private func setupAWSServices() { // Clients will be initialized lazily when first needed - // IMPORTANT: For iOS apps, you need to configure AWS credentials through: - // 1. AWS Cognito (recommended for mobile apps) - // 2. Environment variables (for development) - // 3. Custom credential provider - // - // The new AWS SDK will look for credentials in this order: - // - Environment variables: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_DEFAULT_REGION - // - AWS credential files (if available) transcribeClient = nil s3Client = nil } @@ -115,18 +107,16 @@ class AWSTranscribeService: NSObject, ObservableObject { // Use shared AWS credentials for all services let sharedCredentials = AWSCredentialsManager.shared.credentials - - // Ensure environment variables are set from shared credentials - AWSCredentialsManager.shared.initializeEnvironment() + guard sharedCredentials.isValid else { + throw AWSTranscribeError.configurationMissing + } do { let clientConfig = try await TranscribeClient.TranscribeClientConfig( + awsCredentialIdentityResolver: AWSClientCredentialResolver.staticResolver(credentials: sharedCredentials), region: sharedCredentials.region ) - - // AWS SDK for Swift will automatically use environment variables - // set by AWSCredentialsManager.initializeEnvironment() - + let client = TranscribeClient(config: clientConfig) self.transcribeClient = client return client @@ -142,18 +132,16 @@ class AWSTranscribeService: NSObject, ObservableObject { // Use shared AWS credentials for all services let sharedCredentials = AWSCredentialsManager.shared.credentials - - // Ensure environment variables are set from shared credentials - AWSCredentialsManager.shared.initializeEnvironment() + guard sharedCredentials.isValid else { + throw AWSTranscribeError.configurationMissing + } do { let clientConfig = try await S3Client.S3ClientConfig( + awsCredentialIdentityResolver: AWSClientCredentialResolver.staticResolver(credentials: sharedCredentials), region: sharedCredentials.region ) - - // AWS SDK for Swift will automatically use environment variables - // set by AWSCredentialsManager.initializeEnvironment() - + let client = S3Client(config: clientConfig) self.s3Client = client return client @@ -723,4 +711,4 @@ enum AWSTranscribeError: LocalizedError { return "Invalid transcript format" } } -} \ No newline at end of file +} diff --git a/BisonNotes AI/BisonNotes AI/AppFileProtection.swift b/BisonNotes AI/BisonNotes AI/AppFileProtection.swift new file mode 100644 index 0000000..c0b3397 --- /dev/null +++ b/BisonNotes AI/BisonNotes AI/AppFileProtection.swift @@ -0,0 +1,27 @@ +import Foundation + +enum AppFileProtection { + static let sensitiveFileProtection: FileProtectionType = .complete + + static func apply(to url: URL) { + try? FileManager.default.setAttributes( + [.protectionKey: sensitiveFileProtection], + ofItemAtPath: url.path + ) + } + + static func applyRecursively(to directoryURL: URL) { + apply(to: directoryURL) + + guard let enumerator = FileManager.default.enumerator( + at: directoryURL, + includingPropertiesForKeys: nil + ) else { + return + } + + for case let url as URL in enumerator { + apply(to: url) + } + } +} diff --git a/BisonNotes AI/BisonNotes AI/BackgroundProcessingManager.swift b/BisonNotes AI/BisonNotes AI/BackgroundProcessingManager.swift index b4bb534..ce38475 100644 --- a/BisonNotes AI/BisonNotes AI/BackgroundProcessingManager.swift +++ b/BisonNotes AI/BisonNotes AI/BackgroundProcessingManager.swift @@ -305,9 +305,27 @@ enum JobProcessingStatus: Codable, Equatable { @MainActor class BackgroundProcessingManager: ObservableObject { - + + // Mac Catalyst doesn't expose battery state — UIDevice.batteryLevel returns -1 + // and accessing .batteryState spams "Error retrieving battery status" to the log. + fileprivate static var batteryLevelString: String { + #if targetEnvironment(macCatalyst) + return "n/a (Mac)" + #else + return "\(UIDevice.current.batteryLevel)" + #endif + } + + fileprivate static var batteryStateString: String { + #if targetEnvironment(macCatalyst) + return "n/a (Mac)" + #else + return "\(UIDevice.current.batteryState.rawValue)" + #endif + } + // MARK: - Published Properties - + @Published var activeJobs: [ProcessingJob] = [] @Published var processingStatus: JobProcessingStatus = .ready @Published var currentJob: ProcessingJob? @@ -1187,7 +1205,7 @@ class BackgroundProcessingManager: ObservableObject { // MARK: - Configuration Helpers private func getOpenAIConfig() -> OpenAITranscribeConfig { - let apiKey = UserDefaults.standard.string(forKey: "openAIAPIKey") ?? "" + let apiKey = KeychainSecretStore.shared.string(forKey: KeychainSecretStore.openAIAPIKey) ?? "" let modelString = UserDefaults.standard.string(forKey: "openAIModel") ?? OpenAITranscribeModel.gpt4oMiniTranscribe.rawValue let baseURL = UserDefaults.standard.string(forKey: "openAIBaseURL") ?? "https://api.openai.com/v1" @@ -1201,7 +1219,7 @@ class BackgroundProcessingManager: ObservableObject { } private func getMistralTranscribeConfig() -> MistralTranscribeConfig { - let apiKey = UserDefaults.standard.string(forKey: "mistralAPIKey") ?? "" + let apiKey = KeychainSecretStore.shared.string(forKey: KeychainSecretStore.mistralAPIKey) ?? "" let modelString = UserDefaults.standard.string(forKey: "mistralTranscribeModel") ?? MistralTranscribeModel.voxtralMiniLatest.rawValue let baseURL = UserDefaults.standard.string(forKey: "mistralBaseURL") ?? "https://api.mistral.ai/v1" let diarize = UserDefaults.standard.bool(forKey: "mistralTranscribeDiarize") @@ -1243,15 +1261,13 @@ class BackgroundProcessingManager: ObservableObject { } private func getAWSConfig() -> AWSTranscribeConfig { - let accessKey = UserDefaults.standard.string(forKey: "awsAccessKey") ?? "" - let secretKey = UserDefaults.standard.string(forKey: "awsSecretKey") ?? "" - let region = UserDefaults.standard.string(forKey: "awsRegion") ?? "us-east-1" + let credentials = AWSCredentialsManager.shared.credentials let bucketName = UserDefaults.standard.string(forKey: "awsBucketName") ?? "" return AWSTranscribeConfig( - region: region, - accessKey: accessKey, - secretKey: secretKey, + region: credentials.region, + accessKey: credentials.accessKeyId, + secretKey: credentials.secretAccessKey, bucketName: bucketName ) } @@ -1573,7 +1589,7 @@ class BackgroundProcessingManager: ObservableObject { } private func getOpenAISummarizationConfig() -> OpenAISummarizationConfig { - let apiKey = UserDefaults.standard.string(forKey: "openAIAPIKey") ?? "" + let apiKey = KeychainSecretStore.shared.string(forKey: KeychainSecretStore.openAIAPIKey) ?? "" let modelString = UserDefaults.standard.string(forKey: "openAISummarizationModel") ?? OpenAISummarizationModel.gpt41Mini.rawValue let baseURL = UserDefaults.standard.string(forKey: "openAIBaseURL") ?? "https://api.openai.com/v1" @@ -1751,8 +1767,8 @@ class BackgroundProcessingManager: ObservableObject { - Full Error: \(error) SYSTEM INFO: - - Battery Level: \(UIDevice.current.batteryLevel) - - Battery State: \(UIDevice.current.batteryState.rawValue) + - Battery Level: \(Self.batteryLevelString) + - Battery State: \(Self.batteryStateString) - Available Memory: \(ProcessInfo.processInfo.physicalMemory) ================= @@ -2247,12 +2263,19 @@ class BackgroundProcessingManager: ObservableObject { // MARK: - Background Task Management private func beginBackgroundTask() async { + #if targetEnvironment(macCatalyst) + // Mac apps don't get suspended by the OS, so the iOS background-task + // machinery (UIApplication.beginBackgroundTask, AVAudioSession-backed + // keep-alive audio) is unnecessary here and just spams the log with + // Mach port errors and "task created over 30 seconds ago" warnings. + return + #else // Don't start a new background task if one is already running guard backgroundTaskID == .invalid else { AppLog.shared.backgroundProcessing("Background task already running: \(backgroundTaskID.rawValue)", level: .debug) return } - + // Configure audio session for background processing to get extended time // This is CRITICAL for getting more than 30 seconds of background time do { @@ -2314,17 +2337,22 @@ class BackgroundProcessingManager: ObservableObject { // Start monitoring background time for long operations startBackgroundTimeMonitoring() - + // Start keep-alive audio to prevent app suspension during long tasks (like On-Device LLM) startKeepAliveAudio() } + #endif } private func endBackgroundTask() async { + #if targetEnvironment(macCatalyst) + // beginBackgroundTask is a no-op on Catalyst; nothing to tear down. + return + #else // Stop keep-alive audio stopKeepAliveAudio() - + // Cancel background time monitor first backgroundTimeMonitor?.cancel() backgroundTimeMonitor = nil @@ -2334,7 +2362,7 @@ class BackgroundProcessingManager: ObservableObject { UIApplication.shared.endBackgroundTask(backgroundTaskID) backgroundTaskID = .invalid backgroundTaskStartTime = nil - + // Clean up audio session when background task ends Task { do { @@ -2345,6 +2373,7 @@ class BackgroundProcessingManager: ObservableObject { } } } + #endif } private func handleBackgroundTaskExpiration() async { @@ -2451,7 +2480,9 @@ class BackgroundProcessingManager: ObservableObject { return } } catch { - AppLog.shared.backgroundProcessing("Error requesting notification permission: \(error.localizedDescription)", level: .error) + // On Mac Catalyst this commonly fails until the user enables + // notifications in System Settings — not a real error. + AppLog.shared.backgroundProcessing("Notification permission request failed: \(error.localizedDescription)", level: .debug) return } } else if settings.authorizationStatus != .authorized { diff --git a/BisonNotes AI/BisonNotes AI/BisonNotes AI.entitlements b/BisonNotes AI/BisonNotes AI/BisonNotes AI.entitlements index 7302651..301b206 100644 --- a/BisonNotes AI/BisonNotes AI/BisonNotes AI.entitlements +++ b/BisonNotes AI/BisonNotes AI/BisonNotes AI.entitlements @@ -3,7 +3,7 @@ aps-environment - development + $(APS_ENVIRONMENT) com.apple.developer.icloud-container-identifiers iCloud.$(CFBundleIdentifier) @@ -25,5 +25,7 @@ com.apple.developer.kernel.increased-memory-limit + com.apple.security.device.audio-input + diff --git a/BisonNotes AI/BisonNotes AI/BisonNotesAIApp.swift b/BisonNotes AI/BisonNotes AI/BisonNotesAIApp.swift index 588230a..106c8d6 100644 --- a/BisonNotes AI/BisonNotes AI/BisonNotesAIApp.swift +++ b/BisonNotes AI/BisonNotes AI/BisonNotesAIApp.swift @@ -76,12 +76,7 @@ struct BisonNotesAIApp: App { } else { // Set OpenAI as default for devices with less than 6GB RAM UserDefaults.standard.set("OpenAI", forKey: aiEngineKey) - // Set dummy API key if none exists - if UserDefaults.standard.string(forKey: "openAIAPIKey") == nil { - UserDefaults.standard.set("sk-000000000000", forKey: "openAIAPIKey") - NSLog("✅ Set dummy OpenAI API key for older device") - } - UserDefaults.standard.set(true, forKey: "enableOpenAI") + UserDefaults.standard.set(false, forKey: "enableOpenAI") NSLog("✅ AI engine migrated from '\(currentAIEngine ?? "nil")' to 'OpenAI' (device has <6GB RAM)") } } @@ -96,12 +91,7 @@ struct BisonNotesAIApp: App { } else { // Set OpenAI as default for devices with less than 6GB RAM UserDefaults.standard.set("OpenAI", forKey: transcriptionEngineKey) - // Set dummy API key if none exists - if UserDefaults.standard.string(forKey: "openAIAPIKey") == nil { - UserDefaults.standard.set("sk-000000000000", forKey: "openAIAPIKey") - NSLog("✅ Set dummy OpenAI API key for older device") - } - UserDefaults.standard.set(true, forKey: "enableOpenAI") + UserDefaults.standard.set(false, forKey: "enableOpenAI") NSLog("✅ Transcription engine migrated from '\(currentTranscriptionEngine ?? "nil")' to 'OpenAI' (device has <6GB RAM)") } } @@ -295,10 +285,23 @@ struct BisonNotesAIApp: App { UserDefaults.standard.set(true, forKey: migrationKey) } + private func migrateiCloudSensitiveBackupDefault() { + let migrationKey = "iCloudSensitiveBackupDefaultMigrated_v1.4" + + guard !UserDefaults.standard.bool(forKey: migrationKey) else { + return + } + + UserDefaults.standard.set(false, forKey: "iCloudBackupIncludeSensitiveSettings") + UserDefaults.standard.set(true, forKey: migrationKey) + } + init() { #if DEBUG Self.configureCoverageOutputIfNeeded() #endif + KeychainSecretStore.shared.migrateLegacySecretsFromUserDefaults() + // Log device capabilities on startup logDeviceCapabilities() @@ -310,6 +313,7 @@ struct BisonNotesAIApp: App { migrateWhisperKitToParakeet() migrateOnDeviceLLMNameToOnDeviceAI() migrateRemovedModels() + migrateiCloudSensitiveBackupDefault() setupDarwinNotificationObserver() } @@ -374,13 +378,13 @@ struct BisonNotesAIApp: App { // may be skipped. appDelegate.clearAppBadge(reason: "activation") // Scan for files placed by the Share Extension (Voice Memos, etc.) - scanSharedContainerForImports() + scanSharedContainerForImports(trigger: .pendingToken) // Also scan Documents/Inbox/ for files from "Open In" / document interaction. scanInboxForImportableFiles() } .onReceive(NotificationCenter.default.publisher(for: Notification.Name("ShareExtensionDidSaveFile"))) { _ in NSLog("📎 Darwin notification received from Share Extension") - scanSharedContainerForImports() + scanSharedContainerForImports(trigger: .pendingToken) } } .commands { @@ -434,12 +438,15 @@ struct BisonNotesAIApp: App { /// Handles files opened from the share sheet (e.g. Voice Memos, Files). Imports audio as recordings, text as transcripts. /// Also handles the `bisonnotes://share-import` URL scheme from the Share Extension. private func handleOpenURL(_ url: URL) { - NSLog("📎 handleOpenURL called with: \(url.absoluteString) (scheme: \(url.scheme ?? "nil"))") + NSLog("📎 handleOpenURL called (scheme: \(url.scheme ?? "nil"), host: \(url.host ?? "nil"), file: \(url.isFileURL ? url.lastPathComponent : "none"))") - // Handle custom URL scheme from Share Extension → scan shared container - if url.scheme == "bisonnotes" { + // Handle authenticated custom URL scheme from Share Extension. + if ShareImportAuthorization.isShareImportURL(url) { NSLog("📎 handleOpenURL: Share Extension triggered import via URL scheme") - scanSharedContainerForImports() + scanSharedContainerForImports(trigger: .url(url)) + return + } else if url.scheme == "bisonnotes" { + NSLog("📎 handleOpenURL: rejected unsupported bisonnotes URL") return } @@ -486,18 +493,38 @@ struct BisonNotesAIApp: App { private let appGroupID = "group.bisonnotesai.shared" private let shareInboxFolder = "ShareInbox" + private enum SharedContainerImportTrigger { + case url(URL) + case pendingToken + } + /// Scans the App Group shared container for files placed by the Share Extension /// (e.g. from Voice Memos share sheet). Imports them and cleans up. - private func scanSharedContainerForImports() { + private func scanSharedContainerForImports(trigger: SharedContainerImportTrigger) { guard let containerURL = FileManager.default .containerURL(forSecurityApplicationGroupIdentifier: appGroupID)? .appendingPathComponent(shareInboxFolder) else { return } guard FileManager.default.fileExists(atPath: containerURL.path) else { return } + let authorized: Bool + switch trigger { + case .url(let url): + authorized = ShareImportAuthorization.consumeURLToken(from: url, in: containerURL) + case .pendingToken: + authorized = ShareImportAuthorization.consumePendingToken(in: containerURL) + } + + guard authorized else { + NSLog("📎 Shared container scan skipped: missing or invalid Share Extension import token") + return + } + let files: [URL] do { - files = try FileManager.default.contentsOfDirectory(at: containerURL, includingPropertiesForKeys: nil) + files = try FileManager.default + .contentsOfDirectory(at: containerURL, includingPropertiesForKeys: nil) + .filter { $0.lastPathComponent != ShareImportAuthorization.tokenFileName } } catch { return } diff --git a/BisonNotes AI/BisonNotes AI/ContentView.swift b/BisonNotes AI/BisonNotes AI/ContentView.swift index 95833c0..855c7b6 100644 --- a/BisonNotes AI/BisonNotes AI/ContentView.swift +++ b/BisonNotes AI/BisonNotes AI/ContentView.swift @@ -128,9 +128,22 @@ struct ContentView: View { Text("Apple Intelligence has been removed from the app. Your settings have been automatically updated to use On-Device AI, which provides similar functionality. Please download an AI model to continue using on-device AI processing.") } .sheet(isPresented: $showingOnDeviceLLMSettings) { - NavigationView { + #if targetEnvironment(macCatalyst) + VStack(spacing: 0) { + HStack { + Text("On-Device AI").font(.headline) + Spacer() + Button("Done") { showingOnDeviceLLMSettings = false }.buttonStyle(.bordered) + } + .padding(.horizontal, 16).padding(.vertical, 10) + Divider() + OnDeviceLLMSettingsView() + } + #else + NavigationStack { OnDeviceLLMSettingsView() } + #endif } .alert("Switched to Parakeet", isPresented: $showingWhisperKitSwitchedAlert) { Button("OK") { } @@ -154,9 +167,22 @@ struct ContentView: View { Text("Your transcription engine has been upgraded to Parakeet, a fast and accurate on-device engine. Please download the Parakeet model (~250MB) to continue transcribing audio.") } .sheet(isPresented: $showingFluidAudioSettings) { + #if targetEnvironment(macCatalyst) + VStack(spacing: 0) { + HStack { + Text("On Device Transcription").font(.headline) + Spacer() + Button("Done") { showingFluidAudioSettings = false }.buttonStyle(.bordered) + } + .padding(.horizontal, 16).padding(.vertical, 10) + Divider() + FluidAudioSettingsView() + } + #else NavigationStack { FluidAudioSettingsView() } + #endif } .alert("Download Complete", isPresented: $downloadMonitor.showingCompletionAlert) { Button("OK") { diff --git a/BisonNotes AI/BisonNotes AI/EndpointSecurityPolicy.swift b/BisonNotes AI/BisonNotes AI/EndpointSecurityPolicy.swift new file mode 100644 index 0000000..c25e5c8 --- /dev/null +++ b/BisonNotes AI/BisonNotes AI/EndpointSecurityPolicy.swift @@ -0,0 +1,104 @@ +// +// EndpointSecurityPolicy.swift +// BisonNotes AI +// +// Validates user-configurable AI endpoint URLs before recordings, +// transcripts, or summaries are sent over the network. +// + +import Foundation + +enum EndpointSecurityPolicy { + static let allowInsecurePublicEndpointsKey = "allowInsecurePublicAIEndpoints" + + static func validationMessage( + for endpoint: String, + allowInsecurePublicEndpoints: Bool = UserDefaults.standard.bool(forKey: allowInsecurePublicEndpointsKey) + ) -> String? { + guard let url = URL(string: endpoint.trimmingCharacters(in: .whitespacesAndNewlines)), + let scheme = url.scheme?.lowercased(), + let host = url.host else { + return "Enter a valid endpoint URL." + } + + if scheme == "https" || scheme == "wss" { + return nil + } + + guard scheme == "http" || scheme == "ws" else { + return "Use HTTPS, WSS, HTTP, or WS for endpoint URLs." + } + + if isLocalOrPrivateHost(host) { + return nil + } + + if allowInsecurePublicEndpoints { + return nil + } + + return "Public HTTP endpoints are blocked because audio and transcripts would be sent in cleartext. Use HTTPS, localhost, or a private IP address." + } + + static func isAllowed( + endpoint: String, + allowInsecurePublicEndpoints: Bool = UserDefaults.standard.bool(forKey: allowInsecurePublicEndpointsKey) + ) -> Bool { + validationMessage(for: endpoint, allowInsecurePublicEndpoints: allowInsecurePublicEndpoints) == nil + } + + static func warningMessage(for endpoint: String) -> String? { + guard let url = URL(string: endpoint.trimmingCharacters(in: .whitespacesAndNewlines)), + let scheme = url.scheme?.lowercased(), + let host = url.host else { + return nil + } + + guard scheme == "http" || scheme == "ws" else { + return nil + } + + if isLocalOrPrivateHost(host) { + return "This endpoint uses HTTP. Only use it with services you control on localhost or a private network." + } + + return "This endpoint uses public HTTP. Audio, transcripts, and summaries can be read or modified on the network unless Development Mode is enabled." + } + + private static func isLocalOrPrivateHost(_ host: String) -> Bool { + let normalized = host.trimmingCharacters(in: CharacterSet(charactersIn: "[]")).lowercased() + + if normalized == "localhost" || normalized.hasSuffix(".localhost") || normalized == "::1" { + return true + } + + if let ipv4 = parseIPv4(normalized) { + return isPrivateIPv4(ipv4) + } + + return normalized.hasPrefix("fc") || normalized.hasPrefix("fd") || normalized.hasPrefix("fe80:") + } + + private static func parseIPv4(_ host: String) -> [Int]? { + let parts = host.split(separator: ".") + guard parts.count == 4 else { return nil } + + let octets = parts.compactMap { part -> Int? in + guard let value = Int(part), value >= 0, value <= 255 else { return nil } + return value + } + + return octets.count == 4 ? octets : nil + } + + private static func isPrivateIPv4(_ octets: [Int]) -> Bool { + let first = octets[0] + let second = octets[1] + + return first == 10 + || first == 127 + || (first == 172 && (16...31).contains(second)) + || (first == 192 && second == 168) + || (first == 169 && second == 254) + } +} diff --git a/BisonNotes AI/BisonNotes AI/EnhancedAudioSessionManager.swift b/BisonNotes AI/BisonNotes AI/EnhancedAudioSessionManager.swift index d1f783d..625757f 100644 --- a/BisonNotes AI/BisonNotes AI/EnhancedAudioSessionManager.swift +++ b/BisonNotes AI/BisonNotes AI/EnhancedAudioSessionManager.swift @@ -133,6 +133,11 @@ class EnhancedAudioSessionManager: NSObject, ObservableObject { /// Restore audio session to previous configuration (useful after interruptions) func restoreAudioSession() async throws { + #if targetEnvironment(macCatalyst) + // No AVAudioSession to restore on Mac Catalyst — applyConfiguration is a no-op + // and there is no interruption model. Recording is driven by CoreAudio directly. + return + #else guard let config = currentConfiguration else { // Default to mixed audio if no previous configuration try await configureMixedAudioSession() @@ -147,9 +152,9 @@ class EnhancedAudioSessionManager: NSObject, ObservableObject { for attempt in 1...maxAttempts { do { - // First, try to deactivate the session to clear any lingering state from interruption - // Use best-effort deactivation (don't throw if it fails) + #if !targetEnvironment(macCatalyst) try? session.setActive(false, options: .notifyOthersOnDeactivation) + #endif // Wait with exponential backoff, capped at 2 seconds per attempt // Attempt 1: 0.5s, 2: 1s, 3: 1.5s, 4: 2s, 5-10: 2s each @@ -174,8 +179,9 @@ class EnhancedAudioSessionManager: NSObject, ObservableObject { let audioError = AudioProcessingError.audioSessionConfigurationFailed("Session restoration failed after \(maxAttempts) attempts: \(lastAttemptError?.localizedDescription ?? "unknown error")") lastError = audioError throw audioError + #endif } - + /// Configure standard recording session (fallback for compatibility) func configureStandardRecording() async throws { do { @@ -198,27 +204,30 @@ class EnhancedAudioSessionManager: NSObject, ObservableObject { /// Configure audio session for playback (with mixWithOthers to avoid interfering with music) func configurePlaybackSession() async throws { + #if !targetEnvironment(macCatalyst) do { - // Use .playback category with .mixWithOthers option to not interrupt other audio try session.setCategory(.playback, mode: .default, options: [.mixWithOthers]) try session.setActive(true) - - isMixedAudioEnabled = true // We're mixing with others - isBackgroundRecordingEnabled = false - currentConfiguration = nil // This is a lightweight playback config - isConfigured = true - - AppLog.shared.audioSession("Playback session configured successfully with mixWithOthers") - } catch { let audioError = AudioProcessingError.audioSessionConfigurationFailed("Playback configuration failed: \(error.localizedDescription)") lastError = audioError throw audioError } + #endif + isMixedAudioEnabled = true + isBackgroundRecordingEnabled = false + currentConfiguration = nil + isConfigured = true + AppLog.shared.audioSession("Playback session configured successfully with mixWithOthers") } /// Set preferred audio input device func setPreferredInput(_ input: AVAudioSessionPortDescription) async throws { + #if targetEnvironment(macCatalyst) + // AVAudioSession input selection isn't supported on Mac — input device is + // managed by macOS Sound preferences. Calling this would spam Mach port errors. + return + #else do { try session.setPreferredInput(input) AppLog.shared.audioSession("Preferred input set to: \(input.portName) (\(input.portType.rawValue))") @@ -227,10 +236,14 @@ class EnhancedAudioSessionManager: NSObject, ObservableObject { lastError = audioError throw audioError } + #endif } - + /// Clear the preferred input to let iOS use its default microphone func clearPreferredInput() async throws { + #if targetEnvironment(macCatalyst) + return + #else do { try session.setPreferredInput(nil) AppLog.shared.audioSession("Preferred input cleared, iOS will use default microphone") @@ -239,61 +252,78 @@ class EnhancedAudioSessionManager: NSObject, ObservableObject { lastError = audioError throw audioError } + #endif } - + /// Get available audio inputs func getAvailableInputs() -> [AVAudioSessionPortDescription] { + #if targetEnvironment(macCatalyst) + return [] + #else return session.availableInputs ?? [] + #endif } /// Get the currently active or preferred input func getActiveInput() -> AVAudioSessionPortDescription? { + #if targetEnvironment(macCatalyst) + return nil + #else if let preferredInput = session.preferredInput { return preferredInput } return session.currentRoute.inputs.first + #endif } - + /// Check if mixed audio recording is currently supported func isMixedAudioSupported() -> Bool { - return session.category == .playAndRecord && + #if targetEnvironment(macCatalyst) + // No AVAudioSession on Mac — assume default macOS behavior is fine. + return true + #else + return session.category == .playAndRecord && session.categoryOptions.contains(.mixWithOthers) + #endif } /// Deactivate audio session func deactivateSession() async throws { + #if !targetEnvironment(macCatalyst) do { try session.setActive(false, options: .notifyOthersOnDeactivation) - isConfigured = false - isMixedAudioEnabled = false - isBackgroundRecordingEnabled = false - currentConfiguration = nil - AppLog.shared.audioSession("Audio session deactivated and reset") } catch { let audioError = AudioProcessingError.audioSessionConfigurationFailed("Failed to deactivate session: \(error.localizedDescription)") lastError = audioError throw audioError } + #endif + isConfigured = false + isMixedAudioEnabled = false + isBackgroundRecordingEnabled = false + currentConfiguration = nil + AppLog.shared.audioSession("Audio session deactivated and reset") } // MARK: - Private Methods private func applyConfiguration(_ config: AudioSessionConfig) async throws { + #if !targetEnvironment(macCatalyst) + // On Mac Catalyst, setCategory/setActive talk to mediaserverd via Mach ports + // that don't exist on macOS, flooding the log with "cannot add handler" errors. + // CoreAudio handles input/output directly on Mac without AVAudioSession setup. try session.setCategory(config.category, mode: config.mode, options: config.options) - // Prefer telephony-friendly settings when recording with Bluetooth HFP if config.category == .playAndRecord { - // These are best-effort; if they fail it's okay to continue try? session.setPreferredSampleRate(16000) try? session.setPreferredIOBufferDuration(0.1) } try session.setActive(true, options: []) - - // Additional configuration for background recording + #endif + if config.backgroundRecording { - // Request background audio capability try await requestBackgroundAudioCapability() } } @@ -310,14 +340,23 @@ class EnhancedAudioSessionManager: NSObject, ObservableObject { } private func requestBackgroundAudioCapability() async throws { + #if targetEnvironment(macCatalyst) + // Mac apps don't have iOS background modes — Catalyst recording stays alive + // while the app is running, regardless of UIBackgroundModes config. + return + #else // This would typically involve requesting background app refresh permission // For now, we'll just verify the configuration is correct guard session.category == .playAndRecord else { throw AudioProcessingError.backgroundRecordingNotPermitted } + #endif } private func setupNotificationObservers() { + #if !targetEnvironment(macCatalyst) + // AVAudioSession Mach port handlers don't exist on Mac — skip to avoid + // flooding the log with "cannot add handler" messages. // Audio interruption observer interruptionObserver = NotificationCenter.default.addObserver( forName: AVAudioSession.interruptionNotification, @@ -327,7 +366,7 @@ class EnhancedAudioSessionManager: NSObject, ObservableObject { // Capture the notification data we need before entering Task let userInfo = notification.userInfo let interruptionType = userInfo?[AVAudioSessionInterruptionTypeKey] as? AVAudioSession.InterruptionType - + Task { @MainActor in guard let self = self else { return } // Create a new notification with only the data we need @@ -338,7 +377,7 @@ class EnhancedAudioSessionManager: NSObject, ObservableObject { } } } - + // Route change observer routeChangeObserver = NotificationCenter.default.addObserver( forName: AVAudioSession.routeChangeNotification, @@ -348,7 +387,7 @@ class EnhancedAudioSessionManager: NSObject, ObservableObject { // Capture the notification data we need before entering Task let userInfo = notification.userInfo let routeChangeReason = userInfo?[AVAudioSessionRouteChangeReasonKey] as? AVAudioSession.RouteChangeReason - + Task { @MainActor in guard let self = self else { return } // Create a new notification with only the data we need @@ -363,6 +402,7 @@ class EnhancedAudioSessionManager: NSObject, ObservableObject { } } } + #endif } private func removeNotificationObservers() { @@ -435,6 +475,10 @@ class EnhancedAudioSessionManager: NSObject, ObservableObject { /// Selects Bluetooth HFP input if available, otherwise falls back to built-in mic @MainActor private func autoSelectBestInput() async { + #if targetEnvironment(macCatalyst) + // Input device selection on Mac is handled by macOS Sound preferences. + return + #else guard let inputs = session.availableInputs else { return } if let bluetoothHFP = inputs.first(where: { $0.portType == .bluetoothHFP }) { do { try session.setPreferredInput(bluetoothHFP) } catch { /* best-effort */ } @@ -443,6 +487,7 @@ class EnhancedAudioSessionManager: NSObject, ObservableObject { if let builtInMic = inputs.first(where: { $0.portType == .builtInMic }) { do { try session.setPreferredInput(builtInMic) } catch { /* best-effort */ } } + #endif } } diff --git a/BisonNotes AI/BisonNotes AI/EnhancedLoggingSystem.swift b/BisonNotes AI/BisonNotes AI/EnhancedLoggingSystem.swift index d7ae192..d0631b3 100644 --- a/BisonNotes AI/BisonNotes AI/EnhancedLoggingSystem.swift +++ b/BisonNotes AI/BisonNotes AI/EnhancedLoggingSystem.swift @@ -102,6 +102,7 @@ class AppLog { lines = Array(lines.suffix(Self.maxBufferLines)) } try? lines.joined(separator: "\n").write(to: url, atomically: true, encoding: .utf8) + AppFileProtection.apply(to: url) } } diff --git a/BisonNotes AI/BisonNotes AI/EnhancedTranscriptionManager.swift b/BisonNotes AI/BisonNotes AI/EnhancedTranscriptionManager.swift index f9a530b..7c77b9f 100644 --- a/BisonNotes AI/BisonNotes AI/EnhancedTranscriptionManager.swift +++ b/BisonNotes AI/BisonNotes AI/EnhancedTranscriptionManager.swift @@ -147,7 +147,7 @@ class EnhancedTranscriptionManager: NSObject, ObservableObject { // OpenAI Configuration private var openAIConfig: OpenAITranscribeConfig? { - let apiKey = UserDefaults.standard.string(forKey: "openAIAPIKey") ?? "" + let apiKey = KeychainSecretStore.shared.string(forKey: KeychainSecretStore.openAIAPIKey) ?? "" let modelString = UserDefaults.standard.string(forKey: "openAIModel") ?? OpenAITranscribeModel.gpt4oMiniTranscribe.rawValue let baseURL = UserDefaults.standard.string(forKey: "openAIBaseURL") ?? "https://api.openai.com/v1" @@ -170,7 +170,7 @@ class EnhancedTranscriptionManager: NSObject, ObservableObject { // Mistral Transcribe Configuration private var mistralTranscribeConfig: MistralTranscribeConfig? { - let apiKey = UserDefaults.standard.string(forKey: "mistralAPIKey") ?? "" + let apiKey = KeychainSecretStore.shared.string(forKey: KeychainSecretStore.mistralAPIKey) ?? "" let modelString = UserDefaults.standard.string(forKey: "mistralTranscribeModel") ?? MistralTranscribeModel.voxtralMiniLatest.rawValue let baseURL = UserDefaults.standard.string(forKey: "mistralBaseURL") ?? "https://api.mistral.ai/v1" let diarize = UserDefaults.standard.bool(forKey: "mistralTranscribeDiarize") diff --git a/BisonNotes AI/BisonNotes AI/FileImportManager.swift b/BisonNotes AI/BisonNotes AI/FileImportManager.swift index fe0fbec..dc96db9 100644 --- a/BisonNotes AI/BisonNotes AI/FileImportManager.swift +++ b/BisonNotes AI/BisonNotes AI/FileImportManager.swift @@ -118,6 +118,7 @@ class FileImportManager: NSObject, ObservableObject { // Copy file to documents directory with comprehensive error handling for thumbnail issues do { try FileManager.default.copyItem(at: sourceURL, to: destinationURL) + AppFileProtection.apply(to: destinationURL) } catch { // Check if this is a thumbnail-related error that we can ignore @@ -129,6 +130,7 @@ class FileImportManager: NSObject, ObservableObject { throw ImportError.copyFailed(error.localizedDescription) } } + AppFileProtection.apply(to: destinationURL) // Validate the copied file try validateAudioFile(at: destinationURL) @@ -228,6 +230,7 @@ class FileImportManager: NSObject, ObservableObject { do { try FileManager.default.copyItem(at: sourceURL, to: destinationURL) + AppFileProtection.apply(to: destinationURL) } catch { if error.isThumbnailGenerationError { AppLog.shared.fileManagement("Thumbnail generation warning: \(error.localizedDescription)", level: .debug) @@ -235,6 +238,7 @@ class FileImportManager: NSObject, ObservableObject { throw ImportError.copyFailed(error.localizedDescription) } } + AppFileProtection.apply(to: destinationURL) try validateAudioFile(at: destinationURL) @@ -269,6 +273,7 @@ class FileImportManager: NSObject, ObservableObject { exportSession.outputFileType = .m4a try await exportSession.export(to: destinationURL, as: .m4a) + AppFileProtection.apply(to: destinationURL) // Validate the extracted audio try validateAudioFile(at: destinationURL) @@ -505,4 +510,4 @@ struct ImportResults { return "Imported \(successful) of \(total) files successfully" } } -} \ No newline at end of file +} diff --git a/BisonNotes AI/BisonNotes AI/FluidAudio/FluidAudioManager.swift b/BisonNotes AI/BisonNotes AI/FluidAudio/FluidAudioManager.swift index 5dfd998..f842963 100644 --- a/BisonNotes AI/BisonNotes AI/FluidAudio/FluidAudioManager.swift +++ b/BisonNotes AI/BisonNotes AI/FluidAudio/FluidAudioManager.swift @@ -193,8 +193,7 @@ final class FluidAudioManager: ObservableObject { downloadProgress = 0.95 currentStatus = "Initializing model..." - let manager = AsrManager(config: .default) - try await manager.initialize(models: models) + let manager = AsrManager(config: .default, models: models) asrManager = manager loadedModelVersion = selectedVersion @@ -288,7 +287,8 @@ final class FluidAudioManager: ObservableObject { currentStatus = "Transcribing with Parakeet..." let start = Date() - let result = try await asrManager.transcribe(audioURL, source: .system) + var decoderState = TdtDecoderState.make(decoderLayers: await asrManager.decoderLayerCount) + let result = try await asrManager.transcribe(audioURL, decoderState: &decoderState) // Determine audio duration for accurate segment end time let asset = AVURLAsset(url: audioURL) diff --git a/BisonNotes AI/BisonNotes AI/FutureAIEngines.swift b/BisonNotes AI/BisonNotes AI/FutureAIEngines.swift index afc15ca..67be31b 100644 --- a/BisonNotes AI/BisonNotes AI/FutureAIEngines.swift +++ b/BisonNotes AI/BisonNotes AI/FutureAIEngines.swift @@ -199,7 +199,7 @@ class AWSBedrockEngine: SummarizationEngine, ConnectionTestable { private func updateConfiguration() { // Use unified credentials manager instead of separate UserDefaults keys let credentials = AWSCredentialsManager.shared.credentials - let sessionToken = UserDefaults.standard.string(forKey: "awsBedrockSessionToken") + let sessionToken = KeychainSecretStore.shared.string(forKey: KeychainSecretStore.awsBedrockSessionToken) let storedModelString = UserDefaults.standard.string(forKey: "awsBedrockModel") ?? AWSBedrockModel.claude45Haiku.rawValue // Migrate legacy model identifiers let modelString = AWSBedrockModel.migrate(rawValue: storedModelString) @@ -1151,7 +1151,7 @@ class GoogleAIStudioEngine: SummarizationEngine { init() { // Check if Google AI Studio is enabled and configured - let apiKey = UserDefaults.standard.string(forKey: "googleAIStudioAPIKey") ?? "" + let apiKey = KeychainSecretStore.shared.string(forKey: KeychainSecretStore.googleAIStudioAPIKey) ?? "" let isEnabled = UserDefaults.standard.bool(forKey: "enableGoogleAIStudio") self.isAvailable = !apiKey.isEmpty && isEnabled } diff --git a/BisonNotes AI/BisonNotes AI/GoogleAIStudioService.swift b/BisonNotes AI/BisonNotes AI/GoogleAIStudioService.swift index adf0b44..512a5ae 100644 --- a/BisonNotes AI/BisonNotes AI/GoogleAIStudioService.swift +++ b/BisonNotes AI/BisonNotes AI/GoogleAIStudioService.swift @@ -14,13 +14,15 @@ import SwiftUI class GoogleAIStudioService: ObservableObject { private let logger = Logger(subsystem: Bundle.main.bundleIdentifier ?? "com.bisonnotes.app", category: "GoogleAIStudio") - @AppStorage("googleAIStudioAPIKey") private var apiKey: String = "" @AppStorage("googleAIStudioModel") private var selectedModel: String = "gemini-3-flash-preview" @AppStorage("googleAIStudioTemperature") private var temperature: Double = 0.1 @AppStorage("googleAIStudioMaxTokens") private var maxTokens: Int = 8192 @AppStorage("enableGoogleAIStudio") private var enableGoogleAIStudio: Bool = false private let baseURL = "https://generativelanguage.googleapis.com/v1beta" + private var apiKey: String { + KeychainSecretStore.shared.string(forKey: KeychainSecretStore.googleAIStudioAPIKey) ?? "" + } // MARK: - API Response Models diff --git a/BisonNotes AI/BisonNotes AI/GoogleAIStudioSettingsView.swift b/BisonNotes AI/BisonNotes AI/GoogleAIStudioSettingsView.swift index 272b22c..3469123 100644 --- a/BisonNotes AI/BisonNotes AI/GoogleAIStudioSettingsView.swift +++ b/BisonNotes AI/BisonNotes AI/GoogleAIStudioSettingsView.swift @@ -10,7 +10,7 @@ import os.log struct GoogleAIStudioSettingsView: View { @Environment(\.dismiss) private var dismiss - @AppStorage("googleAIStudioAPIKey") private var apiKey: String = "" + @SecureStorage(KeychainSecretStore.googleAIStudioAPIKey) private var apiKey: String = "" @AppStorage("googleAIStudioModel") private var selectedModel: String = "gemini-3-flash-preview" @AppStorage("googleAIStudioTemperature") private var temperature: Double = 0.1 @AppStorage("googleAIStudioMaxTokens") private var maxTokens: Int = 4096 @@ -30,7 +30,7 @@ struct GoogleAIStudioSettingsView: View { ] var body: some View { - NavigationView { + NavigationStack { Form { Section(header: Text("API Configuration")) { VStack(alignment: .leading, spacing: 8) { @@ -178,4 +178,4 @@ struct GoogleAIStudioSettingsView: View { GoogleAIStudioSettingsView { // Preview callback } -} \ No newline at end of file +} diff --git a/BisonNotes AI/BisonNotes AI/Info.plist b/BisonNotes AI/BisonNotes AI/Info.plist index c48d14f..aa3a602 100644 --- a/BisonNotes AI/BisonNotes AI/Info.plist +++ b/BisonNotes AI/BisonNotes AI/Info.plist @@ -80,7 +80,7 @@ CFBundlePackageType APPL CFBundleShortVersionString - 1.10 + 1.11 CFBundleVersion 1 LSApplicationQueriesSchemes @@ -89,8 +89,6 @@ ITSAppUsesNonExemptEncryption - LSRequiresIPhoneOS - LSSupportsOpeningDocumentsInPlace NSCalendarsUsageDescription @@ -128,7 +126,7 @@ UIRequiresPersistentWiFi UIFileSharingEnabled - + UILaunchStoryboardName LaunchScreen UIMainStoryboardFile diff --git a/BisonNotes AI/BisonNotes AI/IntegrationSelectionView.swift b/BisonNotes AI/BisonNotes AI/IntegrationSelectionView.swift index 638f8a5..00eb96d 100644 --- a/BisonNotes AI/BisonNotes AI/IntegrationSelectionView.swift +++ b/BisonNotes AI/BisonNotes AI/IntegrationSelectionView.swift @@ -18,7 +18,7 @@ struct IntegrationSelectionView: View { @Environment(\.dismiss) private var dismiss var body: some View { - NavigationView { + NavigationStack { VStack(spacing: 24) { // Header VStack(spacing: 8) { diff --git a/BisonNotes AI/BisonNotes AI/KeychainSecretStore.swift b/BisonNotes AI/BisonNotes AI/KeychainSecretStore.swift new file mode 100644 index 0000000..4e8752e --- /dev/null +++ b/BisonNotes AI/BisonNotes AI/KeychainSecretStore.swift @@ -0,0 +1,141 @@ +// +// KeychainSecretStore.swift +// BisonNotes AI +// +// Keychain-backed storage for API keys and cloud credentials. +// + +import Foundation +import Security +import SwiftUI + +final class KeychainSecretStore { + static let shared = KeychainSecretStore() + + static let openAIAPIKey = "openAIAPIKey" + static let openAICompatibleAPIKey = "openAICompatibleAPIKey" + static let googleAIStudioAPIKey = "googleAIStudioAPIKey" + static let mistralAPIKey = "mistralAPIKey" + static let awsCredentials = "AWSCredentials" + static let awsBedrockSessionToken = "awsBedrockSessionToken" + + private static let stringSecretKeys = [ + openAIAPIKey, + openAICompatibleAPIKey, + googleAIStudioAPIKey, + mistralAPIKey, + awsBedrockSessionToken + ] + + private let service: String + + init(service: String = Bundle.main.bundleIdentifier ?? "com.bisonnotes.ai") { + self.service = "\(service).secrets" + } + + func string(forKey key: String) -> String? { + guard let data = data(forKey: key) else { return nil } + return String(data: data, encoding: .utf8) + } + + func setString(_ value: String, forKey key: String) { + let trimmedValue = value.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmedValue.isEmpty else { + delete(forKey: key) + return + } + + guard let data = trimmedValue.data(using: .utf8) else { return } + setData(data, forKey: key) + } + + func data(forKey key: String) -> Data? { + var query = baseQuery(forKey: key) + query[kSecReturnData as String] = true + query[kSecMatchLimit as String] = kSecMatchLimitOne + + var result: AnyObject? + let status = SecItemCopyMatching(query as CFDictionary, &result) + guard status == errSecSuccess else { return nil } + return result as? Data + } + + func setData(_ data: Data, forKey key: String) { + var query = baseQuery(forKey: key) + let attributes: [String: Any] = [ + kSecValueData as String: data, + kSecAttrAccessible as String: kSecAttrAccessibleWhenUnlockedThisDeviceOnly + ] + + let status = SecItemUpdate(query as CFDictionary, attributes as CFDictionary) + guard status != errSecSuccess else { return } + + if status != errSecItemNotFound { + SecItemDelete(query as CFDictionary) + } + + query.merge(attributes) { _, new in new } + SecItemAdd(query as CFDictionary, nil) + } + + func delete(forKey key: String) { + SecItemDelete(baseQuery(forKey: key) as CFDictionary) + } + + func migrateLegacySecretsFromUserDefaults(_ defaults: UserDefaults = .standard) { + for key in Self.stringSecretKeys { + if data(forKey: key) == nil, let legacyValue = defaults.string(forKey: key), !legacyValue.isEmpty { + setString(legacyValue, forKey: key) + } + defaults.removeObject(forKey: key) + } + + if data(forKey: Self.awsCredentials) == nil, let legacyData = defaults.data(forKey: Self.awsCredentials) { + setData(legacyData, forKey: Self.awsCredentials) + } + defaults.removeObject(forKey: Self.awsCredentials) + } + + private func baseQuery(forKey key: String) -> [String: Any] { + [ + kSecClass as String: kSecClassGenericPassword, + kSecAttrService as String: service, + kSecAttrAccount as String: key + ] + } +} + +@propertyWrapper +struct SecureStorage: DynamicProperty { + private let key: String + private let defaultValue: String + @State private var value: String + + init(wrappedValue defaultValue: String, _ key: String) { + self.key = key + self.defaultValue = defaultValue + _value = State(initialValue: KeychainSecretStore.shared.string(forKey: key) ?? defaultValue) + } + + func update() { + let storedValue = KeychainSecretStore.shared.string(forKey: key) ?? defaultValue + if storedValue != value { + value = storedValue + } + } + + var wrappedValue: String { + get { value } + nonmutating set { + value = newValue + KeychainSecretStore.shared.setString(newValue, forKey: key) + } + } + + var projectedValue: Binding { + Binding( + get: { wrappedValue }, + set: { wrappedValue = $0 } + ) + } +} diff --git a/BisonNotes AI/BisonNotes AI/LocationDetailView.swift b/BisonNotes AI/BisonNotes AI/LocationDetailView.swift index 117ab8b..a354631 100644 --- a/BisonNotes AI/BisonNotes AI/LocationDetailView.swift +++ b/BisonNotes AI/BisonNotes AI/LocationDetailView.swift @@ -23,7 +23,7 @@ struct LocationDetailView: View { } var body: some View { - NavigationView { + NavigationStack { VStack(spacing: 0) { // Map view Map(position: .constant(.region(region))) { diff --git a/BisonNotes AI/BisonNotes AI/MLXSwiftEngine.swift b/BisonNotes AI/BisonNotes AI/MLXSwiftEngine.swift index 3c90c36..46f48d7 100644 --- a/BisonNotes AI/BisonNotes AI/MLXSwiftEngine.swift +++ b/BisonNotes AI/BisonNotes AI/MLXSwiftEngine.swift @@ -575,6 +575,30 @@ private actor MLXSwiftService { /// the model weights room to load with buffer for inference overhead. private static let minimumAvailableMemory: UInt64 = 2_500_000_000 + /// Returns the amount of memory the process can safely use before loading. + /// On iOS this is the jetsam-limit headroom; on Mac Catalyst that API returns 0 + /// (no jetsam limits on macOS), so we read host VM stats for actual free RAM. + private static func availableMemoryForModelLoad() -> UInt64 { + #if targetEnvironment(macCatalyst) + var stats = vm_statistics64() + var count = mach_msg_type_number_t(MemoryLayout.size / MemoryLayout.size) + let result = withUnsafeMutablePointer(to: &stats) { ptr -> kern_return_t in + ptr.withMemoryRebound(to: integer_t.self, capacity: Int(count)) { + host_statistics64(mach_host_self(), HOST_VM_INFO64, $0, &count) + } + } + guard result == KERN_SUCCESS else { return 0 } + let pageSize = UInt64(vm_kernel_page_size) + let free = UInt64(stats.free_count) * pageSize + let inactive = UInt64(stats.inactive_count) * pageSize + let speculative = UInt64(stats.speculative_count) * pageSize + let purgeable = UInt64(stats.purgeable_count) * pageSize + return free + inactive + speculative + purgeable + #else + return UInt64(os_proc_available_memory()) + #endif + } + private func loadContainer() async throws -> ModelContainer { let modelId = UserDefaults.standard.string(forKey: MLXSwiftSettingsKeys.modelId) ?? MLXSwiftSettingsKeys.defaultModelId @@ -590,7 +614,7 @@ private actor MLXSwiftService { Memory.clearCache() // Check available memory before loading to avoid jetsam (OOM) kills - let available = os_proc_available_memory() + let available = Self.availableMemoryForModelLoad() AppLog.shared.summarization("[MLXSwift] Available memory before load: \(available / 1_000_000) MB") guard available >= Self.minimumAvailableMemory else { let availableMB = available / 1_000_000 diff --git a/BisonNotes AI/BisonNotes AI/MistralAIEngine.swift b/BisonNotes AI/BisonNotes AI/MistralAIEngine.swift index dabdc1d..eb4c7de 100644 --- a/BisonNotes AI/BisonNotes AI/MistralAIEngine.swift +++ b/BisonNotes AI/BisonNotes AI/MistralAIEngine.swift @@ -46,7 +46,7 @@ class MistralAIEngine: SummarizationEngine, ConnectionTestable { private let deduplicationSimilarityThreshold = 0.8 var isAvailable: Bool { - let apiKey = UserDefaults.standard.string(forKey: "mistralAPIKey") ?? "" + let apiKey = KeychainSecretStore.shared.string(forKey: KeychainSecretStore.mistralAPIKey) ?? "" guard !apiKey.isEmpty else { if PerformanceOptimizer.shouldLogEngineAvailabilityChecks() { AppLogger.shared.verbose("Mistral AI API key not configured", category: "MistralAIEngine") @@ -246,7 +246,7 @@ class MistralAIEngine: SummarizationEngine, ConnectionTestable { /// The configuration is only recreated if settings have actually changed (via equality check), /// preventing unnecessary service recreation and ensuring thread-safe access to latest settings. private func updateConfiguration() { - let apiKey = UserDefaults.standard.string(forKey: "mistralAPIKey") ?? "" + let apiKey = KeychainSecretStore.shared.string(forKey: KeychainSecretStore.mistralAPIKey) ?? "" let modelId = UserDefaults.standard.string(forKey: "mistralModel") ?? MistralAIModel.mistralMedium2508.rawValue let baseURL = UserDefaults.standard.string(forKey: "mistralBaseURL") ?? "https://api.mistral.ai/v1" let temperature = UserDefaults.standard.double(forKey: "mistralTemperature") diff --git a/BisonNotes AI/BisonNotes AI/MistralAISettingsView.swift b/BisonNotes AI/BisonNotes AI/MistralAISettingsView.swift index 7946f4e..2606ebe 100644 --- a/BisonNotes AI/BisonNotes AI/MistralAISettingsView.swift +++ b/BisonNotes AI/BisonNotes AI/MistralAISettingsView.swift @@ -11,7 +11,7 @@ import os.log struct MistralAISettingsView: View { @Environment(\.dismiss) private var dismiss - @AppStorage("mistralAPIKey") private var apiKey: String = "" + @SecureStorage(KeychainSecretStore.mistralAPIKey) private var apiKey: String = "" @AppStorage("mistralBaseURL") private var baseURL: String = "https://api.mistral.ai/v1" @AppStorage("mistralModel") private var selectedModel: String = MistralAIModel.mistralMedium2508.rawValue @AppStorage("mistralTemperature") private var temperature: Double = 0.1 @@ -33,7 +33,7 @@ struct MistralAISettingsView: View { } var body: some View { - NavigationView { + NavigationStack { Form { if apiKey.isEmpty { Section { diff --git a/BisonNotes AI/BisonNotes AI/MistralTranscribeSettingsView.swift b/BisonNotes AI/BisonNotes AI/MistralTranscribeSettingsView.swift index a62efe1..dcf171c 100644 --- a/BisonNotes AI/BisonNotes AI/MistralTranscribeSettingsView.swift +++ b/BisonNotes AI/BisonNotes AI/MistralTranscribeSettingsView.swift @@ -9,7 +9,7 @@ import SwiftUI struct MistralTranscribeSettingsView: View { // Reuse existing Mistral API key from summarization settings - @AppStorage("mistralAPIKey") private var apiKey: String = "" + @SecureStorage(KeychainSecretStore.mistralAPIKey) private var apiKey: String = "" @AppStorage("mistralBaseURL") private var baseURL: String = "https://api.mistral.ai/v1" // Transcription-specific settings @@ -24,7 +24,7 @@ struct MistralTranscribeSettingsView: View { @Environment(\.dismiss) private var dismiss var body: some View { - NavigationView { + NavigationStack { Form { apiConfigurationSection transcriptionSettingsSection diff --git a/BisonNotes AI/BisonNotes AI/Models/AppDataCoordinator.swift b/BisonNotes AI/BisonNotes AI/Models/AppDataCoordinator.swift index 0b8655b..c40e31d 100644 --- a/BisonNotes AI/BisonNotes AI/Models/AppDataCoordinator.swift +++ b/BisonNotes AI/BisonNotes AI/Models/AppDataCoordinator.swift @@ -231,6 +231,7 @@ class AppDataCoordinator: ObservableObject { let tempURL = tempDirectory.appendingPathComponent(syncRequest.filename) try audioData.write(to: tempURL) + AppFileProtection.apply(to: tempURL) // Convert watch location data if available let locationData: LocationData? = syncRequest.locationData?.toLocationData() @@ -271,4 +272,4 @@ class AppDataCoordinator: ObservableObject { func debugDatabaseContents() { coreDataManager.debugDatabaseContents() } -} \ No newline at end of file +} diff --git a/BisonNotes AI/BisonNotes AI/Models/AudioChunkingModels.swift b/BisonNotes AI/BisonNotes AI/Models/AudioChunkingModels.swift index d1ea6d5..f61b67e 100644 --- a/BisonNotes AI/BisonNotes AI/Models/AudioChunkingModels.swift +++ b/BisonNotes AI/BisonNotes AI/Models/AudioChunkingModels.swift @@ -160,7 +160,7 @@ struct AudioFileInfo { let format: String let sampleRate: Double let channels: Int - + static func create(from url: URL) async throws -> AudioFileInfo { AppLog.shared.chunking("AudioFileInfo.create - Analyzing file: \(url.lastPathComponent)", level: .debug) AppLog.shared.chunking("AudioFileInfo.create - Full path: \(url.path)", level: .debug) @@ -229,7 +229,7 @@ struct AudioFileInfo { return audioFileInfo } - private init(url: URL, duration: TimeInterval, fileSize: Int64, format: String, sampleRate: Double, channels: Int) { + init(url: URL, duration: TimeInterval, fileSize: Int64, format: String, sampleRate: Double, channels: Int) { self.url = url self.duration = duration self.fileSize = fileSize diff --git a/BisonNotes AI/BisonNotes AI/Models/AudioModels.swift b/BisonNotes AI/BisonNotes AI/Models/AudioModels.swift index 30c8fcc..7cb8eff 100644 --- a/BisonNotes AI/BisonNotes AI/Models/AudioModels.swift +++ b/BisonNotes AI/BisonNotes AI/Models/AudioModels.swift @@ -14,6 +14,10 @@ public enum AudioQuality: String, CaseIterable, Codable { var settings: [String: Any] { switch self { case .whisperOptimized: + // Used by AVAudioRecorder on iOS. Mac Catalyst recording is + // driven by AVAudioEngine + AVAudioFile (see + // AudioRecorderViewModel+CatalystEngine.swift) which configures + // its own AAC settings using the input node's native sample rate. return [ AVFormatIDKey: Int(kAudioFormatMPEG4AAC), AVSampleRateKey: 22050, // Good for Whisper, compatible with AAC diff --git a/BisonNotes AI/BisonNotes AI/Models/CoreDataManager.swift b/BisonNotes AI/BisonNotes AI/Models/CoreDataManager.swift index 5a90a6a..faca737 100644 --- a/BisonNotes AI/BisonNotes AI/Models/CoreDataManager.swift +++ b/BisonNotes AI/BisonNotes AI/Models/CoreDataManager.swift @@ -1084,6 +1084,7 @@ class CoreDataManager: ObservableObject { let matchingFiles = fileURLs.filter { $0.lastPathComponent == filename } if let newURL = matchingFiles.first { + AppFileProtection.apply(to: newURL) // Update the Core Data entry with the correct relative path recording.recordingURL = urlToRelativePath(newURL) recording.lastModified = Date() @@ -1105,6 +1106,7 @@ class CoreDataManager: ObservableObject { } if let newURL = matchingFilesByName.first { + AppFileProtection.apply(to: newURL) // Update the Core Data entry with the correct relative path recording.recordingURL = urlToRelativePath(newURL) recording.lastModified = Date() diff --git a/BisonNotes AI/BisonNotes AI/Models/EnhancedSummaryData.swift b/BisonNotes AI/BisonNotes AI/Models/EnhancedSummaryData.swift index b67d8fd..11b2360 100644 --- a/BisonNotes AI/BisonNotes AI/Models/EnhancedSummaryData.swift +++ b/BisonNotes AI/BisonNotes AI/Models/EnhancedSummaryData.swift @@ -415,7 +415,7 @@ public struct EnhancedSummaryData: Codable, Identifiable, Sendable { } // Initializer for Core Data conversion that preserves the original ID - init(id: UUID, recordingId: UUID, transcriptId: UUID? = nil, recordingURL: URL, recordingName: String, recordingDate: Date, summary: String, tasks: [TaskItem] = [], reminders: [ReminderItem] = [], titles: [TitleItem] = [], attachments: [SummaryAttachment] = [], userNotes: String? = nil, contentType: ContentType = .general, aiEngine: String = "Unknown", aiModel: String, originalLength: Int, processingTime: TimeInterval = 0, generatedAt: Date? = nil, version: Int = 1, wordCount: Int? = nil, compressionRatio: Double? = nil, confidence: Double? = nil) { + init(id: UUID, recordingId: UUID?, transcriptId: UUID? = nil, recordingURL: URL, recordingName: String, recordingDate: Date, summary: String, tasks: [TaskItem] = [], reminders: [ReminderItem] = [], titles: [TitleItem] = [], attachments: [SummaryAttachment] = [], userNotes: String? = nil, contentType: ContentType = .general, aiEngine: String = "Unknown", aiModel: String, originalLength: Int, processingTime: TimeInterval = 0, generatedAt: Date? = nil, version: Int = 1, wordCount: Int? = nil, compressionRatio: Double? = nil, confidence: Double? = nil) { self.id = id self.recordingId = recordingId self.transcriptId = transcriptId diff --git a/BisonNotes AI/BisonNotes AI/Models/RecordingArchiveService.swift b/BisonNotes AI/BisonNotes AI/Models/RecordingArchiveService.swift index d021bb1..7846116 100644 --- a/BisonNotes AI/BisonNotes AI/Models/RecordingArchiveService.swift +++ b/BisonNotes AI/BisonNotes AI/Models/RecordingArchiveService.swift @@ -277,6 +277,7 @@ class RecordingArchiveService: ObservableObject { coordinator.coordinate(readingItemAt: sourceURL, options: [], error: &coordinatorError) { coordinatedURL in do { try FileManager.default.copyItem(at: coordinatedURL, to: destinationURL) + AppFileProtection.apply(to: destinationURL) didCopy = true } catch { operationError = error @@ -385,8 +386,18 @@ class RecordingArchiveService: ObservableObject { locationObject.setValue(exportedAt, forKey: "lastVerifiedAt") locationObject.setValue(Self.statusAvailable, forKey: "status") + // Persist a security-scoped bookmark so sandbox access survives + // app launches. Mac Catalyst requires the explicit option; on iOS + // the default already retains the picker-granted scope. + let bookmarkOptions: URL.BookmarkCreationOptions = { + #if targetEnvironment(macCatalyst) + return [.withSecurityScope] + #else + return [] + #endif + }() let bookmarkData = try? url.bookmarkData( - options: [.minimalBookmark], + options: bookmarkOptions, includingResourceValuesForKeys: nil, relativeTo: nil ) @@ -474,10 +485,20 @@ class RecordingArchiveService: ObservableObject { private func resolvedArchiveURL(from locationObject: NSManagedObject) throws -> URL { if let bookmarkData = locationObject.value(forKey: "bookmarkData") as? Data { var isStale = false + // Mac Catalyst stores security-scoped bookmarks; resolution must + // pass the matching option for startAccessingSecurityScopedResource() + // to grant access on a subsequent launch. + let resolutionOptions: URL.BookmarkResolutionOptions = { + #if targetEnvironment(macCatalyst) + return [.withoutUI, .withSecurityScope] + #else + return [.withoutUI] + #endif + }() do { let url = try URL( resolvingBookmarkData: bookmarkData, - options: [.withoutUI], + options: resolutionOptions, relativeTo: nil, bookmarkDataIsStale: &isStale ) diff --git a/BisonNotes AI/BisonNotes AI/Models/SummaryAttachmentStore.swift b/BisonNotes AI/BisonNotes AI/Models/SummaryAttachmentStore.swift index 0319a78..a2af927 100644 --- a/BisonNotes AI/BisonNotes AI/Models/SummaryAttachmentStore.swift +++ b/BisonNotes AI/BisonNotes AI/Models/SummaryAttachmentStore.swift @@ -34,6 +34,7 @@ final class SummaryAttachmentStore { let id = UUID() let destinationFolder = attachmentsDirectory(for: summaryId) try fileManager.createDirectory(at: destinationFolder, withIntermediateDirectories: true) + AppFileProtection.applyRecursively(to: storageDirectory(for: summaryId)) let sanitizedName = sanitizeFileName(fileName) let storedFileName = "\(id.uuidString)_\(sanitizedName)" @@ -44,6 +45,7 @@ final class SummaryAttachmentStore { } try fileManager.copyItem(at: sourceURL, to: destinationURL) + AppFileProtection.apply(to: destinationURL) let attributes = try? fileManager.attributesOfItem(atPath: destinationURL.path) let fileSize = (attributes?[.size] as? NSNumber)?.int64Value ?? 0 @@ -102,6 +104,7 @@ final class SummaryAttachmentStore { try fileManager.removeItem(at: newDir) } try fileManager.moveItem(at: oldDir, to: newDir) + AppFileProtection.applyRecursively(to: newDir) } func fileURL(for attachment: SummaryAttachment, summaryId: UUID) -> URL { @@ -111,9 +114,11 @@ final class SummaryAttachmentStore { private func save(_ supplemental: SummarySupplementalData, summaryId: UUID) throws { let directory = storageDirectory(for: summaryId) try fileManager.createDirectory(at: directory, withIntermediateDirectories: true) + AppFileProtection.apply(to: directory) let metadataURL = metadataFileURL(for: summaryId) let data = try encoder.encode(supplemental) try data.write(to: metadataURL, options: .atomic) + AppFileProtection.apply(to: metadataURL) } private func rootDirectory() -> URL { diff --git a/BisonNotes AI/BisonNotes AI/Models/TranscriptionStarter.swift b/BisonNotes AI/BisonNotes AI/Models/TranscriptionStarter.swift new file mode 100644 index 0000000..d5f0e32 --- /dev/null +++ b/BisonNotes AI/BisonNotes AI/Models/TranscriptionStarter.swift @@ -0,0 +1,218 @@ +// +// TranscriptionStarter.swift +// BisonNotes AI +// +// Shared entry point for starting transcription from any UI surface +// (Recordings list, AudioPlayerView, Transcripts tab). Owns the serial +// audio-cleanup queue so heavy cleanup work never runs concurrently. +// + +import Foundation +import SwiftUI + +@MainActor +final class TranscriptionStarter: ObservableObject { + static let shared = TranscriptionStarter() + + /// Recordings waiting for their turn in the cleanup queue. + @Published private(set) var queuedCleanupRecordings: [RecordingEntry] = [] + /// The recording currently undergoing audio cleanup, if any. + @Published private(set) var activeCleaningRecordingId: UUID? + + private var isProcessingCleanupQueue: Bool = false + private let backgroundProcessingManager = BackgroundProcessingManager.shared + private let enhancedTranscriptionManager = EnhancedTranscriptionManager() + + private init() {} + + // MARK: - State queries + + func isCleaning(_ recordingId: UUID) -> Bool { + activeCleaningRecordingId == recordingId + } + + func isQueuedForCleanup(_ recordingId: UUID) -> Bool { + queuedCleanupRecordings.contains { $0.id == recordingId } + } + + /// True when the recording has a queued or processing transcription job in the background manager. + /// Resolves the filename directly from the stored URL string — no disk I/O — so it is safe to + /// call per row at list scale (List previously stalled on Mac Catalyst when this routed through + /// AppDataCoordinator.getAbsoluteURL, which probes FileManager and may save the Core Data context). + func hasActiveTranscriptionJob(for recording: RecordingEntry, appCoordinator: AppDataCoordinator) -> Bool { + guard let filename = filename(for: recording) else { return false } + return backgroundProcessingManager.activeJobs.contains { job in + job.recordingPath == filename && + job.type.isTranscription && + (job.status == .queued || job.status == .processing) + } + } + + /// The current status of the active transcription job for this recording, if any. + func activeTranscriptionJobStatus(for recording: RecordingEntry, appCoordinator: AppDataCoordinator) -> JobProcessingStatus? { + guard let filename = filename(for: recording) else { return nil } + return backgroundProcessingManager.activeJobs.first { job in + job.recordingPath == filename && + job.type.isTranscription && + (job.status == .queued || job.status == .processing) + }?.status + } + + /// Cheap filename derivation from the stored URL string. Handles both legacy absolute URLs and + /// relative paths. Does not touch the file system or Core Data context. + private func filename(for recording: RecordingEntry) -> String? { + guard let stored = recording.recordingURL else { return nil } + if let url = URL(string: stored), url.scheme != nil { + return url.lastPathComponent + } + return (stored as NSString).lastPathComponent + } + + // MARK: - Entry point + + /// Begin transcription for a recording. Caller is responsible for asking the user + /// whether to clean audio first; pass `cleanFirst` accordingly. + func startTranscription(for recording: RecordingEntry, + cleanFirst: Bool, + appCoordinator: AppDataCoordinator) { + guard !hasActiveTranscriptionJob(for: recording, appCoordinator: appCoordinator) else { return } + + if cleanFirst { + queuedCleanupRecordings.append(recording) + processCleanupQueueIfNeeded(appCoordinator: appCoordinator) + } else { + performEnhancedTranscription(for: recording, sourceAudioURL: nil, appCoordinator: appCoordinator) + } + } + + // MARK: - Cleanup queue (serial) + + private func processCleanupQueueIfNeeded(appCoordinator: AppDataCoordinator) { + guard !isProcessingCleanupQueue, !queuedCleanupRecordings.isEmpty else { return } + isProcessingCleanupQueue = true + + let recording = queuedCleanupRecordings.removeFirst() + activeCleaningRecordingId = recording.id + + Task { @MainActor in + defer { + activeCleaningRecordingId = nil + isProcessingCleanupQueue = false + processCleanupQueueIfNeeded(appCoordinator: appCoordinator) + } + + guard let recordingURL = appCoordinator.getAbsoluteURL(for: recording) else { + performEnhancedTranscription(for: recording, sourceAudioURL: nil, appCoordinator: appCoordinator) + return + } + do { + let tempCleanedURL = try await AudioCleanupService.shared.cleanAudio(at: recordingURL) + AppLog.shared.transcription("Cleaned audio created at temp location: \(tempCleanedURL.lastPathComponent)", level: .debug) + + // Copy cleaned file into Documents so ProcessingJob can resolve it. + guard let documentsURL = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask).first else { + AppLog.shared.transcription("Could not access Documents directory, using original file", level: .error) + performEnhancedTranscription(for: recording, sourceAudioURL: nil, appCoordinator: appCoordinator) + return + } + + let cleanedFilename = tempCleanedURL.lastPathComponent + let documentsCleanedURL = documentsURL.appendingPathComponent(cleanedFilename) + try? FileManager.default.removeItem(at: documentsCleanedURL) + try FileManager.default.copyItem(at: tempCleanedURL, to: documentsCleanedURL) + AppLog.shared.transcription("Copied cleaned audio to Documents: \(cleanedFilename)", level: .debug) + await AudioCleanupService.shared.removeTempFile(at: tempCleanedURL) + + performEnhancedTranscription(for: recording, sourceAudioURL: documentsCleanedURL, appCoordinator: appCoordinator) + } catch { + AppLog.shared.transcription("Audio cleanup failed, falling back to original: \(error)", level: .error) + performEnhancedTranscription(for: recording, sourceAudioURL: nil, appCoordinator: appCoordinator) + } + } + } + + // MARK: - Transcription start (BG manager + direct fallback) + + private func performEnhancedTranscription(for recording: RecordingEntry, + sourceAudioURL: URL?, + appCoordinator: AppDataCoordinator) { + Task { @MainActor in + let selectedEngine = TranscriptionEngine( + rawValue: UserDefaults.standard.string(forKey: "selectedTranscriptionEngine") ?? TranscriptionEngine.fluidAudio.rawValue + ) ?? .fluidAudio + + do { + guard let recordingURL = appCoordinator.getAbsoluteURL(for: recording) else { + AppLog.shared.transcription("Invalid recording URL", level: .error) + throw NSError(domain: "Transcription", code: -1, + userInfo: [NSLocalizedDescriptionKey: "Invalid recording URL"]) + } + + try await backgroundProcessingManager.startTranscriptionJob( + recordingURL: recordingURL, + recordingName: recording.recordingName ?? "Unknown Recording", + engine: selectedEngine, + sourceAudioURL: sourceAudioURL + ) + + AppLog.shared.transcription("Transcription job started through BackgroundProcessingManager") + } catch { + AppLog.shared.transcription("Failed to start transcription job: \(error)", level: .error) + + // Fallback to direct transcription if background processing fails. + AppLog.shared.transcription("Falling back to direct transcription...", level: .debug) + do { + let transcriptionURL = sourceAudioURL ?? appCoordinator.getAbsoluteURL(for: recording) + guard let transcriptionURL else { + AppLog.shared.transcription("Invalid recording URL for fallback transcription", level: .error) + return + } + + let result = try await enhancedTranscriptionManager.transcribeAudioFile(at: transcriptionURL, using: selectedEngine) + AppLog.shared.transcription("Transcription result: success=\(result.success), textLength=\(result.fullText.count)", level: .debug) + + if result.success && !result.fullText.isEmpty { + let identityURL = appCoordinator.getAbsoluteURL(for: recording) ?? transcriptionURL + let transcriptData = TranscriptData( + recordingURL: identityURL, + recordingName: recording.recordingName ?? "Unknown Recording", + recordingDate: recording.recordingDate ?? Date(), + segments: result.segments + ) + guard let recordingId = transcriptData.recordingId else { + AppLog.shared.transcription("Transcript data missing recording ID", level: .error) + return + } + let transcriptId = appCoordinator.addTranscript( + for: recordingId, + segments: transcriptData.segments, + speakerMappings: transcriptData.speakerMappings, + engine: transcriptData.engine, + processingTime: transcriptData.processingTime, + confidence: transcriptData.confidence + ) + if transcriptId != nil { + AppLog.shared.transcription("Transcript saved to Core Data with ID: \(transcriptId!)") + } else { + AppLog.shared.transcription("Failed to save transcript to Core Data", level: .error) + } + + NotificationCenter.default.post( + name: NSNotification.Name("TranscriptionCompleted"), object: nil + ) + } else { + AppLog.shared.transcription("Transcription failed or returned empty result", level: .error) + } + } catch { + AppLog.shared.transcription("Fallback transcription also failed: \(error)", level: .error) + } + + // Clean up source audio on fallback-path failure (no BG manager to do it). + if let cleanupURL = sourceAudioURL, cleanupURL.lastPathComponent.hasPrefix("cleaned_") { + try? FileManager.default.removeItem(at: cleanupURL) + AppLog.shared.transcription("Cleaned up source audio file after fallback: \(cleanupURL.lastPathComponent)", level: .debug) + } + } + } + } +} diff --git a/BisonNotes AI/BisonNotes AI/OllamaService.swift b/BisonNotes AI/BisonNotes AI/OllamaService.swift index 111c201..1b88718 100644 --- a/BisonNotes AI/BisonNotes AI/OllamaService.swift +++ b/BisonNotes AI/BisonNotes AI/OllamaService.swift @@ -332,11 +332,18 @@ class OllamaService: ObservableObject { sessionConfig.timeoutIntervalForResource = config.timeoutInterval * 2 self.session = URLSession(configuration: sessionConfig) } + + private func validateEndpoint() throws { + if let message = EndpointSecurityPolicy.validationMessage(for: config.baseURL) { + throw OllamaError.serverError(message) + } + } // MARK: - Connection Management func testConnection() async -> Bool { do { + try validateEndpoint() let url = URL(string: "\(config.baseURL)/api/tags")! AppLog.shared.networking("OllamaService: Testing connection to \(url)", level: .debug) @@ -382,6 +389,8 @@ class OllamaService: ObservableObject { guard isConnected else { throw OllamaError.notConnected } + + try validateEndpoint() let url = URL(string: "\(config.baseURL)/api/tags")! let (data, response) = try await session.data(from: url) @@ -2414,6 +2423,8 @@ class OllamaService: ObservableObject { guard isConnected else { throw OllamaError.notConnected } + + try validateEndpoint() let url = URL(string: "\(config.baseURL)/api/generate")! var request = URLRequest(url: url) @@ -2489,6 +2500,8 @@ class OllamaService: ObservableObject { guard isConnected else { throw OllamaError.notConnected } + + try validateEndpoint() let url = URL(string: "\(config.baseURL)/api/generate")! var request = URLRequest(url: url) @@ -2580,6 +2593,8 @@ class OllamaService: ObservableObject { guard isConnected else { throw OllamaError.notConnected } + + try validateEndpoint() let url = URL(string: "\(config.baseURL)/api/generate")! var request = URLRequest(url: url) diff --git a/BisonNotes AI/BisonNotes AI/OllamaSettingsView.swift b/BisonNotes AI/BisonNotes AI/OllamaSettingsView.swift index d7e3e3d..52097a4 100644 --- a/BisonNotes AI/BisonNotes AI/OllamaSettingsView.swift +++ b/BisonNotes AI/BisonNotes AI/OllamaSettingsView.swift @@ -15,6 +15,7 @@ struct OllamaSettingsView: View { @AppStorage("ollamaTemperature") private var temperature: Double = 0.1 /// Maximum context window the selected model supports @AppStorage("ollamaContextTokens") private var maxContextTokens: Int = 4096 + @AppStorage(EndpointSecurityPolicy.allowInsecurePublicEndpointsKey) private var allowInsecurePublicEndpoints: Bool = false var onConfigurationChanged: (() -> Void)? @@ -52,7 +53,7 @@ struct OllamaSettingsView: View { } var body: some View { - NavigationView { + NavigationStack { Form { headerSection serverConfigurationSection @@ -162,6 +163,8 @@ struct OllamaSettingsView: View { Text("Enter the URL of your Ollama server (e.g., http://localhost)") .font(.caption) .foregroundColor(.secondary) + + endpointSecurityWarning(for: "\(serverURL):\(port)") } } @@ -419,6 +422,20 @@ struct OllamaSettingsView: View { formatter.countStyle = .file return formatter.string(fromByteCount: bytes) } + + @ViewBuilder + private func endpointSecurityWarning(for endpoint: String) -> some View { + if let warning = EndpointSecurityPolicy.warningMessage(for: endpoint) { + Label(warning, systemImage: "exclamationmark.triangle.fill") + .font(.caption) + .foregroundColor(.orange) + } + + if EndpointSecurityPolicy.validationMessage(for: endpoint, allowInsecurePublicEndpoints: false) != nil { + Toggle("Development Mode: Allow Public HTTP", isOn: $allowInsecurePublicEndpoints) + .font(.caption) + } + } } struct OllamaSettingsView_Previews: PreviewProvider { diff --git a/BisonNotes AI/BisonNotes AI/OnDeviceAIDownloadMonitor.swift b/BisonNotes AI/BisonNotes AI/OnDeviceAIDownloadMonitor.swift index f07ed70..b5f1ec1 100644 --- a/BisonNotes AI/BisonNotes AI/OnDeviceAIDownloadMonitor.swift +++ b/BisonNotes AI/BisonNotes AI/OnDeviceAIDownloadMonitor.swift @@ -82,7 +82,9 @@ class OnDeviceAIDownloadMonitor: ObservableObject { do { _ = try await center.requestAuthorization(options: [.alert, .badge, .sound]) } catch { - AppLog.shared.summarization("[OnDeviceAIDownloadMonitor] Error requesting notification permission: \(error)", level: .error) + // On Mac Catalyst the request often fails until the user explicitly + // approves notifications in System Settings; that's not an error. + AppLog.shared.summarization("[OnDeviceAIDownloadMonitor] Notification permission request failed: \(error.localizedDescription)", level: .debug) return } } else if settings.authorizationStatus != .authorized { diff --git a/BisonNotes AI/BisonNotes AI/OnDeviceLLM/OnDeviceLLMEngine.swift b/BisonNotes AI/BisonNotes AI/OnDeviceLLM/OnDeviceLLMEngine.swift index 31d986d..c54540b 100644 --- a/BisonNotes AI/BisonNotes AI/OnDeviceLLM/OnDeviceLLMEngine.swift +++ b/BisonNotes AI/BisonNotes AI/OnDeviceLLM/OnDeviceLLMEngine.swift @@ -56,6 +56,7 @@ class OnDeviceLLMEngine: SummarizationEngine, ConnectionTestable { private let logger = Logger(subsystem: Bundle.main.bundleIdentifier ?? "com.bisonnotes.app", category: "OnDeviceLLMEngine") private var backgroundObserver: NSObjectProtocol? private var foregroundObserver: NSObjectProtocol? + private var terminateObserver: NSObjectProtocol? // MARK: - Initialization @@ -67,6 +68,7 @@ class OnDeviceLLMEngine: SummarizationEngine, ConnectionTestable { deinit { if let backgroundObserver { NotificationCenter.default.removeObserver(backgroundObserver) } if let foregroundObserver { NotificationCenter.default.removeObserver(foregroundObserver) } + if let terminateObserver { NotificationCenter.default.removeObserver(terminateObserver) } } // MARK: - Background Safety @@ -92,6 +94,21 @@ class OnDeviceLLMEngine: SummarizationEngine, ConnectionTestable { AppLog.shared.summarization("[OnDeviceLLMEngine] App entering foreground - resuming GPU inference") self?.service?.setAppBackgrounded(false) } + + // On Mac Catalyst, quitting via the menu calls NSApplication.terminate: → exit(). + // Static C++ destructors for ggml_metal_device then run before Swift deinits, which + // triggers GGML_ASSERT([rsets->data count] == 0) if any Metal command buffers are + // still in flight. Explicitly unloading the model here forces llama_free / + // llama_model_free to run before exit(), releasing all Metal resources cleanly. + terminateObserver = NotificationCenter.default.addObserver( + forName: UIApplication.willTerminateNotification, + object: nil, + queue: .main + ) { [weak self] _ in + AppLog.shared.summarization("[OnDeviceLLMEngine] App will terminate - releasing llama.cpp Metal resources") + self?.service?.unloadModel() + self?.service = nil + } } // MARK: - Configuration diff --git a/BisonNotes AI/BisonNotes AI/OnDeviceLLM/README-LLAMA-SETUP.md b/BisonNotes AI/BisonNotes AI/OnDeviceLLM/README-LLAMA-SETUP.md index 2f32a98..0311043 100644 --- a/BisonNotes AI/BisonNotes AI/OnDeviceLLM/README-LLAMA-SETUP.md +++ b/BisonNotes AI/BisonNotes AI/OnDeviceLLM/README-LLAMA-SETUP.md @@ -154,9 +154,52 @@ Track framework versions here for reference: | Date | llama.cpp Version | Notes | |------|-------------------|-------| +| 2026-05-13 | b9134 | Updated to latest release | | 2026-03-23 | b8495 | Updated from prebuilt release | | 2025-01-10 | b4000+ | Initial setup from OLMoE.swift | +### Mac Catalyst Slice Creation (Required After Each Update) + +The upstream llama.cpp xcframework does NOT include a Mac Catalyst slice. You must create it manually from the macOS arm64 binary: + +```bash +# 1. Extract the arm64 slice from the macOS fat binary +lipo -thin arm64 \ + Frameworks/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/llama \ + -output /tmp/llama-arm64 + +# 2. Create versioned macOS-style framework layout (Catalyst requires this) +CATALYST=Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework +mkdir -p $CATALYST/Versions/A/Resources +mkdir -p $CATALYST/Versions/A/Headers +mkdir -p $CATALYST/Versions/A/Modules + +# 3. Copy headers, modules, and Info.plist from the macOS framework +cp Frameworks/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/* \ + $CATALYST/Versions/A/Headers/ +cp Frameworks/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Modules/* \ + $CATALYST/Versions/A/Modules/ +cp Frameworks/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Resources/Info.plist \ + $CATALYST/Versions/A/Resources/Info.plist + +# 4. Place the thin binary +cp /tmp/llama-arm64 $CATALYST/Versions/A/llama + +# 5. Patch the Mach-O platform header from MACOS to MACCATALYST (CRITICAL) +vtool -set-build-version maccatalyst 14.0 15.5 -replace \ + -output $CATALYST/Versions/A/llama \ + $CATALYST/Versions/A/llama + +# 6. Create symlinks +ln -s A $CATALYST/Versions/Current +ln -s Versions/Current/llama $CATALYST/llama +ln -s Versions/Current/Headers $CATALYST/Headers +ln -s Versions/Current/Modules $CATALYST/Modules +ln -s Versions/Current/Resources $CATALYST/Resources +``` + +**Step 5 is critical** — without the `vtool` patch, the linker warns "built for macOS" and may fail codesigning. + --- ## Quick Reference diff --git a/BisonNotes AI/BisonNotes AI/OpenAI/OpenAISummarizationService.swift b/BisonNotes AI/BisonNotes AI/OpenAI/OpenAISummarizationService.swift index 4d6e1ba..ceefebd 100644 --- a/BisonNotes AI/BisonNotes AI/OpenAI/OpenAISummarizationService.swift +++ b/BisonNotes AI/BisonNotes AI/OpenAI/OpenAISummarizationService.swift @@ -200,6 +200,10 @@ class OpenAISummarizationService: ObservableObject { throw SummarizationError.aiServiceUnavailable(service: "API key is empty") } + if let message = EndpointSecurityPolicy.validationMessage(for: baseURL) { + throw SummarizationError.aiServiceUnavailable(service: message) + } + guard let url = URL(string: "\(baseURL)/models") else { throw SummarizationError.aiServiceUnavailable(service: "Invalid base URL") } @@ -238,6 +242,10 @@ class OpenAISummarizationService: ObservableObject { normalizedBaseURL.removeLast() } + if let message = EndpointSecurityPolicy.validationMessage(for: normalizedBaseURL) { + throw SummarizationError.aiServiceUnavailable(service: message) + } + guard let url = URL(string: "\(normalizedBaseURL)/models") else { throw SummarizationError.aiServiceUnavailable(service: "Invalid base URL: \(baseURL)") } @@ -335,6 +343,11 @@ class OpenAISummarizationService: ObservableObject { throw SummarizationError.aiServiceUnavailable(service: "OpenAI API key format is invalid") } + if let message = EndpointSecurityPolicy.validationMessage(for: config.baseURL) { + AppLog.shared.networking("Blocked insecure OpenAI endpoint: \(config.baseURL)", level: .error) + throw SummarizationError.aiServiceUnavailable(service: message) + } + AppLog.shared.networking("OpenAI API Configuration - Model: \(config.effectiveModelId), BaseURL: \(config.baseURL)", level: .debug) guard let url = URL(string: "\(config.baseURL)/chat/completions") else { @@ -394,4 +407,4 @@ class OpenAISummarizationService: ObservableObject { throw SummarizationError.aiServiceUnavailable(service: "OpenAI API request failed: \(error.localizedDescription)") } } -} \ No newline at end of file +} diff --git a/BisonNotes AI/BisonNotes AI/OpenAISettingsView.swift b/BisonNotes AI/BisonNotes AI/OpenAISettingsView.swift index a1f1057..93bd7cc 100644 --- a/BisonNotes AI/BisonNotes AI/OpenAISettingsView.swift +++ b/BisonNotes AI/BisonNotes AI/OpenAISettingsView.swift @@ -8,9 +8,10 @@ import SwiftUI struct OpenAISettingsView: View { - @AppStorage("openAIAPIKey") private var apiKey: String = "" + @SecureStorage(KeychainSecretStore.openAIAPIKey) private var apiKey: String = "" @AppStorage("openAIModel") private var selectedModel: String = OpenAITranscribeModel.whisper1.rawValue @AppStorage("openAIBaseURL") private var baseURL: String = "https://api.openai.com/v1" + @AppStorage(EndpointSecurityPolicy.allowInsecurePublicEndpointsKey) private var allowInsecurePublicEndpoints: Bool = false @State private var isTestingConnection = false @State private var connectionTestResult: String = "" @@ -25,7 +26,7 @@ struct OpenAISettingsView: View { } var body: some View { - NavigationView { + NavigationStack { Form { Section { VStack(alignment: .leading, spacing: 8) { @@ -105,6 +106,8 @@ struct OpenAISettingsView: View { Text("Use default OpenAI URL or a compatible API endpoint") .font(.caption) .foregroundColor(.secondary) + + endpointSecurityWarning(for: baseURL) } } header: { Text("API Configuration") @@ -235,6 +238,20 @@ struct OpenAISettingsView: View { baseURL = "https://api.openai.com/v1" showingConnectionResult = false } + + @ViewBuilder + private func endpointSecurityWarning(for endpoint: String) -> some View { + if let warning = EndpointSecurityPolicy.warningMessage(for: endpoint) { + Label(warning, systemImage: "exclamationmark.triangle.fill") + .font(.caption) + .foregroundColor(.orange) + } + + if EndpointSecurityPolicy.validationMessage(for: endpoint, allowInsecurePublicEndpoints: false) != nil { + Toggle("Development Mode: Allow Public HTTP", isOn: $allowInsecurePublicEndpoints) + .font(.caption) + } + } } struct FeatureRow: View { @@ -269,4 +286,4 @@ struct OpenAISettingsView_Previews: PreviewProvider { static var previews: some View { OpenAISettingsView() } -} \ No newline at end of file +} diff --git a/BisonNotes AI/BisonNotes AI/OpenAISummarizationEngine.swift b/BisonNotes AI/BisonNotes AI/OpenAISummarizationEngine.swift index ade0098..273d684 100644 --- a/BisonNotes AI/BisonNotes AI/OpenAISummarizationEngine.swift +++ b/BisonNotes AI/BisonNotes AI/OpenAISummarizationEngine.swift @@ -22,7 +22,7 @@ class OpenAISummarizationEngine: SummarizationEngine, ConnectionTestable { var isAvailable: Bool { // Check if API key is configured (unified with transcription) - let apiKey = UserDefaults.standard.string(forKey: "openAIAPIKey") ?? "" + let apiKey = KeychainSecretStore.shared.string(forKey: KeychainSecretStore.openAIAPIKey) ?? "" guard !apiKey.isEmpty else { // Only log if verbose logging is enabled if PerformanceOptimizer.shouldLogEngineAvailabilityChecks() { @@ -184,7 +184,7 @@ class OpenAISummarizationEngine: SummarizationEngine, ConnectionTestable { // MARK: - Configuration Management private func updateConfiguration() { - let apiKey = UserDefaults.standard.string(forKey: "openAIAPIKey") ?? "" + let apiKey = KeychainSecretStore.shared.string(forKey: KeychainSecretStore.openAIAPIKey) ?? "" let modelString = UserDefaults.standard.string(forKey: "openAISummarizationModel") ?? OpenAISummarizationModel.gpt41Mini.rawValue let baseURL = UserDefaults.standard.string(forKey: "openAISummarizationBaseURL") ?? "https://api.openai.com/v1" let temperature = UserDefaults.standard.double(forKey: "openAISummarizationTemperature") @@ -409,7 +409,7 @@ class OpenAICompatibleEngine: SummarizationEngine, ConnectionTestable { var isAvailable: Bool { // Check if API key is configured - let apiKey = UserDefaults.standard.string(forKey: "openAICompatibleAPIKey") ?? "" + let apiKey = KeychainSecretStore.shared.string(forKey: KeychainSecretStore.openAICompatibleAPIKey) ?? "" guard !apiKey.isEmpty else { // Only log if verbose logging is enabled if PerformanceOptimizer.shouldLogEngineAvailabilityChecks() { @@ -559,7 +559,7 @@ class OpenAICompatibleEngine: SummarizationEngine, ConnectionTestable { // MARK: - Configuration Management private func updateConfiguration() { - let apiKey = UserDefaults.standard.string(forKey: "openAICompatibleAPIKey") ?? "" + let apiKey = KeychainSecretStore.shared.string(forKey: KeychainSecretStore.openAICompatibleAPIKey) ?? "" let modelId = UserDefaults.standard.string(forKey: "openAICompatibleModel") ?? "gpt-3.5-turbo" let baseURL = UserDefaults.standard.string(forKey: "openAICompatibleBaseURL") ?? "https://api.openai.com/v1" let temperature = UserDefaults.standard.double(forKey: "openAICompatibleTemperature") diff --git a/BisonNotes AI/BisonNotes AI/OpenAISummarizationSettingsView.swift b/BisonNotes AI/BisonNotes AI/OpenAISummarizationSettingsView.swift index 552414a..1c53e78 100644 --- a/BisonNotes AI/BisonNotes AI/OpenAISummarizationSettingsView.swift +++ b/BisonNotes AI/BisonNotes AI/OpenAISummarizationSettingsView.swift @@ -8,12 +8,13 @@ import SwiftUI struct OpenAISummarizationSettingsView: View { - @AppStorage("openAIAPIKey") private var apiKey: String = "" + @SecureStorage(KeychainSecretStore.openAIAPIKey) private var apiKey: String = "" @AppStorage("openAISummarizationModel") private var selectedModel: String = OpenAISummarizationModel.gpt41Mini.rawValue @AppStorage("openAISummarizationBaseURL") private var baseURL: String = "https://api.openai.com/v1" @AppStorage("openAISummarizationTemperature") private var temperature: Double = 0.1 @AppStorage("openAISummarizationMaxTokens") private var maxTokens: Int = 0 @AppStorage("enableOpenAI") private var enableOpenAI: Bool = true + @AppStorage(EndpointSecurityPolicy.allowInsecurePublicEndpointsKey) private var allowInsecurePublicEndpoints: Bool = false @State private var isTestingConnection = false @State private var connectionTestResult: String = "" @@ -29,7 +30,7 @@ struct OpenAISummarizationSettingsView: View { } var body: some View { - NavigationView { + NavigationStack { Form { authenticationSection apiConfigurationSection @@ -114,6 +115,8 @@ struct OpenAISummarizationSettingsView: View { Text("Default: https://api.openai.com/v1") .font(.caption) .foregroundColor(.secondary) + + endpointSecurityWarning(for: baseURL) } } header: { Text("API Configuration") @@ -240,12 +243,26 @@ struct OpenAISummarizationSettingsView: View { maxTokens = 0 showingConnectionResult = false } + + @ViewBuilder + private func endpointSecurityWarning(for endpoint: String) -> some View { + if let warning = EndpointSecurityPolicy.warningMessage(for: endpoint) { + Label(warning, systemImage: "exclamationmark.triangle.fill") + .font(.caption) + .foregroundColor(.orange) + } + + if EndpointSecurityPolicy.validationMessage(for: endpoint, allowInsecurePublicEndpoints: false) != nil { + Toggle("Development Mode: Allow Public HTTP", isOn: $allowInsecurePublicEndpoints) + .font(.caption) + } + } } // MARK: - OpenAI API Compatible Settings View struct OpenAICompatibleSettingsView: View { - @AppStorage("openAICompatibleAPIKey") private var apiKey: String = "" + @SecureStorage(KeychainSecretStore.openAICompatibleAPIKey) private var apiKey: String = "" @AppStorage("openAICompatibleModel") private var selectedModel: String = "gpt-4o" @AppStorage("openAICompatibleBaseURL") private var baseURL: String = "" @AppStorage("openAICompatibleTemperature") private var temperature: Double = 0.1 @@ -253,6 +270,7 @@ struct OpenAICompatibleSettingsView: View { @AppStorage("enableOpenAICompatible") private var enableOpenAICompatible: Bool = false @AppStorage("openAICompatibleManualFormatOverride") private var manualFormatOverride: Bool = false @AppStorage("openAICompatibleManualFormat") private var manualFormat: String = "string" + @AppStorage(EndpointSecurityPolicy.allowInsecurePublicEndpointsKey) private var allowInsecurePublicEndpoints: Bool = false @State private var isTestingConnection = false @State private var connectionTestResult: String = "" @@ -392,7 +410,7 @@ struct OpenAICompatibleSettingsView: View { } var body: some View { - NavigationView { + NavigationStack { Form { compatibilityGuideSection authenticationSection @@ -647,6 +665,8 @@ struct OpenAICompatibleSettingsView: View { .font(.caption) .foregroundColor(.secondary) } + + endpointSecurityWarning(for: baseURL) } } header: { Text("API Configuration") @@ -1026,6 +1046,20 @@ struct OpenAICompatibleSettingsView: View { Text("Features & Capabilities") } } + + @ViewBuilder + private func endpointSecurityWarning(for endpoint: String) -> some View { + if let warning = EndpointSecurityPolicy.warningMessage(for: endpoint) { + Label(warning, systemImage: "exclamationmark.triangle.fill") + .font(.caption) + .foregroundColor(.orange) + } + + if EndpointSecurityPolicy.validationMessage(for: endpoint, allowInsecurePublicEndpoints: false) != nil { + Toggle("Development Mode: Allow Public HTTP", isOn: $allowInsecurePublicEndpoints) + .font(.caption) + } + } } diff --git a/BisonNotes AI/BisonNotes AI/OpenAITranscribeService.swift b/BisonNotes AI/BisonNotes AI/OpenAITranscribeService.swift index 2fa20fe..b3e5cc3 100644 --- a/BisonNotes AI/BisonNotes AI/OpenAITranscribeService.swift +++ b/BisonNotes AI/BisonNotes AI/OpenAITranscribeService.swift @@ -171,6 +171,10 @@ class OpenAITranscribeService: NSObject, ObservableObject { guard !config.apiKey.isEmpty else { throw OpenAITranscribeError.configurationMissing } + + if let message = EndpointSecurityPolicy.validationMessage(for: config.baseURL) { + throw OpenAITranscribeError.invalidResponse(message) + } // Test the API key by making a simple request to the models endpoint let testURL = URL(string: "\(config.baseURL)/models")! @@ -303,6 +307,10 @@ class OpenAITranscribeService: NSObject, ObservableObject { private func performTranscription(audioData: Data, fileName: String) async throws -> OpenAITranscribeResult { let startTime = Date() + + if let message = EndpointSecurityPolicy.validationMessage(for: config.baseURL) { + throw OpenAITranscribeError.invalidResponse(message) + } // Create multipart form data let boundary = UUID().uuidString diff --git a/BisonNotes AI/BisonNotes AI/PerformanceOptimizer.swift b/BisonNotes AI/BisonNotes AI/PerformanceOptimizer.swift index 5e62cc7..e5bbfd4 100644 --- a/BisonNotes AI/BisonNotes AI/PerformanceOptimizer.swift +++ b/BisonNotes AI/BisonNotes AI/PerformanceOptimizer.swift @@ -130,33 +130,49 @@ class PerformanceOptimizer: ObservableObject, Sendable { // MARK: - Battery Monitoring private func startBatteryMonitoring() { + #if targetEnvironment(macCatalyst) + // No battery API on Mac — leave default battery info (treated as full/charged) + // and skip the polling timer to avoid "Error retrieving battery status" logs. + return + #else UIDevice.current.isBatteryMonitoringEnabled = true - + batteryMonitorTimer = Timer.scheduledTimer(withTimeInterval: 30.0, repeats: true) { [weak self] _ in Task { @MainActor in await self?.updateBatteryInfo() } } - + // Initial battery info update Task { @MainActor in await updateBatteryInfo() } + #endif } - + private func updateBatteryInfo() async { + #if targetEnvironment(macCatalyst) + // Battery state is not available on Mac; only respect Low Power Mode. + let batteryInfo = BatteryInfo( + level: 1.0, + state: .unknown, + isLowPowerMode: ProcessInfo.processInfo.isLowPowerModeEnabled + ) + self.batteryInfo = batteryInfo + await adjustOptimizationLevel() + #else let device = UIDevice.current let batteryInfo = BatteryInfo( level: device.batteryLevel, state: device.batteryState, isLowPowerMode: ProcessInfo.processInfo.isLowPowerModeEnabled ) - + self.batteryInfo = batteryInfo - + // Adjust optimization level based on battery state await adjustOptimizationLevel() - + #endif } private func adjustOptimizationLevel() async { @@ -462,12 +478,14 @@ class PerformanceOptimizer: ObservableObject, Sendable { memoryMonitorTimer?.invalidate() batteryMonitorTimer?.invalidate() optimizationTimer?.invalidate() - + memoryMonitorTimer = nil batteryMonitorTimer = nil optimizationTimer = nil - + + #if !targetEnvironment(macCatalyst) UIDevice.current.isBatteryMonitoringEnabled = false + #endif } // MARK: - Chunked Processing @@ -923,7 +941,7 @@ struct PerformanceMonitorView: View { @Binding var isPresented: Bool var body: some View { - NavigationView { + NavigationStack { ScrollView { VStack(spacing: 20) { // Memory Usage Section diff --git a/BisonNotes AI/BisonNotes AI/Persistence.swift b/BisonNotes AI/BisonNotes AI/Persistence.swift index b76f92d..5bdd724 100644 --- a/BisonNotes AI/BisonNotes AI/Persistence.swift +++ b/BisonNotes AI/BisonNotes AI/Persistence.swift @@ -57,6 +57,12 @@ struct PersistenceController { */ fatalError("Unresolved error \(error), \(error.userInfo)") } + + if let storeURL = storeDescription.url, !inMemory { + AppFileProtection.apply(to: storeURL) + AppFileProtection.apply(to: URL(fileURLWithPath: storeURL.path + "-wal")) + AppFileProtection.apply(to: URL(fileURLWithPath: storeURL.path + "-shm")) + } }) container.viewContext.automaticallyMergesChangesFromParent = true } diff --git a/BisonNotes AI/BisonNotes AI/ShareImportAuthorization.swift b/BisonNotes AI/BisonNotes AI/ShareImportAuthorization.swift new file mode 100644 index 0000000..f4e0229 --- /dev/null +++ b/BisonNotes AI/BisonNotes AI/ShareImportAuthorization.swift @@ -0,0 +1,68 @@ +// +// ShareImportAuthorization.swift +// BisonNotes AI +// + +import Foundation + +enum ShareImportAuthorization { + static let tokenFileName = ".share-import-token" + + static func isShareImportURL(_ url: URL) -> Bool { + url.scheme?.lowercased() == "bisonnotes" + && url.host?.lowercased() == "share-import" + } + + static func token(from url: URL) -> String? { + guard let components = URLComponents(url: url, resolvingAgainstBaseURL: false) else { + return nil + } + + return components.queryItems?.first(where: { $0.name == "token" })?.value + } + + static func consumeURLToken(from url: URL, in inboxURL: URL) -> Bool { + guard isShareImportURL(url), + let candidate = token(from: url), + isValidToken(candidate), + let stored = storedToken(in: inboxURL), + candidate == stored else { + return false + } + + removeToken(in: inboxURL) + return true + } + + static func consumePendingToken(in inboxURL: URL) -> Bool { + guard let stored = storedToken(in: inboxURL), isValidToken(stored) else { + return false + } + + removeToken(in: inboxURL) + return true + } + + static func removeToken(in inboxURL: URL) { + try? FileManager.default.removeItem(at: tokenFileURL(in: inboxURL)) + } + + static func tokenFileURL(in inboxURL: URL) -> URL { + inboxURL.appendingPathComponent(tokenFileName, isDirectory: false) + } + + private static func storedToken(in inboxURL: URL) -> String? { + let tokenURL = tokenFileURL(in: inboxURL) + guard let data = try? Data(contentsOf: tokenURL), + let token = String(data: data, encoding: .utf8)?.trimmingCharacters(in: .whitespacesAndNewlines), + !token.isEmpty else { + return nil + } + + return token + } + + private static func isValidToken(_ token: String) -> Bool { + UUID(uuidString: token) != nil + } +} diff --git a/BisonNotes AI/BisonNotes AI/SummariesView.swift b/BisonNotes AI/BisonNotes AI/SummariesView.swift index a6bf0d7..df0f812 100644 --- a/BisonNotes AI/BisonNotes AI/SummariesView.swift +++ b/BisonNotes AI/BisonNotes AI/SummariesView.swift @@ -327,7 +327,42 @@ struct SummariesView: View { // MARK: - Date Filter Sheet private var dateFilterSheet: some View { - NavigationView { + #if targetEnvironment(macCatalyst) + VStack(spacing: 0) { + HStack { + Button("Cancel") { showDateFilter = false } + Spacer() + Text("Filter by Date").font(.headline) + Spacer() + Button("Apply") { + isDateFilterActive = true + showDateFilter = false + refreshTrigger.toggle() + } + .fontWeight(.semibold) + } + .padding(.horizontal, 16).padding(.vertical, 10) + Divider() + Form { + Section { + DatePicker("From", selection: $dateFilterStart, in: ...Date(), displayedComponents: .date) + DatePicker("To", selection: $dateFilterEnd, in: dateFilterStart...Date(), displayedComponents: .date) + } + if isDateFilterActive { + Section { + Button(role: .destructive) { + isDateFilterActive = false + showDateFilter = false + refreshTrigger.toggle() + } label: { + HStack { Spacer(); Text("Clear Filter"); Spacer() } + } + } + } + } + } + #else + NavigationStack { Form { Section { DatePicker("From", selection: $dateFilterStart, in: ...Date(), displayedComponents: .date) @@ -367,6 +402,7 @@ struct SummariesView: View { } } } + #endif } // MARK: - Recordings List View diff --git a/BisonNotes AI/BisonNotes AI/SummaryDetailView.swift b/BisonNotes AI/BisonNotes AI/SummaryDetailView.swift index 463833a..5624679 100644 --- a/BisonNotes AI/BisonNotes AI/SummaryDetailView.swift +++ b/BisonNotes AI/BisonNotes AI/SummaryDetailView.swift @@ -33,7 +33,6 @@ struct SummaryDetailView: View { @Environment(\.dismiss) private var dismiss @EnvironmentObject var appCoordinator: AppDataCoordinator @State private var locationAddress: String? - @State private var expandedSections: Set = ["summary"] @ObservedObject private var processingManager = BackgroundProcessingManager.shared @State private var isRegenerating = false @State private var regeneratingJobId: UUID? @@ -108,92 +107,51 @@ struct SummaryDetailView: View { } var body: some View { - NavigationView { - content - .navigationTitle("Summary") - .navigationBarTitleDisplayMode(.inline) - .toolbar { - ToolbarItem(placement: .topBarLeading) { - Button { - showingExportFormatPicker = true - } label: { - HStack(spacing: 4) { - if isExporting { - ProgressView() - .scaleEffect(0.8) - if let activeExportFormat { - Text("Exporting \(activeExportFormat.displayName)...") - .font(.caption) - } else { - Text("Exporting...") - .font(.caption) - } - } else { - Image(systemName: "square.and.arrow.up") - Text("Export") - .font(.caption) - } + // NavigationStack { Form } is the only sheet pattern that scrolls reliably + // on Mac Catalyst. See feedback_mac_catalyst_scrollview.md. + NavigationStack { + Form { + Section { locationSection } + Section { headerSection } + Section { summarySection } + Section { tasksSection } + Section { remindersSection } + Section { titlesSection } + Section { attachmentsSection } + Section { dateTimeEditorSection } + Section { metadataSection } + Section { regenerateSection } + } + .navigationTitle("Summary") + .navigationBarTitleDisplayMode(.inline) + .toolbar { + ToolbarItem(placement: .navigationBarLeading) { + Button { + showingExportFormatPicker = true + } label: { + HStack(spacing: 4) { + if isExporting { + ProgressView().scaleEffect(0.8) + Text(activeExportFormat.map { "Exporting \($0.displayName)..." } ?? "Exporting...") + .font(.caption) + } else { + Image(systemName: "square.and.arrow.up") + Text("Export").font(.caption) } } - .disabled(isExporting) - } - - ToolbarItem(placement: .topBarTrailing) { - Button("Done") { - dismiss() - } } + .disabled(isExporting) } + ToolbarItem(placement: .navigationBarTrailing) { + Button("Done") { dismiss() } + } + } } .configurationWarnings( showingTranscriptionWarning: .constant(false), showingAIWarning: $showingAIWarning, - onSettingsRequested: { - // Navigate to settings - you might want to implement navigation to AI settings - // For now, just dismiss the alert - } + onSettingsRequested: {} ) - } - - private var content: some View { - VStack(spacing: 0) { - // Location Section - Shows map or add location option - locationSection - - // Enhanced Summary Content - ScrollView([.vertical], showsIndicators: true) { - VStack(alignment: .leading, spacing: 16) { - // Header Section - headerSection - - // Summary Section (Expandable) - summarySection - - // Tasks Section (Expandable) - tasksSection - - // Reminders Section (Expandable) - remindersSection - - // Titles Section (Expandable) - titlesSection - - // Attachments + Note Section - attachmentsSection - - // Date/Time Editor Section - dateTimeEditorSection - - // Metadata Section (Expandable, moved to bottom) - metadataSection - - // Regenerate Button Section - regenerateSection - } - .padding(.vertical) - .padding(.horizontal, 16) // Apple's recommended margin for text readability - } - } .onAppear { // Debug summary data being displayed AppLog.shared.summarization("SummaryDetailView opened: summary=\(summaryData.summary.count) chars, tasks=\(summaryData.tasks.count), reminders=\(summaryData.reminders.count), titles=\(summaryData.titles.count)", level: .debug) @@ -321,35 +279,34 @@ struct SummaryDetailView: View { handleAttachmentImport(result) } .sheet(isPresented: $showingTextAttachment) { - NavigationView { - ScrollView { - Text(selectedAttachmentText) - .font(.body.monospaced()) - .frame(maxWidth: .infinity, alignment: .leading) - .padding() + NavigationStack { + Form { + Section { + Text(selectedAttachmentText) + .font(.body.monospaced()) + .textSelection(.enabled) + } } .navigationTitle(selectedAttachmentName) .navigationBarTitleDisplayMode(.inline) .toolbar { - ToolbarItem(placement: .topBarTrailing) { - Button("Done") { - showingTextAttachment = false - } + ToolbarItem(placement: .navigationBarTrailing) { + Button("Done") { showingTextAttachment = false } } } } } .sheet(isPresented: $showingPDFAttachment) { if let selectedAttachmentPDFURL { - NavigationView { + // SummaryAttachmentPDFView wraps PDFKit which handles its own + // scrolling natively, so a NavigationStack toolbar is enough. + NavigationStack { SummaryAttachmentPDFView(url: selectedAttachmentPDFURL) .navigationTitle(selectedAttachmentName) .navigationBarTitleDisplayMode(.inline) .toolbar { - ToolbarItem(placement: .topBarTrailing) { - Button("Done") { - showingPDFAttachment = false - } + ToolbarItem(placement: .navigationBarTrailing) { + Button("Done") { showingPDFAttachment = false } } } } @@ -667,11 +624,8 @@ struct SummaryDetailView: View { metadataRow(title: "Processing Time", value: formattedProcessingTime(summaryData.processingTime), icon: "timer") } } - .onTapGesture { - toggleSection("metadata") - } } - + private func metadataRow(title: String, value: String, icon: String, valueColor: Color = .primary) -> some View { HStack { Image(systemName: icon) @@ -711,9 +665,6 @@ struct SummaryDetailView: View { .padding(.top, 4) .textSelection(.enabled) } - .onTapGesture { - toggleSection("summary") - } } // MARK: - Tasks Section @@ -745,11 +696,8 @@ struct SummaryDetailView: View { .padding(.top, 4) } } - .onTapGesture { - toggleSection("tasks") - } } - + // MARK: - Reminders Section private var remindersSection: some View { @@ -779,11 +727,8 @@ struct SummaryDetailView: View { .padding(.top, 4) } } - .onTapGesture { - toggleSection("reminders") - } } - + // MARK: - Titles Section private var titlesSection: some View { @@ -868,9 +813,6 @@ struct SummaryDetailView: View { } } } - .onTapGesture { - toggleSection("titles") - } .sheet(isPresented: $showingTitleSelector) { TitleSelectorView( titles: summaryData.titles, @@ -1091,6 +1033,7 @@ struct SummaryDetailView: View { .background(isRegenerating ? Color.gray : Color.orange) .cornerRadius(10) } + .buttonStyle(.borderless) .disabled(isRegenerating) } @@ -1159,6 +1102,7 @@ struct SummaryDetailView: View { .background(isUpdatingLocation ? Color.gray : Color.green) .cornerRadius(10) } + .buttonStyle(.borderless) .disabled(isUpdatingLocation) } } else { @@ -1168,7 +1112,7 @@ struct SummaryDetailView: View { .font(.caption) .foregroundColor(.secondary) .multilineTextAlignment(.center) - + Button(action: { showingLocationPicker = true }) { @@ -1189,6 +1133,7 @@ struct SummaryDetailView: View { .background(isUpdatingLocation ? Color.gray : Color.green) .cornerRadius(10) } + .buttonStyle(.borderless) .disabled(isUpdatingLocation) } } @@ -1228,6 +1173,7 @@ struct SummaryDetailView: View { .background(Color.red) .cornerRadius(10) } + .buttonStyle(.borderless) } } } @@ -1342,14 +1288,6 @@ struct SummaryDetailView: View { .padding(.top, 4) } - private func toggleSection(_ section: String) { - if expandedSections.contains(section) { - expandedSections.remove(section) - } else { - expandedSections.insert(section) - } - } - private func formatDate(_ date: Date) -> String { return UserPreferences.shared.formatShortDateTime(date) } @@ -1683,34 +1621,10 @@ struct SummaryDetailView: View { private func rebuildSummaryData(userNotes: String?, attachments: [SummaryAttachment]) -> EnhancedSummaryData { let normalizedNotes = userNotes?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty == true ? nil : userNotes - if let recordingId = summaryData.recordingId { - return EnhancedSummaryData( - id: summaryData.id, - recordingId: recordingId, - transcriptId: summaryData.transcriptId, - recordingURL: summaryData.recordingURL, - recordingName: summaryData.recordingName, - recordingDate: summaryData.recordingDate, - summary: summaryData.summary, - tasks: summaryData.tasks, - reminders: summaryData.reminders, - titles: summaryData.titles, - attachments: attachments, - userNotes: normalizedNotes, - contentType: summaryData.contentType, - aiEngine: summaryData.aiEngine, - aiModel: summaryData.aiModel, - originalLength: summaryData.originalLength, - processingTime: summaryData.processingTime, - generatedAt: summaryData.generatedAt, - version: summaryData.version, - wordCount: summaryData.wordCount, - compressionRatio: summaryData.compressionRatio, - confidence: summaryData.confidence - ) - } - return EnhancedSummaryData( + id: summaryData.id, + recordingId: summaryData.recordingId, + transcriptId: summaryData.transcriptId, recordingURL: summaryData.recordingURL, recordingName: summaryData.recordingName, recordingDate: summaryData.recordingDate, @@ -1724,7 +1638,12 @@ struct SummaryDetailView: View { aiEngine: summaryData.aiEngine, aiModel: summaryData.aiModel, originalLength: summaryData.originalLength, - processingTime: summaryData.processingTime + processingTime: summaryData.processingTime, + generatedAt: summaryData.generatedAt, + version: summaryData.version, + wordCount: summaryData.wordCount, + compressionRatio: summaryData.compressionRatio, + confidence: summaryData.confidence ) } @@ -2149,7 +2068,7 @@ struct TitleSelectorView: View { @State private var showingCustomTitleField = false var body: some View { - NavigationView { + NavigationStack { VStack(spacing: 0) { // Header VStack(alignment: .leading, spacing: 8) { @@ -2341,7 +2260,7 @@ struct DateTimeEditorView: View { } var body: some View { - NavigationView { + NavigationStack { VStack(spacing: 0) { // Header VStack(alignment: .leading, spacing: 8) { @@ -2532,7 +2451,7 @@ struct LocationPickerView: View { @State private var searchTask: Task? var body: some View { - NavigationView { + NavigationStack { VStack(spacing: 0) { // Header VStack(alignment: .leading, spacing: 8) { @@ -3549,7 +3468,7 @@ private struct NoteEditorSheet: View { @FocusState private var isFocused: Bool var body: some View { - NavigationView { + NavigationStack { VStack(spacing: 0) { TextEditor(text: $draft) .focused($isFocused) diff --git a/BisonNotes AI/BisonNotes AI/SummaryManager.swift b/BisonNotes AI/BisonNotes AI/SummaryManager.swift index 5563318..fb1d7a6 100644 --- a/BisonNotes AI/BisonNotes AI/SummaryManager.swift +++ b/BisonNotes AI/BisonNotes AI/SummaryManager.swift @@ -154,7 +154,7 @@ class SummaryManager: ObservableObject { func getEnhancedSummary(for recordingURL: URL) -> EnhancedSummaryData? { // Only log if verbose logging is enabled if PerformanceOptimizer.shouldLogEngineInitialization() { - AppLog.shared.summarization("Looking for enhanced summary with URL: \(recordingURL)", level: .debug) + AppLog.shared.summarization("Looking for enhanced summary with file: \(recordingURL.lastPathComponent)", level: .debug) AppLog.shared.summarization("Total enhanced summaries: \(enhancedSummaries.count)", level: .debug) } @@ -1987,7 +1987,7 @@ class SummaryManager: ObservableObject { // Get the recording from Core Data using the coordinator guard let recordingEntry = coordinator.getRecording(url: recordingURL), let recordingId = recordingEntry.id else { - AppLog.shared.summarization("Could not find recording in Core Data for URL: \(recordingURL)", level: .error) + AppLog.shared.summarization("Could not find recording in Core Data for file: \(recordingURL.lastPathComponent)", level: .error) return } diff --git a/BisonNotes AI/BisonNotes AI/TranscriptImportManager.swift b/BisonNotes AI/BisonNotes AI/TranscriptImportManager.swift index f876153..c8f1871 100644 --- a/BisonNotes AI/BisonNotes AI/TranscriptImportManager.swift +++ b/BisonNotes AI/BisonNotes AI/TranscriptImportManager.swift @@ -578,6 +578,7 @@ class TranscriptImportManager: NSObject, ObservableObject { // Create silent audio file programmatically (no microphone needed) try await createSilentAudioFile(at: fileURL) + AppFileProtection.apply(to: fileURL) // Verify file was created guard FileManager.default.fileExists(atPath: fileURL.path) else { diff --git a/BisonNotes AI/BisonNotes AI/TranscriptionSettingsView.swift b/BisonNotes AI/BisonNotes AI/TranscriptionSettingsView.swift index a24fa5e..1131ea4 100644 --- a/BisonNotes AI/BisonNotes AI/TranscriptionSettingsView.swift +++ b/BisonNotes AI/BisonNotes AI/TranscriptionSettingsView.swift @@ -23,7 +23,7 @@ struct TranscriptionSettingsView: View { @Environment(\.dismiss) private var dismiss var body: some View { - NavigationView { + NavigationStack { Form { liveTranscriptionSection fileTranscriptionSection @@ -48,9 +48,22 @@ struct TranscriptionSettingsView: View { WhisperSettingsView() } .sheet(isPresented: $showingFluidAudioSettings) { + #if targetEnvironment(macCatalyst) + VStack(spacing: 0) { + HStack { + Text("On Device Transcription").font(.headline) + Spacer() + Button("Done") { showingFluidAudioSettings = false }.buttonStyle(.bordered) + } + .padding(.horizontal, 16).padding(.vertical, 10) + Divider() + FluidAudioSettingsView() + } + #else NavigationStack { FluidAudioSettingsView() } + #endif } .sheet(isPresented: $showingOpenAISettings) { OpenAISettingsView() diff --git a/BisonNotes AI/BisonNotes AI/ViewModels/AudioRecorderViewModel+Background.swift b/BisonNotes AI/BisonNotes AI/ViewModels/AudioRecorderViewModel+Background.swift index a18bcc4..b758b33 100644 --- a/BisonNotes AI/BisonNotes AI/ViewModels/AudioRecorderViewModel+Background.swift +++ b/BisonNotes AI/BisonNotes AI/ViewModels/AudioRecorderViewModel+Background.swift @@ -13,24 +13,39 @@ import UIKit extension AudioRecorderViewModel { func beginBackgroundTask() { + #if targetEnvironment(macCatalyst) + // Mac apps don't get suspended like iOS — recording stays alive while the + // app process is running. Skip the iOS background-task lifecycle to avoid + // the "Background Task ... was created over 30 seconds ago" warning. + return + #else guard backgroundTask == .invalid else { return } AppLog.shared.backgroundProcessing("Starting background task for recording") backgroundTask = UIApplication.shared.beginBackgroundTask(withName: "Recording") { [weak self] in AppLog.shared.backgroundProcessing("Recording background task expiring", level: .error) self?.endBackgroundTask() } + #endif } func endBackgroundTask() { + #if targetEnvironment(macCatalyst) + return + #else guard backgroundTask != .invalid else { return } AppLog.shared.backgroundProcessing("Ending recording background task") UIApplication.shared.endBackgroundTask(backgroundTask) backgroundTask = .invalid + #endif } // MARK: - Background Time Monitoring func startBackgroundTimeMonitoring() { + #if targetEnvironment(macCatalyst) + // No iOS-style background time limit on Mac — skip polling. + return + #else backgroundTimeMonitor?.invalidate() // Check remaining background time every 30 seconds @@ -53,6 +68,7 @@ extension AudioRecorderViewModel { } } } + #endif } func stopBackgroundTimeMonitoring() { diff --git a/BisonNotes AI/BisonNotes AI/ViewModels/AudioRecorderViewModel+CatalystEngine.swift b/BisonNotes AI/BisonNotes AI/ViewModels/AudioRecorderViewModel+CatalystEngine.swift new file mode 100644 index 0000000..dd020f1 --- /dev/null +++ b/BisonNotes AI/BisonNotes AI/ViewModels/AudioRecorderViewModel+CatalystEngine.swift @@ -0,0 +1,167 @@ +// +// AudioRecorderViewModel+CatalystEngine.swift +// BisonNotes AI +// +// Mac Catalyst recording uses AVAudioEngine + AVAudioFile because +// AVAudioRecorder cannot reliably set up its AAC/PCM converter without an +// AVAudioSession. The engine taps the input node directly, AVAudioFile +// writes AAC/M4A using the buffer's known format, and pause/resume work +// by removing and re-installing the tap on a single file (no segments). +// + +#if targetEnvironment(macCatalyst) + +import Foundation +@preconcurrency import AVFoundation + +extension AudioRecorderViewModel { + + /// Start recording on Mac Catalyst using AVAudioEngine. Writes AAC into a + /// single .m4a file. Throws on setup failure so the caller can surface an + /// error to the user. + func startCatalystEngineRecording(at url: URL) throws { + // Tear down any leftover engine state from a previous run. + stopCatalystEngineRecording(closingFile: false) + + // AVAudioEngine input on Mac Catalyst requires a configured audio + // session — without it the input audio unit fails to initialize + // (AUIOBase Initialize error=-50). Best-effort: setCategory may log + // "cannot add handler" Mach port noise but still installs the + // category that the audio unit needs. + let session = AVAudioSession.sharedInstance() + try? session.setCategory(.playAndRecord, mode: .default, options: [.mixWithOthers]) + try? session.setActive(true) + + let engine = AVAudioEngine() + let inputNode = engine.inputNode + let inputFormat = inputNode.outputFormat(forBus: 0) + + guard inputFormat.sampleRate > 0, inputFormat.channelCount > 0 else { + throw NSError( + domain: "AudioRecorderViewModel.Catalyst", + code: -1, + userInfo: [NSLocalizedDescriptionKey: "Microphone not available — check macOS Sound input settings."] + ) + } + + let outputSettings: [String: Any] = [ + AVFormatIDKey: Int(kAudioFormatMPEG4AAC), + AVSampleRateKey: inputFormat.sampleRate, + AVNumberOfChannelsKey: 1, + AVEncoderBitRateKey: 64000 + ] + let audioFile = try AVAudioFile(forWriting: url, settings: outputSettings) + AppFileProtection.apply(to: url) + + catalystAudioEngine = engine + catalystAudioFile = audioFile + catalystEngineFormat = inputFormat + + installCatalystInputTap() + + engine.prepare() + try engine.start() + } + + /// Pause Catalyst recording: remove the input tap so the file stops + /// receiving samples. The engine and file stay alive so resume can + /// continue writing to the same file. + func pauseCatalystEngineRecording() { + guard let engine = catalystAudioEngine else { return } + engine.inputNode.removeTap(onBus: 0) + } + + /// Resume Catalyst recording: re-install the tap on the same input node, + /// writing into the same AVAudioFile that was opened in `start...`. + func resumeCatalystEngineRecording() throws { + guard catalystAudioEngine != nil, catalystAudioFile != nil else { + throw NSError( + domain: "AudioRecorderViewModel.Catalyst", + code: -2, + userInfo: [NSLocalizedDescriptionKey: "Engine state was lost; cannot resume."] + ) + } + installCatalystInputTap() + } + + /// Fully stop Catalyst recording. Closes the file (via deinit) and tears + /// down the engine. After this returns the file at the original URL is + /// finalized AAC and ready to read. + func stopCatalystEngineRecording(closingFile: Bool = true) { + if let engine = catalystAudioEngine { + engine.inputNode.removeTap(onBus: 0) + if engine.isRunning { + engine.stop() + } + } + catalystAudioEngine = nil + catalystEngineFormat = nil + if closingFile { + catalystAudioFile = nil + // Best-effort: release the playAndRecord category we set in start. + try? AVAudioSession.sharedInstance().setActive(false, options: .notifyOthersOnDeactivation) + } + } + + /// Persist a Catalyst recording to Core Data after `stopRecording()` + /// finishes. Mirrors the iOS path in `audioRecorderDidFinishRecording`, + /// minus the AAC transcode (the file is already AAC). + @MainActor + func finalizeCatalystRecording(at url: URL) async { + guard FileManager.default.fileExists(atPath: url.path) else { + AppLog.shared.recording("Catalyst finalize: recording file is missing at \(url.lastPathComponent)", level: .error) + errorMessage = "Recording was lost — file was not written." + return + } + + saveLocationData(for: url) + + guard let workflowManager = workflowManager else { + AppLog.shared.recording("WorkflowManager not set - Catalyst recording not saved", level: .error) + return + } + + let fileSize = getFileSize(url: url) + let duration = getRecordingDuration(url: url) + let quality = AudioRecorderViewModel.getCurrentAudioQuality() + let displayName = generateAppRecordingDisplayName() + + let recordingId = workflowManager.createRecording( + url: url, + name: displayName, + date: currentRecordingDate(for: url), + fileSize: fileSize, + duration: duration, + quality: quality, + locationData: recordingLocationSnapshot() + ) + AppLog.shared.recording("Catalyst recording created with workflow manager, ID: \(recordingId)") + + self.resetRecordingLocation() + self.recordingStartedAt = nil + self.recordingBeingProcessed = false + } + + private func installCatalystInputTap() { + guard let engine = catalystAudioEngine, + let format = catalystEngineFormat else { return } + + // `format: nil` means "use the input node's native format", which + // is what we already captured. Pass it explicitly so the tap matches + // the file we're writing to. + engine.inputNode.installTap( + onBus: 0, + bufferSize: 4096, + format: format + ) { [weak self] buffer, _ in + guard let file = self?.catalystAudioFile else { return } + do { + try file.write(from: buffer) + } catch { + AppLog.shared.recording("Catalyst engine: file write failed: \(error.localizedDescription)", level: .error) + } + } + } +} + +#endif diff --git a/BisonNotes AI/BisonNotes AI/ViewModels/AudioRecorderViewModel+Interruptions.swift b/BisonNotes AI/BisonNotes AI/ViewModels/AudioRecorderViewModel+Interruptions.swift index 59f6634..ceabd3d 100644 --- a/BisonNotes AI/BisonNotes AI/ViewModels/AudioRecorderViewModel+Interruptions.swift +++ b/BisonNotes AI/BisonNotes AI/ViewModels/AudioRecorderViewModel+Interruptions.swift @@ -174,10 +174,12 @@ extension AudioRecorderViewModel { audioRecorder = try AVAudioRecorder(url: newSegmentURL, settings: settings) audioRecorder?.delegate = self + AppFileProtection.apply(to: newSegmentURL) audioRecorder?.isMeteringEnabled = true // Enable metering for silence detection // Start recording audioRecorder?.record() + AppFileProtection.apply(to: newSegmentURL) // Brief delay to let the session stabilize before verifying try? await Task.sleep(nanoseconds: 150_000_000) // 150ms @@ -279,10 +281,12 @@ extension AudioRecorderViewModel { audioRecorder = try AVAudioRecorder(url: newSegmentURL, settings: settings) audioRecorder?.delegate = self + AppFileProtection.apply(to: newSegmentURL) audioRecorder?.isMeteringEnabled = true // Enable metering for silence detection // Start recording audioRecorder?.record() + AppFileProtection.apply(to: newSegmentURL) // Brief delay to let the session stabilize before verifying try? await Task.sleep(nanoseconds: 150_000_000) // 150ms diff --git a/BisonNotes AI/BisonNotes AI/ViewModels/AudioRecorderViewModel+Location.swift b/BisonNotes AI/BisonNotes AI/ViewModels/AudioRecorderViewModel+Location.swift index a57bc00..1ee244d 100644 --- a/BisonNotes AI/BisonNotes AI/ViewModels/AudioRecorderViewModel+Location.swift +++ b/BisonNotes AI/BisonNotes AI/ViewModels/AudioRecorderViewModel+Location.swift @@ -70,6 +70,7 @@ extension AudioRecorderViewModel { do { let data = try JSONEncoder().encode(locationData) try data.write(to: locationURL) + AppFileProtection.apply(to: locationURL) AppLog.shared.recording("Location data saved for recording") } catch { AppLog.shared.recording("Failed to save location data: \(error)", level: .error) diff --git a/BisonNotes AI/BisonNotes AI/ViewModels/AudioRecorderViewModel+MicrophoneReconnection.swift b/BisonNotes AI/BisonNotes AI/ViewModels/AudioRecorderViewModel+MicrophoneReconnection.swift index 319a27c..7721136 100644 --- a/BisonNotes AI/BisonNotes AI/ViewModels/AudioRecorderViewModel+MicrophoneReconnection.swift +++ b/BisonNotes AI/BisonNotes AI/ViewModels/AudioRecorderViewModel+MicrophoneReconnection.swift @@ -186,9 +186,11 @@ extension AudioRecorderViewModel { audioRecorder = try AVAudioRecorder(url: newSegmentURL, settings: settings) audioRecorder?.delegate = self + AppFileProtection.apply(to: newSegmentURL) audioRecorder?.isMeteringEnabled = true if audioRecorder?.record() == true { + AppFileProtection.apply(to: newSegmentURL) recordingSegments.append(newSegmentURL) recordingURL = newSegmentURL isRecording = true diff --git a/BisonNotes AI/BisonNotes AI/ViewModels/AudioRecorderViewModel+Segments.swift b/BisonNotes AI/BisonNotes AI/ViewModels/AudioRecorderViewModel+Segments.swift index 03c7b05..cef55a1 100644 --- a/BisonNotes AI/BisonNotes AI/ViewModels/AudioRecorderViewModel+Segments.swift +++ b/BisonNotes AI/BisonNotes AI/ViewModels/AudioRecorderViewModel+Segments.swift @@ -87,6 +87,7 @@ extension AudioRecorderViewModel { // Use the modern export API (iOS 18+) try await exportSession.export(to: tempURL, as: .m4a) + AppFileProtection.apply(to: tempURL) AppLog.shared.recording("Successfully merged all segments to temporary file", level: .debug) @@ -103,6 +104,7 @@ extension AudioRecorderViewModel { // Move temp file to final location try fileManager.moveItem(at: tempURL, to: mainURL) + AppFileProtection.apply(to: mainURL) AppLog.shared.recording("Successfully merged all segments") diff --git a/BisonNotes AI/BisonNotes AI/ViewModels/AudioRecorderViewModel+Utilities.swift b/BisonNotes AI/BisonNotes AI/ViewModels/AudioRecorderViewModel+Utilities.swift index ec345bc..79407ae 100644 --- a/BisonNotes AI/BisonNotes AI/ViewModels/AudioRecorderViewModel+Utilities.swift +++ b/BisonNotes AI/BisonNotes AI/ViewModels/AudioRecorderViewModel+Utilities.swift @@ -28,96 +28,94 @@ extension AudioRecorderViewModel: AVAudioRecorderDelegate { } nonisolated func audioRecorderDidFinishRecording(_ recorder: AVAudioRecorder, successfully flag: Bool) { - Task { - await MainActor.run { - // Check if we're still in recording mode - if so, this was an interruption, not a user stop - // In this case, don't save as a finished recording - let the interruption handler deal with it - if isRecording { - AppLog.shared.recording("Recorder finished but still in recording mode - ignoring (interruption will be handled)", level: .debug) - return - } + Task { @MainActor in + // Check if we're still in recording mode - if so, this was an interruption, not a user stop + // In this case, don't save as a finished recording - let the interruption handler deal with it + if isRecording { + AppLog.shared.recording("Recorder finished but still in recording mode - ignoring (interruption will be handled)", level: .debug) + return + } - // Check if recording is already being processed by interruption handler - // But allow processing if app is backgrounding (normal completion scenario) - if recordingBeingProcessed && !appIsBackgrounding { - AppLog.shared.recording("Recording already processed by interruption handler, skipping normal completion", level: .debug) - recordingBeingProcessed = false // Reset flag - return - } + // Check if recording is already being processed by interruption handler + // But allow processing if app is backgrounding (normal completion scenario) + if recordingBeingProcessed && !appIsBackgrounding { + AppLog.shared.recording("Recording already processed by interruption handler, skipping normal completion", level: .debug) + recordingBeingProcessed = false // Reset flag + return + } - // Check if we have multiple segments - if so, the merge will handle saving - if recordingSegments.count > 1 { - AppLog.shared.recording("Multiple segments detected - merge process will handle saving", level: .debug) - return - } + // Check if we have multiple segments - if so, the merge will handle saving + if recordingSegments.count > 1 { + AppLog.shared.recording("Multiple segments detected - merge process will handle saving", level: .debug) + return + } - // Start background task to protect Core Data save operations - beginBackgroundTask() + // Start background task to protect Core Data save operations + beginBackgroundTask() - if flag { - if appIsBackgrounding { - AppLog.shared.recording("Recording finished successfully during backgrounding - processing normally") + if flag { + if appIsBackgrounding { + AppLog.shared.recording("Recording finished successfully during backgrounding - processing normally") + } else { + AppLog.shared.recording("Recording finished successfully") + } + recordingBeingProcessed = true // Set flag to prevent duplicate processing + + if let resolvedRecordingURL = recordingURL { + saveLocationData(for: resolvedRecordingURL) + + // New recordings are already in Whisper-optimized format (16kHz, 64kbps AAC) + AppLog.shared.recording("Recording saved in Whisper-optimized format") + + // Add recording using workflow manager for proper UUID consistency + if let workflowManager = workflowManager { + let fileSize = getFileSize(url: resolvedRecordingURL) + let duration = getRecordingDuration(url: resolvedRecordingURL) + let quality = AudioRecorderViewModel.getCurrentAudioQuality() + + // Create display name for phone recording + let displayName = generateAppRecordingDisplayName() + + // Create recording + let recordingId = workflowManager.createRecording( + url: resolvedRecordingURL, + name: displayName, + date: currentRecordingDate(for: resolvedRecordingURL), + fileSize: fileSize, + duration: duration, + quality: quality, + locationData: recordingLocationSnapshot() + ) + + AppLog.shared.recording("Recording created with workflow manager, ID: \(recordingId)") + + // Watch audio integration removed + self.resetRecordingLocation() + self.recordingStartedAt = nil } else { - AppLog.shared.recording("Recording finished successfully") - } - recordingBeingProcessed = true // Set flag to prevent duplicate processing - - if let recordingURL = recordingURL { - saveLocationData(for: recordingURL) - - // New recordings are already in Whisper-optimized format (16kHz, 64kbps AAC) - AppLog.shared.recording("Recording saved in Whisper-optimized format") - - // Add recording using workflow manager for proper UUID consistency - if let workflowManager = workflowManager { - let fileSize = getFileSize(url: recordingURL) - let duration = getRecordingDuration(url: recordingURL) - let quality = AudioRecorderViewModel.getCurrentAudioQuality() - - // Create display name for phone recording - let displayName = generateAppRecordingDisplayName() - - // Create recording - let recordingId = workflowManager.createRecording( - url: recordingURL, - name: displayName, - date: currentRecordingDate(for: recordingURL), - fileSize: fileSize, - duration: duration, - quality: quality, - locationData: recordingLocationSnapshot() - ) - - AppLog.shared.recording("Recording created with workflow manager, ID: \(recordingId)") - - // Watch audio integration removed - self.resetRecordingLocation() - self.recordingStartedAt = nil - } else { - AppLog.shared.recording("WorkflowManager not set - recording not saved to database", level: .error) - } + AppLog.shared.recording("WorkflowManager not set - recording not saved to database", level: .error) } + } - // Reset processing flag after successful completion - recordingBeingProcessed = false - - // Deactivate audio session to restore high-quality music playback - Task { - try? await enhancedAudioSessionManager.deactivateSession() - } - } else { - errorMessage = "Recording failed" - recordingBeingProcessed = false // Reset flag on failure too + // Reset processing flag after successful completion + recordingBeingProcessed = false - // Also deactivate session on failure - Task { - try? await enhancedAudioSessionManager.deactivateSession() - } + // Deactivate audio session to restore high-quality music playback + Task { + try? await enhancedAudioSessionManager.deactivateSession() } + } else { + errorMessage = "Recording failed" + recordingBeingProcessed = false // Reset flag on failure too - // End background task that protected the Core Data save operation - self.endBackgroundTask() + // Also deactivate session on failure + Task { + try? await enhancedAudioSessionManager.deactivateSession() + } } + + // End background task that protected the Core Data save operation + self.endBackgroundTask() } } } @@ -303,7 +301,9 @@ extension AudioRecorderViewModel { } do { - try data.write(to: recordingTimestampMetadataURL(for: url), options: .atomic) + let metadataURL = recordingTimestampMetadataURL(for: url) + try data.write(to: metadataURL, options: .atomic) + AppFileProtection.apply(to: metadataURL) } catch { AppLog.shared.recording("Failed to write recording timestamp metadata: \(error.localizedDescription)", level: .error) } diff --git a/BisonNotes AI/BisonNotes AI/ViewModels/AudioRecorderViewModel+Warnings.swift b/BisonNotes AI/BisonNotes AI/ViewModels/AudioRecorderViewModel+Warnings.swift index be9da01..e062bb6 100644 --- a/BisonNotes AI/BisonNotes AI/ViewModels/AudioRecorderViewModel+Warnings.swift +++ b/BisonNotes AI/BisonNotes AI/ViewModels/AudioRecorderViewModel+Warnings.swift @@ -92,6 +92,9 @@ extension AudioRecorderViewModel { } // 4. BATTERY CHECK + // Mac Catalyst has no battery API — UIDevice.batteryLevel returns -1 and + // enabling monitoring spams "Error retrieving battery status" to the log. + #if !targetEnvironment(macCatalyst) UIDevice.current.isBatteryMonitoringEnabled = true let batteryLevel = UIDevice.current.batteryLevel @@ -114,6 +117,7 @@ extension AudioRecorderViewModel { isCritical: false ) } + #endif } /// Get available storage on device diff --git a/BisonNotes AI/BisonNotes AI/ViewModels/AudioRecorderViewModel+WatchIntegration.swift b/BisonNotes AI/BisonNotes AI/ViewModels/AudioRecorderViewModel+WatchIntegration.swift index 81b30bd..bb65641 100644 --- a/BisonNotes AI/BisonNotes AI/ViewModels/AudioRecorderViewModel+WatchIntegration.swift +++ b/BisonNotes AI/BisonNotes AI/ViewModels/AudioRecorderViewModel+WatchIntegration.swift @@ -75,6 +75,7 @@ extension AudioRecorderViewModel { let permanentURL = documentsURL.appendingPathComponent(iPhoneStyleFilename) try audioData.write(to: permanentURL) + AppFileProtection.apply(to: permanentURL) // Create Core Data entry guard let appCoordinator = appCoordinator else { @@ -296,6 +297,7 @@ extension AudioRecorderViewModel { let backupURL = documentsURL.appendingPathComponent("watch_backup_\(recordingId).pcm") try watchAudioData.write(to: backupURL) + AppFileProtection.apply(to: backupURL) AppLog.shared.watchConnectivity("Stored watch audio backup") // Optionally store metadata about the backup @@ -312,6 +314,7 @@ extension AudioRecorderViewModel { let metadataData = try JSONSerialization.data(withJSONObject: metadata) try metadataData.write(to: metadataURL) + AppFileProtection.apply(to: metadataURL) } catch { AppLog.shared.watchConnectivity("Failed to store watch audio backup: \(error)", level: .error) @@ -325,6 +328,7 @@ extension AudioRecorderViewModel { let backupURL = documentsURL.appendingPathComponent("phone_backup_\(recordingId).m4a") try FileManager.default.copyItem(at: phoneAudioURL, to: backupURL) + AppFileProtection.apply(to: backupURL) AppLog.shared.watchConnectivity("Stored phone audio backup") } catch { diff --git a/BisonNotes AI/BisonNotes AI/ViewModels/AudioRecorderViewModel.swift b/BisonNotes AI/BisonNotes AI/ViewModels/AudioRecorderViewModel.swift index 348f7ea..0879ff7 100644 --- a/BisonNotes AI/BisonNotes AI/ViewModels/AudioRecorderViewModel.swift +++ b/BisonNotes AI/BisonNotes AI/ViewModels/AudioRecorderViewModel.swift @@ -83,6 +83,16 @@ class AudioRecorderViewModel: NSObject, ObservableObject { // Track last checkpoint time for periodic data flushing var lastCheckpointTime: Date = Date.distantPast var recordingStartedAt: (url: URL, date: Date)? + + #if targetEnvironment(macCatalyst) + // On Mac Catalyst, AVAudioRecorder cannot reliably encode without an + // AVAudioSession. We use AVAudioEngine + AVAudioFile instead — input node + // taps deliver PCM buffers we can write directly to a single file with + // pause/resume implemented by removing/re-installing the tap. + var catalystAudioEngine: AVAudioEngine? + var catalystAudioFile: AVAudioFile? + var catalystEngineFormat: AVAudioFormat? + #endif let checkpointInterval: TimeInterval = 30.0 // Try to checkpoint every 30 seconds let forceCheckpointInterval: TimeInterval = 90.0 // Force checkpoint after 90 seconds even without silence @@ -236,6 +246,10 @@ class AudioRecorderViewModel: NSObject, ObservableObject { // MARK: - Notification Observers func setupNotificationObservers() { + #if !targetEnvironment(macCatalyst) + // AVAudioSession interruption/route notifications use Mach ports that don't + // exist on Mac — registering for them floods the log with "cannot add handler". + // Phone-call interruptions and Bluetooth routing don't apply on Mac anyway. interruptionObserver = NotificationCenter.default.addObserver( forName: AVAudioSession.interruptionNotification, object: nil, @@ -276,6 +290,7 @@ class AudioRecorderViewModel: NSObject, ObservableObject { } } } + #endif willEnterForegroundObserver = NotificationCenter.default.addObserver( forName: UIApplication.willEnterForegroundNotification, @@ -401,55 +416,110 @@ class AudioRecorderViewModel: NSObject, ObservableObject { // MARK: - Core Recording func startRecording() { + AppLog.shared.recording("startRecording: requesting microphone permission") + #if targetEnvironment(macCatalyst) + Task { @MainActor [weak self] in self?.requestMicPermissionAndRecord() } + #else AVAudioApplication.requestRecordPermission { [weak self] granted in DispatchQueue.main.async { guard let self = self else { return } if granted { + AppLog.shared.recording("startRecording: microphone permission granted") Task { do { - // Phase 4: Use background recording for perfect backgrounding support try await self.enhancedAudioSessionManager.configureBackgroundRecording() AppLog.shared.recording("Background recording session configured") await self.applySelectedInputToSession() } catch { - AppLog.shared.recording("Failed to configure background recording session: \(error)", level: .error) + AppLog.shared.recording("Failed to configure audio session: \(error)", level: .error) + await MainActor.run { + self.errorMessage = "Failed to set up audio: \(error.localizedDescription)" + } return } - await MainActor.run { self.setupRecording() } } } else { + AppLog.shared.recording("startRecording: microphone permission denied", level: .error) self.errorMessage = "Microphone permission denied" } } } + #endif + } + + #if targetEnvironment(macCatalyst) + @MainActor + private func requestMicPermissionAndRecord() { + let status = AVAudioApplication.shared.recordPermission + AppLog.shared.recording("Mac mic permission status: \(status.rawValue)") + switch status { + case .granted: + didReceiveMicrophonePermission(granted: true) + case .undetermined: + AVAudioApplication.requestRecordPermission { [weak self] granted in + Task { @MainActor [weak self] in + AppLog.shared.recording("Mac mic permission result: \(granted)") + self?.didReceiveMicrophonePermission(granted: granted) + } + } + case .denied: + AppLog.shared.recording("Mac mic permission denied", level: .error) + errorMessage = "Microphone access is denied. Open System Settings → Privacy & Security → Microphone and enable BisonNotes AI, then try again." + @unknown default: + AVAudioApplication.requestRecordPermission { [weak self] granted in + Task { @MainActor [weak self] in + self?.didReceiveMicrophonePermission(granted: granted) + } + } + } } + @MainActor + private func didReceiveMicrophonePermission(granted: Bool) { + guard granted else { + errorMessage = "Microphone permission denied" + return + } + AppLog.shared.recording("startRecording: microphone permission granted") + // Skip AVAudioSession.setCategory/setActive on Mac — those calls communicate + // with mediaserverd via Mach ports that don't exist on macOS, flooding the + // log. AVAudioRecorder uses the default CoreAudio input directly on Mac. + setupRecording() + } + #endif + func startBackgroundRecording() { + AppLog.shared.recording("startBackgroundRecording: requesting microphone permission") + #if targetEnvironment(macCatalyst) + Task { @MainActor [weak self] in self?.requestMicPermissionAndRecord() } + #else AVAudioApplication.requestRecordPermission { [weak self] granted in DispatchQueue.main.async { guard let self = self else { return } if granted { + AppLog.shared.recording("startBackgroundRecording: microphone permission granted") Task { do { try await self.enhancedAudioSessionManager.configureBackgroundRecording() await self.applySelectedInputToSession() } catch { - AppLog.shared.recording("Failed to configure background recording session: \(error)", level: .error) + AppLog.shared.recording("Failed to configure audio session: \(error)", level: .error) return } - await MainActor.run { self.setupRecording() } } } else { + AppLog.shared.recording("startBackgroundRecording: microphone permission denied", level: .error) self.errorMessage = "Microphone permission denied" } } } + #endif } func setupRecording() { @@ -477,13 +547,20 @@ class AudioRecorderViewModel: NSObject, ObservableObject { return } - // Use Whisper-optimized quality for all recordings - let selectedQuality = AudioQuality.whisperOptimized - let settings = selectedQuality.settings - do { + #if targetEnvironment(macCatalyst) + // Catalyst: drive recording with AVAudioEngine + AVAudioFile so we + // bypass AVAudioRecorder's broken converter setup. The file is AAC + // from the start — no post-stop transcode needed. + try startCatalystEngineRecording(at: audioFilename) + #else + // Use Whisper-optimized quality for all recordings + let selectedQuality = AudioQuality.whisperOptimized + let settings = selectedQuality.settings + audioRecorder = try AVAudioRecorder(url: audioFilename, settings: settings) audioRecorder?.delegate = self + AppFileProtection.apply(to: audioFilename) // Enable metering for silence detection (used for smart checkpoints) audioRecorder?.isMeteringEnabled = true @@ -494,8 +571,11 @@ class AudioRecorderViewModel: NSObject, ObservableObject { // No background task needed here - audio background mode keeps recording alive audioRecorder?.record() + AppFileProtection.apply(to: audioFilename) + #endif isRecording = true + recordingState = .recording recordingTime = 0 lastCheckpointTime = Date() // Initialize checkpoint time to now @@ -523,6 +603,67 @@ class AudioRecorderViewModel: NSObject, ObservableObject { } } + /// True when an active recording has been paused via `pauseRecording()`. + var isPaused: Bool { + if case .paused = recordingState { return true } + return false + } + + /// Pause an active recording. On iOS the AVAudioRecorder is paused + /// (file stays open). On Mac Catalyst the AVAudioEngine input tap is + /// removed so no more samples reach the file. In both cases + /// `resumeRecording()` continues writing to the same file. + /// No-op if not recording, already paused, or in live transcription mode. + func pauseRecording() { + guard isRecording, !isPaused else { return } + guard !isUsingLiveTranscription else { + AppLog.shared.recording("Pause requested in live transcription mode — ignoring (use stop instead)", level: .debug) + return + } + + #if targetEnvironment(macCatalyst) + guard catalystAudioEngine != nil else { return } + pauseCatalystEngineRecording() + #else + guard let recorder = audioRecorder else { return } + recorder.pause() + #endif + + stopRecordingTimer() + recordingState = .paused + AppLog.shared.recording("Recording paused at \(Int(recordingTime))s") + notifyWatchOfRecordingStateChange() + } + + /// Resume a paused recording. Continues writing to the same file on both + /// iOS (AVAudioRecorder.record()) and Mac Catalyst (re-installing the + /// AVAudioEngine tap). + func resumeRecording() { + guard isPaused else { return } + + #if targetEnvironment(macCatalyst) + do { + try resumeCatalystEngineRecording() + } catch { + AppLog.shared.recording("Catalyst resume failed: \(error.localizedDescription)", level: .error) + errorMessage = "Could not resume recording: \(error.localizedDescription)" + return + } + #else + guard let recorder = audioRecorder else { return } + guard recorder.record() else { + AppLog.shared.recording("Failed to resume paused recording — recorder.record() returned false", level: .error) + errorMessage = "Could not resume recording. Please stop and start a new recording." + return + } + #endif + + recordingState = .recording + startRecordingTimer() + AppLog.shared.recording("Recording resumed at \(Int(recordingTime))s") + notifyWatchOfRecordingStateChange() + } + func stopRecording() { // Handle live transcription path if isUsingLiveTranscription, let service = liveTranscriptionService { @@ -547,8 +688,17 @@ class AudioRecorderViewModel: NSObject, ObservableObject { return } + #if targetEnvironment(macCatalyst) + // On Catalyst, recording is driven by AVAudioEngine + AVAudioFile. + // Tear it down here, then run the save flow manually since there's + // no AVAudioRecorder delegate to fire it. + let catalystFinalURL = recordingURL + stopCatalystEngineRecording() + #endif + audioRecorder?.stop() isRecording = false + recordingState = .idle stopRecordingTimer() audioRecorder = nil @@ -570,7 +720,18 @@ class AudioRecorderViewModel: NSObject, ObservableObject { await mergeRecordingSegments() } } else { + #if targetEnvironment(macCatalyst) + if let url = catalystFinalURL { + AppLog.shared.recording("Recording finished successfully (Catalyst engine)") + Task { @MainActor in + await self.finalizeCatalystRecording(at: url) + } + } else { + AppLog.shared.recording("Catalyst stop: no recording URL — nothing to save", level: .error) + } + #else AppLog.shared.recording("Recording has single segment, no merge needed", level: .debug) + #endif } // Deactivate audio session to restore high-quality music playback @@ -689,6 +850,14 @@ class AudioRecorderViewModel: NSObject, ObservableObject { // while the app is in the background, and we need to detect that. // NOTE: In live transcription mode, audioRecorder is nil so this block is // safely skipped — LiveTranscriptionService manages its own AVAudioEngine. + #if targetEnvironment(macCatalyst) + // On Mac Catalyst there is no AVAudioSession interruption model and + // AVAudioRecorder.isRecording is unreliable; the recovery flow that + // reactivates the session and rebuilds the recorder is a no-op here + // and surfaces a misleading "Microphone became unavailable" error. + // Trust that record() succeeded and only stop on explicit user action. + self.performSmartCheckpoint() + #else if self.isRecording, let recorder = self.audioRecorder, !recorder.isRecording { if self.isInInterruption { // We're in an interruption - wait for it to end rather than trying to resume now @@ -718,6 +887,7 @@ class AudioRecorderViewModel: NSObject, ObservableObject { // Perform smart checkpoint that waits for silence self.performSmartCheckpoint() } + #endif self.recordingTime += 1 // Sync live transcript text when live transcription is active diff --git a/BisonNotes AI/BisonNotes AI/Views/AITextView.swift b/BisonNotes AI/BisonNotes AI/Views/AITextView.swift index 9a9eaff..ceb7f91 100644 --- a/BisonNotes AI/BisonNotes AI/Views/AITextView.swift +++ b/BisonNotes AI/BisonNotes AI/Views/AITextView.swift @@ -77,7 +77,9 @@ struct AITextView: View { baseURL: nil, syntaxExtensions: [] ) + #if !targetEnvironment(macCatalyst) .textual.textSelection(.enabled) + #endif .frame(maxWidth: .infinity, alignment: .leading) } diff --git a/BisonNotes AI/BisonNotes AI/Views/AcknowledgementsView.swift b/BisonNotes AI/BisonNotes AI/Views/AcknowledgementsView.swift index c687425..96ad2bc 100644 --- a/BisonNotes AI/BisonNotes AI/Views/AcknowledgementsView.swift +++ b/BisonNotes AI/BisonNotes AI/Views/AcknowledgementsView.swift @@ -10,145 +10,77 @@ struct AcknowledgementsView: View { @State private var showTransitiveDependencies = false var body: some View { - NavigationView { - ScrollView { - VStack(alignment: .leading, spacing: 20) { - headerSection - directDependenciesSection - forksSection - transitiveDependenciesSection - footerSection - } - .padding() - } - .navigationBarTitleDisplayMode(.inline) - .toolbar { - ToolbarItem(placement: .topBarLeading) { - Button("Done") { - dismiss() - } - .fontWeight(.medium) + NavigationStack { + Form { + Section { + Text("BisonNotes AI is built on the shoulders of outstanding open-source projects.") + .font(.subheadline) + .foregroundColor(.secondary) } - } - } - } - - private var headerSection: some View { - VStack(alignment: .leading, spacing: 8) { - Text("Acknowledgements") - .font(.largeTitle) - .fontWeight(.bold) - .foregroundColor(.primary) - - Text("BisonNotes AI is built on the shoulders of outstanding open-source projects.") - .font(.subheadline) - .foregroundColor(.secondary) - } - } - - private var directDependenciesSection: some View { - VStack(alignment: .leading, spacing: 12) { - Text("Direct Dependencies") - .font(.headline) - .foregroundColor(.primary) - ForEach(directDependencies) { dependency in - VStack(alignment: .leading, spacing: 8) { - HStack { - Text(dependency.name) - .font(.body) - .fontWeight(.semibold) - .foregroundColor(.primary) - Spacer() - Text(dependency.license) - .font(.caption) - .fontWeight(.medium) - .foregroundColor(.secondary) + Section("Direct Dependencies") { + ForEach(directDependencies) { dependency in + VStack(alignment: .leading, spacing: 6) { + HStack { + Text(dependency.name) + .fontWeight(.semibold) + Spacer() + Text(dependency.license) + .font(.caption) + .foregroundColor(.secondary) + } + Text(dependency.description) + .font(.caption) + .foregroundColor(.secondary) + .fixedSize(horizontal: false, vertical: true) + Link(dependency.linkText, destination: dependency.url) + .font(.caption) + } + .padding(.vertical, 4) } - - Text(dependency.description) - .font(.caption) - .foregroundColor(.secondary) - .fixedSize(horizontal: false, vertical: true) - - Link(dependency.linkText, destination: dependency.url) - .font(.caption) } - .padding(12) - .background( - RoundedRectangle(cornerRadius: 10) - .fill(Color(.systemGray6)) - ) - } - } - } - - private var forksSection: some View { - VStack(alignment: .leading, spacing: 12) { - Text("Forked Repositories") - .font(.headline) - .foregroundColor(.primary) - - Link("github.com/bisonbet", destination: URL(string: "https://github.com/bisonbet")!) - .font(.caption) - VStack(alignment: .leading, spacing: 8) { - ForEach(forkedRepositories) { repository in - VStack(alignment: .leading, spacing: 4) { - Link(repository.name, destination: repository.url) - .font(.body) - Text(repository.description) - .font(.caption) - .foregroundColor(.secondary) + Section("Forked Repositories") { + Link("github.com/bisonbet", destination: URL(string: "https://github.com/bisonbet")!) + .font(.caption) + ForEach(forkedRepositories) { repository in + VStack(alignment: .leading, spacing: 4) { + Link(repository.name, destination: repository.url) + Text(repository.description) + .font(.caption) + .foregroundColor(.secondary) + } + .padding(.vertical, 4) } - .padding(10) - .background( - RoundedRectangle(cornerRadius: 8) - .fill(Color(.systemGray6)) - ) } - } - } - } - private var transitiveDependenciesSection: some View { - VStack(alignment: .leading, spacing: 10) { - DisclosureGroup("Transitive Dependencies", isExpanded: $showTransitiveDependencies) { - VStack(alignment: .leading, spacing: 14) { - ForEach(transitiveDependencyGroups) { group in - VStack(alignment: .leading, spacing: 6) { - Text(group.title) - .font(.subheadline) - .fontWeight(.semibold) - .foregroundColor(.primary) - - ForEach(group.projects) { project in - Link(project.name, destination: project.url) - .font(.caption) + Section { + DisclosureGroup("Transitive Dependencies", isExpanded: $showTransitiveDependencies) { + ForEach(transitiveDependencyGroups) { group in + VStack(alignment: .leading, spacing: 6) { + Text(group.title) + .font(.subheadline) + .fontWeight(.semibold) + ForEach(group.projects) { project in + Link(project.name, destination: project.url) + .font(.caption) + } } + .padding(.vertical, 4) } } + } footer: { + Text("These are brought in through AWS SDK and other direct dependencies. All dependencies are MIT or Apache 2.0 licensed. See each project repository for full terms.") + } + } + .navigationTitle("Acknowledgements") + .navigationBarTitleDisplayMode(.inline) + .toolbar { + ToolbarItem(placement: .navigationBarTrailing) { + Button("Done") { dismiss() } } - .padding(.top, 6) } - .font(.headline) - - Text("These are brought in through AWS SDK and other direct dependencies.") - .font(.caption) - .foregroundColor(.secondary) } - .padding(12) - .background( - RoundedRectangle(cornerRadius: 10) - .fill(Color(.systemGray6)) - ) - } - - private var footerSection: some View { - Text("All dependencies are MIT or Apache 2.0 licensed. See each project repository for full terms.") - .font(.caption) - .foregroundColor(.secondary) - .textSelection(.enabled) } } diff --git a/BisonNotes AI/BisonNotes AI/Views/ArchiveConfirmationView.swift b/BisonNotes AI/BisonNotes AI/Views/ArchiveConfirmationView.swift index 1b1e13f..ad02cb3 100644 --- a/BisonNotes AI/BisonNotes AI/Views/ArchiveConfirmationView.swift +++ b/BisonNotes AI/BisonNotes AI/Views/ArchiveConfirmationView.swift @@ -17,7 +17,7 @@ struct ArchiveConfirmationView: View { let onCancel: () -> Void var body: some View { - NavigationView { + NavigationStack { VStack(spacing: 20) { // Header VStack(spacing: 8) { diff --git a/BisonNotes AI/BisonNotes AI/Views/AudioPlayerView.swift b/BisonNotes AI/BisonNotes AI/Views/AudioPlayerView.swift index 0ab14d1..696f4ff 100644 --- a/BisonNotes AI/BisonNotes AI/Views/AudioPlayerView.swift +++ b/BisonNotes AI/BisonNotes AI/Views/AudioPlayerView.swift @@ -22,6 +22,14 @@ struct AudioPlayerView: View { @State private var audioExportURL: URL? @State private var audioExportError: String? + // Transcript-from-recording flow + @ObservedObject private var transcriptionStarter = TranscriptionStarter.shared + @ObservedObject private var backgroundProcessingManager = BackgroundProcessingManager.shared + @State private var showingAudioCleanupPrompt = false + @State private var recordingPendingTranscription: RecordingEntry? + @State private var selectedRecordingForTranscript: RecordingEntry? + @State private var transcriptStateRefresh = false + var body: some View { VStack(spacing: 20) { HStack { @@ -50,7 +58,9 @@ struct AudioPlayerView: View { Text("Date: \(recording.dateString)") .font(.subheadline) .foregroundColor(.secondary) - + + transcriptActionRow + Spacer() // Audio scrubber with progress and seek functionality @@ -131,6 +141,42 @@ struct AudioPlayerView: View { ShareSheet(activityItems: [audioExportURL]) } } + .sheet(item: $selectedRecordingForTranscript) { entry in + if let recordingId = entry.id, + let transcript = appCoordinator.getTranscriptData(for: recordingId) { + EditableTranscriptView(recording: entry, transcript: transcript, transcriptManager: TranscriptManager.shared) + .environmentObject(appCoordinator) + } else { + TranscriptDetailView(recording: entry, transcriptText: "") + .environmentObject(appCoordinator) + } + } + .confirmationDialog( + "Clean Audio Before Transcribing?", + isPresented: $showingAudioCleanupPrompt, + titleVisibility: .visible + ) { + Button("Clean & Transcribe") { + if let pending = recordingPendingTranscription { + recordingPendingTranscription = nil + transcriptionStarter.startTranscription(for: pending, cleanFirst: true, appCoordinator: appCoordinator) + } + } + Button("Transcribe As-Is") { + if let pending = recordingPendingTranscription { + recordingPendingTranscription = nil + transcriptionStarter.startTranscription(for: pending, cleanFirst: false, appCoordinator: appCoordinator) + } + } + Button("Cancel", role: .cancel) { + recordingPendingTranscription = nil + } + } message: { + Text("Cleaning reduces static and normalizes volume, which can improve transcription accuracy. The original audio file is not changed.") + } + .onReceive(NotificationCenter.default.publisher(for: NSNotification.Name("TranscriptionCompleted"))) { _ in + transcriptStateRefresh.toggle() + } .onAppear { AppLog.shared.recording("AudioPlayerView appeared", level: .debug) editableTitle = recording.name @@ -165,6 +211,48 @@ struct AudioPlayerView: View { } } + /// "Generate Transcript" action shown in the player. Disappears once a transcript exists — + /// users edit transcripts from the Transcripts tab. Resolves the RecordingEntry on demand + /// so state reflects the latest Core Data state (post-transcription, post-rename). + @ViewBuilder + private var transcriptActionRow: some View { + if let entry = appCoordinator.getRecording(url: recording.url), + entry.transcript == nil { + let recordingId = entry.id ?? UUID() + let isProcessing = transcriptionStarter.isCleaning(recordingId) + || transcriptionStarter.isQueuedForCleanup(recordingId) + || transcriptionStarter.hasActiveTranscriptionJob(for: entry, appCoordinator: appCoordinator) + + Button(action: { + if !isProcessing { + recordingPendingTranscription = entry + showingAudioCleanupPrompt = true + } + }) { + HStack(spacing: 8) { + if isProcessing { + ProgressView() + .scaleEffect(0.8) + .tint(.white) + Text("Transcribing…") + } else { + Image(systemName: "text.bubble") + Text("Generate Transcript") + } + } + .font(.subheadline) + .padding(.horizontal, 14) + .padding(.vertical, 8) + .background(isProcessing ? Color.orange : Color.accentColor) + .foregroundColor(.white) + .cornerRadius(10) + } + .buttonStyle(.plain) + .disabled(isProcessing) + .id("transcript-action-\(recordingId)-\(isProcessing)-\(transcriptStateRefresh)") + } + } + private func setupAudio() { AppLog.shared.recording("AudioPlayerView setupAudio called", level: .debug) diff --git a/BisonNotes AI/BisonNotes AI/Views/BackgroundProcessingView.swift b/BisonNotes AI/BisonNotes AI/Views/BackgroundProcessingView.swift index 9d6b81d..7d728d0 100644 --- a/BisonNotes AI/BisonNotes AI/Views/BackgroundProcessingView.swift +++ b/BisonNotes AI/BisonNotes AI/Views/BackgroundProcessingView.swift @@ -10,10 +10,11 @@ import SwiftUI struct BackgroundProcessingView: View { @ObservedObject private var processingManager = BackgroundProcessingManager.shared @State private var selectedJob: ProcessingJob? + @Environment(\.dismiss) private var dismiss var body: some View { let _ = AppLog.shared.backgroundProcessing("BackgroundProcessingView body: activeJobs.count = \(processingManager.activeJobs.count)", level: .debug) - NavigationView { + NavigationStack { VStack(spacing: 0) { // Header with overall status headerSection @@ -30,6 +31,9 @@ struct BackgroundProcessingView: View { .navigationTitle("Background Processing") .navigationBarTitleDisplayMode(.large) .toolbar { + ToolbarItem(placement: .navigationBarLeading) { + Button("Done") { dismiss() } + } ToolbarItem(placement: .navigationBarTrailing) { Menu { Button("Cleanup Completed Jobs") { @@ -437,7 +441,7 @@ struct JobDetailView: View { @State private var timer: Timer? var body: some View { - NavigationView { + NavigationStack { ScrollView { VStack(alignment: .leading, spacing: 20) { // Job header diff --git a/BisonNotes AI/BisonNotes AI/Views/CombineRecordingsView.swift b/BisonNotes AI/BisonNotes AI/Views/CombineRecordingsView.swift index 71c29bd..2ff56dd 100644 --- a/BisonNotes AI/BisonNotes AI/Views/CombineRecordingsView.swift +++ b/BisonNotes AI/BisonNotes AI/Views/CombineRecordingsView.swift @@ -41,7 +41,7 @@ struct CombineRecordingsView: View { } var body: some View { - NavigationView { + NavigationStack { ScrollView { VStack(alignment: .leading, spacing: 24) { headerSection diff --git a/BisonNotes AI/BisonNotes AI/Views/DataMigrationView.swift b/BisonNotes AI/BisonNotes AI/Views/DataMigrationView.swift index 0d29e1c..0fd9cb9 100644 --- a/BisonNotes AI/BisonNotes AI/Views/DataMigrationView.swift +++ b/BisonNotes AI/BisonNotes AI/Views/DataMigrationView.swift @@ -58,7 +58,7 @@ struct DataMigrationView: View { } var body: some View { - NavigationView { + NavigationStack { ScrollView { VStack(spacing: 30) { headerSection diff --git a/BisonNotes AI/BisonNotes AI/Views/EnhancedDeleteDialog.swift b/BisonNotes AI/BisonNotes AI/Views/EnhancedDeleteDialog.swift index 95f0bcb..f27fb7a 100644 --- a/BisonNotes AI/BisonNotes AI/Views/EnhancedDeleteDialog.swift +++ b/BisonNotes AI/BisonNotes AI/Views/EnhancedDeleteDialog.swift @@ -17,7 +17,7 @@ struct EnhancedDeleteDialog: View { @Environment(\.dismiss) private var dismiss var body: some View { - NavigationView { + NavigationStack { VStack(spacing: 20) { // Header VStack(spacing: 8) { diff --git a/BisonNotes AI/BisonNotes AI/Views/MistralOnboardingView.swift b/BisonNotes AI/BisonNotes AI/Views/MistralOnboardingView.swift index 368bd21..86726de 100644 --- a/BisonNotes AI/BisonNotes AI/Views/MistralOnboardingView.swift +++ b/BisonNotes AI/BisonNotes AI/Views/MistralOnboardingView.swift @@ -13,7 +13,7 @@ import UIKit struct MistralOnboardingView: View { @Environment(\.dismiss) private var dismiss - @AppStorage("mistralAPIKey") private var mistralAPIKey: String = "" + @SecureStorage(KeychainSecretStore.mistralAPIKey) private var mistralAPIKey: String = "" @AppStorage("mistralModel") private var mistralModel: String = MistralAIModel.mistralMedium2508.rawValue @AppStorage("mistralBaseURL") private var mistralBaseURL: String = "https://api.mistral.ai/v1" @AppStorage("mistralTemperature") private var mistralTemperature: Double = 0.1 @@ -38,7 +38,7 @@ struct MistralOnboardingView: View { private let totalSteps = 5 var body: some View { - NavigationView { + NavigationStack { VStack(spacing: 0) { stepIndicator .padding(.top, 12) @@ -68,6 +68,14 @@ struct MistralOnboardingView: View { #endif } } + .onChange(of: showingSafari) { _, isShowing in + #if targetEnvironment(macCatalyst) + if isShowing, let url = safariURL { + UIApplication.shared.open(url) + showingSafari = false + } + #endif + } } } diff --git a/BisonNotes AI/BisonNotes AI/Views/OnDeviceAIDownloadView.swift b/BisonNotes AI/BisonNotes AI/Views/OnDeviceAIDownloadView.swift index 92820a6..0521d52 100644 --- a/BisonNotes AI/BisonNotes AI/Views/OnDeviceAIDownloadView.swift +++ b/BisonNotes AI/BisonNotes AI/Views/OnDeviceAIDownloadView.swift @@ -26,18 +26,21 @@ struct OnDeviceAIDownloadView: View { } var body: some View { - NavigationView { + NavigationStack { ScrollView { VStack(spacing: 24) { confirmationView } .padding(.horizontal, 24) .padding(.top, 24) - .padding(.bottom, 40) // Extra bottom padding for safe scrolling + .padding(.bottom, 40) } .navigationTitle("Download Models") .navigationBarTitleDisplayMode(.inline) } + #if targetEnvironment(macCatalyst) + .frame(minWidth: 520, minHeight: 640) + #endif .onAppear { // Set On-Device LLM to use Granite Micro (recommended for 6GB+ devices) // Do this BEFORE any access to selectedModel to prevent migration loops diff --git a/BisonNotes AI/BisonNotes AI/Views/PreferencesView.swift b/BisonNotes AI/BisonNotes AI/Views/PreferencesView.swift index a99d1ac..e5ae5e2 100644 --- a/BisonNotes AI/BisonNotes AI/Views/PreferencesView.swift +++ b/BisonNotes AI/BisonNotes AI/Views/PreferencesView.swift @@ -13,7 +13,7 @@ struct PreferencesView: View { @State private var showingTimeFormatExample = false var body: some View { - NavigationView { + NavigationStack { ScrollView { VStack(alignment: .leading, spacing: 24) { headerSection diff --git a/BisonNotes AI/BisonNotes AI/Views/RecordingsListView.swift b/BisonNotes AI/BisonNotes AI/Views/RecordingsListView.swift index a391a40..bd9a47c 100644 --- a/BisonNotes AI/BisonNotes AI/Views/RecordingsListView.swift +++ b/BisonNotes AI/BisonNotes AI/Views/RecordingsListView.swift @@ -74,8 +74,16 @@ struct RecordingsListView: View { @State private var dateFilterEnd: Date = Date() @State private var isDateFilterActive = false + // Transcript-from-recording flow + @ObservedObject private var transcriptionStarter = TranscriptionStarter.shared + @ObservedObject private var backgroundProcessingManager = BackgroundProcessingManager.shared + @State private var showingAudioCleanupPrompt = false + @State private var recordingPendingTranscription: RecordingEntry? + @State private var selectedRecordingForTranscript: RecordingEntry? + @State private var transcriptStateRefresh = false + var body: some View { - NavigationView { + NavigationStack { VStack { // Custom header HStack { @@ -344,6 +352,39 @@ struct RecordingsListView: View { } message: { Text(archiveRestoreError ?? "Unknown error") } + .sheet(item: $selectedRecordingForTranscript) { entry in + if let recordingId = entry.id, + let transcript = appCoordinator.getTranscriptData(for: recordingId) { + EditableTranscriptView(recording: entry, transcript: transcript, transcriptManager: TranscriptManager.shared) + .environmentObject(appCoordinator) + } else { + TranscriptDetailView(recording: entry, transcriptText: "") + .environmentObject(appCoordinator) + } + } + .confirmationDialog( + "Clean Audio Before Transcribing?", + isPresented: $showingAudioCleanupPrompt, + titleVisibility: .visible + ) { + Button("Clean & Transcribe") { + if let recording = recordingPendingTranscription { + recordingPendingTranscription = nil + transcriptionStarter.startTranscription(for: recording, cleanFirst: true, appCoordinator: appCoordinator) + } + } + Button("Transcribe As-Is") { + if let recording = recordingPendingTranscription { + recordingPendingTranscription = nil + transcriptionStarter.startTranscription(for: recording, cleanFirst: false, appCoordinator: appCoordinator) + } + } + Button("Cancel", role: .cancel) { + recordingPendingTranscription = nil + } + } message: { + Text("Cleaning reduces static and normalizes volume, which can improve transcription accuracy. The original audio file is not changed.") + } } .onAppear { refreshFileRelationships() @@ -365,6 +406,10 @@ struct RecordingsListView: View { loadRecordings() refreshFileRelationships() } + .onReceive(NotificationCenter.default.publisher(for: NSNotification.Name("TranscriptionCompleted"))) { _ in + transcriptStateRefresh.toggle() + loadRecordings() + } } @@ -469,7 +514,40 @@ struct RecordingsListView: View { } private var dateFilterSheet: some View { - NavigationView { + #if targetEnvironment(macCatalyst) + VStack(spacing: 0) { + HStack { + Button("Cancel") { showDateFilter = false } + Spacer() + Text("Filter by Date").font(.headline) + Spacer() + Button("Apply") { + isDateFilterActive = true + showDateFilter = false + } + .fontWeight(.semibold) + } + .padding(.horizontal, 16).padding(.vertical, 10) + Divider() + Form { + Section { + DatePicker("From", selection: $dateFilterStart, in: ...Date(), displayedComponents: .date) + DatePicker("To", selection: $dateFilterEnd, in: dateFilterStart...Date(), displayedComponents: .date) + } + if isDateFilterActive { + Section { + Button(role: .destructive) { + isDateFilterActive = false + showDateFilter = false + } label: { + HStack { Spacer(); Text("Clear Filter"); Spacer() } + } + } + } + } + } + #else + NavigationStack { Form { Section { DatePicker("From", selection: $dateFilterStart, in: ...Date(), displayedComponents: .date) @@ -507,6 +585,7 @@ struct RecordingsListView: View { } } } + #endif } private func recordingsListView(_ filtered: [AudioRecordingFile]) -> some View { @@ -525,6 +604,7 @@ struct RecordingsListView: View { } private func recordingRow(for recording: AudioRecordingFile) -> some View { + VStack(alignment: .leading, spacing: 8) { HStack { // Selection checkbox (if in selection mode) if isSelectionMode { @@ -686,7 +766,7 @@ struct RecordingsListView: View { } .buttonStyle(PlainButtonStyle()) } - + Button(action: { deletionData.recordingToDelete = recording deleteRecording(recording) @@ -698,8 +778,73 @@ struct RecordingsListView: View { .buttonStyle(PlainButtonStyle()) } } + + transcriptActionButton(for: recording) + } .padding(.vertical, 4) } + + /// Labeled "Generate Transcript" action shown beneath each recording row. + /// Hidden for archived recordings with no local audio. Hidden once a transcript exists — + /// users edit the transcript from the Transcripts tab. + /// - In progress → orange pill with spinner + phase label. + /// - Otherwise → accent "Generate Transcript" pill that opens the cleanup confirmation dialog. + @ViewBuilder + private func transcriptActionButton(for recording: AudioRecordingFile) -> some View { + if recording.hasLocalAudio, + let entry = appCoordinator.getRecording(url: recording.url), + entry.transcript == nil { + let recordingId = entry.id ?? UUID() + let isCleaning = transcriptionStarter.isCleaning(recordingId) + let isQueuedForCleanup = transcriptionStarter.isQueuedForCleanup(recordingId) + let jobStatus = transcriptionStarter.activeTranscriptionJobStatus(for: entry, appCoordinator: appCoordinator) + let hasActiveJob = jobStatus != nil + let isProcessing = isCleaning || isQueuedForCleanup || hasActiveJob + + Button(action: { + if !isProcessing { + recordingPendingTranscription = entry + showingAudioCleanupPrompt = true + } + }) { + HStack(spacing: 8) { + if isProcessing { + ProgressView() + .scaleEffect(0.8) + .tint(.white) + Text(transcriptProcessingLabel(isCleaning: isCleaning, + isQueuedForCleanup: isQueuedForCleanup, + jobStatus: jobStatus)) + } else { + Image(systemName: "text.bubble") + Text("Generate Transcript") + } + } + .font(.subheadline) + .padding(.horizontal, 14) + .padding(.vertical, 8) + .background(isProcessing ? Color.orange : Color.accentColor) + .foregroundColor(.white) + .cornerRadius(10) + } + .buttonStyle(.plain) + .disabled(isProcessing) + .accessibilityLabel("Generate Transcript") + .id("transcript-\(recordingId)-\(isProcessing)-\(transcriptStateRefresh)") + } + } + + private func transcriptProcessingLabel(isCleaning: Bool, + isQueuedForCleanup: Bool, + jobStatus: JobProcessingStatus?) -> String { + if isCleaning { return "Cleaning Audio…" } + if isQueuedForCleanup { return "Queued…" } + switch jobStatus { + case .queued: return "Queued…" + case .processing: return "Transcribing…" + default: return "Processing…" + } + } // MARK: - Search and Date Filtering @@ -1156,7 +1301,7 @@ struct RecordingsListView: View { } private var archiveOlderThanSheet: some View { - NavigationView { + NavigationStack { VStack(spacing: 20) { Image(systemName: "calendar.badge.clock") .font(.system(size: 40)) diff --git a/BisonNotes AI/BisonNotes AI/Views/RecordingsView.swift b/BisonNotes AI/BisonNotes AI/Views/RecordingsView.swift index 43d3974..ababdb8 100644 --- a/BisonNotes AI/BisonNotes AI/Views/RecordingsView.swift +++ b/BisonNotes AI/BisonNotes AI/Views/RecordingsView.swift @@ -22,7 +22,102 @@ struct RecordingsView: View { @State private var showingRecordingsList = false @State private var showingBackgroundProcessing = false @State private var showingHelpDocumentation = false - + @State private var showingRecorderError = false + @State private var recorderErrorMessage = "" + + // MARK: - Recording Controls + + @ViewBuilder + private var recordingTimerView: some View { + HStack(spacing: 8) { + Text(recorderVM.formatTime(recorderVM.recordingTime)) + .font(.title) + .fontWeight(.bold) + .foregroundColor(recorderVM.isPaused ? .secondary : .accentColor) + .monospacedDigit() + if recorderVM.isPaused { + Text("Paused") + .font(.subheadline) + .fontWeight(.semibold) + .foregroundColor(.secondary) + .padding(.horizontal, 10) + .padding(.vertical, 4) + .background(Capsule().fill(Color.secondary.opacity(0.15))) + } + } + } + + @ViewBuilder + private var recordingControls: some View { + HStack(spacing: 12) { + recordingActionButton( + title: recorderVM.isPaused ? "Resume" : "Pause", + systemImage: recorderVM.isPaused ? "play.circle.fill" : "pause.circle.fill", + tint: .accentColor + ) { + if recorderVM.isPaused { + recorderVM.resumeRecording() + } else { + recorderVM.pauseRecording() + } + } + + recordingActionButton( + title: "Stop", + systemImage: "stop.circle.fill", + tint: .red + ) { + recorderVM.stopRecording() + } + } + .padding(.horizontal, 40) + } + + @ViewBuilder + private var startRecordingButton: some View { + Button(action: { recorderVM.startRecording() }) { + HStack { + Image(systemName: "mic.circle.fill").font(.title) + Text("Start Recording") + .font(.title2) + .fontWeight(.semibold) + } + .foregroundColor(.white) + .padding() + .frame(maxWidth: .infinity) + .background( + RoundedRectangle(cornerRadius: 16) + .fill(Color.accentColor) + .shadow(color: .accentColor.opacity(0.3), radius: 8, x: 0, y: 4) + ) + .padding(.horizontal, 40) + } + } + + private func recordingActionButton( + title: String, + systemImage: String, + tint: Color, + action: @escaping () -> Void + ) -> some View { + Button(action: action) { + HStack { + Image(systemName: systemImage).font(.title) + Text(title) + .font(.title2) + .fontWeight(.semibold) + } + .foregroundColor(.white) + .padding() + .frame(maxWidth: .infinity) + .background( + RoundedRectangle(cornerRadius: 16) + .fill(tint) + .shadow(color: tint.opacity(0.3), radius: 8, x: 0, y: 4) + ) + } + } + var body: some View { GeometryReader { geometry in VStack(spacing: 0) { @@ -69,40 +164,12 @@ struct RecordingsView: View { VStack(spacing: 16) { if recorderVM.isRecording { - Text(recorderVM.formatTime(recorderVM.recordingTime)) - .font(.title) - .fontWeight(.bold) - .foregroundColor(.accentColor) - .monospacedDigit() + recordingTimerView + recordingControls + } else { + startRecordingButton } - Button(action: { - if recorderVM.isRecording { - recorderVM.stopRecording() - } else { - recorderVM.startRecording() - } - }) { - HStack { - Image(systemName: recorderVM.isRecording ? "stop.circle.fill" : "mic.circle.fill") - .font(.title) - Text(recorderVM.isRecording ? "Stop Recording" : "Start Recording") - .font(.title2) - .fontWeight(.semibold) - } - .foregroundColor(.white) - .padding() - .frame(maxWidth: .infinity) - .background( - RoundedRectangle(cornerRadius: 16) - .fill(recorderVM.isRecording ? Color.red : Color.accentColor) - .shadow(color: recorderVM.isRecording ? .red.opacity(0.3) : .accentColor.opacity(0.3), radius: 8, x: 0, y: 4) - ) - .padding(.horizontal, 40) - } - .scaleEffect(recorderVM.isRecording ? 1.05 : 1.0) - .animation(.easeInOut(duration: 0.2), value: recorderVM.isRecording) - Button(action: { showingRecordingsList = true }) { @@ -311,6 +378,18 @@ struct RecordingsView: View { } } } + .onChange(of: recorderVM.errorMessage) { _, message in + if let message, !message.isEmpty { + recorderErrorMessage = message + showingRecorderError = true + recorderVM.errorMessage = nil + } + } + .alert("Recording Error", isPresented: $showingRecorderError) { + Button("OK", role: .cancel) {} + } message: { + Text(recorderErrorMessage) + } } } diff --git a/BisonNotes AI/BisonNotes AI/Views/SettingsView.swift b/BisonNotes AI/BisonNotes AI/Views/SettingsView.swift index 23aaba0..aff86f8 100644 --- a/BisonNotes AI/BisonNotes AI/Views/SettingsView.swift +++ b/BisonNotes AI/BisonNotes AI/Views/SettingsView.swift @@ -10,6 +10,7 @@ import AVFoundation import CoreLocation struct SettingsView: View { + @Environment(\.dismiss) private var dismiss @EnvironmentObject var recorderVM: AudioRecorderViewModel @EnvironmentObject var appCoordinator: AppDataCoordinator @StateObject private var regenerationManager: SummaryRegenerationManager @@ -34,7 +35,7 @@ struct SettingsView: View { @AppStorage("WatchBatteryAware") private var watchBatteryAware: Bool = true @AppStorage("iCloudBackupIncludeAudioFiles") private var iCloudBackupIncludeAudioFiles: Bool = false @AppStorage("iCloudBackupIncludeSettings") private var iCloudBackupIncludeSettings: Bool = true - @AppStorage("iCloudBackupIncludeSensitiveSettings") private var iCloudBackupIncludeSensitiveSettings: Bool = true + @AppStorage("iCloudBackupIncludeSensitiveSettings") private var iCloudBackupIncludeSensitiveSettings: Bool = false @AppStorage(OnDeviceLLMModelInfo.SettingsKeys.enableExperimentalModels) private var enableExperimentalModels = false @AppStorage(ComedyMode.SettingsKeys.enabled) private var comedyModeEnabled = false @AppStorage(ComedyMode.SettingsKeys.style) private var comedyModeStyle = "snarky" @@ -52,38 +53,44 @@ struct SettingsView: View { } var body: some View { - ScrollView { - VStack(alignment: .leading, spacing: 20) { - headerSection - preferencesSection - aiEngineSection - transcriptionSection - microphoneSection - advancedSection - debugSection - - - Spacer(minLength: 40) + // NavigationStack { Form } is the only sheet pattern that scrolls reliably + // on Mac Catalyst — Form is UITableView-backed, ScrollView is not. See + // feedback_mac_catalyst_scrollview.md for the diagnostic that confirmed this. + NavigationStack { + Form { + preferencesSection + aiEngineSection + transcriptionSection + microphoneSection + locationSection + iCloudSyncSection + comedyModeSection + experimentalSection + debugTroubleshootingSection + aboutSection + } + .navigationTitle("Settings") + .navigationBarTitleDisplayMode(.inline) + .toolbar { + ToolbarItem(placement: .navigationBarTrailing) { + Button("Done") { dismiss() } } - .frame(maxWidth: 700) - .frame(maxWidth: .infinity) } + } .alert("Regeneration Complete", isPresented: $regenerationManager.showingRegenerationAlert) { Button("OK") { regenerationManager.showingRegenerationAlert = false } } message: { - Text("Regeneration completed successfully") // Use default message since regenerationAlertMessage doesn't exist + Text("Regeneration completed successfully") } .onAppear { refreshEngineStatuses() - // Fetch available microphone inputs Task { await recorderVM.fetchInputs() } } .onChange(of: selectedAIEngine) { _, newEngine in - // Immediately update the SummaryManager when user changes AI engine selection SummaryManager.shared.setEngine(newEngine) AppLog.shared.log("SettingsView: Updated AI engine to '\(newEngine)'", level: .debug, category: .general) } @@ -91,38 +98,28 @@ struct SettingsView: View { OnDeviceLLMDownloadManager.shared.refreshModelStatus() if !newValue { - // Disable the MLX engine itself so it no longer appears available UserDefaults.standard.set(false, forKey: MLXSwiftSettingsKeys.enabled) - // Migrate away from any experimental on-device model that is no longer available let currentModelId = UserDefaults.standard.string(forKey: OnDeviceLLMModelInfo.SettingsKeys.selectedModelId) ?? "" if !OnDeviceLLMModelInfo.availableModels.contains(where: { $0.id == currentModelId }) { if let firstAvailable = OnDeviceLLMModelInfo.availableModels.first { UserDefaults.standard.set(firstAvailable.id, forKey: OnDeviceLLMModelInfo.SettingsKeys.selectedModelId) } - // If availableModels is empty (e.g. <6GB device), leave the stored ID as-is; - // it will be re-used if experimental is re-enabled later. } - // Determine the best fallback engine when on-device AI has no usable models. - // On <6GB devices, all on-device models are experimental, so both MLX and - // On-Device AI become unusable when experimental mode is disabled. let onDeviceHasModels = !OnDeviceLLMModelInfo.availableModels.isEmpty let fallbackEngine: String if onDeviceHasModels { fallbackEngine = AIEngineType.onDeviceLLM.rawValue } else { - // <6GB device: no non-experimental on-device models remain; prefer Apple Native fallbackEngine = AIEngineType.appleNative.rawValue } - // Switch away from MLX Swift if selectedAIEngine == AIEngineType.mlxSwift.rawValue { selectedAIEngine = fallbackEngine SummaryManager.shared.setEngine(fallbackEngine) } - // Switch away from On-Device AI if it no longer has any usable models if selectedAIEngine == AIEngineType.onDeviceLLM.rawValue && !onDeviceHasModels { selectedAIEngine = fallbackEngine SummaryManager.shared.setEngine(fallbackEngine) @@ -175,85 +172,35 @@ struct SettingsView: View { } } - private var headerSection: some View { - VStack(alignment: .leading, spacing: 12) { - HStack { - Text("Settings") - .font(.largeTitle) - .fontWeight(.bold) - .foregroundColor(.primary) - - Spacer() - } - .padding(.top, 20) - .padding(.horizontal, 24) - } - } - private var preferencesSection: some View { - VStack(alignment: .leading, spacing: 16) { - Text("Preferences") - .font(.headline) - .foregroundColor(.primary) - .padding(.horizontal, 24) - - Button(action: { + Section("Preferences") { + Button { showingPreferences = true - }) { + } label: { HStack { Image(systemName: "slider.horizontal.3") - .font(.title2) .foregroundColor(.indigo) - .frame(width: 32) - - VStack(alignment: .leading, spacing: 4) { + .frame(width: 28) + VStack(alignment: .leading, spacing: 2) { Text("Display Preferences") - .font(.body) - .fontWeight(.medium) .foregroundColor(.primary) - Text("Time format and display options") .font(.caption) .foregroundColor(.secondary) } - Spacer() - Image(systemName: "chevron.right") .font(.caption) .foregroundColor(.secondary) } - .padding(.horizontal, 24) - .padding(.vertical, 12) - .background( - RoundedRectangle(cornerRadius: 8) - .fill(Color.indigo.opacity(0.1)) - ) + .contentShape(Rectangle()) } - .buttonStyle(PlainButtonStyle()) - .padding(.horizontal, 24) + .buttonStyle(.plain) } } private var microphoneSection: some View { - VStack(alignment: .leading, spacing: 16) { - HStack { - Text("Microphone Selection") - .font(.headline) - .foregroundColor(.primary) - Spacer() - Button(action: { - Task { - await recorderVM.fetchInputs() - } - }) { - Image(systemName: "arrow.clockwise") - .font(.title3) - .foregroundColor(.accentColor) - } - } - .padding(.horizontal, 24) - + Section { if recorderVM.availableInputs.isEmpty { HStack { Image(systemName: "exclamationmark.triangle") @@ -261,767 +208,407 @@ struct SettingsView: View { Text("No microphones found.") .foregroundColor(.secondary) } - .padding(.horizontal, 24) } else { - microphonePicker - } - } - } - - private var microphonePicker: some View { - VStack(alignment: .leading, spacing: 12) { - ForEach(recorderVM.availableInputs, id: \.uid) { input in - HStack { - Button(action: { + ForEach(recorderVM.availableInputs, id: \.uid) { input in + Button { recorderVM.selectedInput = input recorderVM.setPreferredInput() - }) { + } label: { HStack { Image(systemName: recorderVM.selectedInput?.uid == input.uid ? "largecircle.fill.circle" : "circle") .foregroundColor(recorderVM.selectedInput?.uid == input.uid ? .blue : .gray) - .font(.title2) - VStack(alignment: .leading, spacing: 2) { Text(input.portName) - .font(.body) - .fontWeight(.medium) .foregroundColor(.primary) - Text(input.portType.rawValue) .font(.caption) .foregroundColor(.secondary) } - Spacer() } + .contentShape(Rectangle()) } - .buttonStyle(PlainButtonStyle()) + .buttonStyle(.plain) + } + } + } header: { + HStack { + Text("Microphone Selection") + Spacer() + Button { + Task { await recorderVM.fetchInputs() } + } label: { + Image(systemName: "arrow.clockwise") } - .padding(.horizontal, 24) - .padding(.vertical, 8) - .background( - RoundedRectangle(cornerRadius: 8) - .fill(recorderVM.selectedInput?.uid == input.uid ? Color.blue.opacity(0.1) : Color.clear) - ) } } } private var aiEngineSection: some View { - VStack(alignment: .leading, spacing: 16) { - Text("AI Processing") - .font(.headline) - .foregroundColor(.primary) - .padding(.horizontal, 24) - - VStack(alignment: .leading, spacing: 8) { - HStack { - Text("Current Engine:") - .font(.body) - .foregroundColor(.secondary) - Text(selectedAIEngine) - .font(.body) - .fontWeight(.medium) - .foregroundColor(.primary) - } - - // Engine status indicator - HStack { - Text("Status:") - .font(.body) - .foregroundColor(.secondary) - - // TODO: Update to use new Core Data system - // let engineStatus = appCoordinator.registryManager.getEngineAvailabilityStatus()[selectedAIEngine] - let statusColor: Color = .green // Temporary: assume available - let statusText = "Available" // Temporary: assume available - - HStack(spacing: 4) { - Circle() - .fill(statusColor) - .frame(width: 8, height: 8) - Text(statusText) - .font(.body) - .fontWeight(.medium) - .foregroundColor(statusColor) - } - } + Section("AI Processing") { + HStack { + Text("Current Engine") + Spacer() + Text(selectedAIEngine) + .foregroundColor(.secondary) } - .padding(.horizontal, 24) - HStack { - Button(action: { - showingAISettings = true - }) { - HStack { - Text("Configure AI Engines") - Spacer() - Image(systemName: "arrow.right") - } - .padding(.horizontal, 24) - .padding(.vertical, 12) - .background( - RoundedRectangle(cornerRadius: 8) - .fill(Color.blue.opacity(0.1)) - ) - .foregroundColor(.blue) + Text("Status") + Spacer() + HStack(spacing: 4) { + Circle() + .fill(.green) + .frame(width: 8, height: 8) + Text("Available") + .foregroundColor(.secondary) } - - Button(action: { - // TODO: Update to use new Core Data system - // Task { - // await appCoordinator.registryManager.refreshEngineAvailability() - // } - }) { - Image(systemName: "arrow.clockwise") + } + Button { + showingAISettings = true + } label: { + HStack { + Text("Configure AI Engines") + Spacer() + Image(systemName: "chevron.right") .font(.caption) - .foregroundColor(.blue) + .foregroundColor(.secondary) } - .padding(.horizontal, 8) + .contentShape(Rectangle()) } - .padding(.horizontal, 24) - + .buttonStyle(.plain) } } - + private var transcriptionSection: some View { - VStack(alignment: .leading, spacing: 16) { - Text("Transcription Engine") - .font(.headline) - .foregroundColor(.primary) - .padding(.horizontal, 24) - + Section("Transcription Engine") { HStack { - Text("Current Engine:") - .font(.body) - .foregroundColor(.secondary) + Text("Current Engine") + Spacer() Text(TranscriptionEngine(rawValue: selectedTranscriptionEngine)?.rawValue ?? "On Device") - .font(.body) - .fontWeight(.medium) - .foregroundColor(.primary) + .foregroundColor(.secondary) } - .padding(.horizontal, 24) - - Button(action: { + Button { showingTranscriptionSettings = true - }) { + } label: { HStack { Text("Configure Transcription") Spacer() - Image(systemName: "arrow.right") + Image(systemName: "chevron.right") + .font(.caption) + .foregroundColor(.secondary) } - .padding(.horizontal, 24) - .padding(.vertical, 12) - .background( - RoundedRectangle(cornerRadius: 8) - .fill(Color.purple.opacity(0.1)) - ) - .foregroundColor(.purple) + .contentShape(Rectangle()) } - .padding(.horizontal, 24) + .buttonStyle(.plain) } } - private var advancedSection: some View { - VStack(alignment: .leading, spacing: 16) { - Text("Advanced Settings") - .font(.headline) - .foregroundColor(.primary) - .padding(.horizontal, 24) - - VStack(alignment: .leading, spacing: 8) { - // Location Services - VStack(spacing: 8) { - HStack { - VStack(alignment: .leading, spacing: 4) { - Text("Location Services") - .font(.body) - .foregroundColor(.primary) - Text("Capture location data with recordings") - .font(.caption) - .foregroundColor(.secondary) - } - Spacer() - Toggle("", isOn: Binding( - get: { recorderVM.isLocationTrackingEnabled }, - set: { newValue in - recorderVM.toggleLocationTracking(newValue) - } - )) - .labelsHidden() - } - - // Location status indicator - if recorderVM.isLocationTrackingEnabled { - HStack { - Image(systemName: locationStatusIcon) - .font(.caption) - .foregroundColor(locationStatusColor) - Text(locationStatusText) - .font(.caption) - .foregroundColor(locationStatusColor) - Spacer() - } - } + private var locationSection: some View { + Section { + Toggle("Location Services", isOn: Binding( + get: { recorderVM.isLocationTrackingEnabled }, + set: { recorderVM.toggleLocationTracking($0) } + )) + if recorderVM.isLocationTrackingEnabled { + HStack { + Image(systemName: locationStatusIcon) + .foregroundColor(locationStatusColor) + Text(locationStatusText) + .font(.caption) + .foregroundColor(locationStatusColor) } - .padding(.horizontal, 24) - .padding(.vertical, 12) - .background( - Rectangle() - .fill(Color(.systemGray6)) - .opacity(0.3) - ) - - // iCloud Sync - VStack(spacing: 8) { - HStack { - VStack(alignment: .leading, spacing: 4) { - Text("iCloud Sync") - .font(.body) - .foregroundColor(.primary) - Text("Sync summaries, transcripts, and settings across your devices") - .font(.caption) - .foregroundColor(.secondary) - } - Spacer() - Toggle("", isOn: $iCloudManager.isEnabled) - .labelsHidden() - } + } + } footer: { + Text("Capture location data with recordings") + } + } + + private var iCloudSyncSection: some View { + Section { + Toggle("Enable iCloud Sync", isOn: $iCloudManager.isEnabled) + + if iCloudManager.isEnabled { + Toggle("Include audio files in backup", isOn: $iCloudBackupIncludeAudioFiles) + Toggle("Include app settings", isOn: $iCloudBackupIncludeSettings) + Toggle("Include sensitive settings", isOn: $iCloudBackupIncludeSensitiveSettings) + .disabled(!iCloudBackupIncludeSettings) + Text("API keys and AWS credentials stay in Keychain and are never included in iCloud settings backups. Leave sensitive settings off unless you explicitly want eligible future sensitive preferences copied to iCloud.") + .font(.caption) + .foregroundColor(.secondary) + + Button { + Task { await backupAllDataToiCloud() } + } label: { + Label("Backup Now", systemImage: "icloud.and.arrow.up") + } + .disabled(isRunningCloudBackupAction) + + Button { + Task { await restoreAllDataFromiCloud() } + } label: { + Label("Restore From iCloud", systemImage: "arrow.down.doc") + .foregroundColor(.green) + } + .disabled(isRunningCloudBackupAction) - // Show sync status + if isRunningCloudBackupAction { HStack { - Text("Status:") + ProgressView() + .progressViewStyle(.circular) + .scaleEffect(0.8) + Text("Working…") .font(.caption) .foregroundColor(.secondary) - Spacer() - HStack(spacing: 4) { - Circle() - .fill(iCloudManager.isEnabled ? Color.green : Color.gray) - .frame(width: 8, height: 8) - Text(iCloudManager.isEnabled ? "Enabled" : "Disabled") - .font(.caption) - .foregroundColor(iCloudManager.isEnabled ? .green : .gray) - } } - .padding(.top, 4) - - if iCloudManager.isEnabled { - VStack(alignment: .leading, spacing: 8) { - Toggle("Include audio files in backup", isOn: $iCloudBackupIncludeAudioFiles) - .font(.caption) - - Toggle("Include app settings", isOn: $iCloudBackupIncludeSettings) - .font(.caption) - - Toggle("Include API keys and credentials (iCloud encrypted)", isOn: $iCloudBackupIncludeSensitiveSettings) - .font(.caption) - .disabled(!iCloudBackupIncludeSettings) - - if isRunningCloudBackupAction { - ProgressView() - .progressViewStyle(.circular) - .scaleEffect(0.8) - } + } - HStack(spacing: 8) { - Button(action: { - Task { - await backupAllDataToiCloud() - } - }) { - HStack(spacing: 4) { - Image(systemName: "icloud.and.arrow.up") - Text("Backup Now") - } - .font(.caption) - .foregroundColor(.blue) - .padding(.horizontal, 10) - .padding(.vertical, 6) - .background( - RoundedRectangle(cornerRadius: 6) - .fill(Color.blue.opacity(0.1)) - ) - } - .disabled(isRunningCloudBackupAction) + if !cloudBackupActionMessage.isEmpty { + Text(cloudBackupActionMessage) + .font(.caption) + .foregroundColor(cloudBackupActionIsError ? .red : .secondary) + } + } - Button(action: { - Task { - await restoreAllDataFromiCloud() - } - }) { - HStack(spacing: 4) { - Image(systemName: "arrow.down.doc") - Text("Restore") - } - .font(.caption) - .foregroundColor(.green) - .padding(.horizontal, 10) - .padding(.vertical, 6) - .background( - RoundedRectangle(cornerRadius: 6) - .fill(Color.green.opacity(0.1)) - ) - } - .disabled(isRunningCloudBackupAction) + if !iCloudManager.pendingConflicts.isEmpty { + ForEach(iCloudManager.pendingConflicts, id: \.summaryId) { conflict in + VStack(alignment: .leading, spacing: 6) { + Text(conflict.localSummary.recordingName) + .font(.caption) + Text("Modified on different devices") + .font(.caption2) + .foregroundColor(.secondary) + HStack { + Button("Use Local") { + Task { try? await iCloudManager.resolveConflict(conflict, useLocal: true) } } - - if !cloudBackupActionMessage.isEmpty { - Text(cloudBackupActionMessage) - .font(.caption2) - .foregroundColor(cloudBackupActionIsError ? .red : .secondary) + .buttonStyle(.bordered) + Button("Use Cloud") { + Task { try? await iCloudManager.resolveConflict(conflict, useLocal: false) } } + .buttonStyle(.bordered) + .tint(.green) } - .padding(.top, 4) } + } + } - // Show conflicts if any exist - if !iCloudManager.pendingConflicts.isEmpty { - VStack(alignment: .leading, spacing: 8) { - Text("Sync Conflicts (\(iCloudManager.pendingConflicts.count))") - .font(.caption) - .fontWeight(.medium) - .foregroundColor(.orange) - - ForEach(iCloudManager.pendingConflicts, id: \.summaryId) { conflict in - HStack { - VStack(alignment: .leading, spacing: 2) { - Text(conflict.localSummary.recordingName) - .font(.caption) - .foregroundColor(.primary) - Text("Modified on different devices") - .font(.caption2) - .foregroundColor(.secondary) - } - - Spacer() - - HStack(spacing: 8) { - Button("Use Local") { - Task { - try? await iCloudManager.resolveConflict(conflict, useLocal: true) - } - } - .font(.caption2) - .foregroundColor(.blue) - .padding(.horizontal, 8) - .padding(.vertical, 4) - .background( - RoundedRectangle(cornerRadius: 4) - .fill(Color.blue.opacity(0.1)) - ) + if let error = iCloudManager.lastError { + Text("Error: \(error)") + .font(.caption) + .foregroundColor(.red) + } - Button("Use Cloud") { - Task { - try? await iCloudManager.resolveConflict(conflict, useLocal: false) + if !iCloudManager.isEnabled { + Button { + Task { + do { + let cloudSummaries = try await iCloudManager.fetchSummariesFromiCloud(forRecovery: true) + + let localSummaries = appCoordinator.coreDataManager.getAllSummaries() + let localSummaryIds = Set(localSummaries.compactMap { $0.id }) + let cloudOnlySummaries = cloudSummaries.filter { !localSummaryIds.contains($0.id) } + + if !cloudOnlySummaries.isEmpty { + await MainActor.run { + let alert = UIAlertController( + title: "iCloud Data Found", + message: "We found \(cloudOnlySummaries.count) summaries in your iCloud that aren't on this device. Would you like to download them?", + preferredStyle: .alert + ) + alert.addAction(UIAlertAction(title: "Cancel", style: .cancel)) + alert.addAction(UIAlertAction(title: "Download", style: .default) { _ in + Task { + do { + let count = try await iCloudManager.downloadSummariesFromCloud(appCoordinator: appCoordinator, forRecovery: true) + AppLog.shared.log("Downloaded \(count) summaries from iCloud", category: .general) + } catch { + AppLog.shared.log("Failed to download summaries: \(error)", level: .error, category: .general) } } - .font(.caption2) - .foregroundColor(.green) - .padding(.horizontal, 8) - .padding(.vertical, 4) - .background( - RoundedRectangle(cornerRadius: 4) - .fill(Color.green.opacity(0.1)) - ) + }) + if let windowScene = UIApplication.shared.connectedScenes.first as? UIWindowScene, + let rootViewController = windowScene.windows.first?.rootViewController { + rootViewController.present(alert, animated: true) } } - .padding(.vertical, 4) - } - } - .padding(.top, 4) - } - - // Show errors if any exist - if let error = iCloudManager.lastError { - Text("Error: \(error)") - .font(.caption) - .foregroundColor(.red) - .padding(.top, 4) - } - - if !iCloudManager.isEnabled { - Button(action: { - Task { - do { - // Check for iCloud data using recovery flag that works even when sync is disabled - let cloudSummaries = try await iCloudManager.fetchSummariesFromiCloud(forRecovery: true) - - let localSummaries = appCoordinator.coreDataManager.getAllSummaries() - let localSummaryIds = Set(localSummaries.compactMap { $0.id }) - let cloudOnlySummaries = cloudSummaries.filter { !localSummaryIds.contains($0.id) } - - if !cloudOnlySummaries.isEmpty { - // Show alert asking user if they want to download - await MainActor.run { - let alert = UIAlertController( - title: "iCloud Data Found", - message: "We found \(cloudOnlySummaries.count) summaries in your iCloud that aren't on this device. Would you like to download them?", - preferredStyle: .alert - ) - - alert.addAction(UIAlertAction(title: "Cancel", style: .cancel)) - alert.addAction(UIAlertAction(title: "Download", style: .default) { _ in - Task { - do { - let count = try await iCloudManager.downloadSummariesFromCloud(appCoordinator: appCoordinator, forRecovery: true) - AppLog.shared.log("Downloaded \(count) summaries from iCloud", category: .general) - } catch { - AppLog.shared.log("Failed to download summaries: \(error)", level: .error, category: .general) - } - } - }) - - if let windowScene = UIApplication.shared.connectedScenes.first as? UIWindowScene, - let rootViewController = windowScene.windows.first?.rootViewController { - rootViewController.present(alert, animated: true) - } - } - } else { - // Show message that no cloud data was found - await MainActor.run { - let alert = UIAlertController( - title: "No iCloud Data", - message: "No summaries were found in your iCloud account.", - preferredStyle: .alert - ) - - alert.addAction(UIAlertAction(title: "OK", style: .default)) - - if let windowScene = UIApplication.shared.connectedScenes.first as? UIWindowScene, - let rootViewController = windowScene.windows.first?.rootViewController { - rootViewController.present(alert, animated: true) - } - } - } - } catch { - AppLog.shared.log("Failed to check for iCloud data: \(error)", level: .error, category: .general) - await MainActor.run { - let alert = UIAlertController( - title: "Check Failed", - message: "Could not check for iCloud data: \(error.localizedDescription)", - preferredStyle: .alert - ) - - alert.addAction(UIAlertAction(title: "OK", style: .default)) - - if let windowScene = UIApplication.shared.connectedScenes.first as? UIWindowScene, - let rootViewController = windowScene.windows.first?.rootViewController { - rootViewController.present(alert, animated: true) - } + } else { + await MainActor.run { + let alert = UIAlertController( + title: "No iCloud Data", + message: "No summaries were found in your iCloud account.", + preferredStyle: .alert + ) + alert.addAction(UIAlertAction(title: "OK", style: .default)) + if let windowScene = UIApplication.shared.connectedScenes.first as? UIWindowScene, + let rootViewController = windowScene.windows.first?.rootViewController { + rootViewController.present(alert, animated: true) } } } - }) { - HStack { - Image(systemName: "icloud.and.arrow.down") - Text("Check for iCloud Data") + } catch { + AppLog.shared.log("Failed to check for iCloud data: \(error)", level: .error, category: .general) + await MainActor.run { + let alert = UIAlertController( + title: "Check Failed", + message: "Could not check for iCloud data: \(error.localizedDescription)", + preferredStyle: .alert + ) + alert.addAction(UIAlertAction(title: "OK", style: .default)) + if let windowScene = UIApplication.shared.connectedScenes.first as? UIWindowScene, + let rootViewController = windowScene.windows.first?.rootViewController { + rootViewController.present(alert, animated: true) + } } - .font(.caption) - .foregroundColor(.blue) - .padding(.horizontal, 12) - .padding(.vertical, 6) - .background( - RoundedRectangle(cornerRadius: 6) - .fill(Color.blue.opacity(0.1)) - ) } - .padding(.top, 8) } + } label: { + Label("Check for iCloud Data", systemImage: "icloud.and.arrow.down") } - .padding(.horizontal, 24) - .padding(.vertical, 12) - .background( - Rectangle() - .fill(Color(.systemGray6)) - .opacity(0.3) - ) } + } header: { + HStack { + Text("iCloud Sync") + Spacer() + HStack(spacing: 4) { + Circle() + .fill(iCloudManager.isEnabled ? Color.green : Color.gray) + .frame(width: 8, height: 8) + Text(iCloudManager.isEnabled ? "Enabled" : "Disabled") + .font(.caption) + } + } + } footer: { + Text("Sync summaries, transcripts, and settings across your devices") } } - private var debugSection: some View { - VStack(alignment: .leading, spacing: 16) { - // Comedy Mode - VStack(spacing: 8) { - HStack { - VStack(alignment: .leading, spacing: 4) { - Text("Comedy Mode") - .font(.body) - .foregroundColor(.primary) - Text("Make AI summaries entertaining with a comedic twist. All information is preserved — just delivered with flair.") - .font(.caption) - .foregroundColor(.secondary) - } - Spacer() - Toggle("", isOn: $comedyModeEnabled) - .labelsHidden() + private var comedyModeSection: some View { + Section { + Toggle("Comedy Mode", isOn: $comedyModeEnabled) + if comedyModeEnabled { + Picker("Style", selection: $comedyModeStyle) { + Text("Snarky — dry wit & sarcasm").tag("snarky") + Text("Funny — goofy & absurd").tag("funny") } + } + } footer: { + Text("Make AI summaries entertaining with a comedic twist. All information is preserved — just delivered with flair.") + } + } - if comedyModeEnabled { - VStack(alignment: .leading, spacing: 8) { - Text("Comedy Style") - .font(.caption) - .fontWeight(.medium) - .foregroundColor(.secondary) - - HStack(spacing: 12) { - Button(action: { comedyModeStyle = "snarky" }) { - HStack(spacing: 6) { - Image(systemName: comedyModeStyle == "snarky" ? "checkmark.circle.fill" : "circle") - .foregroundColor(comedyModeStyle == "snarky" ? .orange : .secondary) - VStack(alignment: .leading, spacing: 2) { - Text("Snarky") - .font(.subheadline) - .fontWeight(.medium) - Text("Dry wit & sarcasm") - .font(.caption2) - .foregroundColor(.secondary) - } - } - .padding(.horizontal, 12) - .padding(.vertical, 8) - .frame(maxWidth: .infinity, alignment: .leading) - .background( - RoundedRectangle(cornerRadius: 8) - .fill(comedyModeStyle == "snarky" ? Color.orange.opacity(0.15) : Color(.systemGray5).opacity(0.5)) - ) - .overlay( - RoundedRectangle(cornerRadius: 8) - .stroke(comedyModeStyle == "snarky" ? Color.orange.opacity(0.5) : Color.clear, lineWidth: 1) - ) - } - .buttonStyle(PlainButtonStyle()) + private var experimentalSection: some View { + Section { + Toggle("Experimental summary models & MLX AI engine", isOn: $enableExperimentalModels) + } footer: { + Text("Allow experimental local summary models and show the MLX Swift AI engine in AI settings. These models are unreliable and may produce empty summaries. For devices with <6GB RAM, this enables on-device AI with only LFM 2.5 available.") + } + } - Button(action: { comedyModeStyle = "funny" }) { - HStack(spacing: 6) { - Image(systemName: comedyModeStyle == "funny" ? "checkmark.circle.fill" : "circle") - .foregroundColor(comedyModeStyle == "funny" ? .purple : .secondary) - VStack(alignment: .leading, spacing: 2) { - Text("Funny") - .font(.subheadline) - .fontWeight(.medium) - Text("Goofy & absurd") - .font(.caption2) - .foregroundColor(.secondary) - } - } - .padding(.horizontal, 12) - .padding(.vertical, 8) - .frame(maxWidth: .infinity, alignment: .leading) - .background( - RoundedRectangle(cornerRadius: 8) - .fill(comedyModeStyle == "funny" ? Color.purple.opacity(0.15) : Color(.systemGray5).opacity(0.5)) - ) - .overlay( - RoundedRectangle(cornerRadius: 8) - .stroke(comedyModeStyle == "funny" ? Color.purple.opacity(0.5) : Color.clear, lineWidth: 1) - ) - } - .buttonStyle(PlainButtonStyle()) - } - } - .padding(.top, 4) + private var debugTroubleshootingSection: some View { + Section("Debug & Troubleshooting") { + HStack { + VStack(alignment: .leading, spacing: 2) { + Text("Total Recordings Storage") + Text("Space used by audio recordings") + .font(.caption) + .foregroundColor(.secondary) } + Spacer() + Text(totalRecordingsStorageString) + .foregroundColor(.secondary) } - .padding(.horizontal, 24) - .padding(.vertical, 12) - .background( - Rectangle() - .fill(Color(.systemGray6)) - .opacity(0.3) - ) - // Experimental On-Device AI Models - VStack(spacing: 8) { + Button { + showingBackgroundProcessing = true + } label: { HStack { - VStack(alignment: .leading, spacing: 4) { - Text("Enable experimental summary models and MLX AI engine") - .font(.body) + VStack(alignment: .leading, spacing: 2) { + Text("Background Processing") .foregroundColor(.primary) - Text("Allow experimental local summary models and show the MLX Swift AI engine in AI settings. These models are unreliable and may produce empty summaries. For devices with <6GB RAM, this enables on-device AI with only LFM 2.5 available.") + Text("Manage transcription and summarization jobs") .font(.caption) .foregroundColor(.secondary) } Spacer() - Toggle("", isOn: $enableExperimentalModels) - .labelsHidden() + Image(systemName: "chevron.right") + .font(.caption) + .foregroundColor(.secondary) } + .contentShape(Rectangle()) } - .padding(.horizontal, 24) - .padding(.vertical, 12) - .background( - Rectangle() - .fill(Color(.systemGray6)) - .opacity(0.3) - ) + .buttonStyle(.plain) - // Debug & Troubleshooting - Text("Debug & Troubleshooting") - .font(.headline) - .foregroundColor(.primary) - .padding(.horizontal, 24) + Button { + logExportError = nil + withAnimation(.easeInOut(duration: 0.2)) { isPreparingLogs = true } - VStack(alignment: .leading, spacing: 8) { - // Storage Information - HStack { - VStack(alignment: .leading, spacing: 4) { - Text("Total Recordings Storage") - .font(.body) - .foregroundColor(.primary) - Text("Space used by audio recordings") - .font(.caption) - .foregroundColor(.secondary) + Task { + do { + let url = try await Task.detached(priority: .userInitiated) { + try await LogExporter.exportLogs() + }.value + + await MainActor.run { + LogEmailPresenter.shared.presentLogEmail( + logFileURL: url, + onPresented: { + withAnimation(.easeInOut(duration: 0.2)) { isPreparingLogs = false } + } + ) { + withAnimation(.easeInOut(duration: 0.2)) { isPreparingLogs = false } + } + } + } catch { + await MainActor.run { + withAnimation(.easeInOut(duration: 0.2)) { isPreparingLogs = false } + logExportError = error.localizedDescription + } } - Spacer() - Text(totalRecordingsStorageString) - .font(.body) - .fontWeight(.medium) - .foregroundColor(.primary) } - .padding(.horizontal, 24) - .padding(.vertical, 12) - .background( - Rectangle() - .fill(Color(.systemGray6)) - .opacity(0.3) - ) - - // Background Processing + } label: { HStack { - VStack(alignment: .leading, spacing: 4) { - Text("Background Processing") - .font(.body) + VStack(alignment: .leading, spacing: 2) { + Text("Export Diagnostic Logs") .foregroundColor(.primary) - Text("Manage transcription and summarization jobs") + Text("Email logs to developer for troubleshooting") .font(.caption) .foregroundColor(.secondary) } Spacer() - Button(action: { - showingBackgroundProcessing = true - }) { - HStack { - Text("Manage Jobs") - Image(systemName: "arrow.right") - } - .font(.caption) - .foregroundColor(.blue) - .padding(.horizontal, 12) - .padding(.vertical, 6) - .background( - RoundedRectangle(cornerRadius: 6) - .fill(Color.blue.opacity(0.1)) - ) - } - } - .padding(.horizontal, 24) - .padding(.vertical, 12) - .background( - Rectangle() - .fill(Color(.systemGray6)) - .opacity(0.3) - ) - - // Export Diagnostic Logs - VStack(spacing: 8) { - HStack { - VStack(alignment: .leading, spacing: 4) { - Text("Export Diagnostic Logs") - .font(.body) - .foregroundColor(.primary) - Text("Email logs to developer for troubleshooting") - .font(.caption) - .foregroundColor(.secondary) - } - Spacer() - Button(action: { - logExportError = nil - withAnimation(.easeInOut(duration: 0.2)) { isPreparingLogs = true } - - Task { - do { - let url = try await Task.detached(priority: .userInitiated) { - try await LogExporter.exportLogs() - }.value - - await MainActor.run { - LogEmailPresenter.shared.presentLogEmail( - logFileURL: url, - onPresented: { - withAnimation(.easeInOut(duration: 0.2)) { isPreparingLogs = false } - } - ) { - // Overlay is already gone; called on mail-sheet dismiss - withAnimation(.easeInOut(duration: 0.2)) { isPreparingLogs = false } - } - } - } catch { - await MainActor.run { - withAnimation(.easeInOut(duration: 0.2)) { isPreparingLogs = false } - logExportError = error.localizedDescription - } - } - } - }) { - HStack { - Text("Send Logs") - Image(systemName: "envelope") - } + if isPreparingLogs { + ProgressView().scaleEffect(0.8) + } else { + Image(systemName: "envelope") .font(.caption) - .foregroundColor(.blue) - .padding(.horizontal, 12) - .padding(.vertical, 6) - .background( - RoundedRectangle(cornerRadius: 6) - .fill(Color.blue.opacity(0.1)) - ) - } - .disabled(isPreparingLogs) - .opacity(isPreparingLogs ? 0.5 : 1.0) - } - if let logExportError { - Text("Error: \(logExportError)") - .font(.caption) - .foregroundColor(.red) + .foregroundColor(.secondary) } } - .padding(.horizontal, 24) - .padding(.vertical, 12) - .background( - Rectangle() - .fill(Color(.systemGray6)) - .opacity(0.3) - ) + .contentShape(Rectangle()) } - .background( - RoundedRectangle(cornerRadius: 12) - .fill(Color(.systemBackground)) - .shadow(color: Color.black.opacity(0.05), radius: 1, x: 0, y: 1) - .overlay( - RoundedRectangle(cornerRadius: 12) - .stroke(Color(.systemGray5), lineWidth: 0.5) - ) - .opacity(0.3) - ) + .buttonStyle(.plain) + .disabled(isPreparingLogs) - // About - Text("About") - .font(.headline) - .foregroundColor(.primary) - .padding(.horizontal, 24) + if let logExportError { + Text("Error: \(logExportError)") + .font(.caption) + .foregroundColor(.red) + } + } + } - Button(action: { + private var aboutSection: some View { + Section("About") { + Button { showingAcknowledgements = true - }) { + } label: { HStack { Image(systemName: "hand.raised.fill") .foregroundColor(.indigo) VStack(alignment: .leading, spacing: 2) { Text("Acknowledgements") - .font(.body) .foregroundColor(.primary) Text("Open-source projects and licenses") .font(.caption) @@ -1032,34 +619,15 @@ struct SettingsView: View { .font(.caption) .foregroundColor(.secondary) } - .padding(.horizontal, 12) - .padding(.vertical, 10) - .background( - RoundedRectangle(cornerRadius: 8) - .fill(Color.indigo.opacity(0.1)) - ) + .contentShape(Rectangle()) } - .buttonStyle(PlainButtonStyle()) - .padding(.horizontal, 24) + .buttonStyle(.plain) - // Advanced Troubleshooting - Button(action: { + Button(role: .destructive) { showingTroubleshootingWarning = true - }) { - HStack { - Image(systemName: "wrench.and.screwdriver") - Text("Advanced Troubleshooting") - } - .font(.caption) - .foregroundColor(.white) - .padding(.horizontal, 12) - .padding(.vertical, 6) - .background( - RoundedRectangle(cornerRadius: 6) - .fill(Color.orange) - ) + } label: { + Label("Advanced Troubleshooting", systemImage: "wrench.and.screwdriver") } - .padding(.horizontal, 24) .alert("Warning", isPresented: $showingTroubleshootingWarning) { Button("Cancel", role: .cancel) { } Button("OK") { @@ -1189,7 +757,7 @@ struct SettingsView: View { let settingsText: String if result.settingsBackedUp { - settingsText = result.includedSensitiveSettings ? "settings + encrypted keys" : "settings" + settingsText = result.includedSensitiveSettings ? "settings + sensitive settings" : "settings" } else { settingsText = "no settings" } @@ -1229,7 +797,7 @@ struct SettingsView: View { let settingsText: String if result.settingsRestored { - settingsText = result.includedSensitiveSettings ? "settings + encrypted keys" : "settings" + settingsText = result.includedSensitiveSettings ? "settings + sensitive settings" : "settings" } else { settingsText = "no settings" } diff --git a/BisonNotes AI/BisonNotes AI/Views/SimpleSettingsView.swift b/BisonNotes AI/BisonNotes AI/Views/SimpleSettingsView.swift index e4dcf4f..ce400a8 100644 --- a/BisonNotes AI/BisonNotes AI/Views/SimpleSettingsView.swift +++ b/BisonNotes AI/BisonNotes AI/Views/SimpleSettingsView.swift @@ -60,7 +60,7 @@ struct SimpleSettingsView: View { @State private var showingHelpDocumentation = false @State private var showingOnDeviceAIDownload = false @State private var showingMistralOnboarding = false - + var body: some View { AdaptiveNavigationWrapper { ScrollView { @@ -122,24 +122,31 @@ struct SimpleSettingsView: View { } } .sheet(isPresented: $showingAdvancedSettings) { - NavigationView { - SettingsView() - .environmentObject(recorderVM) - .environmentObject(appCoordinator) - .navigationBarTitleDisplayMode(.inline) - .toolbar { - ToolbarItem(placement: .navigationBarTrailing) { - Button("Done") { - showingAdvancedSettings = false - } - } - } - } + // SettingsView provides its own NavigationStack and Done toolbar. + SettingsView() + .environmentObject(recorderVM) + .environmentObject(appCoordinator) } .sheet(isPresented: $showingOnDeviceLLMSettings) { + #if targetEnvironment(macCatalyst) + VStack(spacing: 0) { + HStack { + Text("On-Device AI Settings") + .font(.headline) + Spacer() + Button("Done") { showingOnDeviceLLMSettings = false } + .buttonStyle(.bordered) + } + .padding(.horizontal, 16) + .padding(.vertical, 10) + Divider() + OnDeviceLLMSettingsView() + } + #else NavigationStack { OnDeviceLLMSettingsView() } + #endif } .sheet(isPresented: $showingHelpDocumentation) { #if !targetEnvironment(macCatalyst) @@ -616,7 +623,7 @@ struct SimpleSettingsView: View { private func saveConfiguration() { // For Mistral, launch the onboarding wizard only if no API key exists yet if selectedOption == .mistralAI { - let existingKey = UserDefaults.standard.string(forKey: "mistralAPIKey") ?? "" + let existingKey = KeychainSecretStore.shared.string(forKey: KeychainSecretStore.mistralAPIKey) ?? "" if existingKey.isEmpty { showingMistralOnboarding = true return @@ -676,25 +683,32 @@ struct SimpleSettingsView: View { UserDefaults.standard.set(TranscriptionEngine.fluidAudio.rawValue, forKey: "selectedTranscriptionEngine") UserDefaults.standard.set(true, forKey: FluidAudioModelInfo.SettingsKeys.enableFluidAudio) - // Set AI engine to On-Device AI for summaries unless the user already chose - // another on-device summary engine from advanced settings. + // Honor any local AI engine the user has already chosen (On-Device LLM, + // MLX, or Apple Intelligence) along with its selected model. Only fall + // back to the On-Device LLM + Granite default if no local engine is set. let currentAI = UserDefaults.standard.string(forKey: "SelectedAIEngine") - if currentAI == AIEngineType.mlxSwift.rawValue { - UserDefaults.standard.set(true, forKey: MLXSwiftSettingsKeys.enabled) - } else if currentAI != AIEngineType.appleNative.rawValue { + let localEngines: Set = [ + AIEngineType.onDeviceLLM.rawValue, + AIEngineType.mlxSwift.rawValue, + AIEngineType.appleNative.rawValue, + ] + + if let currentAI, localEngines.contains(currentAI) { + if currentAI == AIEngineType.mlxSwift.rawValue { + UserDefaults.standard.set(true, forKey: MLXSwiftSettingsKeys.enabled) + } else if currentAI == AIEngineType.onDeviceLLM.rawValue { + UserDefaults.standard.set(true, forKey: OnDeviceLLMModelInfo.SettingsKeys.enableOnDeviceLLM) + } + } else { UserDefaults.standard.set(AIEngineType.onDeviceLLM.rawValue, forKey: "SelectedAIEngine") - } - - // Enable On-Device LLM - UserDefaults.standard.set(true, forKey: OnDeviceLLMModelInfo.SettingsKeys.enableOnDeviceLLM) - - // Set On-Device LLM to use Granite Micro (recommended for 6GB+ devices) - // This prevents migration warnings when selectedModel is accessed - let deviceRAM = DeviceCapabilities.totalRAMInGB - if deviceRAM >= 6.0 { - UserDefaults.standard.set(OnDeviceLLMModelInfo.granite4Micro.id, forKey: OnDeviceLLMModelInfo.SettingsKeys.selectedModelId) - } else if deviceRAM >= 8.0 { - UserDefaults.standard.set(OnDeviceLLMModelInfo.granite4H.id, forKey: OnDeviceLLMModelInfo.SettingsKeys.selectedModelId) + UserDefaults.standard.set(true, forKey: OnDeviceLLMModelInfo.SettingsKeys.enableOnDeviceLLM) + + let deviceRAM = DeviceCapabilities.totalRAMInGB + if deviceRAM >= 6.0 { + UserDefaults.standard.set(OnDeviceLLMModelInfo.granite4Micro.id, forKey: OnDeviceLLMModelInfo.SettingsKeys.selectedModelId) + } else if deviceRAM >= 8.0 { + UserDefaults.standard.set(OnDeviceLLMModelInfo.granite4H.id, forKey: OnDeviceLLMModelInfo.SettingsKeys.selectedModelId) + } } } diff --git a/BisonNotes AI/BisonNotes AI/Views/TranscriptViews.swift b/BisonNotes AI/BisonNotes AI/Views/TranscriptViews.swift index 01ba47a..9f1b7ed 100644 --- a/BisonNotes AI/BisonNotes AI/Views/TranscriptViews.swift +++ b/BisonNotes AI/BisonNotes AI/Views/TranscriptViews.swift @@ -18,8 +18,6 @@ struct TranscriptsView: View { @State private var recordings: [(recording: RecordingEntry, transcript: TranscriptData?)] = [] @State private var importedTranscripts: [(recording: RecordingEntry, transcript: TranscriptData?)] = [] @State private var selectedRecording: RecordingEntry? - @State private var isGeneratingTranscript = false - @State private var generatingTranscriptRecording: RecordingEntry? @State private var showingAudioCleanupPrompt = false @State private var recordingPendingTranscription: RecordingEntry? @State private var selectedLocationData: LocationData? @@ -35,12 +33,10 @@ struct TranscriptsView: View { @State private var dateFilterStart: Date = Calendar.current.date(byAdding: .month, value: -1, to: Date()) ?? Date() @State private var dateFilterEnd: Date = Date() @State private var isDateFilterActive = false - /// The recording currently undergoing audio cleanup - @State private var activeCleaningRecordingId: UUID? - /// Recordings queued for audio cleanup (waiting for the active one to finish) - @State private var queuedCleanupRecordings: [RecordingEntry] = [] - /// True while the cleanup queue is being processed - @State private var isProcessingCleanupQueue = false + /// Shared service that owns the serial audio-cleanup queue and transcription start. + @ObservedObject private var transcriptionStarter = TranscriptionStarter.shared + /// Recordings whose summary generation we kicked off and are still awaiting completion. + @State private var generatingSummaryRecordingIds: Set = [] var body: some View { AdaptiveNavigationWrapper { @@ -50,8 +46,12 @@ struct TranscriptsView: View { if let recordingId = recording.id, let transcript = appCoordinator.getTranscriptData(for: recordingId) { EditableTranscriptView(recording: recording, transcript: transcript, transcriptManager: TranscriptManager.shared) + .environmentObject(appCoordinator) + .environmentObject(recorderVM) } else { TranscriptDetailView(recording: recording, transcriptText: "") + .environmentObject(appCoordinator) + .environmentObject(recorderVM) } } .sheet(item: $selectedLocationData) { locationData in @@ -65,13 +65,13 @@ struct TranscriptsView: View { Button("Clean & Transcribe") { if let recording = recordingPendingTranscription { recordingPendingTranscription = nil - proceedWithTranscription(for: recording, cleanFirst: true) + transcriptionStarter.startTranscription(for: recording, cleanFirst: true, appCoordinator: appCoordinator) } } Button("Transcribe As-Is") { if let recording = recordingPendingTranscription { recordingPendingTranscription = nil - proceedWithTranscription(for: recording, cleanFirst: false) + transcriptionStarter.startTranscription(for: recording, cleanFirst: false, appCoordinator: appCoordinator) } } Button("Cancel", role: .cancel) { @@ -238,7 +238,42 @@ struct TranscriptsView: View { } private var dateFilterSheet: some View { - NavigationView { + #if targetEnvironment(macCatalyst) + VStack(spacing: 0) { + HStack { + Button("Cancel") { showDateFilter = false } + Spacer() + Text("Filter by Date").font(.headline) + Spacer() + Button("Apply") { + isDateFilterActive = true + showDateFilter = false + refreshTrigger.toggle() + } + .fontWeight(.semibold) + } + .padding(.horizontal, 16).padding(.vertical, 10) + Divider() + Form { + Section { + DatePicker("From", selection: $dateFilterStart, in: ...Date(), displayedComponents: .date) + DatePicker("To", selection: $dateFilterEnd, in: dateFilterStart...Date(), displayedComponents: .date) + } + if isDateFilterActive { + Section { + Button(role: .destructive) { + isDateFilterActive = false + showDateFilter = false + refreshTrigger.toggle() + } label: { + HStack { Spacer(); Text("Clear Filter"); Spacer() } + } + } + } + } + } + #else + NavigationStack { Form { Section { DatePicker("From", selection: $dateFilterStart, in: ...Date(), displayedComponents: .date) @@ -278,31 +313,65 @@ struct TranscriptsView: View { } } } + #endif } private func transcriptsListView(_ filtered: [(recording: RecordingEntry, transcript: TranscriptData?)], _ filteredImported: [(recording: RecordingEntry, transcript: TranscriptData?)]) -> some View { - // For preview mode, show only first 3 items total across all sections + #if targetEnvironment(macCatalyst) + // On Mac Catalyst, the preview-+-NavigationLink-to-More pattern wedges the responder chain + // (destination renders but becomes unresponsive). Render everything inline instead; + // List handles virtualization for arbitrary item counts. + return List { + if !filtered.isEmpty { + Section(header: Text("Audio Transcripts")) { + ForEach(filtered, id: \.recording.id) { recordingData in + recordingRowView(recordingData) + } + } + } + + if !filteredImported.isEmpty { + Section(header: + HStack { + Text("Imported Transcripts") + Spacer() + Text("\(filteredImported.count)") + .font(.caption) + .foregroundColor(.secondary) + } + ) { + ForEach(filteredImported, id: \.recording.id) { recordingData in + importedTranscriptRowView(recordingData) + } + .onDelete { indexSet in + deleteImportedTranscripts(at: indexSet, in: filteredImported) + } + } + } + } + .id("list-\(isDateFilterActive)-\(dateFilterStart)-\(dateFilterEnd)-\(searchText)") + #else + // iOS / iPadOS: original preview list with "More" navigation to the full list page. let recentRecordings = Array(filtered.prefix(3)) let recentImportedTranscripts = Array(filteredImported.prefix(3)) return List { - // Audio Recordings with Transcripts if !filtered.isEmpty { Section(header: Text("Audio Transcripts")) { - // Show first 3 items with their section headers ForEach(recentRecordings, id: \.recording.id) { recordingData in recordingRowView(recordingData) } if filtered.count > recentRecordings.count { - NavigationLink(destination: audioRecordingsFullListView) { + NavigationLink { + audioRecordingsFullListView + } label: { moreRowView(remainingCount: filtered.count - recentRecordings.count) } } } } - // Imported Transcripts (with delete functionality) if !filteredImported.isEmpty { Section(header: HStack { @@ -321,7 +390,9 @@ struct TranscriptsView: View { } if filteredImported.count > recentImportedTranscripts.count { - NavigationLink(destination: importedTranscriptsFullListView) { + NavigationLink { + importedTranscriptsFullListView + } label: { moreRowView(remainingCount: filteredImported.count - recentImportedTranscripts.count) } } @@ -329,6 +400,7 @@ struct TranscriptsView: View { } } .id("list-\(isDateFilterActive)-\(dateFilterStart)-\(dateFilterEnd)-\(searchText)") + #endif } private var audioRecordingsFullListView: some View { @@ -378,7 +450,6 @@ struct TranscriptsView: View { importedTranscriptRowView((recording: itemWithDate.recording, transcript: itemWithDate.transcript)) } .onDelete { indexSet in - // Map local indexSet to global importedTranscripts array let itemsToDelete = indexSet.map { sectionData.items[$0] } for item in itemsToDelete { deleteImportedTranscript((recording: item.recording, transcript: item.transcript)) @@ -400,13 +471,14 @@ struct TranscriptsView: View { } .foregroundColor(.accentColor) } - + private func recordingRowView(_ recordingData: (recording: RecordingEntry, transcript: TranscriptData?)) -> some View { VStack(alignment: .leading, spacing: 8) { HStack { recordingInfoView(recordingData) Spacer() transcriptButtonView(recordingData) + summaryButtonView(recordingData) } } .padding(.vertical, 4) @@ -455,8 +527,10 @@ struct TranscriptsView: View { Text(UserPreferences.shared.formatMediumDateTime(recordingData.recording.recordingDate ?? Date())) .font(.caption) .foregroundColor(.secondary) - if let recordingURL = appCoordinator.getAbsoluteURL(for: recordingData.recording), - let locationData = loadLocationDataForRecording(url: recordingURL) { + // Cheap attribute reads only — getAbsoluteURL would probe FileManager (and possibly + // save the Core Data context) per row, which stalled Mac Catalyst on long lists. + if let locationData = appCoordinator.coreDataManager.getLocationData(for: recordingData.recording), + let recordingURL = appCoordinator.getStoredURL(for: recordingData.recording) { locationButtonView(locationData, recordingURL: recordingURL) } } @@ -480,11 +554,11 @@ struct TranscriptsView: View { private func transcriptButtonView(_ recordingData: (recording: RecordingEntry, transcript: TranscriptData?)) -> some View { let hasTranscript = recordingData.transcript != nil - let hasActiveJob = hasActiveTranscriptionJob(for: recordingData.recording) + let hasActiveJob = transcriptionStarter.hasActiveTranscriptionJob(for: recordingData.recording, appCoordinator: appCoordinator) let recordingId = recordingData.recording.id ?? UUID() - let isCleaning = activeCleaningRecordingId == recordingId - let isQueuedForCleanup = queuedCleanupRecordings.contains { $0.id == recordingId } - let jobStatus = activeTranscriptionJobStatus(for: recordingData.recording) + let isCleaning = transcriptionStarter.isCleaning(recordingId) + let isQueuedForCleanup = transcriptionStarter.isQueuedForCleanup(recordingId) + let jobStatus = transcriptionStarter.activeTranscriptionJobStatus(for: recordingData.recording, appCoordinator: appCoordinator) let isProcessing = isCleaning || isQueuedForCleanup || hasActiveJob return Button(action: { @@ -544,35 +618,83 @@ struct TranscriptsView: View { } } - /// Check if a recording has an active transcription job (queued or processing) - private func hasActiveTranscriptionJob(for recording: RecordingEntry) -> Bool { - guard let recordingURL = appCoordinator.getAbsoluteURL(for: recording) else { - return false - } - let filename = recordingURL.lastPathComponent + /// Second button in each row: visible only when a transcript exists and no summary does. + /// Once a summary is created, the button disappears — the user regenerates from inside + /// the existing summary detail view. + @ViewBuilder + private func summaryButtonView(_ recordingData: (recording: RecordingEntry, transcript: TranscriptData?)) -> some View { + let recording = recordingData.recording + let hasTranscript = recordingData.transcript != nil + // Read the cheap status attribute rather than faulting recording.summary on every row. + let status = recording.summaryStatus + let hasSummary = status == ProcessingStatus.completed.rawValue - return backgroundProcessingManager.activeJobs.contains { job in - // Match by filename and check if job is active (queued or processing) - job.recordingPath == filename && - job.type.isTranscription && - (job.status == .queued || job.status == .processing) + if let recordingId = recording.id, hasTranscript, !hasSummary { + let isGenerating = generatingSummaryRecordingIds.contains(recordingId) + || status == ProcessingStatus.processing.rawValue + + Button(action: { + if !isGenerating { + generateSummary(for: recording) + } + }) { + HStack(spacing: 6) { + if isGenerating { + ProgressView() + .scaleEffect(0.7) + .tint(.white) + Text("Generating…") + .font(.caption2) + } else { + Image(systemName: "doc.text.magnifyingglass") + Text("Generate Summary") + } + } + .font(.caption) + .padding(.horizontal, 12) + .padding(.vertical, 6) + .background(isGenerating ? Color.orange : Color.purple) + .foregroundColor(.white) + .cornerRadius(8) + } + .buttonStyle(.plain) + .disabled(isGenerating) } } - /// Returns the current status of the active transcription job for this recording, if any - private func activeTranscriptionJobStatus(for recording: RecordingEntry) -> JobProcessingStatus? { - guard let recordingURL = appCoordinator.getAbsoluteURL(for: recording) else { - return nil + private func generateSummary(for recording: RecordingEntry) { + guard let recordingId = recording.id else { return } + AppLog.shared.summarization("generateSummary called from TranscriptsView row", level: .debug) + generatingSummaryRecordingIds.insert(recordingId) + + let selectedEngine = UserDefaults.standard.string(forKey: "SelectedAIEngine") ?? "On-Device AI" + let selectedModel = UserDefaults.standard.string(forKey: "SelectedAIModel") + let recordingURL: URL + if let absoluteURL = appCoordinator.getAbsoluteURL(for: recording) { + recordingURL = absoluteURL + } else { + recordingURL = URL(fileURLWithPath: recording.recordingURL ?? "") } - let filename = recordingURL.lastPathComponent + let recordingName = recording.recordingName ?? "Unknown Recording" - return backgroundProcessingManager.activeJobs.first { job in - job.recordingPath == filename && - job.type.isTranscription && - (job.status == .queued || job.status == .processing) - }?.status + Task { + do { + try await BackgroundProcessingManager.shared.startSummarizationJob( + recordingURL: recordingURL, + recordingName: recordingName, + engine: selectedEngine, + modelName: selectedModel + ) + AppLog.shared.summarization("Summary job queued from TranscriptsView row") + } catch { + AppLog.shared.summarization("Failed to queue summary job from TranscriptsView row: \(error)", level: .error) + await MainActor.run { + _ = generatingSummaryRecordingIds.remove(recordingId) + } + } + } } - + // MARK: - Search and Date Filtering private var filteredRecordings: [(recording: RecordingEntry, transcript: TranscriptData?)] { @@ -815,183 +937,14 @@ struct TranscriptsView: View { private func generateTranscript(for recording: RecordingEntry) { - // Check if this specific recording already has an active job - guard !hasActiveTranscriptionJob(for: recording) else { return } + // Skip if this recording already has a queued or processing transcription job. + guard !transcriptionStarter.hasActiveTranscriptionJob(for: recording, appCoordinator: appCoordinator) else { return } - // Show the audio cleanup prompt before starting transcription + // Ask the user whether to clean audio first; the dialog buttons route to TranscriptionStarter. recordingPendingTranscription = recording showingAudioCleanupPrompt = true } - private func proceedWithTranscription(for recording: RecordingEntry, cleanFirst: Bool) { - // Job tracking is now handled by BackgroundProcessingManager.activeJobs - // No longer need global isGeneratingTranscript flag - - if cleanFirst { - // Add to the cleanup queue and process serially (one cleanup at a time) - queuedCleanupRecordings.append(recording) - processCleanupQueueIfNeeded() - } else { - self.performEnhancedTranscription(for: recording) - } - } - - /// Processes the cleanup queue one recording at a time to avoid concurrent heavy audio processing - private func processCleanupQueueIfNeeded() { - guard !isProcessingCleanupQueue, !queuedCleanupRecordings.isEmpty else { return } - isProcessingCleanupQueue = true - - // Take the next recording from the front of the queue - let recording = queuedCleanupRecordings.removeFirst() - activeCleaningRecordingId = recording.id - - Task { - defer { - // Clear active cleaning state and process next in queue - activeCleaningRecordingId = nil - isProcessingCleanupQueue = false - processCleanupQueueIfNeeded() - } - - guard let recordingURL = appCoordinator.getAbsoluteURL(for: recording) else { - self.performEnhancedTranscription(for: recording) - return - } - do { - let tempCleanedURL = try await AudioCleanupService.shared.cleanAudio(at: recordingURL) - AppLog.shared.transcription("Cleaned audio created at temp location: \(tempCleanedURL.lastPathComponent)", level: .debug) - - // Copy cleaned file to Documents directory so the job can find it - // (ProcessingJob resolves paths relative to Documents directory) - guard let documentsURL = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask).first else { - AppLog.shared.transcription("Could not access Documents directory, using original file", level: .error) - self.performEnhancedTranscription(for: recording) - return - } - - let cleanedFilename = tempCleanedURL.lastPathComponent - let documentsCleanedURL = documentsURL.appendingPathComponent(cleanedFilename) - - // Remove any existing file at destination - try? FileManager.default.removeItem(at: documentsCleanedURL) - - // Copy from temp to Documents - try FileManager.default.copyItem(at: tempCleanedURL, to: documentsCleanedURL) - AppLog.shared.transcription("Copied cleaned audio to Documents: \(cleanedFilename)", level: .debug) - - // Clean up temp file immediately since we've copied it to Documents - await AudioCleanupService.shared.removeTempFile(at: tempCleanedURL) - - // Pass the original recording URL for identity, cleaned URL as source audio. - // BackgroundProcessingManager will delete the cleaned file when the job finishes. - self.performEnhancedTranscription(for: recording, sourceAudioURL: documentsCleanedURL) - } catch { - AppLog.shared.transcription("Audio cleanup failed, falling back to original: \(error)", level: .error) - self.performEnhancedTranscription(for: recording) - } - } - } - - private func performEnhancedTranscription(for recording: RecordingEntry, sourceAudioURL: URL? = nil) { - // Progress is now shown inline on the button, no modal needed - - Task { - // Use the selected transcription engine - let selectedEngine = TranscriptionEngine(rawValue: UserDefaults.standard.string(forKey: "selectedTranscriptionEngine") ?? TranscriptionEngine.fluidAudio.rawValue) ?? .fluidAudio - - do { - // Always resolve the original recording URL for identity - guard let recordingURL = appCoordinator.getAbsoluteURL(for: recording) else { - AppLog.shared.transcription("Invalid recording URL", level: .error) - throw NSError(domain: "Transcription", code: -1, userInfo: [NSLocalizedDescriptionKey: "Invalid recording URL"]) - } - - // Start transcription job through BackgroundProcessingManager. - // The original recordingURL is used for identity/Core Data; sourceAudioURL - // (if set) is the cleaned audio file the job will actually read from. - // BackgroundProcessingManager handles deleting the source audio file when done. - try await backgroundProcessingManager.startTranscriptionJob( - recordingURL: recordingURL, - recordingName: recording.recordingName ?? "Unknown Recording", - engine: selectedEngine, - sourceAudioURL: sourceAudioURL - ) - - AppLog.shared.transcription("Transcription job started through BackgroundProcessingManager") - - } catch { - AppLog.shared.transcription("Failed to start transcription job: \(error)", level: .error) - - // Fallback to direct transcription if background processing fails - AppLog.shared.transcription("Falling back to direct transcription...", level: .debug) - do { - // Use source audio URL (cleaned audio) if available, otherwise resolve from Core Data - let transcriptionURL = sourceAudioURL ?? appCoordinator.getAbsoluteURL(for: recording) - guard let transcriptionURL else { - AppLog.shared.transcription("Invalid recording URL for fallback transcription", level: .error) - return - } - - let result = try await enhancedTranscriptionManager.transcribeAudioFile(at: transcriptionURL, using: selectedEngine) - - AppLog.shared.transcription("Transcription result: success=\(result.success), textLength=\(result.fullText.count)", level: .debug) - - if result.success && !result.fullText.isEmpty { - AppLog.shared.transcription("Creating transcript data...", level: .debug) - // Use the original recording URL for identity, not the cleaned URL - let identityURL = appCoordinator.getAbsoluteURL(for: recording) ?? transcriptionURL - let transcriptData = TranscriptData( - recordingURL: identityURL, - recordingName: recording.recordingName ?? "Unknown Recording", - recordingDate: recording.recordingDate ?? Date(), - segments: result.segments - ) - - // Save the transcript using Core Data - let appCoordinator = appCoordinator - guard let recordingId = transcriptData.recordingId else { - AppLog.shared.transcription("Transcript data missing recording ID", level: .error) - return - } - let transcriptId = appCoordinator.addTranscript( - for: recordingId, - segments: transcriptData.segments, - speakerMappings: transcriptData.speakerMappings, - engine: transcriptData.engine, - processingTime: transcriptData.processingTime, - confidence: transcriptData.confidence - ) - if transcriptId != nil { - AppLog.shared.transcription("Transcript saved to Core Data with ID: \(transcriptId!)") - } else { - AppLog.shared.transcription("Failed to save transcript to Core Data", level: .error) - } - - // Force UI refresh to update button states - self.forceRefreshUI() - } else { - AppLog.shared.transcription("Transcription failed or returned empty result", level: .error) - } - } catch { - AppLog.shared.transcription("Fallback transcription also failed: \(error)", level: .error) - } - - // Clean up source audio on fallback path failure (no BackgroundProcessingManager to do it) - if let cleanupURL = sourceAudioURL, cleanupURL.lastPathComponent.hasPrefix("cleaned_") { - try? FileManager.default.removeItem(at: cleanupURL) - AppLog.shared.transcription("Cleaned up source audio file after fallback: \(cleanupURL.lastPathComponent)", level: .debug) - } - } - - await MainActor.run { - AppLog.shared.transcription("Transcription process completed", level: .debug) - - // Refresh the recordings list to show the new transcript - self.loadRecordings() - } - } - } - private func setupTranscriptionCompletionCallback() { // Capture the transcription manager for the notification handler let transcriptionManager = enhancedTranscriptionManager @@ -1134,6 +1087,10 @@ struct EditableTranscriptView: View { @State private var showingSaveErrorAlert = false @State private var showingSpeakerEditor = false @State private var saveErrorMessage = "" + @State private var isGeneratingSummary = false + @State private var showSummarySheet = false + @State private var summaryGenerationError: String? + @State private var summaryStateRefresh = false @StateObject private var enhancedTranscriptionManager = EnhancedTranscriptionManager() @ObservedObject private var backgroundProcessingManager = BackgroundProcessingManager.shared @@ -1158,101 +1115,87 @@ struct EditableTranscriptView: View { } var body: some View { - NavigationView { - VStack(spacing: 0) { - - // Transcript Content - ScrollView { - VStack(alignment: .leading, spacing: 16) { - recordingTitleEditor + // NavigationStack { Form } is the only sheet pattern that scrolls reliably + // on Mac Catalyst. See feedback_mac_catalyst_scrollview.md. + NavigationStack { + Form { + Section { + recordingTitleEditor + } - if editedSegments.isEmpty { - VStack(spacing: 16) { - Image(systemName: "doc.text") - .font(.system(size: 48)) - .foregroundColor(.gray) - Text("No transcript content available") - .font(.title2) - .foregroundColor(.secondary) - Text("Transcript segments: \(editedSegments.count)") - .font(.caption) - .foregroundColor(.gray) - } - .frame(maxWidth: .infinity, maxHeight: .infinity) - .padding() - } else { - LazyVStack(alignment: .leading, spacing: 16) { - if !uniqueSpeakers.isEmpty { - Button(action: { showingSpeakerEditor = true }) { - HStack { - Image(systemName: "person.2.fill") - Text("Edit Speakers (\(uniqueSpeakers.count))") - Spacer() - Image(systemName: "chevron.right") - .font(.caption) - } - .font(.subheadline) - .fontWeight(.medium) - .padding(.horizontal, 16) - .padding(.vertical, 10) - .background(Color.purple.opacity(0.1)) + if editedSegments.isEmpty { + Section { + VStack(spacing: 16) { + Image(systemName: "doc.text") + .font(.system(size: 48)) + .foregroundColor(.gray) + Text("No transcript content available") + .font(.title3) + .foregroundColor(.secondary) + Text("Transcript segments: \(editedSegments.count)") + .font(.caption) + .foregroundColor(.gray) + } + .frame(maxWidth: .infinity) + .padding(.vertical, 24) + } + } else { + if !uniqueSpeakers.isEmpty { + Section { + Button { + showingSpeakerEditor = true + } label: { + HStack { + Image(systemName: "person.2.fill") .foregroundColor(.purple) - .cornerRadius(10) - } - } - - ForEach(Array(editedSegments.enumerated()), id: \.offset) { index, segment in - TranscriptSegmentView(segment: $editedSegments[index], speakerMappings: speakerMappings) + Text("Edit Speakers (\(uniqueSpeakers.count))") + .foregroundColor(.primary) + Spacer() + Image(systemName: "chevron.right") + .font(.caption) + .foregroundColor(.secondary) } + .contentShape(Rectangle()) } - .padding(.horizontal, 16) - .padding(.vertical, 12) - .id("transcript-\(editedSegments.count)-\(editedSegments.first?.text.prefix(10).hashValue ?? 0)") + .buttonStyle(.plain) + } + } + + Section("Segments") { + ForEach(Array(editedSegments.enumerated()), id: \.offset) { index, segment in + TranscriptSegmentView(segment: $editedSegments[index], speakerMappings: speakerMappings) } } + .id("transcript-\(editedSegments.count)-\(editedSegments.first?.text.prefix(10).hashValue ?? 0)") } - .frame(maxWidth: .infinity, maxHeight: .infinity) - - // Rerun Transcription Button - VStack(spacing: 12) { - Divider() - .padding(.horizontal, 16) - - Button(action: { + + summarySection + + Section { + Button { showingRerunAlert = true - }) { + } label: { HStack { if isRerunningTranscription { - ProgressView() - .scaleEffect(0.8) - .tint(.white) + ProgressView().scaleEffect(0.8) Text("Rerunning Transcription...") } else { Image(systemName: "arrow.clockwise") Text("Rerun Transcription") } + Spacer() } - .font(.body) - .fontWeight(.medium) - .foregroundColor(.white) - .frame(maxWidth: .infinity) - .padding(.vertical, 12) - .background(isRerunningTranscription ? Color.orange : Color.blue) - .cornerRadius(10) + .contentShape(Rectangle()) } + .buttonStyle(.plain) .disabled(isRerunningTranscription) - .padding(.horizontal, 16) - .padding(.bottom, 16) } } - .frame(maxWidth: .infinity, maxHeight: .infinity) .navigationTitle("Edit Transcript") .navigationBarTitleDisplayMode(.inline) .toolbar { ToolbarItem(placement: .navigationBarLeading) { - Button("Cancel") { - dismiss() - } + Button("Cancel") { dismiss() } } ToolbarItem(placement: .navigationBarTrailing) { Button("Save") { @@ -1265,76 +1208,84 @@ struct EditableTranscriptView: View { .fontWeight(.semibold) } } - .alert("Rerun Transcription", isPresented: $showingRerunAlert) { - Button("Cancel", role: .cancel) { } - Button("Rerun", role: .destructive) { - rerunTranscription() - } - } message: { - Text("This will replace the current transcript with a new transcription using the currently configured transcription service. This action cannot be undone.") + } + .alert("Rerun Transcription", isPresented: $showingRerunAlert) { + Button("Cancel", role: .cancel) { } + Button("Rerun", role: .destructive) { + rerunTranscription() } - .alert("Transcript Saved", isPresented: $showingSaveSuccessAlert) { - Button("OK") { - showingSaveSuccessAlert = false - dismiss() - } - } message: { - Text("Your transcript changes have been saved.") + } message: { + Text("This will replace the current transcript with a new transcription using the currently configured transcription service. This action cannot be undone.") + } + .alert("Transcript Saved", isPresented: $showingSaveSuccessAlert) { + Button("OK") { + showingSaveSuccessAlert = false + dismiss() } - .alert("Save Failed", isPresented: $showingSaveErrorAlert) { - Button("OK", role: .cancel) { - showingSaveErrorAlert = false - } - } message: { - Text(saveErrorMessage) - } - .alert("Rename Failed", isPresented: Binding( - get: { recordingRenameError != nil }, - set: { if !$0 { recordingRenameError = nil } } - )) { - Button("OK", role: .cancel) { - recordingRenameError = nil - } - } message: { - Text(recordingRenameError ?? "Unknown error") + } message: { + Text("Your transcript changes have been saved.") + } + .alert("Save Failed", isPresented: $showingSaveErrorAlert) { + Button("OK", role: .cancel) { + showingSaveErrorAlert = false } - .sheet(isPresented: $showingSpeakerEditor) { - SpeakerEditingView( - speakerIds: uniqueSpeakers, - speakerMappings: $speakerMappings - ) + } message: { + Text(saveErrorMessage) + } + .alert("Rename Failed", isPresented: Binding( + get: { recordingRenameError != nil }, + set: { if !$0 { recordingRenameError = nil } } + )) { + Button("OK", role: .cancel) { + recordingRenameError = nil } - .onReceive(NotificationCenter.default.publisher(for: NSNotification.Name("TranscriptionRerunCompleted"))) { notification in - // Handle transcription rerun completion - if let userInfo = notification.userInfo, - let notificationURL = userInfo["recordingURL"] as? URL, - let segments = userInfo["segments"] as? [TranscriptSegment], - let recordingURL = appCoordinator.getAbsoluteURL(for: recording), - notificationURL == recordingURL { - - AppLog.shared.transcription("Received transcription rerun completion notification", level: .debug) - - // Save the new transcript to Core Data first (this will replace the existing transcript) - saveNewTranscriptToCoreData(segments: segments) - - isRerunningTranscription = false + } message: { + Text(recordingRenameError ?? "Unknown error") + } + .sheet(isPresented: $showingSpeakerEditor) { + SpeakerEditingView( + speakerIds: uniqueSpeakers, + speakerMappings: $speakerMappings + ) + } + .sheet(isPresented: $showSummarySheet) { + summarySheetContent + } + .alert("Unable to Generate Summary", isPresented: Binding( + get: { summaryGenerationError != nil }, + set: { if !$0 { summaryGenerationError = nil } } + )) { + Button("OK", role: .cancel) { summaryGenerationError = nil } + } message: { + Text(summaryGenerationError ?? "Unknown error") + } + .onReceive(NotificationCenter.default.publisher(for: NSNotification.Name("SummaryCreated"))) { _ in + isGeneratingSummary = false + summaryStateRefresh.toggle() + } + .onReceive(NotificationCenter.default.publisher(for: NSNotification.Name("SummaryDeleted"))) { _ in + summaryStateRefresh.toggle() + } + .onReceive(NotificationCenter.default.publisher(for: NSNotification.Name("TranscriptionRerunCompleted"))) { notification in + if let userInfo = notification.userInfo, + let notificationURL = userInfo["recordingURL"] as? URL, + let segments = userInfo["segments"] as? [TranscriptSegment], + let recordingURL = appCoordinator.getAbsoluteURL(for: recording), + notificationURL == recordingURL { - AppLog.shared.transcription("Transcript UI updated with rerun results from notification") - - // Force the parent view to refresh by posting a notification - NotificationCenter.default.post(name: NSNotification.Name("TranscriptReplacementCompleted"), object: nil) - } - } - .onReceive(NotificationCenter.default.publisher(for: NSNotification.Name("TranscriptionCompleted"))) { _ in - // Refresh transcript data from Core Data when transcription is completed - refreshTranscriptFromCoreData() - } - .onAppear { - // Always refresh transcript data when the view appears to ensure we have the latest content - refreshTranscriptFromCoreData() + AppLog.shared.transcription("Received transcription rerun completion notification", level: .debug) + saveNewTranscriptToCoreData(segments: segments) + isRerunningTranscription = false + AppLog.shared.transcription("Transcript UI updated with rerun results from notification") + NotificationCenter.default.post(name: NSNotification.Name("TranscriptReplacementCompleted"), object: nil) } } - .frame(maxWidth: .infinity, maxHeight: .infinity) + .onReceive(NotificationCenter.default.publisher(for: NSNotification.Name("TranscriptionCompleted"))) { _ in + refreshTranscriptFromCoreData() + } + .onAppear { + refreshTranscriptFromCoreData() + } } private var recordingTitleEditor: some View { @@ -1344,7 +1295,112 @@ struct EditableTranscriptView: View { isSaving: isUpdatingRecordingName, onSave: renameRecordingFromTranscript ) - .padding(.horizontal, 16) + } + + @ViewBuilder + private var summarySection: some View { + if let recordingId = recording.id { + let hasSummary = appCoordinator.getSummary(for: recordingId) != nil + let isProcessing = isGeneratingSummary + || recording.summaryStatus == ProcessingStatus.processing.rawValue + + Section { + Button { + if hasSummary { + showSummarySheet = true + } else if !isProcessing { + generateSummary() + } + } label: { + HStack { + if isProcessing { + ProgressView().scaleEffect(0.8) + Text("Generating Summary…") + } else if hasSummary { + Image(systemName: "doc.text.fill") + .foregroundColor(.blue) + Text("View Summary") + } else { + Image(systemName: "doc.text.magnifyingglass") + Text("Generate Summary") + } + Spacer() + } + .contentShape(Rectangle()) + } + .buttonStyle(.plain) + .disabled(isProcessing) + } + .id("summary-section-\(recordingId)-\(hasSummary)-\(isProcessing)-\(summaryStateRefresh)") + } + } + + @ViewBuilder + private var summarySheetContent: some View { + if let recordingId = recording.id, + let enhancedSummary = appCoordinator.getCompleteRecordingData(id: recordingId)?.summary { + SummaryDetailView( + recording: RecordingFile( + url: appCoordinator.getAbsoluteURL(for: recording) ?? URL(fileURLWithPath: ""), + name: recording.recordingName ?? "Unknown", + date: recording.recordingDate ?? Date(), + duration: recording.duration, + locationData: appCoordinator.coreDataManager.getLocationData(for: recording) + ), + summaryData: enhancedSummary + ) + .environmentObject(appCoordinator) + } else { + VStack(spacing: 16) { + Image(systemName: "doc.text.magnifyingglass") + .font(.system(size: 50)) + .foregroundColor(.secondary) + Text("Summary Not Available") + .font(.title2) + .fontWeight(.semibold) + Text("A summary for this recording could not be found.") + .foregroundColor(.secondary) + .multilineTextAlignment(.center) + .padding(.horizontal) + } + } + } + + private func generateSummary() { + AppLog.shared.summarization("generateSummary called from EditableTranscriptView", level: .debug) + isGeneratingSummary = true + + let selectedEngine = UserDefaults.standard.string(forKey: "SelectedAIEngine") ?? "On-Device AI" + let selectedModel = UserDefaults.standard.string(forKey: "SelectedAIModel") + let recordingURL: URL + if let absoluteURL = appCoordinator.getAbsoluteURL(for: recording) { + recordingURL = absoluteURL + } else { + recordingURL = URL(fileURLWithPath: recording.recordingURL ?? "") + } + let recordingName = recording.recordingName ?? "Unknown Recording" + + Task { + do { + try await BackgroundProcessingManager.shared.startSummarizationJob( + recordingURL: recordingURL, + recordingName: recordingName, + engine: selectedEngine, + modelName: selectedModel + ) + AppLog.shared.summarization("Summary job queued from EditableTranscriptView") + } catch { + AppLog.shared.summarization("Failed to queue summary job from EditableTranscriptView: \(error)", level: .error) + await MainActor.run { + if let summarizationError = error as? SummarizationError { + summaryGenerationError = summarizationError.localizedDescription + } else { + summaryGenerationError = "Failed to start summary: \(error.localizedDescription)" + } + isGeneratingSummary = false + } + } + } } private func saveTranscript() -> Bool { @@ -1738,66 +1794,89 @@ struct SpeakerEditingView: View { } var body: some View { - NavigationView { - Form { - Section(header: Text("Rename Speakers"), footer: Text("Enter a name for each speaker. Changes apply to the entire transcript and are used in AI summaries.")) { - ForEach(speakerIds, id: \.self) { speakerId in - HStack(spacing: 12) { - let hash = abs(speakerId.hashValue) - let color = Self.speakerColors[hash % Self.speakerColors.count] - - Circle() - .fill(color) - .frame(width: 10, height: 10) - - VStack(alignment: .leading, spacing: 2) { - Text(defaultName(for: speakerId)) - .font(.caption2) - .foregroundColor(.secondary) - - TextField(defaultName(for: speakerId), text: binding(for: speakerId)) - .textFieldStyle(RoundedBorderTextFieldStyle()) - .autocapitalization(.words) - } - } + #if targetEnvironment(macCatalyst) + VStack(spacing: 0) { + HStack { + Button("Cancel") { dismiss() } + Spacer() + Text("Edit Speakers").font(.headline) + Spacer() + Button("Apply") { + applyNames() + } + .fontWeight(.semibold) + } + .padding(.horizontal, 16).padding(.vertical, 10) + Divider() + speakerForm + } + #else + NavigationStack { + speakerForm + .navigationTitle("Edit Speakers") + .navigationBarTitleDisplayMode(.inline) + .toolbar { + ToolbarItem(placement: .navigationBarLeading) { + Button("Cancel") { dismiss() } + } + ToolbarItem(placement: .navigationBarTrailing) { + Button("Apply") { applyNames() } + .fontWeight(.semibold) } } + } + #endif + } - if speakerMappings.values.contains(where: { !$0.isEmpty }) { - Section { - Button("Clear All Names", role: .destructive) { - for id in speakerIds { - editingNames[id] = "" - } + private var speakerForm: some View { + Form { + Section(header: Text("Rename Speakers"), footer: Text("Enter a name for each speaker. Changes apply to the entire transcript and are used in AI summaries.")) { + ForEach(speakerIds, id: \.self) { speakerId in + HStack(spacing: 12) { + let hash = abs(speakerId.hashValue) + let color = Self.speakerColors[hash % Self.speakerColors.count] + + Circle() + .fill(color) + .frame(width: 10, height: 10) + + VStack(alignment: .leading, spacing: 2) { + Text(defaultName(for: speakerId)) + .font(.caption2) + .foregroundColor(.secondary) + + TextField(defaultName(for: speakerId), text: binding(for: speakerId)) + .textFieldStyle(RoundedBorderTextFieldStyle()) + .autocapitalization(.words) } } } } - .navigationTitle("Edit Speakers") - .navigationBarTitleDisplayMode(.inline) - .toolbar { - ToolbarItem(placement: .navigationBarLeading) { - Button("Cancel") { dismiss() } - } - ToolbarItem(placement: .navigationBarTrailing) { - Button("Apply") { - // Write non-empty names to speakerMappings - var newMappings: [String: String] = [:] - for (id, name) in editingNames { - let trimmed = name.trimmingCharacters(in: .whitespacesAndNewlines) - if !trimmed.isEmpty { - newMappings[id] = trimmed - } + + if speakerMappings.values.contains(where: { !$0.isEmpty }) { + Section { + Button("Clear All Names", role: .destructive) { + for id in speakerIds { + editingNames[id] = "" } - speakerMappings = newMappings - dismiss() } - .fontWeight(.semibold) } } } } + private func applyNames() { + var newMappings: [String: String] = [:] + for (id, name) in editingNames { + let trimmed = name.trimmingCharacters(in: .whitespacesAndNewlines) + if !trimmed.isEmpty { + newMappings[id] = trimmed + } + } + speakerMappings = newMappings + dismiss() + } + private func defaultName(for speakerId: String) -> String { if speakerId.hasPrefix("speaker_") { let num = speakerId.dropFirst("speaker_".count) @@ -1822,32 +1901,35 @@ struct TranscriptDetailView: View { @State private var locationAddress: String? var body: some View { - NavigationView { - VStack { + // NavigationStack { Form } is the only sheet pattern that scrolls reliably + // on Mac Catalyst. See feedback_mac_catalyst_scrollview.md. + NavigationStack { + Form { if transcriptText.isEmpty { - VStack(spacing: 16) { - ProgressView() - .scaleEffect(1.5) - Text("Generating transcript...") - .font(.headline) - .foregroundColor(.primary) + Section { + HStack { + Spacer() + VStack(spacing: 16) { + ProgressView().scaleEffect(1.5) + Text("Generating transcript...") + .font(.headline) + } + Spacer() + } + .padding(.vertical, 40) } - .frame(maxWidth: .infinity, maxHeight: .infinity) } else { - ScrollView { - VStack(alignment: .leading, spacing: 16) { + Section { + VStack(alignment: .leading, spacing: 6) { Text(recording.recordingName ?? "Unknown Recording") - .font(.title2) + .font(.title3) .fontWeight(.bold) - .foregroundColor(.primary) - Text(UserPreferences.shared.formatMediumDateTime(recording.recordingDate ?? Date())) .font(.caption) .foregroundColor(.secondary) - if let recordingURL = appCoordinator.getAbsoluteURL(for: recording), let locationData = TranscriptsView.loadLocationDataForRecording(url: recordingURL) { - HStack { + HStack(spacing: 4) { Image(systemName: "location.fill") .font(.caption) .foregroundColor(.accentColor) @@ -1856,15 +1938,15 @@ struct TranscriptDetailView: View { .foregroundColor(.accentColor) } } - - Divider() - - Text(transcriptText) - .font(.body) - .foregroundColor(.primary) - .lineSpacing(4) } - .padding() + .padding(.vertical, 4) + } + + Section { + Text(transcriptText) + .font(.body) + .lineSpacing(4) + .textSelection(.enabled) } } } @@ -1872,9 +1954,7 @@ struct TranscriptDetailView: View { .navigationBarTitleDisplayMode(.inline) .toolbar { ToolbarItem(placement: .navigationBarTrailing) { - Button("Done") { - dismiss() - } + Button("Done") { dismiss() } } } .onAppear { diff --git a/BisonNotes AI/BisonNotes AI/WhisperService.swift b/BisonNotes AI/BisonNotes AI/WhisperService.swift index b997622..e4bcb5a 100644 --- a/BisonNotes AI/BisonNotes AI/WhisperService.swift +++ b/BisonNotes AI/BisonNotes AI/WhisperService.swift @@ -184,6 +184,9 @@ class WhisperService: ObservableObject { do { // For REST API, always use HTTP regardless of what user entered let restBaseURL = config.restAPIBaseURL + if let message = EndpointSecurityPolicy.validationMessage(for: restBaseURL) { + throw WhisperError.serverError(message) + } let testURL = URL(string: "\(restBaseURL)/asr")! AppLog.shared.transcription("Testing REST API connection to: \(testURL)", level: .debug) @@ -497,6 +500,9 @@ class WhisperService: ObservableObject { // Create multipart form data request let boundary = UUID().uuidString let restBaseURL = config.restAPIBaseURL + if let message = EndpointSecurityPolicy.validationMessage(for: restBaseURL) { + throw WhisperError.serverError(message) + } var request = URLRequest(url: URL(string: "\(restBaseURL)/asr")!) request.httpMethod = "POST" request.setValue("multipart/form-data; boundary=\(boundary)", forHTTPHeaderField: "Content-Type") @@ -757,6 +763,9 @@ class WhisperService: ObservableObject { // Create multipart form data request for language detection let boundary = UUID().uuidString let restBaseURL = config.restAPIBaseURL + if let message = EndpointSecurityPolicy.validationMessage(for: restBaseURL) { + throw WhisperError.serverError(message) + } var request = URLRequest(url: URL(string: "\(restBaseURL)/detect-language")!) request.httpMethod = "POST" request.setValue("multipart/form-data; boundary=\(boundary)", forHTTPHeaderField: "Content-Type") diff --git a/BisonNotes AI/BisonNotes AI/WhisperSettingsView.swift b/BisonNotes AI/BisonNotes AI/WhisperSettingsView.swift index 9cd35d9..91ee956 100644 --- a/BisonNotes AI/BisonNotes AI/WhisperSettingsView.swift +++ b/BisonNotes AI/BisonNotes AI/WhisperSettingsView.swift @@ -12,6 +12,7 @@ struct WhisperSettingsView: View { @AppStorage("whisperPort") private var port: Int = 9000 @AppStorage("whisperProtocol") private var protocolString: String = WhisperProtocol.rest.rawValue @AppStorage("enableWhisper") private var enableWhisper: Bool = false + @AppStorage(EndpointSecurityPolicy.allowInsecurePublicEndpointsKey) private var allowInsecurePublicEndpoints: Bool = false private var selectedProtocol: WhisperProtocol { get { WhisperProtocol(rawValue: protocolString) ?? .rest } @@ -49,7 +50,7 @@ struct WhisperSettingsView: View { } var body: some View { - NavigationView { + NavigationStack { Form { Section(header: Text("Whisper Service")) { VStack(alignment: .leading, spacing: 8) { @@ -174,6 +175,8 @@ struct WhisperSettingsView: View { "The URL of your Whisper server (e.g., http://localhost, http://192.168.1.100)") .font(.caption) .foregroundColor(.secondary) + + endpointSecurityWarning(for: endpointForSecurityWarning) } VStack(alignment: .leading, spacing: 8) { @@ -348,6 +351,18 @@ struct WhisperSettingsView: View { private var isConfigurationValid: Bool { !serverURL.isEmpty && port > 0 && port <= 65535 } + + private var endpointForSecurityWarning: String { + if selectedProtocol == .rest { + return WhisperConfig(serverURL: serverURL, port: port, whisperProtocol: .rest).restAPIBaseURL + } + + if serverURL.hasPrefix("ws://") || serverURL.hasPrefix("wss://") || serverURL.hasPrefix("http://") || serverURL.hasPrefix("https://") { + return "\(serverURL):\(port)" + } + + return "ws://\(serverURL):\(port)" + } private func updateWhisperService() { AppLog.shared.general("WhisperSettingsView: Updating service with protocol: \(selectedProtocol.rawValue)") @@ -376,6 +391,20 @@ struct WhisperSettingsView: View { } } } + + @ViewBuilder + private func endpointSecurityWarning(for endpoint: String) -> some View { + if let warning = EndpointSecurityPolicy.warningMessage(for: endpoint) { + Label(warning, systemImage: "exclamationmark.triangle.fill") + .font(.caption) + .foregroundColor(.orange) + } + + if EndpointSecurityPolicy.validationMessage(for: endpoint, allowInsecurePublicEndpoints: false) != nil { + Toggle("Development Mode: Allow Public HTTP", isOn: $allowInsecurePublicEndpoints) + .font(.caption) + } + } } struct WhisperInstructionRow: View { @@ -412,4 +441,4 @@ struct WhisperSettingsView_Previews: PreviewProvider { static var previews: some View { WhisperSettingsView() } -} \ No newline at end of file +} diff --git a/BisonNotes AI/BisonNotes AI/iCloudStorageManager.swift b/BisonNotes AI/BisonNotes AI/iCloudStorageManager.swift index fd112a8..078b8ca 100644 --- a/BisonNotes AI/BisonNotes AI/iCloudStorageManager.swift +++ b/BisonNotes AI/BisonNotes AI/iCloudStorageManager.swift @@ -1686,7 +1686,7 @@ class iCloudStorageManager: ObservableObject { : true // default: on let includeSensitive = defaults.object(forKey: "iCloudBackupIncludeSensitiveSettings") != nil ? defaults.bool(forKey: "iCloudBackupIncludeSensitiveSettings") - : true // default: on + : false // default: off for sensitive values let options = CloudBackupOptions( includeAudioFiles: includeAudio, @@ -2525,7 +2525,6 @@ extension iCloudStorageManager { "WatchAutoSync", "WatchBatteryAware", "isLocationTrackingEnabled", - "openAIAPIKey", "openAIModel", "openAIBaseURL", "openAISummarizationModel", @@ -2533,7 +2532,6 @@ extension iCloudStorageManager { "openAISummarizationTemperature", "openAISummarizationMaxTokens", "enableOpenAI", - "openAICompatibleAPIKey", "openAICompatibleModel", "openAICompatibleBaseURL", "openAICompatibleTemperature", @@ -2541,12 +2539,10 @@ extension iCloudStorageManager { "enableOpenAICompatible", "openAICompatibleManualFormatOverride", "openAICompatibleManualFormat", - "googleAIStudioAPIKey", "googleAIStudioModel", "googleAIStudioTemperature", "googleAIStudioMaxTokens", "enableGoogleAIStudio", - "mistralAPIKey", "mistralBaseURL", "mistralModel", "mistralTemperature", @@ -2558,8 +2554,6 @@ extension iCloudStorageManager { "mistralTranscribeLanguage", "awsBucketName", "enableAWSTranscribe", - "AWSCredentials", - "awsBedrockSessionToken", "awsBedrockModel", "awsBedrockTemperature", "awsBedrockMaxTokens", @@ -3253,7 +3247,7 @@ extension iCloudStorageManager { } try context.save() - AWSCredentialsManager.shared.initializeEnvironment() + AWSCredentialsManager.shared.clearCredentialEnvironment() await MainActor.run { self.lastSyncDate = Date() @@ -4186,12 +4180,41 @@ extension iCloudStorageManager { continue } + if applyLegacySensitiveSetting(rawValue, forKey: key) { + defaults.removeObject(forKey: key) + continue + } + defaults.set(rawValue, forKey: key) } defaults.synchronize() } + private func applyLegacySensitiveSetting(_ rawValue: Any, forKey key: String) -> Bool { + switch key { + case KeychainSecretStore.openAIAPIKey, + KeychainSecretStore.openAICompatibleAPIKey, + KeychainSecretStore.googleAIStudioAPIKey, + KeychainSecretStore.mistralAPIKey, + KeychainSecretStore.awsBedrockSessionToken: + guard let value = rawValue as? String else { return true } + KeychainSecretStore.shared.setString(value, forKey: key) + return true + case KeychainSecretStore.awsCredentials: + guard let data = rawValue as? Data else { return true } + if let credentials = try? JSONDecoder().decode(AWSCredentials.self, from: data) { + AWSCredentialsManager.shared.updateCredentials(credentials) + } else { + KeychainSecretStore.shared.setData(data, forKey: key) + AWSCredentialsManager.shared.clearCredentialEnvironment() + } + return true + default: + return false + } + } + private func isSensitiveSettingKey(_ key: String) -> Bool { let lowercase = key.lowercased() // Substring match for unambiguous credential fragments (e.g. "apikey", "secret") diff --git a/BisonNotes AI/BisonNotes AITests/EndpointSecurityPolicyTests.swift b/BisonNotes AI/BisonNotes AITests/EndpointSecurityPolicyTests.swift new file mode 100644 index 0000000..a68a9e5 --- /dev/null +++ b/BisonNotes AI/BisonNotes AITests/EndpointSecurityPolicyTests.swift @@ -0,0 +1,38 @@ +// +// EndpointSecurityPolicyTests.swift +// BisonNotes AITests +// + +import XCTest +@testable import BisonNotes_AI + +final class EndpointSecurityPolicyTests: XCTestCase { + func testAllowsEncryptedPublicEndpoints() { + XCTAssertNil(EndpointSecurityPolicy.validationMessage(for: "https://api.openai.com/v1")) + XCTAssertNil(EndpointSecurityPolicy.validationMessage(for: "wss://example.com/socket")) + } + + func testAllowsLocalAndPrivateHTTP() { + XCTAssertNil(EndpointSecurityPolicy.validationMessage(for: "http://localhost:11434")) + XCTAssertNil(EndpointSecurityPolicy.validationMessage(for: "http://127.0.0.1:9000")) + XCTAssertNil(EndpointSecurityPolicy.validationMessage(for: "http://10.0.0.5:9000")) + XCTAssertNil(EndpointSecurityPolicy.validationMessage(for: "http://172.16.10.5:9000")) + XCTAssertNil(EndpointSecurityPolicy.validationMessage(for: "http://192.168.1.20:9000")) + } + + func testBlocksPublicHTTPByDefault() { + let message = EndpointSecurityPolicy.validationMessage(for: "http://example.com/v1") + + XCTAssertNotNil(message) + XCTAssertTrue(message?.contains("Public HTTP endpoints are blocked") == true) + } + + func testAllowsPublicHTTPOnlyWhenDevelopmentModeIsEnabled() { + XCTAssertNil( + EndpointSecurityPolicy.validationMessage( + for: "http://example.com/v1", + allowInsecurePublicEndpoints: true + ) + ) + } +} diff --git a/BisonNotes AI/BisonNotes AITests/KeychainSecretStoreTests.swift b/BisonNotes AI/BisonNotes AITests/KeychainSecretStoreTests.swift new file mode 100644 index 0000000..9f8e137 --- /dev/null +++ b/BisonNotes AI/BisonNotes AITests/KeychainSecretStoreTests.swift @@ -0,0 +1,74 @@ +// +// KeychainSecretStoreTests.swift +// BisonNotes AITests +// + +import XCTest +@testable import BisonNotes_AI + +final class KeychainSecretStoreTests: XCTestCase { + private var store: KeychainSecretStore! + private var defaults: UserDefaults! + private var suiteName: String! + + override func setUpWithError() throws { + suiteName = "KeychainSecretStoreTests.\(UUID().uuidString)" + defaults = UserDefaults(suiteName: suiteName) + store = KeychainSecretStore(service: "com.bisonnotes.tests.\(UUID().uuidString)") + } + + override func tearDownWithError() throws { + [ + KeychainSecretStore.openAIAPIKey, + KeychainSecretStore.openAICompatibleAPIKey, + KeychainSecretStore.googleAIStudioAPIKey, + KeychainSecretStore.mistralAPIKey, + KeychainSecretStore.awsBedrockSessionToken, + KeychainSecretStore.awsCredentials + ].forEach { store.delete(forKey: $0) } + + defaults.removePersistentDomain(forName: suiteName) + } + + func testMigratesStringSecretsFromUserDefaultsToKeychain() throws { + defaults.set("sk-test-openai", forKey: KeychainSecretStore.openAIAPIKey) + defaults.set("mistral-test-key", forKey: KeychainSecretStore.mistralAPIKey) + + store.migrateLegacySecretsFromUserDefaults(defaults) + + XCTAssertNil(defaults.string(forKey: KeychainSecretStore.openAIAPIKey)) + XCTAssertNil(defaults.string(forKey: KeychainSecretStore.mistralAPIKey)) + XCTAssertEqual(store.string(forKey: KeychainSecretStore.openAIAPIKey), "sk-test-openai") + XCTAssertEqual(store.string(forKey: KeychainSecretStore.mistralAPIKey), "mistral-test-key") + } + + func testMigratesAWSCredentialsBlobFromUserDefaultsToKeychain() throws { + let credentials = AWSCredentials( + accessKeyId: "AKIATEST", + secretAccessKey: "aws-secret-test", + region: "us-west-2" + ) + let encoded = try JSONEncoder().encode(credentials) + defaults.set(encoded, forKey: KeychainSecretStore.awsCredentials) + + store.migrateLegacySecretsFromUserDefaults(defaults) + + XCTAssertNil(defaults.data(forKey: KeychainSecretStore.awsCredentials)) + + let migratedData = try XCTUnwrap(store.data(forKey: KeychainSecretStore.awsCredentials)) + let migratedCredentials = try JSONDecoder().decode(AWSCredentials.self, from: migratedData) + XCTAssertEqual(migratedCredentials, credentials) + } + + func testEmptyStringDeletesKeychainSecret() { + store.setString("temporary-secret", forKey: KeychainSecretStore.googleAIStudioAPIKey) + XCTAssertEqual(store.string(forKey: KeychainSecretStore.googleAIStudioAPIKey), "temporary-secret") + + store.setString("updated-secret", forKey: KeychainSecretStore.googleAIStudioAPIKey) + XCTAssertEqual(store.string(forKey: KeychainSecretStore.googleAIStudioAPIKey), "updated-secret") + + store.setString("", forKey: KeychainSecretStore.googleAIStudioAPIKey) + + XCTAssertNil(store.string(forKey: KeychainSecretStore.googleAIStudioAPIKey)) + } +} diff --git a/BisonNotes AI/BisonNotes AITests/ShareImportAuthorizationTests.swift b/BisonNotes AI/BisonNotes AITests/ShareImportAuthorizationTests.swift new file mode 100644 index 0000000..65bb1ed --- /dev/null +++ b/BisonNotes AI/BisonNotes AITests/ShareImportAuthorizationTests.swift @@ -0,0 +1,57 @@ +// +// ShareImportAuthorizationTests.swift +// BisonNotes AITests +// + +import XCTest +@testable import BisonNotes_AI + +final class ShareImportAuthorizationTests: XCTestCase { + private var inboxURL: URL! + + override func setUpWithError() throws { + inboxURL = FileManager.default.temporaryDirectory + .appendingPathComponent("ShareImportAuthorizationTests") + .appendingPathComponent(UUID().uuidString, isDirectory: true) + try FileManager.default.createDirectory(at: inboxURL, withIntermediateDirectories: true) + } + + override func tearDownWithError() throws { + try? FileManager.default.removeItem(at: inboxURL) + } + + func testRejectsShareImportURLWithoutStoredToken() { + let token = UUID().uuidString + let url = URL(string: "bisonnotes://share-import?token=\(token)")! + + XCTAssertFalse(ShareImportAuthorization.consumeURLToken(from: url, in: inboxURL)) + } + + func testConsumesMatchingURLTokenOnce() throws { + let token = UUID().uuidString + let tokenData = try XCTUnwrap(token.data(using: .utf8)) + try tokenData.write(to: ShareImportAuthorization.tokenFileURL(in: inboxURL), options: .atomic) + let url = URL(string: "bisonnotes://share-import?token=\(token)")! + + XCTAssertTrue(ShareImportAuthorization.consumeURLToken(from: url, in: inboxURL)) + XCTAssertFalse(ShareImportAuthorization.consumeURLToken(from: url, in: inboxURL)) + } + + func testRejectsUnsupportedSchemeHostEvenWithToken() throws { + let token = UUID().uuidString + let tokenData = try XCTUnwrap(token.data(using: .utf8)) + try tokenData.write(to: ShareImportAuthorization.tokenFileURL(in: inboxURL), options: .atomic) + let url = URL(string: "bisonnotes://settings?token=\(token)")! + + XCTAssertFalse(ShareImportAuthorization.consumeURLToken(from: url, in: inboxURL)) + } + + func testConsumesPendingTokenForActivationScan() throws { + let token = UUID().uuidString + let tokenData = try XCTUnwrap(token.data(using: .utf8)) + try tokenData.write(to: ShareImportAuthorization.tokenFileURL(in: inboxURL), options: .atomic) + + XCTAssertTrue(ShareImportAuthorization.consumePendingToken(in: inboxURL)) + XCTAssertFalse(ShareImportAuthorization.consumePendingToken(in: inboxURL)) + } +} diff --git a/BisonNotes AI/BisonNotes AITests/TestHelpers.swift b/BisonNotes AI/BisonNotes AITests/TestHelpers.swift index 45621ed..35e8d49 100644 --- a/BisonNotes AI/BisonNotes AITests/TestHelpers.swift +++ b/BisonNotes AI/BisonNotes AITests/TestHelpers.swift @@ -22,6 +22,7 @@ struct TestHelpers { channels: Int = 2 ) -> AudioFileInfo { return AudioFileInfo( + url: URL(fileURLWithPath: "/test/audio.\(format)"), duration: duration, fileSize: fileSize, format: format, @@ -279,7 +280,7 @@ class MockBackgroundProcessingManager: BackgroundProcessingManager { var mockJobExecutionSuccess = true var mockJobError: Error? - override func startTranscription(_ job: ProcessingJob) async throws { + func startTranscription(_ job: ProcessingJob) async throws { if !mockJobExecutionSuccess { throw mockJobError ?? AudioProcessingError.backgroundProcessingFailed("Mock failure") } @@ -309,9 +310,12 @@ class MockiCloudStorageManager: iCloudStorageManager { set { mockIsEnabled = newValue } } - override func enableiCloudSync() async throws { + override func enableiCloudSync() async { if !mockSyncSuccess { - throw mockSyncError ?? AudioProcessingError.iCloudSyncFailed("Mock failure") + let message = mockSyncError?.localizedDescription ?? "Mock failure" + self.lastError = message + self.syncStatus = .failed(message) + return } await MainActor.run { @@ -320,7 +324,7 @@ class MockiCloudStorageManager: iCloudStorageManager { } } - override func disableiCloudSync() async throws { + override func disableiCloudSync() async { await MainActor.run { self.isEnabled = false self.syncStatus = .idle @@ -333,7 +337,7 @@ class MockiCloudStorageManager: iCloudStorageManager { extension TestHelpers { /// Asserts that a job has the expected status - static func assertJobStatus(_ job: ProcessingJob, expected: ProcessingStatus) { + static func assertJobStatus(_ job: ProcessingJob, expected: JobProcessingStatus) { assert(job.status == expected, "Expected job status \(expected), but got \(job.status)") } @@ -412,4 +416,4 @@ extension TestHelpers { static func simulateNetworkRecovery() -> NetworkStatus { return .available } -} \ No newline at end of file +} diff --git a/BisonNotes AI/BisonNotes Share/ShareViewController.swift b/BisonNotes AI/BisonNotes Share/ShareViewController.swift index 05e0d01..159f7e3 100644 --- a/BisonNotes AI/BisonNotes Share/ShareViewController.swift +++ b/BisonNotes AI/BisonNotes Share/ShareViewController.swift @@ -96,8 +96,19 @@ class ShareViewController: UIViewController { group.notify(queue: .main) { [weak self] in NSLog("📎 Share Extension: done, saved \(processedCount) file(s)") + guard processedCount > 0 else { + self?.completeRequest() + return + } + + guard let importToken = self?.createImportToken() else { + NSLog("❌ Share Extension: could not create import token") + self?.completeRequest() + return + } + // Open the main app so it imports immediately instead of waiting for user to switch. - self?.openMainApp { + self?.openMainApp(importToken: importToken) { self?.completeRequest() } } @@ -172,6 +183,7 @@ class ShareViewController: UIViewController { do { try FileManager.default.copyItem(at: url, to: destination) + applyFileProtection(to: destination) NSLog("✅ Share Extension: saved file with extension: \(url.pathExtension)") return true } catch { @@ -180,6 +192,35 @@ class ShareViewController: UIViewController { } } + private func createImportToken() -> String? { + guard let containerURL = FileManager.default + .containerURL(forSecurityApplicationGroupIdentifier: appGroupID) else { + return nil + } + + let inboxURL = containerURL.appendingPathComponent(shareInboxFolder) + + do { + try FileManager.default.createDirectory(at: inboxURL, withIntermediateDirectories: true) + let token = UUID().uuidString + let tokenURL = inboxURL.appendingPathComponent(".share-import-token") + guard let tokenData = token.data(using: .utf8) else { return nil } + try tokenData.write(to: tokenURL, options: .atomic) + applyFileProtection(to: tokenURL) + return token + } catch { + NSLog("❌ Share Extension: cannot create import token: \(error)") + return nil + } + } + + private func applyFileProtection(to url: URL) { + try? FileManager.default.setAttributes( + [.protectionKey: FileProtectionType.completeUntilFirstUserAuthentication], + ofItemAtPath: url.path + ) + } + // MARK: - Open Main App /// Attempts to open the main app via its custom URL scheme so it imports @@ -191,8 +232,8 @@ class ShareViewController: UIViewController { /// 2. Fall back to the responder chain (in case direct access fails). /// 3. Post a Darwin notification (works when app is already backgrounded). /// 4. (Implicit) The main app scans the shared container on `didBecomeActive`. - private func openMainApp(completion: @escaping () -> Void) { - guard let url = URL(string: "bisonnotes://share-import") else { + private func openMainApp(importToken: String, completion: @escaping () -> Void) { + guard let url = URL(string: "bisonnotes://share-import?token=\(importToken)") else { completion() return } diff --git a/CLAUDE.md b/CLAUDE.md index 67f05f5..a18136d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -121,6 +121,62 @@ The app includes comprehensive AWS Bedrock integration (`AWS/AWSBedrockModels.sw - **Cross-Region Inference Profiles**: Claude and Llama models use `us.*`, `global.*`, `eu.*` prefixes for cross-region routing - Cross-region profiles provide ~10% cost savings and higher throughput by routing requests to available regions +### Mac Catalyst Build Notes + +The `MacOS-Catalyst` branch adds Mac Catalyst support. Several manual steps were required for the vendored `llama.xcframework` and must be repeated any time the framework is rebuilt or updated: + +#### llama.xcframework Catalyst Slice + +The upstream llama.cpp xcframework does not ship a Mac Catalyst slice. The `ios-arm64-maccatalyst` slice in `Frameworks/llama.xcframework/` was created manually: + +```bash +# 1. Extract the arm64 slice from the macOS fat binary +lipo -thin arm64 \ + Frameworks/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/llama \ + -output /tmp/llama-arm64 + +# 2. Create versioned macOS-style framework layout (required — Catalyst does not use shallow bundles) +CATALYST=Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework +mkdir -p $CATALYST/Versions/A/Resources +mkdir -p $CATALYST/Versions/A/Headers +mkdir -p $CATALYST/Versions/A/Modules + +# 3. Copy headers, modules, and Info.plist from the macOS framework +cp Frameworks/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/* \ + $CATALYST/Versions/A/Headers/ +cp Frameworks/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Modules/* \ + $CATALYST/Versions/A/Modules/ +cp Frameworks/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Resources/Info.plist \ + $CATALYST/Versions/A/Resources/Info.plist + +# 4. Place the thin binary +cp /tmp/llama-arm64 $CATALYST/Versions/A/llama + +# 5. Patch the Mach-O platform header from MACOS to MACCATALYST +vtool -set-build-version maccatalyst 14.0 15.5 -replace \ + -output $CATALYST/Versions/A/llama \ + $CATALYST/Versions/A/llama + +# 6. Create Versions/Current symlink and top-level symlinks +ln -s A $CATALYST/Versions/Current +ln -s Versions/Current/llama $CATALYST/llama +ln -s Versions/Current/Headers $CATALYST/Headers +ln -s Versions/Current/Modules $CATALYST/Modules +ln -s Versions/Current/Resources $CATALYST/Resources +``` + +Then add the `ios-arm64-maccatalyst` entry to `Frameworks/llama.xcframework/Info.plist` (see existing entry in that file for the format). + +Step 5 is critical — without the `vtool` patch, the linker warns "built for macOS" and may fail codesigning. + +#### textual (MarkdownUI) Catalyst Fix + +The `bisonbet/textual` fork has Mac Catalyst guards applied (commit `0c2c3b5`). On Mac Catalyst `canImport(AppKit)` is true, which caused the package to take the AppKit path and fail. The fix adds `&& !targetEnvironment(macCatalyst)` to AppKit checks in: +- `Sources/Textual/Internal/Font/PlatformFont.swift` +- `Sources/Textual/Internal/Helpers/PlatformImage.swift` + +If textual is rebased from upstream, reapply these guards. + ### Background Processing For long-running operations, use `BackgroundProcessingManager` to queue jobs and track progress. diff --git a/Frameworks/llama.xcframework/Info.plist b/Frameworks/llama.xcframework/Info.plist index 7e23b2c..03cc563 100644 --- a/Frameworks/llama.xcframework/Info.plist +++ b/Frameworks/llama.xcframework/Info.plist @@ -10,18 +10,15 @@ DebugSymbolsPath dSYMs LibraryIdentifier - tvos-arm64_x86_64-simulator + xros-arm64 LibraryPath llama.framework SupportedArchitectures arm64 - x86_64 SupportedPlatform - tvos - SupportedPlatformVariant - simulator + xros BinaryPath @@ -29,7 +26,7 @@ DebugSymbolsPath dSYMs LibraryIdentifier - tvos-arm64 + ios-arm64 LibraryPath llama.framework SupportedArchitectures @@ -37,26 +34,23 @@ arm64 SupportedPlatform - tvos + ios BinaryPath - llama.framework/llama - DebugSymbolsPath - dSYMs + llama.framework/Versions/A/llama LibraryIdentifier - ios-arm64_x86_64-simulator + ios-arm64-maccatalyst LibraryPath llama.framework SupportedArchitectures arm64 - x86_64 SupportedPlatform ios SupportedPlatformVariant - simulator + maccatalyst BinaryPath @@ -64,18 +58,15 @@ DebugSymbolsPath dSYMs LibraryIdentifier - xros-arm64_x86_64-simulator + tvos-arm64 LibraryPath llama.framework SupportedArchitectures arm64 - x86_64 SupportedPlatform - xros - SupportedPlatformVariant - simulator + tvos BinaryPath @@ -100,15 +91,18 @@ DebugSymbolsPath dSYMs LibraryIdentifier - ios-arm64 + xros-arm64_x86_64-simulator LibraryPath llama.framework SupportedArchitectures arm64 + x86_64 SupportedPlatform - ios + xros + SupportedPlatformVariant + simulator BinaryPath @@ -116,15 +110,37 @@ DebugSymbolsPath dSYMs LibraryIdentifier - xros-arm64 + tvos-arm64_x86_64-simulator LibraryPath llama.framework SupportedArchitectures arm64 + x86_64 SupportedPlatform - xros + tvos + SupportedPlatformVariant + simulator + + + BinaryPath + llama.framework/llama + DebugSymbolsPath + dSYMs + LibraryIdentifier + ios-arm64_x86_64-simulator + LibraryPath + llama.framework + SupportedArchitectures + + arm64 + x86_64 + + SupportedPlatform + ios + SupportedPlatformVariant + simulator CFBundlePackageType diff --git a/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Headers b/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Headers new file mode 120000 index 0000000..a177d2a --- /dev/null +++ b/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Headers @@ -0,0 +1 @@ +Versions/Current/Headers \ No newline at end of file diff --git a/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Modules b/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Modules new file mode 120000 index 0000000..5736f31 --- /dev/null +++ b/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Modules @@ -0,0 +1 @@ +Versions/Current/Modules \ No newline at end of file diff --git a/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Resources b/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Resources new file mode 120000 index 0000000..953ee36 --- /dev/null +++ b/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Resources @@ -0,0 +1 @@ +Versions/Current/Resources \ No newline at end of file diff --git a/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Versions/A/Headers/ggml-alloc.h b/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Versions/A/Headers/ggml-alloc.h new file mode 100644 index 0000000..78aa059 --- /dev/null +++ b/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Versions/A/Headers/ggml-alloc.h @@ -0,0 +1,85 @@ +#pragma once + +#include "ggml.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct ggml_backend_buffer_type * ggml_backend_buffer_type_t; +typedef struct ggml_backend_buffer * ggml_backend_buffer_t; +typedef struct ggml_backend * ggml_backend_t; + +// Tensor allocator +struct ggml_tallocr { + ggml_backend_buffer_t buffer; + void * base; + size_t alignment; + size_t offset; +}; + +GGML_API struct ggml_tallocr ggml_tallocr_new(ggml_backend_buffer_t buffer); +GGML_API enum ggml_status ggml_tallocr_alloc(struct ggml_tallocr * talloc, struct ggml_tensor * tensor); + +// Graph allocator +/* + Example usage: + ggml_gallocr_t galloc = ggml_gallocr_new(ggml_backend_cpu_buffer_type()); + + // optional: create a worst-case graph and reserve the buffers to avoid reallocations + ggml_gallocr_reserve(galloc, build_graph(max_batch)); + + // allocate the graph + struct ggml_cgraph * graph = build_graph(batch); + ggml_gallocr_alloc_graph(galloc, graph); + + printf("compute buffer size: %zu bytes\n", ggml_gallocr_get_buffer_size(galloc, 0)); + + // evaluate the graph + ggml_backend_graph_compute(backend, graph); +*/ + +// special tensor flags for use with the graph allocator: +// ggml_set_input(): all input tensors are allocated at the beginning of the graph in non-overlapping addresses +// ggml_set_output(): output tensors are never freed and never overwritten + +typedef struct ggml_gallocr * ggml_gallocr_t; + +GGML_API ggml_gallocr_t ggml_gallocr_new(ggml_backend_buffer_type_t buft); +GGML_API ggml_gallocr_t ggml_gallocr_new_n(ggml_backend_buffer_type_t * bufts, int n_bufs); +GGML_API void ggml_gallocr_free(ggml_gallocr_t galloc); + +// pre-allocate buffers from a measure graph - does not allocate or modify the graph +// call with a worst-case graph to avoid buffer reallocations +// not strictly required for single buffer usage: ggml_gallocr_alloc_graph will reallocate the buffers automatically if needed +// returns false if the buffer allocation failed +// ggml_gallocr_resrve_n_size writes the buffer sizes per galloc buffer that would be allocated by ggml_gallocr_reserve_n to sizes +GGML_API bool ggml_gallocr_reserve(ggml_gallocr_t galloc, struct ggml_cgraph * graph); +GGML_API void ggml_gallocr_reserve_n_size( + ggml_gallocr_t galloc, + struct ggml_cgraph * graph, + const int * node_buffer_ids, + const int * leaf_buffer_ids, + size_t * sizes); +GGML_API bool ggml_gallocr_reserve_n( + ggml_gallocr_t galloc, + struct ggml_cgraph * graph, + const int * node_buffer_ids, + const int * leaf_buffer_ids); + +// automatic reallocation if the topology changes when using a single buffer +// returns false if using multiple buffers and a re-allocation is needed (call ggml_gallocr_reserve_n first to set the node buffers) +GGML_API bool ggml_gallocr_alloc_graph(ggml_gallocr_t galloc, struct ggml_cgraph * graph); + +GGML_API size_t ggml_gallocr_get_buffer_size(ggml_gallocr_t galloc, int buffer_id); + +// Utils +// Create a buffer and allocate all the tensors in a ggml_context +// ggml_backend_alloc_ctx_tensors_from_buft_size returns the size of the buffer that would be allocated by ggml_backend_alloc_ctx_tensors_from_buft +GGML_API size_t ggml_backend_alloc_ctx_tensors_from_buft_size(struct ggml_context * ctx, ggml_backend_buffer_type_t buft); +GGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors_from_buft(struct ggml_context * ctx, ggml_backend_buffer_type_t buft); +GGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors(struct ggml_context * ctx, ggml_backend_t backend); + +#ifdef __cplusplus +} +#endif diff --git a/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Versions/A/Headers/ggml-backend.h b/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Versions/A/Headers/ggml-backend.h new file mode 100644 index 0000000..b6f7373 --- /dev/null +++ b/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Versions/A/Headers/ggml-backend.h @@ -0,0 +1,431 @@ +#pragma once + +#include "ggml.h" +#include "ggml-alloc.h" + +#ifdef GGML_BACKEND_SHARED +# if defined(_WIN32) && !defined(__MINGW32__) +# ifdef GGML_BACKEND_BUILD +# define GGML_BACKEND_API __declspec(dllexport) extern +# else +# define GGML_BACKEND_API __declspec(dllimport) extern +# endif +# else +# define GGML_BACKEND_API __attribute__ ((visibility ("default"))) extern +# endif +#else +# define GGML_BACKEND_API extern +#endif + +#ifdef __cplusplus +extern "C" { +#endif + + typedef struct ggml_backend_buffer_type * ggml_backend_buffer_type_t; + typedef struct ggml_backend_buffer * ggml_backend_buffer_t; + typedef struct ggml_backend_event * ggml_backend_event_t; + typedef struct ggml_backend * ggml_backend_t; + typedef void * ggml_backend_graph_plan_t; + typedef struct ggml_backend_reg * ggml_backend_reg_t; + typedef struct ggml_backend_device * ggml_backend_dev_t; + + + // + // Backend buffer type + // + + GGML_API const char * ggml_backend_buft_name (ggml_backend_buffer_type_t buft); + GGML_API ggml_backend_buffer_t ggml_backend_buft_alloc_buffer (ggml_backend_buffer_type_t buft, size_t size); + GGML_API size_t ggml_backend_buft_get_alignment (ggml_backend_buffer_type_t buft); + GGML_API size_t ggml_backend_buft_get_max_size (ggml_backend_buffer_type_t buft); + GGML_API size_t ggml_backend_buft_get_alloc_size(ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor); + GGML_API bool ggml_backend_buft_is_host (ggml_backend_buffer_type_t buft); + GGML_API ggml_backend_dev_t ggml_backend_buft_get_device (ggml_backend_buffer_type_t buft); + + // + // Backend buffer + // + + enum ggml_backend_buffer_usage { + GGML_BACKEND_BUFFER_USAGE_ANY = 0, + GGML_BACKEND_BUFFER_USAGE_WEIGHTS = 1, + GGML_BACKEND_BUFFER_USAGE_COMPUTE = 2, + }; + + GGML_API const char * ggml_backend_buffer_name (ggml_backend_buffer_t buffer); + GGML_API void ggml_backend_buffer_free (ggml_backend_buffer_t buffer); + GGML_API void * ggml_backend_buffer_get_base (ggml_backend_buffer_t buffer); + GGML_API size_t ggml_backend_buffer_get_size (ggml_backend_buffer_t buffer); + GGML_API enum ggml_status ggml_backend_buffer_init_tensor (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor); + GGML_API size_t ggml_backend_buffer_get_alignment (ggml_backend_buffer_t buffer); + GGML_API size_t ggml_backend_buffer_get_max_size (ggml_backend_buffer_t buffer); + GGML_API size_t ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor); + GGML_API void ggml_backend_buffer_clear (ggml_backend_buffer_t buffer, uint8_t value); + GGML_API bool ggml_backend_buffer_is_host (ggml_backend_buffer_t buffer); + GGML_API void ggml_backend_buffer_set_usage (ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage); + GGML_API enum ggml_backend_buffer_usage ggml_backend_buffer_get_usage (ggml_backend_buffer_t buffer); + GGML_API ggml_backend_buffer_type_t ggml_backend_buffer_get_type (ggml_backend_buffer_t buffer); + GGML_API void ggml_backend_buffer_reset (ggml_backend_buffer_t buffer); + + // tensor copy between different backends + GGML_API void ggml_backend_tensor_copy(const struct ggml_tensor * src, struct ggml_tensor * dst); + + // + // Backend (stream) + // + + GGML_API ggml_guid_t ggml_backend_guid(ggml_backend_t backend); + GGML_API const char * ggml_backend_name(ggml_backend_t backend); + GGML_API void ggml_backend_free(ggml_backend_t backend); + + GGML_API ggml_backend_buffer_type_t ggml_backend_get_default_buffer_type(ggml_backend_t backend); + GGML_API ggml_backend_buffer_t ggml_backend_alloc_buffer(ggml_backend_t backend, size_t size); + GGML_API size_t ggml_backend_get_alignment(ggml_backend_t backend); + GGML_API size_t ggml_backend_get_max_size(ggml_backend_t backend); + + GGML_API void ggml_backend_tensor_set_async (ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_get_async (ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_set_2d_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); + GGML_API void ggml_backend_tensor_get_2d_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); + + // "offset" refers to the offset in tensor->data for setting/getting data + GGML_API void ggml_backend_tensor_set ( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_get (const struct ggml_tensor * tensor, void * data, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_set_2d( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); + GGML_API void ggml_backend_tensor_get_2d(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); + GGML_API void ggml_backend_tensor_memset( struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size); + + GGML_API void ggml_backend_synchronize(ggml_backend_t backend); + + GGML_API ggml_backend_graph_plan_t ggml_backend_graph_plan_create(ggml_backend_t backend, struct ggml_cgraph * cgraph); + GGML_API void ggml_backend_graph_plan_free (ggml_backend_t backend, ggml_backend_graph_plan_t plan); + + GGML_API enum ggml_status ggml_backend_graph_plan_compute (ggml_backend_t backend, ggml_backend_graph_plan_t plan); + GGML_API enum ggml_status ggml_backend_graph_compute (ggml_backend_t backend, struct ggml_cgraph * cgraph); + GGML_API enum ggml_status ggml_backend_graph_compute_async(ggml_backend_t backend, struct ggml_cgraph * cgraph); + + // NOTE: will be removed, use device version instead + GGML_API bool ggml_backend_supports_op(ggml_backend_t backend, const struct ggml_tensor * op); + GGML_API bool ggml_backend_supports_buft(ggml_backend_t backend, ggml_backend_buffer_type_t buft); + GGML_API bool ggml_backend_offload_op(ggml_backend_t backend, const struct ggml_tensor * op); + + // asynchronous copy + // the copy is performed after all the currently queued operations in backend_src + // backend_dst will wait for the copy to complete before performing other operations + // automatic fallback to sync copy if async is not supported + GGML_API void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, const struct ggml_tensor * src, struct ggml_tensor * dst); + + GGML_API ggml_backend_dev_t ggml_backend_get_device(ggml_backend_t backend); + + // + // Events + // + + GGML_API ggml_backend_event_t ggml_backend_event_new(ggml_backend_dev_t device); + GGML_API void ggml_backend_event_free(ggml_backend_event_t event); + GGML_API void ggml_backend_event_record(ggml_backend_event_t event, ggml_backend_t backend); + GGML_API void ggml_backend_event_synchronize(ggml_backend_event_t event); + GGML_API void ggml_backend_event_wait(ggml_backend_t backend, ggml_backend_event_t event); + + // + // Backend device + // + + enum ggml_backend_dev_type { + // CPU device using system memory + GGML_BACKEND_DEVICE_TYPE_CPU, + // GPU device using dedicated memory + GGML_BACKEND_DEVICE_TYPE_GPU, + // integrated GPU device using host memory + GGML_BACKEND_DEVICE_TYPE_IGPU, + // accelerator devices intended to be used together with the CPU backend (e.g. BLAS or AMX) + GGML_BACKEND_DEVICE_TYPE_ACCEL, + // "meta" device wrapping multiple other devices for tensor parallelism + GGML_BACKEND_DEVICE_TYPE_META, + }; + + // functionality supported by the device + struct ggml_backend_dev_caps { + // asynchronous operations + bool async; + // pinned host buffer + bool host_buffer; + // creating buffers from host ptr + bool buffer_from_host_ptr; + // event synchronization + bool events; + }; + + // all the device properties + struct ggml_backend_dev_props { + // device name + const char * name; + // device description + const char * description; + // device free memory in bytes + size_t memory_free; + // device total memory in bytes + size_t memory_total; + // device type + enum ggml_backend_dev_type type; + // device id + // for PCI devices, this should be the lower-case PCI bus id formatted as "domain:bus:device.function" (e.g. "0000:c1:00.0") + // if the id is unknown, this should be NULL + const char * device_id; + // device capabilities + struct ggml_backend_dev_caps caps; + }; + + GGML_API const char * ggml_backend_dev_name(ggml_backend_dev_t device); + GGML_API const char * ggml_backend_dev_description(ggml_backend_dev_t device); + GGML_API void ggml_backend_dev_memory(ggml_backend_dev_t device, size_t * free, size_t * total); + GGML_API enum ggml_backend_dev_type ggml_backend_dev_type(ggml_backend_dev_t device); + GGML_API void ggml_backend_dev_get_props(ggml_backend_dev_t device, struct ggml_backend_dev_props * props); + GGML_API ggml_backend_reg_t ggml_backend_dev_backend_reg(ggml_backend_dev_t device); + GGML_API ggml_backend_t ggml_backend_dev_init(ggml_backend_dev_t device, const char * params); + GGML_API ggml_backend_buffer_type_t ggml_backend_dev_buffer_type(ggml_backend_dev_t device); + GGML_API ggml_backend_buffer_type_t ggml_backend_dev_host_buffer_type(ggml_backend_dev_t device); + GGML_API ggml_backend_buffer_t ggml_backend_dev_buffer_from_host_ptr(ggml_backend_dev_t device, void * ptr, size_t size, size_t max_tensor_size); + + GGML_API bool ggml_backend_dev_supports_op(ggml_backend_dev_t device, const struct ggml_tensor * op); + GGML_API bool ggml_backend_dev_supports_buft(ggml_backend_dev_t device, ggml_backend_buffer_type_t buft); + GGML_API bool ggml_backend_dev_offload_op(ggml_backend_dev_t device, const struct ggml_tensor * op); + + // + // Backend (reg) + // + + GGML_API const char * ggml_backend_reg_name(ggml_backend_reg_t reg); + GGML_API size_t ggml_backend_reg_dev_count(ggml_backend_reg_t reg); + GGML_API ggml_backend_dev_t ggml_backend_reg_dev_get(ggml_backend_reg_t reg, size_t index); + GGML_API void * ggml_backend_reg_get_proc_address(ggml_backend_reg_t reg, const char * name); + + // Common functions that may be obtained using ggml_backend_reg_get_proc_address + + // Context management and operations for faster communication between backends, used for tensor parallelism (meta backend) + typedef void * (*ggml_backend_comm_init_t)(ggml_backend_t * backends, size_t n_backends); + typedef void (*ggml_backend_comm_free_t)(void * comm_ctx); + typedef bool (*ggml_backend_comm_allreduce_tensor_t)(void * comm_ctx, struct ggml_tensor ** tensors); + + // Split buffer type for tensor parallelism (old) + typedef ggml_backend_buffer_type_t (*ggml_backend_split_buffer_type_t)(int main_device, const float * tensor_split); + // Set the number of threads for the backend + typedef void (*ggml_backend_set_n_threads_t)(ggml_backend_t backend, int n_threads); + // Get additional buffer types provided by the device (returns a NULL-terminated array) + typedef ggml_backend_buffer_type_t * (*ggml_backend_dev_get_extra_bufts_t)(ggml_backend_dev_t device); + // Set the abort callback for the backend + typedef void (*ggml_backend_set_abort_callback_t)(ggml_backend_t backend, ggml_abort_callback abort_callback, void * abort_callback_data); + // Get a list of feature flags supported by the backend (returns a NULL-terminated array) + struct ggml_backend_feature { + const char * name; + const char * value; + }; + typedef struct ggml_backend_feature * (*ggml_backend_get_features_t)(ggml_backend_reg_t reg); + + // + // Backend registry + // + + GGML_API void ggml_backend_register(ggml_backend_reg_t reg); + + GGML_API void ggml_backend_device_register(ggml_backend_dev_t device); + + // Backend (reg) enumeration + GGML_API size_t ggml_backend_reg_count(void); + GGML_API ggml_backend_reg_t ggml_backend_reg_get(size_t index); + GGML_API ggml_backend_reg_t ggml_backend_reg_by_name(const char * name); + + // Device enumeration + GGML_API size_t ggml_backend_dev_count(void); + GGML_API ggml_backend_dev_t ggml_backend_dev_get(size_t index); + GGML_API ggml_backend_dev_t ggml_backend_dev_by_name(const char * name); + GGML_API ggml_backend_dev_t ggml_backend_dev_by_type(enum ggml_backend_dev_type type); + + // Direct backend (stream) initialization + // = ggml_backend_dev_init(ggml_backend_dev_by_name(name), params) + GGML_API ggml_backend_t ggml_backend_init_by_name(const char * name, const char * params); + // = ggml_backend_dev_init(ggml_backend_dev_by_type(type), params) + GGML_API ggml_backend_t ggml_backend_init_by_type(enum ggml_backend_dev_type type, const char * params); + // = ggml_backend_dev_init(ggml_backend_dev_by_type(GPU) OR ggml_backend_dev_by_type(CPU), NULL) + GGML_API ggml_backend_t ggml_backend_init_best(void); + + // Load a backend from a dynamic library and register it + GGML_API ggml_backend_reg_t ggml_backend_load(const char * path); + // Unload a backend if loaded dynamically and unregister it + GGML_API void ggml_backend_unload(ggml_backend_reg_t reg); + // Load all known backends from dynamic libraries + GGML_API void ggml_backend_load_all(void); + GGML_API void ggml_backend_load_all_from_path(const char * dir_path); + + // + // Backend scheduler + // + + // The backend scheduler allows for multiple backend devices to be used together + // Handles compute buffer allocation, assignment of tensors to backends, and copying of tensors between backends + // The backends are selected based on: + // - the backend that supports the operation + // - the location of the pre-allocated tensors (e.g. the weights) + /* + Example usage: + + // operations that use tensors allocated in a buffer with USAGE_WEIGHTS will be assigned + // preferably to run on the same backend as the buffer + ggml_backend_buffer_set_usage(buf_weights, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + + sched = ggml_backend_sched_new({backend_gpu, backend_gpu2, backend_cpu}, NULL, num_backends, GGML_DEFAULT_GRAPH_SIZE, false, true); + + // initialize buffers from a max size graph (optional) + reserve_graph = build_graph(sched, max_batch_size); + + // manually assign nodes to a backend (optional, should not be needed in most cases) + struct ggml_tensor * node = ggml_mul_mat(ctx, ...); + ggml_backend_sched_set_tensor_backend(sched, node, backend_gpu); + + ggml_backend_sched_reserve(sched, reserve_graph); + + // compute + graph = build_graph(sched); // the graph and its tensors are single-use in terms of allocation, multi-use in terms of computation + for (int i = 0; i < 10; ++i) { + ggml_backend_sched_graph_compute(sched, graph); // on the first iteration the graph is allocated automatically + } + + // if there are graph inputs: + graph = build_graph(sched); // get a new graph that is not allocated (the metadata for the old graph is freed once ggml_free is called) + ggml_backend_sched_reset(sched); // clear the allocation of the previous graph + ggml_backend_sched_alloc_graph(sched, graph); // explicitly allocate the new graph but do not execute it + ggml_backend_tensor_set(input_tensor, ...); // copy data to the newly allocated graph tensors + ggml_backend_sched_graph_compute(sched, graph); // execute the graph + + // as an alternative to the above it is also possible to assign the inputs to a dedicated context and + // allocate them statically via ggml_backend_alloc_ctx_tensors + } + */ + + typedef struct ggml_backend_sched * ggml_backend_sched_t; + + // Evaluation callback for each node in the graph (set with ggml_backend_sched_set_eval_callback) + // when ask == true, the scheduler wants to know if the user wants to observe this node + // this allows the scheduler to batch nodes together in order to evaluate them in a single call + // + // when ask == false, the scheduler is passing the node tensor to the user for observation + // if the user returns false, the scheduler will cancel the graph compute + // + typedef bool (*ggml_backend_sched_eval_callback)(struct ggml_tensor * t, bool ask, void * user_data); + + // Initialize a backend scheduler, backends with low index are given priority over backends with high index + GGML_API ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size, bool parallel, bool op_offload); + GGML_API void ggml_backend_sched_free(ggml_backend_sched_t sched); + + // Initialize backend buffers from a measure graph + GGML_API void ggml_backend_sched_reserve_size(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph, size_t * sizes); + GGML_API bool ggml_backend_sched_reserve(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph); // returns success + + GGML_API int ggml_backend_sched_get_n_backends(ggml_backend_sched_t sched); + GGML_API ggml_backend_t ggml_backend_sched_get_backend(ggml_backend_sched_t sched, int i); + + // Get the number of splits of the last graph + GGML_API int ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched); + GGML_API int ggml_backend_sched_get_n_copies(ggml_backend_sched_t sched); + + GGML_API ggml_backend_buffer_type_t ggml_backend_sched_get_buffer_type(ggml_backend_sched_t sched, ggml_backend_t backend); + GGML_API size_t ggml_backend_sched_get_buffer_size(ggml_backend_sched_t sched, ggml_backend_t backend); + + GGML_API void ggml_backend_sched_set_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend); + GGML_API ggml_backend_t ggml_backend_sched_get_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node); + + // Split graph without allocating it + GGML_API void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph); + + // Allocate and compute graph on the backend scheduler + GGML_API bool ggml_backend_sched_alloc_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph); // returns success + GGML_API enum ggml_status ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph); + GGML_API enum ggml_status ggml_backend_sched_graph_compute_async(ggml_backend_sched_t sched, struct ggml_cgraph * graph); + GGML_API void ggml_backend_sched_synchronize(ggml_backend_sched_t sched); + + // Reset all assignments and allocators - must be called before changing the node backends or allocating a new graph. + // This in effect deallocates all tensors that were previously allocated and leaves them with dangling pointers. + // The correct way to use this API is to discard the deallocated tensors and create new ones. + GGML_API void ggml_backend_sched_reset(ggml_backend_sched_t sched); + + // Set a callback to be called for each resulting node during graph compute + GGML_API void ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backend_sched_eval_callback callback, void * user_data); + + // + // Meta backend + // + +#define GGML_BACKEND_META_MAX_DEVICES 16 + + enum ggml_backend_meta_split_axis { + // tensor split by tensor dimensions: + GGML_BACKEND_SPLIT_AXIS_0 = 0, + GGML_BACKEND_SPLIT_AXIS_1 = 1, + GGML_BACKEND_SPLIT_AXIS_2 = 2, + GGML_BACKEND_SPLIT_AXIS_3 = 3, + + GGML_BACKEND_SPLIT_AXIS_MIRRORED = 10, // all values on all backends + GGML_BACKEND_SPLIT_AXIS_PARTIAL = 11, // each backend has a partial sum + + // for internal bookkeeping only: + GGML_BACKEND_SPLIT_AXIS_NONE = 98, + GGML_BACKEND_SPLIT_AXIS_UNKNOWN = 99, + }; + GGML_API const char * ggml_backend_meta_split_axis_name(enum ggml_backend_meta_split_axis split_axis); + + struct ggml_backend_meta_split_state { + enum ggml_backend_meta_split_axis axis; + + // for tensors with axis >= 0 && axis < GGML_MAX_DIMS: + // - each device has a slice of the tensor along the split axis + // - most tensors have n_segments == 1 and a contiguous slice of the tensor data + // - some tensors have an inhomogenenous data layout along the split axis, + // those tensors are divided into segments which are each individually split across devices + // - ne has one entry per segment and device that add up to ggml_tensor::ne for that axis, + // the outer/inner loops are over segments/devices like [seg0_dev0, seg0_dev1, seg1_dev0, seg1_dev1], + // - for example, a transformer may have a fused QKV matrix rather than 3 matrices, those would be 3 separate segments + // that each need to be split individually across devices so that each device gets a slice of Q, K, and V + int64_t ne[16*GGML_BACKEND_META_MAX_DEVICES]; + uint32_t n_segments; + }; + + // function to assign split states for statically allocated tensors, compute tensor split states will be assigned to be compatible: + typedef struct ggml_backend_meta_split_state(*ggml_backend_meta_get_split_state_t)(const struct ggml_tensor * tensor, void * userdata); + + // create a new meta device from "simple" devices, meta buffer type/buffer/backend is then derived from this: + // TODO: this looks a bit strange - a backend API creates a device. I think we should try + // express this as a backend registry functionality instead + GGML_API ggml_backend_dev_t ggml_backend_meta_device( + ggml_backend_dev_t * devs, size_t n_devs, ggml_backend_meta_get_split_state_t get_split_state, void * get_split_state_ud); + + // + // Utils + // + + struct ggml_backend_graph_copy { + ggml_backend_buffer_t buffer; + struct ggml_context * ctx_allocated; + struct ggml_context * ctx_unallocated; + struct ggml_cgraph * graph; + }; + + // Copy a graph to a different backend + GGML_API struct ggml_backend_graph_copy ggml_backend_graph_copy(ggml_backend_t backend, struct ggml_cgraph * graph); + GGML_API void ggml_backend_graph_copy_free(struct ggml_backend_graph_copy copy); + + typedef bool (*ggml_backend_eval_callback)(int node_index, struct ggml_tensor * t1, struct ggml_tensor * t2, void * user_data); + + // Compare the output of two backends + GGML_API bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t backend2, struct ggml_cgraph * graph, ggml_backend_eval_callback callback, void * user_data, struct ggml_tensor const * const * test_nodes, size_t num_test_nodes); + + // Tensor initialization + GGML_API enum ggml_status ggml_backend_tensor_alloc(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, void * addr); + GGML_API enum ggml_status ggml_backend_view_init(struct ggml_tensor * tensor); + + // CPU buffer types are always available + GGML_API ggml_backend_buffer_t ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size); + GGML_API ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void); + +#ifdef __cplusplus +} +#endif diff --git a/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Versions/A/Headers/ggml-blas.h b/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Versions/A/Headers/ggml-blas.h new file mode 100644 index 0000000..87a81b3 --- /dev/null +++ b/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Versions/A/Headers/ggml-blas.h @@ -0,0 +1,25 @@ +#pragma once + +#include "ggml.h" +#include "ggml-backend.h" + + +#ifdef __cplusplus +extern "C" { +#endif + +// backend API +GGML_BACKEND_API ggml_backend_t ggml_backend_blas_init(void); + +GGML_BACKEND_API bool ggml_backend_is_blas(ggml_backend_t backend); + +// number of threads used for conversion to float +// for openblas and blis, this will also set the number of threads used for blas operations +GGML_BACKEND_API void ggml_backend_blas_set_n_threads(ggml_backend_t backend_blas, int n_threads); + +GGML_BACKEND_API ggml_backend_reg_t ggml_backend_blas_reg(void); + + +#ifdef __cplusplus +} +#endif diff --git a/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Versions/A/Headers/ggml-cpu.h b/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Versions/A/Headers/ggml-cpu.h new file mode 100644 index 0000000..e3e067c --- /dev/null +++ b/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Versions/A/Headers/ggml-cpu.h @@ -0,0 +1,151 @@ +#pragma once + +#include "ggml.h" +#include "ggml-backend.h" + +#ifdef __cplusplus +extern "C" { +#endif + + // the compute plan that needs to be prepared for ggml_graph_compute() + // since https://github.com/ggml-org/ggml/issues/287 + struct ggml_cplan { + size_t work_size; // size of work buffer, calculated by `ggml_graph_plan()` + uint8_t * work_data; // work buffer, to be allocated by caller before calling to `ggml_graph_compute()` + + int n_threads; + struct ggml_threadpool * threadpool; + + // abort ggml_graph_compute when true + ggml_abort_callback abort_callback; + void * abort_callback_data; + + // use only reference implementations + bool use_ref; + }; + + // numa strategies + enum ggml_numa_strategy { + GGML_NUMA_STRATEGY_DISABLED = 0, + GGML_NUMA_STRATEGY_DISTRIBUTE = 1, + GGML_NUMA_STRATEGY_ISOLATE = 2, + GGML_NUMA_STRATEGY_NUMACTL = 3, + GGML_NUMA_STRATEGY_MIRROR = 4, + GGML_NUMA_STRATEGY_COUNT + }; + + GGML_BACKEND_API void ggml_numa_init(enum ggml_numa_strategy numa); // call once for better performance on NUMA systems + GGML_BACKEND_API bool ggml_is_numa(void); // true if init detected that system has >1 NUMA node + + GGML_BACKEND_API struct ggml_tensor * ggml_new_i32(struct ggml_context * ctx, int32_t value); + GGML_BACKEND_API struct ggml_tensor * ggml_new_f32(struct ggml_context * ctx, float value); + + GGML_BACKEND_API struct ggml_tensor * ggml_set_i32 (struct ggml_tensor * tensor, int32_t value); + GGML_BACKEND_API struct ggml_tensor * ggml_set_f32 (struct ggml_tensor * tensor, float value); + + GGML_BACKEND_API int32_t ggml_get_i32_1d(const struct ggml_tensor * tensor, int i); + GGML_BACKEND_API void ggml_set_i32_1d(const struct ggml_tensor * tensor, int i, int32_t value); + + GGML_BACKEND_API int32_t ggml_get_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3); + GGML_BACKEND_API void ggml_set_i32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3, int32_t value); + + GGML_BACKEND_API float ggml_get_f32_1d(const struct ggml_tensor * tensor, int i); + GGML_BACKEND_API void ggml_set_f32_1d(const struct ggml_tensor * tensor, int i, float value); + + GGML_BACKEND_API float ggml_get_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3); + GGML_BACKEND_API void ggml_set_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2, int i3, float value); + + GGML_BACKEND_API struct ggml_threadpool * ggml_threadpool_new (struct ggml_threadpool_params * params); + GGML_BACKEND_API void ggml_threadpool_free (struct ggml_threadpool * threadpool); + GGML_BACKEND_API int ggml_threadpool_get_n_threads (struct ggml_threadpool * threadpool); + GGML_BACKEND_API void ggml_threadpool_pause (struct ggml_threadpool * threadpool); + GGML_BACKEND_API void ggml_threadpool_resume (struct ggml_threadpool * threadpool); + + // ggml_graph_plan() has to be called before ggml_graph_compute() + // when plan.work_size > 0, caller must allocate memory for plan.work_data + GGML_BACKEND_API struct ggml_cplan ggml_graph_plan( + const struct ggml_cgraph * cgraph, + int n_threads, /* = GGML_DEFAULT_N_THREADS */ + struct ggml_threadpool * threadpool /* = NULL */ ); + GGML_BACKEND_API enum ggml_status ggml_graph_compute(struct ggml_cgraph * cgraph, struct ggml_cplan * cplan); + + // same as ggml_graph_compute() but the work data is allocated as a part of the context + // note: the drawback of this API is that you must have ensured that the context has enough memory for the work data + GGML_BACKEND_API enum ggml_status ggml_graph_compute_with_ctx(struct ggml_context * ctx, struct ggml_cgraph * cgraph, int n_threads); + + // + // system info + // + + // x86 + GGML_BACKEND_API int ggml_cpu_has_sse3 (void); + GGML_BACKEND_API int ggml_cpu_has_ssse3 (void); + GGML_BACKEND_API int ggml_cpu_has_avx (void); + GGML_BACKEND_API int ggml_cpu_has_avx_vnni (void); + GGML_BACKEND_API int ggml_cpu_has_avx2 (void); + GGML_BACKEND_API int ggml_cpu_has_bmi2 (void); + GGML_BACKEND_API int ggml_cpu_has_f16c (void); + GGML_BACKEND_API int ggml_cpu_has_fma (void); + GGML_BACKEND_API int ggml_cpu_has_avx512 (void); + GGML_BACKEND_API int ggml_cpu_has_avx512_vbmi(void); + GGML_BACKEND_API int ggml_cpu_has_avx512_vnni(void); + GGML_BACKEND_API int ggml_cpu_has_avx512_bf16(void); + GGML_BACKEND_API int ggml_cpu_has_amx_int8 (void); + // ARM + GGML_BACKEND_API int ggml_cpu_has_neon (void); + GGML_BACKEND_API int ggml_cpu_has_arm_fma (void); + GGML_BACKEND_API int ggml_cpu_has_fp16_va (void); + GGML_BACKEND_API int ggml_cpu_has_dotprod (void); + GGML_BACKEND_API int ggml_cpu_has_matmul_int8(void); + GGML_BACKEND_API int ggml_cpu_has_sve (void); + GGML_BACKEND_API int ggml_cpu_get_sve_cnt (void); // sve vector length in bytes + GGML_BACKEND_API int ggml_cpu_has_sme (void); + // other + GGML_BACKEND_API int ggml_cpu_has_riscv_v (void); + GGML_BACKEND_API int ggml_cpu_get_rvv_vlen (void); // risc-v vector length in bytes + GGML_BACKEND_API int ggml_cpu_has_vsx (void); + GGML_BACKEND_API int ggml_cpu_has_vxe (void); + GGML_BACKEND_API int ggml_cpu_has_wasm_simd (void); + GGML_BACKEND_API int ggml_cpu_has_llamafile (void); + + // Internal types and functions exposed for tests and benchmarks + + typedef void (*ggml_vec_dot_t) (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT x, size_t bx, + const void * GGML_RESTRICT y, size_t by, int nrc); + + struct ggml_type_traits_cpu { + ggml_from_float_t from_float; + ggml_vec_dot_t vec_dot; + enum ggml_type vec_dot_type; + int64_t nrows; // number of rows to process simultaneously + }; + + GGML_BACKEND_API const struct ggml_type_traits_cpu * ggml_get_type_traits_cpu(enum ggml_type type); + + GGML_BACKEND_API void ggml_cpu_init(void); + + // + // CPU backend + // + + GGML_BACKEND_API ggml_backend_t ggml_backend_cpu_init(void); + + GGML_BACKEND_API bool ggml_backend_is_cpu (ggml_backend_t backend); + GGML_BACKEND_API void ggml_backend_cpu_set_n_threads (ggml_backend_t backend_cpu, int n_threads); + GGML_BACKEND_API void ggml_backend_cpu_set_threadpool (ggml_backend_t backend_cpu, ggml_threadpool_t threadpool); + GGML_BACKEND_API void ggml_backend_cpu_set_abort_callback(ggml_backend_t backend_cpu, ggml_abort_callback abort_callback, void * abort_callback_data); + + GGML_BACKEND_API void ggml_backend_cpu_set_use_ref(ggml_backend_t backend_cpu, bool use_ref); + + GGML_BACKEND_API ggml_backend_reg_t ggml_backend_cpu_reg(void); + + GGML_BACKEND_API void ggml_cpu_fp32_to_fp32(const float *, float *, int64_t); + GGML_BACKEND_API void ggml_cpu_fp32_to_i32 (const float *, int32_t *, int64_t); + GGML_BACKEND_API void ggml_cpu_fp32_to_fp16(const float *, ggml_fp16_t *, int64_t); + GGML_BACKEND_API void ggml_cpu_fp16_to_fp32(const ggml_fp16_t *, float *, int64_t); + GGML_BACKEND_API void ggml_cpu_fp32_to_bf16(const float *, ggml_bf16_t *, int64_t); + GGML_BACKEND_API void ggml_cpu_bf16_to_fp32(const ggml_bf16_t *, float *, int64_t); + +#ifdef __cplusplus +} +#endif diff --git a/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Versions/A/Headers/ggml-metal.h b/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Versions/A/Headers/ggml-metal.h new file mode 100644 index 0000000..433838f --- /dev/null +++ b/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Versions/A/Headers/ggml-metal.h @@ -0,0 +1,61 @@ +// Note: this description is outdated +// +// An interface allowing to compute ggml_cgraph with Metal +// +// This is a fully functional interface that extends ggml with GPU support for Apple devices. +// A similar interface can be created for other GPU backends (e.g. Vulkan, CUDA, etc.) +// +// How it works? +// +// As long as your program can create and evaluate a ggml_cgraph on the CPU, you can use this +// interface to evaluate the same graph on the GPU. Instead of using ggml_graph_compute(), you +// use ggml_metal_graph_compute() (or ggml_vulkan_graph_compute(), etc.) +// +// You only need to make sure that all memory buffers that you used during the graph creation +// are mapped to the device memory with the ggml_metal_add_buffer() function. This mapping is +// used during the graph evaluation to determine the arguments of the compute kernels. +// +// Synchronization between device and host memory (for example for input and output tensors) +// is done with the ggml_metal_set_tensor() and ggml_metal_get_tensor() functions. +// + +#pragma once + +#include "ggml.h" +#include "ggml-backend.h" + +#include +#include + +struct ggml_tensor; +struct ggml_cgraph; + +#ifdef __cplusplus +extern "C" { +#endif + +// +// backend API +// user-code should use only these functions +// + +// TODO: remove in the future +GGML_BACKEND_API ggml_backend_t ggml_backend_metal_init(void); + +GGML_BACKEND_API bool ggml_backend_is_metal(ggml_backend_t backend); + +GGML_BACKEND_API void ggml_backend_metal_set_abort_callback(ggml_backend_t backend, ggml_abort_callback abort_callback, void * user_data); + +// helper to check if the device supports a specific family +// ideally, the user code should be doing these checks +// ref: https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf +GGML_BACKEND_API bool ggml_backend_metal_supports_family(ggml_backend_t backend, int family); + +// capture all command buffers committed the next time `ggml_backend_graph_compute` is called +GGML_BACKEND_API void ggml_backend_metal_capture_next_compute(ggml_backend_t backend); + +GGML_BACKEND_API ggml_backend_reg_t ggml_backend_metal_reg(void); + +#ifdef __cplusplus +} +#endif diff --git a/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Versions/A/Headers/ggml-opt.h b/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Versions/A/Headers/ggml-opt.h new file mode 100644 index 0000000..1c2ed79 --- /dev/null +++ b/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Versions/A/Headers/ggml-opt.h @@ -0,0 +1,256 @@ +// This file contains functionality for training models using GGML. +// It is not strictly needed vs. just vanilla GGML but it provides a more high-level interface for common needs such as datasets. +// At the bottom of this file especially there are relatively high-level functions that are suitable use or adaptation in user code. +// +// Module maintainer: Johannes Gäßler (@JohannesGaessler, johannesg@5d6.de) + +#pragma once + +#include "ggml.h" +#include "ggml-backend.h" + +#include + +#ifdef __cplusplus +extern "C" { +#endif + + struct ggml_opt_dataset; + struct ggml_opt_context; + struct ggml_opt_result; + + typedef struct ggml_opt_dataset * ggml_opt_dataset_t; + typedef struct ggml_opt_context * ggml_opt_context_t; + typedef struct ggml_opt_result * ggml_opt_result_t; + + // ====== Loss ====== + + // built-in loss types, i.e. the built-in quantities minimized by the optimizer + // custom loss types can be defined via mean or sum which simply reduce the outputs for all datapoints to a single value + enum ggml_opt_loss_type { + GGML_OPT_LOSS_TYPE_MEAN, + GGML_OPT_LOSS_TYPE_SUM, + GGML_OPT_LOSS_TYPE_CROSS_ENTROPY, + GGML_OPT_LOSS_TYPE_MEAN_SQUARED_ERROR, + }; + + // ====== Dataset ====== + + GGML_API ggml_opt_dataset_t ggml_opt_dataset_init( + enum ggml_type type_data, // the type for the internal data tensor + enum ggml_type type_label, // the type for the internal labels tensor + int64_t ne_datapoint, // number of elements per datapoint + int64_t ne_label, // number of elements per label + int64_t ndata, // total number of datapoints/labels + int64_t ndata_shard); // number of datapoints/labels per shard (unit at which the dataset is shuffled/copied) + GGML_API void ggml_opt_dataset_free(ggml_opt_dataset_t dataset); + + // get underlying tensors that store the data + GGML_API int64_t ggml_opt_dataset_ndata (ggml_opt_dataset_t dataset); + GGML_API struct ggml_tensor * ggml_opt_dataset_data (ggml_opt_dataset_t dataset); // shape = [ne_datapoint, ndata] + GGML_API struct ggml_tensor * ggml_opt_dataset_labels(ggml_opt_dataset_t dataset); // shape = [nd_label, ndata] + + // shuffle idata first datapoints from dataset with RNG from opt_ctx, shuffle all datapoints if idata is negative + GGML_API void ggml_opt_dataset_shuffle(ggml_opt_context_t opt_ctx, ggml_opt_dataset_t dataset, int64_t idata); + + // get batch at position ibatch from dataset and copy the data to data_batch and labels_batch + GGML_API void ggml_opt_dataset_get_batch( + ggml_opt_dataset_t dataset, + struct ggml_tensor * data_batch, // shape = [ne_datapoint, ndata_batch] + struct ggml_tensor * labels_batch, // shape = [ne_label, ndata_batch] + int64_t ibatch); + GGML_API void ggml_opt_dataset_get_batch_host( + ggml_opt_dataset_t dataset, + void * data_batch, + size_t nb_data_batch, + void * labels_batch, + int64_t ibatch); + + // ====== Model / Context ====== + + enum ggml_opt_build_type { + GGML_OPT_BUILD_TYPE_FORWARD = 10, + GGML_OPT_BUILD_TYPE_GRAD = 20, + GGML_OPT_BUILD_TYPE_OPT = 30, + }; + + enum ggml_opt_optimizer_type { + GGML_OPT_OPTIMIZER_TYPE_ADAMW, + GGML_OPT_OPTIMIZER_TYPE_SGD, + + GGML_OPT_OPTIMIZER_TYPE_COUNT + }; + + // parameters that control which optimizer is used and how said optimizer tries to find the minimal loss + struct ggml_opt_optimizer_params { + struct { + float alpha; // learning rate + float beta1; // first AdamW momentum + float beta2; // second AdamW momentum + float eps; // epsilon for numerical stability + float wd; // weight decay - 0.0f to disable + } adamw; + struct { + float alpha; // learning rate + float wd; // weight decay + } sgd; + }; + + // callback to calculate optimizer parameters prior to a backward pass + // userdata can be used to pass arbitrary data + typedef struct ggml_opt_optimizer_params (*ggml_opt_get_optimizer_params)(void * userdata); + + // returns the default optimizer params (constant, hard-coded values) + // userdata is not used + GGML_API struct ggml_opt_optimizer_params ggml_opt_get_default_optimizer_params(void * userdata); + + // casts userdata to ggml_opt_optimizer_params and returns it + GGML_API struct ggml_opt_optimizer_params ggml_opt_get_constant_optimizer_params(void * userdata); + + // parameters for initializing a new optimization context + struct ggml_opt_params { + ggml_backend_sched_t backend_sched; // defines which backends are used to construct the compute graphs + + // by default the forward graph needs to be reconstructed for each eval + // if ctx_compute, inputs, and outputs are set the graphs are instead allocated statically + struct ggml_context * ctx_compute; + struct ggml_tensor * inputs; + struct ggml_tensor * outputs; + + enum ggml_opt_loss_type loss_type; + enum ggml_opt_build_type build_type; + + int32_t opt_period; // after how many gradient accumulation steps an optimizer step should be done + + ggml_opt_get_optimizer_params get_opt_pars; // callback for calculating optimizer parameters + void * get_opt_pars_ud; // userdata for calculating optimizer parameters + + // only GGML_OPT_OPTIMIZER_TYPE_ADAMW needs m, v momenta per parameter tensor + enum ggml_opt_optimizer_type optimizer; + }; + + // get parameters for an optimization context with defaults set where possible + // parameters for which no sensible defaults exist are supplied as arguments to this function + GGML_API struct ggml_opt_params ggml_opt_default_params( + ggml_backend_sched_t backend_sched, + enum ggml_opt_loss_type loss_type); + + GGML_API ggml_opt_context_t ggml_opt_init(struct ggml_opt_params params); + GGML_API void ggml_opt_free(ggml_opt_context_t opt_ctx); + + // set gradients to zero, initialize loss, and optionally reset the optimizer + GGML_API void ggml_opt_reset(ggml_opt_context_t opt_ctx, bool optimizer); + + GGML_API bool ggml_opt_static_graphs(ggml_opt_context_t opt_ctx); // whether the graphs are allocated_statically + + // get underlying tensors that store data + // if not using static graphs these pointers become invalid with the next call to ggml_opt_alloc + GGML_API struct ggml_tensor * ggml_opt_inputs( ggml_opt_context_t opt_ctx); // forward graph input tensor + GGML_API struct ggml_tensor * ggml_opt_outputs( ggml_opt_context_t opt_ctx); // forward graph output tensor + GGML_API struct ggml_tensor * ggml_opt_labels( ggml_opt_context_t opt_ctx); // labels to compare outputs against + GGML_API struct ggml_tensor * ggml_opt_loss( ggml_opt_context_t opt_ctx); // scalar tensor that contains the loss + GGML_API struct ggml_tensor * ggml_opt_pred( ggml_opt_context_t opt_ctx); // predictions made by outputs + GGML_API struct ggml_tensor * ggml_opt_ncorrect(ggml_opt_context_t opt_ctx); // number of matching predictions between outputs and labels + + // get the gradient accumulator for a node from the forward graph + GGML_API struct ggml_tensor * ggml_opt_grad_acc(ggml_opt_context_t opt_ctx, struct ggml_tensor * node); + + GGML_API enum ggml_opt_optimizer_type ggml_opt_context_optimizer_type(ggml_opt_context_t); //TODO consistent naming scheme + + GGML_API const char * ggml_opt_optimizer_name(enum ggml_opt_optimizer_type); + + // ====== Optimization Result ====== + + GGML_API ggml_opt_result_t ggml_opt_result_init(void); + GGML_API void ggml_opt_result_free(ggml_opt_result_t result); + GGML_API void ggml_opt_result_reset(ggml_opt_result_t result); + + // get data from result, uncertainties are optional and can be ignored by passing NULL + GGML_API void ggml_opt_result_ndata( ggml_opt_result_t result, int64_t * ndata); // writes 1 value, number of datapoints + GGML_API void ggml_opt_result_loss( ggml_opt_result_t result, double * loss, double * unc); // writes 1 value + GGML_API void ggml_opt_result_pred( ggml_opt_result_t result, int32_t * pred); // writes ndata values + GGML_API void ggml_opt_result_accuracy(ggml_opt_result_t result, double * accuracy, double * unc); // writes 1 value + + // ====== Computation ====== + + // if not using static graphs, this function must be called prior to ggml_opt_alloc + GGML_API void ggml_opt_prepare_alloc( + ggml_opt_context_t opt_ctx, + struct ggml_context * ctx_compute, + struct ggml_cgraph * gf, + struct ggml_tensor * inputs, + struct ggml_tensor * outputs); + + // allocate the next graph for evaluation, either forward or forward + backward + // must be called exactly once prior to calling ggml_opt_eval + GGML_API void ggml_opt_alloc(ggml_opt_context_t opt_ctx, bool backward); + + // do forward pass, increment result if not NULL, do backward pass if allocated + GGML_API void ggml_opt_eval(ggml_opt_context_t opt_ctx, ggml_opt_result_t result); + + // ############################################################################ + // ## The high-level functions start here. They do not depend on any private ## + // ## functions or structs and can be copied to and adapted for user code. ## + // ############################################################################ + + // ====== Intended Usage ====== + // + // 1. Select the appropriate loss for your problem. + // 2. Create a dataset and set the data for the "data" tensor. Also set the "labels" tensor if your loss needs them. + // Setting the shard size to 1 will be fine, it's the granularity with which data is shuffled/loaded (bigger values are faster). + // 3. Create a GGML graph for your model with no_alloc == true. Use two separate contexts for the tensors. + // The first context should contain the model parameters and inputs and be allocated statically in user code. + // The second context should contain all other tensors and will be (re)allocated automatically. + // Due to this automated allocation the data of the second context is not defined when accessed in user code. + // Note that the second dimension of the inputs/outputs are interpreted as the number of datapoints in those tensors. + // 4. Call ggml_opt_fit. If you need more control you can use ggml_opt_epoch instead. + + // signature for a callback while evaluating opt_ctx on dataset, called after an evaluation + typedef void (*ggml_opt_epoch_callback)( + bool train, // true after training evaluation, false after validation evaluation + ggml_opt_context_t opt_ctx, + ggml_opt_dataset_t dataset, + ggml_opt_result_t result, // result associated with the dataset subsection + int64_t ibatch, // number of batches that have been evaluated so far + int64_t ibatch_max, // total number of batches in this dataset subsection + int64_t t_start_us); // time at which the evaluation on the dataset subsection was started + + // do training on front of dataset, do evaluation only on back of dataset + GGML_API void ggml_opt_epoch( + ggml_opt_context_t opt_ctx, + ggml_opt_dataset_t dataset, + ggml_opt_result_t result_train, // result to increment during training, ignored if NULL + ggml_opt_result_t result_eval, // result to increment during evaluation, ignored if NULL + int64_t idata_split, // data index at which to split training and evaluation + ggml_opt_epoch_callback callback_train, + ggml_opt_epoch_callback callback_eval); + + // callback that prints a progress bar on stderr + GGML_API void ggml_opt_epoch_callback_progress_bar( + bool train, + ggml_opt_context_t opt_ctx, + ggml_opt_dataset_t dataset, + ggml_opt_result_t result, + int64_t ibatch, + int64_t ibatch_max, + int64_t t_start_us); + + // fit model defined by inputs and outputs to dataset + GGML_API void ggml_opt_fit( + ggml_backend_sched_t backend_sched, // backend scheduler for constructing the compute graphs + struct ggml_context * ctx_compute, // context with temporarily allocated tensors to calculate the outputs + struct ggml_tensor * inputs, // input tensor with shape [ne_datapoint, ndata_batch] + struct ggml_tensor * outputs, // output tensor, must have shape [ne_label, ndata_batch] if labels are used + ggml_opt_dataset_t dataset, // dataset with data and optionally also labels + enum ggml_opt_loss_type loss_type, // loss to minimize + enum ggml_opt_optimizer_type optimizer, // sgd or adamw + ggml_opt_get_optimizer_params get_opt_pars, // callback to get optimizer params, userdata is pointer to epoch (of type int64_t) + int64_t nepoch, // how many times the dataset should be iterated over + int64_t nbatch_logical, // datapoints optimizer step, must be a multiple of ndata_batch in inputs/outputs + float val_split, // fraction of the dataset to use for validation, must be in [0.0f, 1.0f) + bool silent); // whether or not info prints to stderr should be suppressed + + +#ifdef __cplusplus +} +#endif diff --git a/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Versions/A/Headers/ggml.h b/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Versions/A/Headers/ggml.h new file mode 100644 index 0000000..3357a0d --- /dev/null +++ b/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Versions/A/Headers/ggml.h @@ -0,0 +1,2840 @@ +#pragma once + +// +// GGML Tensor Library +// +// This documentation is still a work in progress. +// If you wish some specific topics to be covered, feel free to drop a comment: +// +// https://github.com/ggml-org/whisper.cpp/issues/40 +// +// ## Overview +// +// This library implements: +// +// - a set of tensor operations +// - automatic differentiation +// - basic optimization algorithms +// +// The aim of this library is to provide a minimalistic approach for various machine learning tasks. This includes, +// but is not limited to, the following: +// +// - linear regression +// - support vector machines +// - neural networks +// +// The library allows the user to define a certain function using the available tensor operations. This function +// definition is represented internally via a computation graph. Each tensor operation in the function definition +// corresponds to a node in the graph. Having the computation graph defined, the user can choose to compute the +// function's value and/or its gradient with respect to the input variables. Optionally, the function can be optimized +// using one of the available optimization algorithms. +// +// For example, here we define the function: f(x) = a*x^2 + b +// +// { +// struct ggml_init_params params = { +// .mem_size = 16*1024*1024, +// .mem_buffer = NULL, +// }; +// +// // memory allocation happens here +// struct ggml_context * ctx = ggml_init(params); +// +// struct ggml_tensor * x = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1); +// +// ggml_set_param(ctx, x); // x is an input variable +// +// struct ggml_tensor * a = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1); +// struct ggml_tensor * b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 1); +// struct ggml_tensor * x2 = ggml_mul(ctx, x, x); +// struct ggml_tensor * f = ggml_add(ctx, ggml_mul(ctx, a, x2), b); +// +// ... +// } +// +// Notice that the function definition above does not involve any actual computation. The computation is performed only +// when the user explicitly requests it. For example, to compute the function's value at x = 2.0: +// +// { +// ... +// +// struct ggml_cgraph * gf = ggml_new_graph(ctx); +// ggml_build_forward_expand(gf, f); +// +// // set the input variable and parameter values +// ggml_set_f32(x, 2.0f); +// ggml_set_f32(a, 3.0f); +// ggml_set_f32(b, 4.0f); +// +// ggml_graph_compute_with_ctx(ctx, &gf, n_threads); +// +// printf("f = %f\n", ggml_get_f32_1d(f, 0)); +// +// ... +// } +// +// The actual computation is performed in the ggml_graph_compute() function. +// +// The ggml_new_tensor_...() functions create new tensors. They are allocated in the memory buffer provided to the +// ggml_init() function. You have to be careful not to exceed the memory buffer size. Therefore, you have to know +// in advance how much memory you need for your computation. Alternatively, you can allocate a large enough memory +// and after defining the computation graph, call the ggml_used_mem() function to find out how much memory was +// actually needed. +// +// The ggml_set_param() function marks a tensor as an input variable. This is used by the automatic +// differentiation and optimization algorithms. +// +// The described approach allows to define the function graph once and then compute its forward or backward graphs +// multiple times. All computations will use the same memory buffer allocated in the ggml_init() function. This way +// the user can avoid the memory allocation overhead at runtime. +// +// The library supports multi-dimensional tensors - up to 4 dimensions. The FP16 and FP32 data types are first class +// citizens, but in theory the library can be extended to support FP8 and integer data types. +// +// Each tensor operation produces a new tensor. Initially the library was envisioned to support only the use of unary +// and binary operations. Most of the available operations fall into one of these two categories. With time, it became +// clear that the library needs to support more complex operations. The way to support these operations is not clear +// yet, but a few examples are demonstrated in the following operations: +// +// - ggml_permute() +// - ggml_conv_1d_1s() +// - ggml_conv_1d_2s() +// +// For each tensor operator, the library implements a forward and backward computation function. The forward function +// computes the output tensor value given the input tensor values. The backward function computes the adjoint of the +// input tensors given the adjoint of the output tensor. For a detailed explanation of what this means, take a +// calculus class, or watch the following video: +// +// What is Automatic Differentiation? +// https://www.youtube.com/watch?v=wG_nF1awSSY +// +// +// ## Tensor data (struct ggml_tensor) +// +// The tensors are stored in memory via the ggml_tensor struct. The structure provides information about the size of +// the tensor, the data type, and the memory buffer where the tensor data is stored. Additionally, it contains +// pointers to the "source" tensors - i.e. the tensors that were used to compute the current tensor. For example: +// +// { +// struct ggml_tensor * c = ggml_add(ctx, a, b); +// +// assert(c->src[0] == a); +// assert(c->src[1] == b); +// } +// +// The multi-dimensional tensors are stored in row-major order. The ggml_tensor struct contains fields for the +// number of elements in each dimension ("ne") as well as the number of bytes ("nb", a.k.a. stride). This allows +// to store tensors that are not contiguous in memory, which is useful for operations such as transposition and +// permutation. All tensor operations have to take the stride into account and not assume that the tensor is +// contiguous in memory. +// +// The data of the tensor is accessed via the "data" pointer. For example: +// +// { +// const int nx = 2; +// const int ny = 3; +// +// struct ggml_tensor * a = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, nx, ny); +// +// for (int y = 0; y < ny; y++) { +// for (int x = 0; x < nx; x++) { +// *(float *) ((char *) a->data + y*a->nb[1] + x*a->nb[0]) = x + y; +// } +// } +// +// ... +// } +// +// Alternatively, there are helper functions, such as ggml_get_f32_1d() and ggml_set_f32_1d() that can be used. +// +// ## The matrix multiplication operator (ggml_mul_mat) +// +// TODO +// +// +// ## Multi-threading +// +// TODO +// +// +// ## Overview of ggml.c +// +// TODO +// +// +// ## SIMD optimizations +// +// TODO +// +// +// ## Debugging ggml +// +// TODO +// +// + +#ifdef GGML_SHARED +# if defined(_WIN32) && !defined(__MINGW32__) +# ifdef GGML_BUILD +# define GGML_API __declspec(dllexport) extern +# else +# define GGML_API __declspec(dllimport) extern +# endif +# else +# define GGML_API __attribute__ ((visibility ("default"))) extern +# endif +#else +# define GGML_API extern +#endif + +// TODO: support for clang +#ifdef __GNUC__ +# define GGML_DEPRECATED(func, hint) func __attribute__((deprecated(hint))) +#elif defined(_MSC_VER) +# define GGML_DEPRECATED(func, hint) __declspec(deprecated(hint)) func +#else +# define GGML_DEPRECATED(func, hint) func +#endif + +#ifndef __GNUC__ +# define GGML_ATTRIBUTE_FORMAT(...) +#elif defined(__MINGW32__) && !defined(__clang__) +# define GGML_ATTRIBUTE_FORMAT(...) __attribute__((format(gnu_printf, __VA_ARGS__))) +#else +# define GGML_ATTRIBUTE_FORMAT(...) __attribute__((format(printf, __VA_ARGS__))) +#endif + +#if defined(_WIN32) && !defined(_WIN32_WINNT) +# define _WIN32_WINNT 0x0A00 +#endif + +#include +#include +#include +#include + +#define GGML_FILE_MAGIC 0x67676d6c // "ggml" +#define GGML_FILE_VERSION 2 + +#define GGML_QNT_VERSION 2 // bump this on quantization format changes +#define GGML_QNT_VERSION_FACTOR 1000 // do not change this + +#define GGML_MAX_DIMS 4 +#define GGML_MAX_PARAMS 2048 +#define GGML_MAX_SRC 10 +#define GGML_MAX_N_THREADS 512 +#define GGML_MAX_OP_PARAMS 64 + +#ifndef GGML_MAX_NAME +# define GGML_MAX_NAME 64 +#endif + +#define GGML_DEFAULT_N_THREADS 4 +#define GGML_DEFAULT_GRAPH_SIZE 2048 + +#if UINTPTR_MAX == 0xFFFFFFFF + #define GGML_MEM_ALIGN 4 +#elif defined(__EMSCRIPTEN__) +// emscripten uses max_align_t == 8, so we need GGML_MEM_ALIGN == 8 for 64-bit wasm. +// (for 32-bit wasm, the first conditional is true and GGML_MEM_ALIGN stays 4.) +// ref: https://github.com/ggml-org/llama.cpp/pull/18628 + #define GGML_MEM_ALIGN 8 +#else + #define GGML_MEM_ALIGN 16 +#endif + +#define GGML_EXIT_SUCCESS 0 +#define GGML_EXIT_ABORTED 1 + +// TODO: convert to enum https://github.com/ggml-org/llama.cpp/pull/16187#discussion_r2388538726 +#define GGML_ROPE_TYPE_NORMAL 0 +#define GGML_ROPE_TYPE_NEOX 2 +#define GGML_ROPE_TYPE_MROPE 8 +#define GGML_ROPE_TYPE_VISION 24 +#define GGML_ROPE_TYPE_IMROPE 40 // binary: 101000 + +#define GGML_MROPE_SECTIONS 4 + +#define GGML_UNUSED(x) (void)(x) +#ifdef __CUDACC__ +template +__host__ __device__ constexpr inline void ggml_unused_vars_impl(Args&&...) noexcept {} +#define GGML_UNUSED_VARS(...) ggml_unused_vars_impl(__VA_ARGS__) +#else +#define GGML_UNUSED_VARS(...) do { (void)sizeof((__VA_ARGS__, 0)); } while(0) +#endif // __CUDACC__ + +#define GGML_PAD(x, n) (((x) + (n) - 1) & ~((n) - 1)) + +#ifndef NDEBUG +# define GGML_UNREACHABLE() do { fprintf(stderr, "statement should be unreachable\n"); abort(); } while(0) +#elif defined(__GNUC__) +# define GGML_UNREACHABLE() __builtin_unreachable() +#elif defined(_MSC_VER) +# define GGML_UNREACHABLE() __assume(0) +#else +# define GGML_UNREACHABLE() ((void) 0) +#endif + +#ifdef __cplusplus +# define GGML_NORETURN [[noreturn]] +#elif defined(_MSC_VER) +# define GGML_NORETURN __declspec(noreturn) +#else +# define GGML_NORETURN _Noreturn +#endif + +#define GGML_ABORT(...) ggml_abort(__FILE__, __LINE__, __VA_ARGS__) +#define GGML_ASSERT(x) if (!(x)) GGML_ABORT("GGML_ASSERT(%s) failed", #x) + +// used to copy the number of elements and stride in bytes of tensors into local variables. +// main purpose is to reduce code duplication and improve readability. +// +// example: +// +// GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne); +// GGML_TENSOR_LOCALS(size_t, nb1, src1, nb); +// +#define GGML_TENSOR_LOCALS_1(type, prefix, pointer, array) \ + const type prefix##0 = (pointer) ? (pointer)->array[0] : 0; \ + GGML_UNUSED(prefix##0); +#define GGML_TENSOR_LOCALS_2(type, prefix, pointer, array) \ + GGML_TENSOR_LOCALS_1 (type, prefix, pointer, array) \ + const type prefix##1 = (pointer) ? (pointer)->array[1] : 0; \ + GGML_UNUSED(prefix##1); +#define GGML_TENSOR_LOCALS_3(type, prefix, pointer, array) \ + GGML_TENSOR_LOCALS_2 (type, prefix, pointer, array) \ + const type prefix##2 = (pointer) ? (pointer)->array[2] : 0; \ + GGML_UNUSED(prefix##2); +#define GGML_TENSOR_LOCALS(type, prefix, pointer, array) \ + GGML_TENSOR_LOCALS_3 (type, prefix, pointer, array) \ + const type prefix##3 = (pointer) ? (pointer)->array[3] : 0; \ + GGML_UNUSED(prefix##3); + +#define GGML_TENSOR_UNARY_OP_LOCALS \ + GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \ + GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \ + GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \ + GGML_TENSOR_LOCALS(size_t, nb, dst, nb) + +#define GGML_TENSOR_BINARY_OP_LOCALS \ + GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \ + GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \ + GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \ + GGML_TENSOR_LOCALS(size_t, nb1, src1, nb) \ + GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \ + GGML_TENSOR_LOCALS(size_t, nb, dst, nb) + +#define GGML_TENSOR_TERNARY_OP_LOCALS \ + GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \ + GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \ + GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \ + GGML_TENSOR_LOCALS(size_t, nb1, src1, nb) \ + GGML_TENSOR_LOCALS(int64_t, ne2, src2, ne) \ + GGML_TENSOR_LOCALS(size_t, nb2, src2, nb) \ + GGML_TENSOR_LOCALS(int64_t, ne, dst, ne) \ + GGML_TENSOR_LOCALS(size_t, nb, dst, nb) + +#define GGML_TENSOR_BINARY_OP_LOCALS01 \ + GGML_TENSOR_LOCALS(int64_t, ne0, src0, ne) \ + GGML_TENSOR_LOCALS(size_t, nb0, src0, nb) \ + GGML_TENSOR_LOCALS(int64_t, ne1, src1, ne) \ + GGML_TENSOR_LOCALS(size_t, nb1, src1, nb) + +#ifdef __cplusplus +extern "C" { +#endif + + // Function type used in fatal error callbacks + typedef void (*ggml_abort_callback_t)(const char * error_message); + + // Set the abort callback (passing null will restore original abort functionality: printing a message to stdout) + // Returns the old callback for chaining + GGML_API ggml_abort_callback_t ggml_set_abort_callback(ggml_abort_callback_t callback); + + GGML_NORETURN GGML_ATTRIBUTE_FORMAT(3, 4) + GGML_API void ggml_abort(const char * file, int line, const char * fmt, ...); + + enum ggml_status { + GGML_STATUS_ALLOC_FAILED = -2, + GGML_STATUS_FAILED = -1, + GGML_STATUS_SUCCESS = 0, + GGML_STATUS_ABORTED = 1, + }; + + // get ggml_status name string + GGML_API const char * ggml_status_to_string(enum ggml_status status); + + // ieee 754-2008 half-precision float16 + // todo: make this not an integral type + typedef uint16_t ggml_fp16_t; + GGML_API float ggml_fp16_to_fp32(ggml_fp16_t); + GGML_API ggml_fp16_t ggml_fp32_to_fp16(float); + GGML_API void ggml_fp16_to_fp32_row(const ggml_fp16_t *, float *, int64_t); + GGML_API void ggml_fp32_to_fp16_row(const float *, ggml_fp16_t *, int64_t); + + // google brain half-precision bfloat16 + typedef struct { uint16_t bits; } ggml_bf16_t; + GGML_API ggml_bf16_t ggml_fp32_to_bf16(float); + GGML_API float ggml_bf16_to_fp32(ggml_bf16_t); // consider just doing << 16 + GGML_API void ggml_bf16_to_fp32_row(const ggml_bf16_t *, float *, int64_t); + GGML_API void ggml_fp32_to_bf16_row_ref(const float *, ggml_bf16_t *, int64_t); + GGML_API void ggml_fp32_to_bf16_row(const float *, ggml_bf16_t *, int64_t); + + struct ggml_object; + struct ggml_context; + struct ggml_cgraph; + + // NOTE: always add types at the end of the enum to keep backward compatibility + enum ggml_type { + GGML_TYPE_F32 = 0, + GGML_TYPE_F16 = 1, + GGML_TYPE_Q4_0 = 2, + GGML_TYPE_Q4_1 = 3, + // GGML_TYPE_Q4_2 = 4, support has been removed + // GGML_TYPE_Q4_3 = 5, support has been removed + GGML_TYPE_Q5_0 = 6, + GGML_TYPE_Q5_1 = 7, + GGML_TYPE_Q8_0 = 8, + GGML_TYPE_Q8_1 = 9, + GGML_TYPE_Q2_K = 10, + GGML_TYPE_Q3_K = 11, + GGML_TYPE_Q4_K = 12, + GGML_TYPE_Q5_K = 13, + GGML_TYPE_Q6_K = 14, + GGML_TYPE_Q8_K = 15, + GGML_TYPE_IQ2_XXS = 16, + GGML_TYPE_IQ2_XS = 17, + GGML_TYPE_IQ3_XXS = 18, + GGML_TYPE_IQ1_S = 19, + GGML_TYPE_IQ4_NL = 20, + GGML_TYPE_IQ3_S = 21, + GGML_TYPE_IQ2_S = 22, + GGML_TYPE_IQ4_XS = 23, + GGML_TYPE_I8 = 24, + GGML_TYPE_I16 = 25, + GGML_TYPE_I32 = 26, + GGML_TYPE_I64 = 27, + GGML_TYPE_F64 = 28, + GGML_TYPE_IQ1_M = 29, + GGML_TYPE_BF16 = 30, + // GGML_TYPE_Q4_0_4_4 = 31, support has been removed from gguf files + // GGML_TYPE_Q4_0_4_8 = 32, + // GGML_TYPE_Q4_0_8_8 = 33, + GGML_TYPE_TQ1_0 = 34, + GGML_TYPE_TQ2_0 = 35, + // GGML_TYPE_IQ4_NL_4_4 = 36, + // GGML_TYPE_IQ4_NL_4_8 = 37, + // GGML_TYPE_IQ4_NL_8_8 = 38, + GGML_TYPE_MXFP4 = 39, // MXFP4 (1 block) + GGML_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale) + GGML_TYPE_Q1_0 = 41, + GGML_TYPE_COUNT = 42, + }; + + // precision + enum ggml_prec { + GGML_PREC_DEFAULT = 0, // stored as ggml_tensor.op_params, 0 by default + GGML_PREC_F32 = 10, + }; + + // op hint + enum ggml_op_hint { + GGML_HINT_NONE = 0, + GGML_HINT_SRC0_IS_HADAMARD = 1, + }; + + // model file types + enum ggml_ftype { + GGML_FTYPE_UNKNOWN = -1, + GGML_FTYPE_ALL_F32 = 0, + GGML_FTYPE_MOSTLY_F16 = 1, // except 1d tensors + GGML_FTYPE_MOSTLY_Q4_0 = 2, // except 1d tensors + GGML_FTYPE_MOSTLY_Q4_1 = 3, // except 1d tensors + GGML_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16 + GGML_FTYPE_MOSTLY_Q8_0 = 7, // except 1d tensors + GGML_FTYPE_MOSTLY_Q5_0 = 8, // except 1d tensors + GGML_FTYPE_MOSTLY_Q5_1 = 9, // except 1d tensors + GGML_FTYPE_MOSTLY_Q2_K = 10, // except 1d tensors + GGML_FTYPE_MOSTLY_Q3_K = 11, // except 1d tensors + GGML_FTYPE_MOSTLY_Q4_K = 12, // except 1d tensors + GGML_FTYPE_MOSTLY_Q5_K = 13, // except 1d tensors + GGML_FTYPE_MOSTLY_Q6_K = 14, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ2_XXS = 15, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ2_XS = 16, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ3_XXS = 17, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ1_S = 18, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ4_NL = 19, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ3_S = 20, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ2_S = 21, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ4_XS = 22, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ1_M = 23, // except 1d tensors + GGML_FTYPE_MOSTLY_BF16 = 24, // except 1d tensors + GGML_FTYPE_MOSTLY_MXFP4 = 25, // except 1d tensors + GGML_FTYPE_MOSTLY_NVFP4 = 26, // except 1d tensors + GGML_FTYPE_MOSTLY_Q1_0 = 27, // except 1d tensors + }; + + // available tensor operations: + enum ggml_op { + GGML_OP_NONE = 0, + + GGML_OP_DUP, + GGML_OP_ADD, + GGML_OP_ADD_ID, + GGML_OP_ADD1, + GGML_OP_ACC, + GGML_OP_SUB, + GGML_OP_MUL, + GGML_OP_DIV, + GGML_OP_SQR, + GGML_OP_SQRT, + GGML_OP_LOG, + GGML_OP_SIN, + GGML_OP_COS, + GGML_OP_SUM, + GGML_OP_SUM_ROWS, + GGML_OP_CUMSUM, + GGML_OP_MEAN, + GGML_OP_ARGMAX, + GGML_OP_COUNT_EQUAL, + GGML_OP_REPEAT, + GGML_OP_REPEAT_BACK, + GGML_OP_CONCAT, + GGML_OP_SILU_BACK, + GGML_OP_NORM, // normalize + GGML_OP_RMS_NORM, + GGML_OP_RMS_NORM_BACK, + GGML_OP_GROUP_NORM, + GGML_OP_L2_NORM, + + GGML_OP_MUL_MAT, + GGML_OP_MUL_MAT_ID, + GGML_OP_OUT_PROD, + + GGML_OP_SCALE, + GGML_OP_SET, + GGML_OP_CPY, + GGML_OP_CONT, + GGML_OP_RESHAPE, + GGML_OP_VIEW, + GGML_OP_PERMUTE, + GGML_OP_TRANSPOSE, + GGML_OP_GET_ROWS, + GGML_OP_GET_ROWS_BACK, + GGML_OP_SET_ROWS, + GGML_OP_DIAG, + GGML_OP_DIAG_MASK_INF, + GGML_OP_DIAG_MASK_ZERO, + GGML_OP_SOFT_MAX, + GGML_OP_SOFT_MAX_BACK, + GGML_OP_ROPE, + GGML_OP_ROPE_BACK, + GGML_OP_CLAMP, + GGML_OP_CONV_TRANSPOSE_1D, + GGML_OP_IM2COL, + GGML_OP_IM2COL_BACK, + GGML_OP_IM2COL_3D, + GGML_OP_CONV_2D, + GGML_OP_CONV_3D, + GGML_OP_CONV_2D_DW, + GGML_OP_CONV_TRANSPOSE_2D, + GGML_OP_POOL_1D, + GGML_OP_POOL_2D, + GGML_OP_POOL_2D_BACK, + GGML_OP_UPSCALE, + GGML_OP_PAD, + GGML_OP_PAD_REFLECT_1D, + GGML_OP_ROLL, + GGML_OP_ARANGE, + GGML_OP_TIMESTEP_EMBEDDING, + GGML_OP_ARGSORT, + GGML_OP_TOP_K, + GGML_OP_LEAKY_RELU, + GGML_OP_TRI, + GGML_OP_FILL, + + GGML_OP_FLASH_ATTN_EXT, + GGML_OP_FLASH_ATTN_BACK, + GGML_OP_SSM_CONV, + GGML_OP_SSM_SCAN, + GGML_OP_WIN_PART, + GGML_OP_WIN_UNPART, + GGML_OP_GET_REL_POS, + GGML_OP_ADD_REL_POS, + GGML_OP_RWKV_WKV6, + GGML_OP_GATED_LINEAR_ATTN, + GGML_OP_RWKV_WKV7, + GGML_OP_SOLVE_TRI, + GGML_OP_GATED_DELTA_NET, + + GGML_OP_UNARY, + + GGML_OP_MAP_CUSTOM1, + GGML_OP_MAP_CUSTOM2, + GGML_OP_MAP_CUSTOM3, + + GGML_OP_CUSTOM, + + GGML_OP_CROSS_ENTROPY_LOSS, + GGML_OP_CROSS_ENTROPY_LOSS_BACK, + GGML_OP_OPT_STEP_ADAMW, + GGML_OP_OPT_STEP_SGD, + + GGML_OP_GLU, + + GGML_OP_COUNT, + }; + + enum ggml_unary_op { + GGML_UNARY_OP_ABS, + GGML_UNARY_OP_SGN, + GGML_UNARY_OP_NEG, + GGML_UNARY_OP_STEP, + GGML_UNARY_OP_TANH, + GGML_UNARY_OP_ELU, + GGML_UNARY_OP_RELU, + GGML_UNARY_OP_SIGMOID, + GGML_UNARY_OP_GELU, + GGML_UNARY_OP_GELU_QUICK, + GGML_UNARY_OP_SILU, + GGML_UNARY_OP_HARDSWISH, + GGML_UNARY_OP_HARDSIGMOID, + GGML_UNARY_OP_EXP, + GGML_UNARY_OP_EXPM1, + GGML_UNARY_OP_SOFTPLUS, + GGML_UNARY_OP_GELU_ERF, + GGML_UNARY_OP_XIELU, + GGML_UNARY_OP_FLOOR, + GGML_UNARY_OP_CEIL, + GGML_UNARY_OP_ROUND, + GGML_UNARY_OP_TRUNC, + + GGML_UNARY_OP_COUNT, + }; + + enum ggml_glu_op { + GGML_GLU_OP_REGLU, + GGML_GLU_OP_GEGLU, + GGML_GLU_OP_SWIGLU, + GGML_GLU_OP_SWIGLU_OAI, + GGML_GLU_OP_GEGLU_ERF, + GGML_GLU_OP_GEGLU_QUICK, + + GGML_GLU_OP_COUNT, + }; + + enum ggml_object_type { + GGML_OBJECT_TYPE_TENSOR, + GGML_OBJECT_TYPE_GRAPH, + GGML_OBJECT_TYPE_WORK_BUFFER + }; + + enum ggml_log_level { + GGML_LOG_LEVEL_NONE = 0, + GGML_LOG_LEVEL_DEBUG = 1, + GGML_LOG_LEVEL_INFO = 2, + GGML_LOG_LEVEL_WARN = 3, + GGML_LOG_LEVEL_ERROR = 4, + GGML_LOG_LEVEL_CONT = 5, // continue previous log + }; + + // this tensor... + enum ggml_tensor_flag { + GGML_TENSOR_FLAG_INPUT = 1, // ...is an input for the GGML compute graph + GGML_TENSOR_FLAG_OUTPUT = 2, // ...is an output for the GGML compute graph + GGML_TENSOR_FLAG_PARAM = 4, // ...contains trainable parameters + GGML_TENSOR_FLAG_LOSS = 8, // ...defines loss for numerical optimization (multiple loss tensors add up) + GGML_TENSOR_FLAG_COMPUTE = 16, // ...must be computed + }; + + enum ggml_tri_type { + GGML_TRI_TYPE_UPPER_DIAG = 0, + GGML_TRI_TYPE_UPPER = 1, + GGML_TRI_TYPE_LOWER_DIAG = 2, + GGML_TRI_TYPE_LOWER = 3 + }; + + struct ggml_init_params { + // memory pool + size_t mem_size; // bytes + void * mem_buffer; // if NULL, memory will be allocated internally + bool no_alloc; // don't allocate memory for the tensor data + }; + + // n-dimensional tensor + struct ggml_tensor { + enum ggml_type type; + + struct ggml_backend_buffer * buffer; + + int64_t ne[GGML_MAX_DIMS]; // number of elements + size_t nb[GGML_MAX_DIMS]; // stride in bytes: + // nb[0] = ggml_type_size(type) + // nb[1] = nb[0] * (ne[0] / ggml_blck_size(type)) + padding + // nb[i] = nb[i-1] * ne[i-1] + + // compute data + enum ggml_op op; + + // op params - allocated as int32_t for alignment + int32_t op_params[GGML_MAX_OP_PARAMS / sizeof(int32_t)]; + + int32_t flags; + + struct ggml_tensor * src[GGML_MAX_SRC]; + + // source tensor and offset for views + struct ggml_tensor * view_src; + size_t view_offs; + + void * data; + + char name[GGML_MAX_NAME]; + + void * extra; // extra things e.g. for ggml-cuda.cu + + char padding[8]; + }; + + static const size_t GGML_TENSOR_SIZE = sizeof(struct ggml_tensor); + + // Abort callback + // If not NULL, called before ggml computation + // If it returns true, the computation is aborted + typedef bool (*ggml_abort_callback)(void * data); + + + // + // GUID + // + + // GUID types + typedef uint8_t ggml_guid[16]; + typedef ggml_guid * ggml_guid_t; + + GGML_API bool ggml_guid_matches(ggml_guid_t guid_a, ggml_guid_t guid_b); + + // misc + + GGML_API const char * ggml_version(void); + GGML_API const char * ggml_commit(void); + + GGML_API void ggml_time_init(void); // call this once at the beginning of the program + GGML_API int64_t ggml_time_ms(void); + GGML_API int64_t ggml_time_us(void); + GGML_API int64_t ggml_cycles(void); + GGML_API int64_t ggml_cycles_per_ms(void); + + // accepts a UTF-8 path, even on Windows + GGML_API FILE * ggml_fopen(const char * fname, const char * mode); + + GGML_API void ggml_print_object (const struct ggml_object * obj); + GGML_API void ggml_print_objects(const struct ggml_context * ctx); + + GGML_API int64_t ggml_nelements (const struct ggml_tensor * tensor); + GGML_API int64_t ggml_nrows (const struct ggml_tensor * tensor); + GGML_API size_t ggml_nbytes (const struct ggml_tensor * tensor); + GGML_API size_t ggml_nbytes_pad(const struct ggml_tensor * tensor); // same as ggml_nbytes() but padded to GGML_MEM_ALIGN + + GGML_API int64_t ggml_blck_size(enum ggml_type type); + GGML_API size_t ggml_type_size(enum ggml_type type); // size in bytes for all elements in a block + GGML_API size_t ggml_row_size (enum ggml_type type, int64_t ne); // size in bytes for all elements in a row + + GGML_DEPRECATED( + GGML_API double ggml_type_sizef(enum ggml_type type), // ggml_type_size()/ggml_blck_size() as float + "use ggml_row_size() instead"); + + GGML_API const char * ggml_type_name(enum ggml_type type); + GGML_API const char * ggml_op_name (enum ggml_op op); + GGML_API const char * ggml_op_symbol(enum ggml_op op); + + GGML_API const char * ggml_unary_op_name(enum ggml_unary_op op); + GGML_API const char * ggml_glu_op_name(enum ggml_glu_op op); + GGML_API const char * ggml_op_desc(const struct ggml_tensor * t); // unary or op name + + GGML_API size_t ggml_element_size(const struct ggml_tensor * tensor); + + GGML_API bool ggml_is_quantized(enum ggml_type type); + + // TODO: temporary until model loading of ggml examples is refactored + GGML_API enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype); + + GGML_API bool ggml_is_transposed(const struct ggml_tensor * tensor); + GGML_API bool ggml_is_permuted (const struct ggml_tensor * tensor); + GGML_API bool ggml_is_empty (const struct ggml_tensor * tensor); + GGML_API bool ggml_is_view (const struct ggml_tensor * tensor); + GGML_API bool ggml_is_scalar (const struct ggml_tensor * tensor); + GGML_API bool ggml_is_vector (const struct ggml_tensor * tensor); + GGML_API bool ggml_is_matrix (const struct ggml_tensor * tensor); + GGML_API bool ggml_is_3d (const struct ggml_tensor * tensor); + GGML_API int ggml_n_dims (const struct ggml_tensor * tensor); // returns 1 for scalars + + // returns whether the tensor elements can be iterated over with a flattened index (no gaps, no permutation) + GGML_API bool ggml_is_contiguous (const struct ggml_tensor * tensor); + GGML_API bool ggml_is_contiguous_0(const struct ggml_tensor * tensor); // same as ggml_is_contiguous() + GGML_API bool ggml_is_contiguous_1(const struct ggml_tensor * tensor); // contiguous for dims >= 1 + GGML_API bool ggml_is_contiguous_2(const struct ggml_tensor * tensor); // contiguous for dims >= 2 + + // returns whether the tensor elements are allocated as one contiguous block of memory (no gaps, but permutation ok) + GGML_API bool ggml_is_contiguously_allocated(const struct ggml_tensor * tensor); + + // true for tensor that is stored in memory as CxWxHxN and has been permuted to WxHxCxN + GGML_API bool ggml_is_contiguous_channels(const struct ggml_tensor * tensor); + + // true if the elements in dimension 0 are contiguous, or there is just 1 block of elements + GGML_API bool ggml_is_contiguous_rows(const struct ggml_tensor * tensor); + + GGML_API bool ggml_are_same_shape (const struct ggml_tensor * t0, const struct ggml_tensor * t1); + GGML_API bool ggml_are_same_stride(const struct ggml_tensor * t0, const struct ggml_tensor * t1); + + GGML_API bool ggml_can_repeat(const struct ggml_tensor * t0, const struct ggml_tensor * t1); + + // use this to compute the memory overhead of a tensor + GGML_API size_t ggml_tensor_overhead(void); + + GGML_API bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbytes); + + // main + + GGML_API struct ggml_context * ggml_init (struct ggml_init_params params); + GGML_API void ggml_reset(struct ggml_context * ctx); + GGML_API void ggml_free (struct ggml_context * ctx); + + GGML_API size_t ggml_used_mem(const struct ggml_context * ctx); + + GGML_API bool ggml_get_no_alloc(struct ggml_context * ctx); + GGML_API void ggml_set_no_alloc(struct ggml_context * ctx, bool no_alloc); + + GGML_API void * ggml_get_mem_buffer (const struct ggml_context * ctx); + GGML_API size_t ggml_get_mem_size (const struct ggml_context * ctx); + GGML_API size_t ggml_get_max_tensor_size(const struct ggml_context * ctx); + + GGML_API struct ggml_tensor * ggml_new_tensor( + struct ggml_context * ctx, + enum ggml_type type, + int n_dims, + const int64_t *ne); + + GGML_API struct ggml_tensor * ggml_new_tensor_1d( + struct ggml_context * ctx, + enum ggml_type type, + int64_t ne0); + + GGML_API struct ggml_tensor * ggml_new_tensor_2d( + struct ggml_context * ctx, + enum ggml_type type, + int64_t ne0, + int64_t ne1); + + GGML_API struct ggml_tensor * ggml_new_tensor_3d( + struct ggml_context * ctx, + enum ggml_type type, + int64_t ne0, + int64_t ne1, + int64_t ne2); + + GGML_API struct ggml_tensor * ggml_new_tensor_4d( + struct ggml_context * ctx, + enum ggml_type type, + int64_t ne0, + int64_t ne1, + int64_t ne2, + int64_t ne3); + + GGML_API void * ggml_new_buffer(struct ggml_context * ctx, size_t nbytes); + + GGML_API struct ggml_tensor * ggml_dup_tensor (struct ggml_context * ctx, const struct ggml_tensor * src); + GGML_API struct ggml_tensor * ggml_view_tensor(struct ggml_context * ctx, struct ggml_tensor * src); + + // Context tensor enumeration and lookup + GGML_API struct ggml_tensor * ggml_get_first_tensor(const struct ggml_context * ctx); + GGML_API struct ggml_tensor * ggml_get_next_tensor (const struct ggml_context * ctx, struct ggml_tensor * tensor); + GGML_API struct ggml_tensor * ggml_get_tensor(struct ggml_context * ctx, const char * name); + + // Converts a flat index into coordinates + GGML_API void ggml_unravel_index(const struct ggml_tensor * tensor, int64_t i, int64_t * i0, int64_t * i1, int64_t * i2, int64_t * i3); + + GGML_API enum ggml_unary_op ggml_get_unary_op(const struct ggml_tensor * tensor); + GGML_API enum ggml_glu_op ggml_get_glu_op(const struct ggml_tensor * tensor); + + GGML_API void * ggml_get_data (const struct ggml_tensor * tensor); + GGML_API float * ggml_get_data_f32(const struct ggml_tensor * tensor); + + GGML_API const char * ggml_get_name (const struct ggml_tensor * tensor); + GGML_API struct ggml_tensor * ggml_set_name ( struct ggml_tensor * tensor, const char * name); + GGML_ATTRIBUTE_FORMAT(2, 3) + GGML_API struct ggml_tensor * ggml_format_name( struct ggml_tensor * tensor, const char * fmt, ...); + + // Tensor flags + GGML_API void ggml_set_input(struct ggml_tensor * tensor); + GGML_API void ggml_set_output(struct ggml_tensor * tensor); + GGML_API void ggml_set_param(struct ggml_tensor * tensor); + GGML_API void ggml_set_loss(struct ggml_tensor * tensor); + + // + // operations on tensors with backpropagation + // + + GGML_API struct ggml_tensor * ggml_dup( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_dup_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_add( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_add_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_add_cast( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + enum ggml_type type); + + // dst[i0, i1, i2] = a[i0, i1, i2] + b[i0, ids[i1, i2]] + GGML_API struct ggml_tensor * ggml_add_id( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * ids); + + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_add1( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b), + "use ggml_add instead"); + + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_add1_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b), + "use ggml_add_inplace instead"); + + // dst = a + // view(dst, nb1, nb2, nb3, offset) += b + // return dst + GGML_API struct ggml_tensor * ggml_acc( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t nb1, + size_t nb2, + size_t nb3, + size_t offset); + + GGML_API struct ggml_tensor * ggml_acc_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t nb1, + size_t nb2, + size_t nb3, + size_t offset); + + GGML_API struct ggml_tensor * ggml_sub( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_sub_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_mul( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_mul_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_div( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_div_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_sqr( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sqr_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sqrt( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sqrt_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_log( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_log_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_expm1( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_expm1_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_softplus( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_softplus_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sin( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sin_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_cos( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_cos_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // return scalar + GGML_API struct ggml_tensor * ggml_sum( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // sums along rows, with input shape [a,b,c,d] return shape [1,b,c,d] + GGML_API struct ggml_tensor * ggml_sum_rows( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_cumsum( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // mean along rows + GGML_API struct ggml_tensor * ggml_mean( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // argmax along rows + GGML_API struct ggml_tensor * ggml_argmax( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // count number of equal elements in a and b + GGML_API struct ggml_tensor * ggml_count_equal( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // if a is the same shape as b, and a is not parameter, return a + // otherwise, return a new tensor: repeat(a) to fit in b + GGML_API struct ggml_tensor * ggml_repeat( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // repeat a to the specified shape + GGML_API struct ggml_tensor * ggml_repeat_4d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2, + int64_t ne3); + + // sums repetitions in a into shape of b + GGML_API struct ggml_tensor * ggml_repeat_back( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); // sum up values that are adjacent in dims > 0 instead of repeated with same stride + + // concat a and b along dim + // used in stable-diffusion + GGML_API struct ggml_tensor * ggml_concat( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int dim); + + GGML_API struct ggml_tensor * ggml_abs( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_abs_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sgn( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sgn_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_neg( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_neg_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_step( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_step_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_tanh( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_tanh_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_elu( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_elu_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_relu( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_leaky_relu( + struct ggml_context * ctx, + struct ggml_tensor * a, float negative_slope, bool inplace); + + GGML_API struct ggml_tensor * ggml_relu_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sigmoid( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_sigmoid_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_gelu( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_gelu_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // GELU using erf (error function) when possible + // some backends may fallback to approximation based on Abramowitz and Stegun formula + GGML_API struct ggml_tensor * ggml_gelu_erf( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_gelu_erf_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_gelu_quick( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_gelu_quick_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_silu( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_silu_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // a - x + // b - dy + GGML_API struct ggml_tensor * ggml_silu_back( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // hardswish(x) = x * relu6(x + 3) / 6 + GGML_API struct ggml_tensor * ggml_hardswish( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // hardsigmoid(x) = relu6(x + 3) / 6 + GGML_API struct ggml_tensor * ggml_hardsigmoid( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_exp( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_exp_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_floor( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_floor_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_ceil( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_ceil_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_round( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_round_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + /** + * Truncates the fractional part of each element in the tensor (towards zero). + * For example: trunc(3.7) = 3.0, trunc(-2.9) = -2.0 + * Similar to std::trunc in C/C++. + */ + + GGML_API struct ggml_tensor * ggml_trunc( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_trunc_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + + + // xIELU activation function + // x = x * (c_a(alpha_n) + c_b(alpha_p, beta) * sigmoid(beta * x)) + eps * (x > 0) + // where c_a = softplus and c_b(a, b) = softplus(a) + b are constraining functions + // that constrain the positive and negative source alpha values respectively + GGML_API struct ggml_tensor * ggml_xielu( + struct ggml_context * ctx, + struct ggml_tensor * a, + float alpha_n, + float alpha_p, + float beta, + float eps); + + // gated linear unit ops + // A: n columns, r rows, + // result is n / 2 columns, r rows, + // expects gate in second half of row, unless swapped is true + GGML_API struct ggml_tensor * ggml_glu( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_glu_op op, + bool swapped); + + GGML_API struct ggml_tensor * ggml_reglu( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_reglu_swapped( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_geglu( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_geglu_swapped( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_swiglu( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_swiglu_swapped( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_geglu_erf( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_geglu_erf_swapped( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_geglu_quick( + struct ggml_context * ctx, + struct ggml_tensor * a); + + GGML_API struct ggml_tensor * ggml_geglu_quick_swapped( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // A: n columns, r rows, + // B: n columns, r rows, + GGML_API struct ggml_tensor * ggml_glu_split( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + enum ggml_glu_op op); + + GGML_API struct ggml_tensor * ggml_reglu_split( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_geglu_split( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_swiglu_split( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_geglu_erf_split( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_geglu_quick_split( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + GGML_API struct ggml_tensor * ggml_swiglu_oai( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + float alpha, + float limit); + + // normalize along rows + GGML_API struct ggml_tensor * ggml_norm( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps); + + GGML_API struct ggml_tensor * ggml_norm_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps); + + GGML_API struct ggml_tensor * ggml_rms_norm( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps); + + GGML_API struct ggml_tensor * ggml_rms_norm_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps); + + // group normalize along ne0*ne1*n_groups + // used in stable-diffusion + GGML_API struct ggml_tensor * ggml_group_norm( + struct ggml_context * ctx, + struct ggml_tensor * a, + int n_groups, + float eps); + + GGML_API struct ggml_tensor * ggml_group_norm_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + int n_groups, + float eps); + + // l2 normalize along rows + // used in rwkv v7 + GGML_API struct ggml_tensor * ggml_l2_norm( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps); + + GGML_API struct ggml_tensor * ggml_l2_norm_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + float eps); + + // a - x + // b - dy + GGML_API struct ggml_tensor * ggml_rms_norm_back( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + float eps); + + // A: k columns, n rows => [ne03, ne02, n, k] + // B: k columns, m rows (i.e. we transpose it internally) => [ne03 * x, ne02 * y, m, k] + // result is n columns, m rows => [ne03 * x, ne02 * y, m, n] + GGML_API struct ggml_tensor * ggml_mul_mat( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // change the precision of a matrix multiplication + // set to GGML_PREC_F32 for higher precision (useful for phi-2) + GGML_API void ggml_mul_mat_set_prec( + struct ggml_tensor * a, + enum ggml_prec prec); + + // change the hint of a matrix multiplication + GGML_API void ggml_mul_mat_set_hint( + struct ggml_tensor * a, + enum ggml_op_hint hint); + + // indirect matrix multiplication + GGML_API struct ggml_tensor * ggml_mul_mat_id( + struct ggml_context * ctx, + struct ggml_tensor * as, + struct ggml_tensor * b, + struct ggml_tensor * ids); + + // A: m columns, n rows, + // B: p columns, n rows, + // result is m columns, p rows + GGML_API struct ggml_tensor * ggml_out_prod( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // + // operations on tensors without backpropagation + // + + GGML_API struct ggml_tensor * ggml_scale( + struct ggml_context * ctx, + struct ggml_tensor * a, + float s); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_scale_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + float s); + + // x = s * a + b + GGML_API struct ggml_tensor * ggml_scale_bias( + struct ggml_context * ctx, + struct ggml_tensor * a, + float s, + float b); + + GGML_API struct ggml_tensor * ggml_scale_bias_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + float s, + float b); + + // b -> view(a,offset,nb1,nb2,3), return modified a + GGML_API struct ggml_tensor * ggml_set( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t nb1, + size_t nb2, + size_t nb3, + size_t offset); // in bytes + + // b -> view(a,offset,nb1,nb2,3), return view(a) + GGML_API struct ggml_tensor * ggml_set_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t nb1, + size_t nb2, + size_t nb3, + size_t offset); // in bytes + + GGML_API struct ggml_tensor * ggml_set_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t offset); // in bytes + + GGML_API struct ggml_tensor * ggml_set_1d_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t offset); // in bytes + + // b -> view(a,offset,nb1,nb2,3), return modified a + GGML_API struct ggml_tensor * ggml_set_2d( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t nb1, + size_t offset); // in bytes + + // b -> view(a,offset,nb1,nb2,3), return view(a) + GGML_API struct ggml_tensor * ggml_set_2d_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + size_t nb1, + size_t offset); // in bytes + + // a -> b, return view(b) + GGML_API struct ggml_tensor * ggml_cpy( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // note: casting from f32 to i32 will discard the fractional part + GGML_API struct ggml_tensor * ggml_cast( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_type type); + + // make contiguous + GGML_API struct ggml_tensor * ggml_cont( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // make contiguous, with new shape + GGML_API struct ggml_tensor * ggml_cont_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0); + + GGML_API struct ggml_tensor * ggml_cont_2d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1); + + GGML_API struct ggml_tensor * ggml_cont_3d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2); + + GGML_API struct ggml_tensor * ggml_cont_4d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2, + int64_t ne3); + + // return view(a), b specifies the new shape + // TODO: when we start computing gradient, make a copy instead of view + GGML_API struct ggml_tensor * ggml_reshape( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // return view(a) + // TODO: when we start computing gradient, make a copy instead of view + GGML_API struct ggml_tensor * ggml_reshape_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0); + + GGML_API struct ggml_tensor * ggml_reshape_2d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1); + + // return view(a) + // TODO: when we start computing gradient, make a copy instead of view + GGML_API struct ggml_tensor * ggml_reshape_3d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2); + + GGML_API struct ggml_tensor * ggml_reshape_4d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2, + int64_t ne3); + + // offset in bytes + GGML_API struct ggml_tensor * ggml_view_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + size_t offset); + + GGML_API struct ggml_tensor * ggml_view_2d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + size_t nb1, // row stride in bytes + size_t offset); + + GGML_API struct ggml_tensor * ggml_view_3d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2, + size_t nb1, // row stride in bytes + size_t nb2, // slice stride in bytes + size_t offset); + + GGML_API struct ggml_tensor * ggml_view_4d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2, + int64_t ne3, + size_t nb1, // row stride in bytes + size_t nb2, // slice stride in bytes + size_t nb3, + size_t offset); + + GGML_API struct ggml_tensor * ggml_permute( + struct ggml_context * ctx, + struct ggml_tensor * a, + int axis0, + int axis1, + int axis2, + int axis3); + + // alias for ggml_permute(ctx, a, 1, 0, 2, 3) + GGML_API struct ggml_tensor * ggml_transpose( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // supports 4D a: + // a [n_embd, ne1, ne2, ne3] + // b I32 [n_rows, ne2, ne3, 1] + // + // return [n_embd, n_rows, ne2, ne3] + GGML_API struct ggml_tensor * ggml_get_rows( + struct ggml_context * ctx, + struct ggml_tensor * a, // data + struct ggml_tensor * b); // row indices + + GGML_API struct ggml_tensor * ggml_get_rows_back( + struct ggml_context * ctx, + struct ggml_tensor * a, // gradients of ggml_get_rows result + struct ggml_tensor * b, // row indices + struct ggml_tensor * c); // data for ggml_get_rows, only used for its shape + + // a TD [n_embd, ne1, ne2, ne3] + // b TS [n_embd, n_rows, ne02, ne03] | ne02 == ne2, ne03 == ne3 + // c I64 [n_rows, ne11, ne12, 1] | c[i] in [0, ne1) + // + // undefined behavior if destination rows overlap + // + // broadcast: + // ne2 % ne11 == 0 + // ne3 % ne12 == 0 + // + // return view(a) + GGML_API struct ggml_tensor * ggml_set_rows( + struct ggml_context * ctx, + struct ggml_tensor * a, // destination + struct ggml_tensor * b, // source + struct ggml_tensor * c); // row indices + + GGML_API struct ggml_tensor * ggml_diag( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // set elements above the diagonal to -INF + GGML_API struct ggml_tensor * ggml_diag_mask_inf( + struct ggml_context * ctx, + struct ggml_tensor * a, + int n_past); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_diag_mask_inf_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + int n_past); + + // set elements above the diagonal to 0 + GGML_API struct ggml_tensor * ggml_diag_mask_zero( + struct ggml_context * ctx, + struct ggml_tensor * a, + int n_past); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_diag_mask_zero_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + int n_past); + + GGML_API struct ggml_tensor * ggml_soft_max( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_soft_max_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a); + + // a [ne0, ne01, ne02, ne03] + // mask [ne0, ne11, ne12, ne13] | ne11 >= ne01, F16 or F32, optional + // + // broadcast: + // ne02 % ne12 == 0 + // ne03 % ne13 == 0 + // + // fused soft_max(a*scale + mask*(ALiBi slope)) + // max_bias = 0.0f for no ALiBi + GGML_API struct ggml_tensor * ggml_soft_max_ext( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * mask, + float scale, + float max_bias); + + GGML_API struct ggml_tensor * ggml_soft_max_ext_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * mask, + float scale, + float max_bias); + + GGML_API void ggml_soft_max_add_sinks( + struct ggml_tensor * a, + struct ggml_tensor * sinks); + + GGML_API struct ggml_tensor * ggml_soft_max_ext_back( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + float scale, + float max_bias); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_soft_max_ext_back_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + float scale, + float max_bias); + + // rotary position embedding + // if (mode & 1) - skip n_past elements (NOT SUPPORTED) + // if (mode & GGML_ROPE_TYPE_NEOX) - GPT-NeoX style + // + // b is an int32 vector with size a->ne[2], it contains the positions + GGML_API struct ggml_tensor * ggml_rope( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int n_dims, + int mode); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_rope_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int n_dims, + int mode); + + // RoPE operations with extended options + // a is the input tensor to apply RoPE to, shape [n_embd, n_head, n_token] + // b is an int32 vector with size n_token + // c is freq factors (e.g. phi3-128k), (optional) + // mode can be GGML_ROPE_TYPE_NORMAL or NEOX; for MROPE and VISION mode, use ggml_rope_multi + // + // pseudo-code for computing theta: + // for i in [0, n_dims/2): + // theta[i] = b[i] * powf(freq_base, -2.0 * i / n_dims); + // theta[i] = theta[i] / c[i]; # if c is provided, divide theta by c + // theta[i] = rope_yarn(theta[i], ...); # note: theta = theta * freq_scale is applied here + // + // other params are used by YaRN RoPE scaling, these default values will disable YaRN: + // freq_scale = 1.0f + // ext_factor = 0.0f + // attn_factor = 1.0f + // beta_fast = 0.0f + // beta_slow = 0.0f + // + // example: + // (marking: c = cos, s = sin, 0 = unrotated) + // given a single head with size = 8 --> [00000000] + // GGML_ROPE_TYPE_NORMAL n_dims = 4 --> [cscs0000] + // GGML_ROPE_TYPE_NORMAL n_dims = 8 --> [cscscscs] + // GGML_ROPE_TYPE_NEOX n_dims = 4 --> [ccss0000] + // GGML_ROPE_TYPE_NEOX n_dims = 8 --> [ccccssss] + GGML_API struct ggml_tensor * ggml_rope_ext( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + int n_dims, + int mode, + int n_ctx_orig, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow); + + // multi-dimensional RoPE, for Qwen-VL and similar vision models + // mode can be either VISION, MROPE, IMROPE, cannot be combined with NORMAL or NEOX + // sections specify how many dimensions to rotate in each section: + // section length is equivalent to number of cos/sin pairs, NOT the number of dims + // (i.e. sum of 4 sections are expected to be n_dims/2) + // last sections can be 0, means ignored + // all other options are identical to ggml_rope_ext + // + // important note: + // - NEOX ordering is automatically applied and cannot be disabled for MROPE and VISION + // if you need normal ordering, there are 2 methods: + // (1) split the tensor manually using ggml_view + // (2) permute the weight upon conversion + // - for VISION, n_dims must be head_size/2 + // + // example M-RoPE: + // given sections = [t=4, y=2, x=2, 0] + // given a single head with size = 18 --> [000000000000000000] + // GGML_ROPE_TYPE_MROPE n_dims = 16 --> [ttttyyxxttttyyxx00] (cos/sin are applied in NEOX ordering) + // GGML_ROPE_TYPE_IMROPE n_dims = 16 --> [ttyxttyxttyxttyx00] (interleaved M-RoPE, still NEOX ordering) + // note: the theta for each dim is computed the same way as ggml_rope_ext, no matter the section + // in other words, idx used for theta: [0123456789... until n_dims/2], not reset for each section + // + // example vision RoPE: + // given sections = [y=4, x=4, 0, 0] (last 2 sections are ignored) + // given a single head with size = 8 --> [00000000] + // GGML_ROPE_TYPE_VISION n_dims = 4 --> [yyyyxxxx] + // other values of n_dims are untested and is undefined behavior + // note: unlike MROPE, the theta for each dim is computed differently for each section + // in other words, idx used for theta: [0123] for y section, then [0123] for x section + GGML_API struct ggml_tensor * ggml_rope_multi( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + int n_dims, + int sections[GGML_MROPE_SECTIONS], + int mode, + int n_ctx_orig, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow); + + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_rope_ext_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + int n_dims, + int mode, + int n_ctx_orig, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow); + + GGML_API struct ggml_tensor * ggml_rope_multi_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + int n_dims, + int sections[GGML_MROPE_SECTIONS], + int mode, + int n_ctx_orig, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow); + + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_rope_custom( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int n_dims, + int mode, + int n_ctx_orig, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow), + "use ggml_rope_ext instead"); + + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_rope_custom_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int n_dims, + int mode, + int n_ctx_orig, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow), + "use ggml_rope_ext_inplace instead"); + + // compute correction dims for YaRN RoPE scaling + GGML_API void ggml_rope_yarn_corr_dims( + int n_dims, int n_ctx_orig, float freq_base, float beta_fast, float beta_slow, float dims[2]); + + // rotary position embedding backward, i.e compute dx from dy + // a - dy + GGML_API struct ggml_tensor * ggml_rope_ext_back( + struct ggml_context * ctx, + struct ggml_tensor * a, // gradients of ggml_rope result + struct ggml_tensor * b, // positions + struct ggml_tensor * c, // freq factors + int n_dims, + int mode, + int n_ctx_orig, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow); + + GGML_API struct ggml_tensor * ggml_rope_multi_back( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + int n_dims, + int sections[4], + int mode, + int n_ctx_orig, + float freq_base, + float freq_scale, + float ext_factor, + float attn_factor, + float beta_fast, + float beta_slow); + + + // clamp + // in-place, returns view(a) + GGML_API struct ggml_tensor * ggml_clamp( + struct ggml_context * ctx, + struct ggml_tensor * a, + float min, + float max); + + // im2col + // converts data into a format that effectively results in a convolution when combined with matrix multiplication + GGML_API struct ggml_tensor * ggml_im2col( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // data + int s0, // stride dimension 0 + int s1, // stride dimension 1 + int p0, // padding dimension 0 + int p1, // padding dimension 1 + int d0, // dilation dimension 0 + int d1, // dilation dimension 1 + bool is_2D, + enum ggml_type dst_type); + + GGML_API struct ggml_tensor * ggml_im2col_back( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // gradient of im2col output + int64_t * ne, // shape of im2col input + int s0, // stride dimension 0 + int s1, // stride dimension 1 + int p0, // padding dimension 0 + int p1, // padding dimension 1 + int d0, // dilation dimension 0 + int d1, // dilation dimension 1 + bool is_2D); + + GGML_API struct ggml_tensor * ggml_conv_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // data + int s0, // stride + int p0, // padding + int d0); // dilation + + // conv_1d with padding = half + // alias for ggml_conv_1d(a, b, s, a->ne[0]/2, d) + GGML_API struct ggml_tensor* ggml_conv_1d_ph( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // data + int s, // stride + int d); // dilation + + // depthwise + // TODO: this is very likely wrong for some cases! - needs more testing + GGML_API struct ggml_tensor * ggml_conv_1d_dw( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // data + int s0, // stride + int p0, // padding + int d0); // dilation + + GGML_API struct ggml_tensor * ggml_conv_1d_dw_ph( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // data + int s0, // stride + int d0); // dilation + + GGML_API struct ggml_tensor * ggml_conv_transpose_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // data + int s0, // stride + int p0, // padding + int d0); // dilation + + GGML_API struct ggml_tensor * ggml_conv_2d( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // data + int s0, // stride dimension 0 + int s1, // stride dimension 1 + int p0, // padding dimension 0 + int p1, // padding dimension 1 + int d0, // dilation dimension 0 + int d1); // dilation dimension 1 + + GGML_API struct ggml_tensor * ggml_im2col_3d( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int64_t IC, + int s0, // stride width + int s1, // stride height + int s2, // stride depth + int p0, // padding width + int p1, // padding height + int p2, // padding depth + int d0, // dilation width + int d1, // dilation height + int d2, // dilation depth + enum ggml_type dst_type); + + // a: [OC*IC, KD, KH, KW] + // b: [N*IC, ID, IH, IW] + // result: [N*OC, OD, OH, OW] + GGML_API struct ggml_tensor * ggml_conv_3d( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int64_t IC, + int s0, // stride width + int s1, // stride height + int s2, // stride depth + int p0, // padding width + int p1, // padding height + int p2, // padding depth + int d0, // dilation width + int d1, // dilation height + int d2 // dilation depth + ); + + // kernel size is a->ne[0] x a->ne[1] + // stride is equal to kernel size + // padding is zero + // example: + // a: 16 16 3 768 + // b: 1024 1024 3 1 + // res: 64 64 768 1 + // used in sam + GGML_API struct ggml_tensor * ggml_conv_2d_sk_p0( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // kernel size is a->ne[0] x a->ne[1] + // stride is 1 + // padding is half + // example: + // a: 3 3 256 256 + // b: 64 64 256 1 + // res: 64 64 256 1 + // used in sam + GGML_API struct ggml_tensor * ggml_conv_2d_s1_ph( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b); + + // depthwise (via im2col and mul_mat) + GGML_API struct ggml_tensor * ggml_conv_2d_dw( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel + struct ggml_tensor * b, // data + int s0, // stride dimension 0 + int s1, // stride dimension 1 + int p0, // padding dimension 0 + int p1, // padding dimension 1 + int d0, // dilation dimension 0 + int d1); // dilation dimension 1 + + // Depthwise 2D convolution + // may be faster than ggml_conv_2d_dw, but not available in all backends + // a: KW KH 1 C convolution kernel + // b: W H C N input data + // res: W_out H_out C N + GGML_API struct ggml_tensor * ggml_conv_2d_dw_direct( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int stride0, + int stride1, + int pad0, + int pad1, + int dilation0, + int dilation1); + + GGML_API struct ggml_tensor * ggml_conv_transpose_2d_p0( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + int stride); + + GGML_API struct ggml_tensor * ggml_conv_2d_direct( + struct ggml_context * ctx, + struct ggml_tensor * a, // convolution kernel [KW, KH, IC, OC] + struct ggml_tensor * b, // input data [W, H, C, N] + int s0, // stride dimension 0 + int s1, // stride dimension 1 + int p0, // padding dimension 0 + int p1, // padding dimension 1 + int d0, // dilation dimension 0 + int d1); // dilation dimension 1 + + GGML_API struct ggml_tensor * ggml_conv_3d_direct( + struct ggml_context * ctx, + struct ggml_tensor * a, // kernel [KW, KH, KD, IC * OC] + struct ggml_tensor * b, // input [W, H, D, C * N] + int s0, // stride + int s1, + int s2, + int p0, // padding + int p1, + int p2, + int d0, // dilation + int d1, + int d2, + int n_channels, + int n_batch, + int n_channels_out); + + enum ggml_op_pool { + GGML_OP_POOL_MAX, + GGML_OP_POOL_AVG, + GGML_OP_POOL_COUNT, + }; + + GGML_API struct ggml_tensor * ggml_pool_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_op_pool op, + int k0, // kernel size + int s0, // stride + int p0); // padding + + // the result will have 2*p0 padding for the first dimension + // and 2*p1 padding for the second dimension + GGML_API struct ggml_tensor * ggml_pool_2d( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_op_pool op, + int k0, + int k1, + int s0, + int s1, + float p0, + float p1); + + GGML_API struct ggml_tensor * ggml_pool_2d_back( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * af, // "a"/input used in forward pass + enum ggml_op_pool op, + int k0, + int k1, + int s0, + int s1, + float p0, + float p1); + + enum ggml_scale_mode { + GGML_SCALE_MODE_NEAREST = 0, + GGML_SCALE_MODE_BILINEAR = 1, + GGML_SCALE_MODE_BICUBIC = 2, + + GGML_SCALE_MODE_COUNT + }; + + enum ggml_scale_flag { + GGML_SCALE_FLAG_ALIGN_CORNERS = (1 << 8), + GGML_SCALE_FLAG_ANTIALIAS = (1 << 9), + }; + + // interpolate + // multiplies ne0 and ne1 by scale factor + GGML_API struct ggml_tensor * ggml_upscale( + struct ggml_context * ctx, + struct ggml_tensor * a, + int scale_factor, + enum ggml_scale_mode mode); + + // interpolate + // interpolate scale to specified dimensions + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_upscale_ext( + struct ggml_context * ctx, + struct ggml_tensor * a, + int ne0, + int ne1, + int ne2, + int ne3, + enum ggml_scale_mode mode), + "use ggml_interpolate instead"); + + // Up- or downsamples the input to the specified size. + // 2D scale modes (eg. bilinear) are applied to the first two dimensions. + GGML_API struct ggml_tensor * ggml_interpolate( + struct ggml_context * ctx, + struct ggml_tensor * a, + int64_t ne0, + int64_t ne1, + int64_t ne2, + int64_t ne3, + uint32_t mode); // ggml_scale_mode [ | ggml_scale_flag...] + + // pad each dimension with zeros: [x, ..., x] -> [x, ..., x, 0, ..., 0] + GGML_API struct ggml_tensor * ggml_pad( + struct ggml_context * ctx, + struct ggml_tensor * a, + int p0, + int p1, + int p2, + int p3); + + // pad each dimension with values on the other side of the torus (looping around) + GGML_API struct ggml_tensor * ggml_pad_circular( + struct ggml_context * ctx, + struct ggml_tensor * a, + int p0, + int p1, + int p2, + int p3); + + GGML_API struct ggml_tensor * ggml_pad_ext( + struct ggml_context * ctx, + struct ggml_tensor * a, + int lp0, + int rp0, + int lp1, + int rp1, + int lp2, + int rp2, + int lp3, + int rp3 + ); + + // pad each dimension with values on the other side of the torus (looping around) + GGML_API struct ggml_tensor * ggml_pad_ext_circular( + struct ggml_context * ctx, + struct ggml_tensor * a, + int lp0, + int rp0, + int lp1, + int rp1, + int lp2, + int rp2, + int lp3, + int rp3); + + // pad each dimension with reflection: [a, b, c, d] -> [b, a, b, c, d, c] + GGML_API struct ggml_tensor * ggml_pad_reflect_1d( + struct ggml_context * ctx, + struct ggml_tensor * a, + int p0, + int p1); + + // Move tensor elements by an offset given for each dimension. Elements that + // are shifted beyond the last position are wrapped around to the beginning. + GGML_API struct ggml_tensor * ggml_roll( + struct ggml_context * ctx, + struct ggml_tensor * a, + int shift0, + int shift1, + int shift2, + int shift3); + + // Convert matrix into a triangular one (upper, strict upper, lower or strict lower) by writing + // zeroes everywhere outside the masked area + GGML_API struct ggml_tensor * ggml_tri( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_tri_type type); + + // Fill tensor a with constant c + GGML_API struct ggml_tensor * ggml_fill( + struct ggml_context * ctx, + struct ggml_tensor * a, + float c); + + GGML_API struct ggml_tensor * ggml_fill_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + float c); + + // Ref: https://github.com/CompVis/stable-diffusion/blob/main/ldm/modules/diffusionmodules/util.py#L151 + // timesteps: [N,] + // return: [N, dim] + GGML_API struct ggml_tensor * ggml_timestep_embedding( + struct ggml_context * ctx, + struct ggml_tensor * timesteps, + int dim, + int max_period); + + // sort rows + enum ggml_sort_order { + GGML_SORT_ORDER_ASC, + GGML_SORT_ORDER_DESC, + }; + + GGML_API struct ggml_tensor * ggml_argsort( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_sort_order order); + + // similar to ggml_top_k but implemented as `argsort` + `view` + GGML_API struct ggml_tensor * ggml_argsort_top_k( + struct ggml_context * ctx, + struct ggml_tensor * a, + int k); + + // top k elements per row + // note: the resulting top k indices are in no particular order + GGML_API struct ggml_tensor * ggml_top_k( + struct ggml_context * ctx, + struct ggml_tensor * a, + int k); + + GGML_API struct ggml_tensor * ggml_arange( + struct ggml_context * ctx, + float start, + float stop, + float step); + + // q: [n_embd_k, n_batch, n_head, ne3 ] + // k: [n_embd_k, n_kv, n_head_kv, ne3 ] + // v: [n_embd_v, n_kv, n_head_kv, ne3 ] !! not transposed !! + // mask: [n_kv, n_batch, ne32, ne33] + // res: [n_embd_v, n_head, n_batch, ne3 ] !! permuted !! + // + // broadcast: + // n_head % n_head_kv == 0 + // n_head % ne32 == 0 + // ne3 % ne33 == 0 + // + GGML_API struct ggml_tensor * ggml_flash_attn_ext( + struct ggml_context * ctx, + struct ggml_tensor * q, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * mask, + float scale, + float max_bias, + float logit_softcap); + + GGML_API void ggml_flash_attn_ext_set_prec( + struct ggml_tensor * a, + enum ggml_prec prec); + + GGML_API enum ggml_prec ggml_flash_attn_ext_get_prec( + const struct ggml_tensor * a); + + GGML_API void ggml_flash_attn_ext_add_sinks( + struct ggml_tensor * a, + struct ggml_tensor * sinks); + + // TODO: needs to be adapted to ggml_flash_attn_ext + GGML_API struct ggml_tensor * ggml_flash_attn_back( + struct ggml_context * ctx, + struct ggml_tensor * q, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * d, + bool masked); + + GGML_API struct ggml_tensor * ggml_ssm_conv( + struct ggml_context * ctx, + struct ggml_tensor * sx, + struct ggml_tensor * c); + + GGML_API struct ggml_tensor * ggml_ssm_scan( + struct ggml_context * ctx, + struct ggml_tensor * s, + struct ggml_tensor * x, + struct ggml_tensor * dt, + struct ggml_tensor * A, + struct ggml_tensor * B, + struct ggml_tensor * C, + struct ggml_tensor * ids); + + // partition into non-overlapping windows with padding if needed + // example: + // a: 768 64 64 1 + // w: 14 + // res: 768 14 14 25 + // used in sam + GGML_API struct ggml_tensor * ggml_win_part( + struct ggml_context * ctx, + struct ggml_tensor * a, + int w); + + // reverse of ggml_win_part + // used in sam + GGML_API struct ggml_tensor * ggml_win_unpart( + struct ggml_context * ctx, + struct ggml_tensor * a, + int w0, + int h0, + int w); + + GGML_API struct ggml_tensor * ggml_unary( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_unary_op op); + + GGML_API struct ggml_tensor * ggml_unary_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + enum ggml_unary_op op); + + // used in sam + GGML_API struct ggml_tensor * ggml_get_rel_pos( + struct ggml_context * ctx, + struct ggml_tensor * a, + int qh, + int kh); + + // used in sam + GGML_API struct ggml_tensor * ggml_add_rel_pos( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * pw, + struct ggml_tensor * ph); + + GGML_API struct ggml_tensor * ggml_add_rel_pos_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * pw, + struct ggml_tensor * ph); + + GGML_API struct ggml_tensor * ggml_rwkv_wkv6( + struct ggml_context * ctx, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * r, + struct ggml_tensor * tf, + struct ggml_tensor * td, + struct ggml_tensor * state); + + GGML_API struct ggml_tensor * ggml_gated_linear_attn( + struct ggml_context * ctx, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * q, + struct ggml_tensor * g, + struct ggml_tensor * state, + float scale); + + GGML_API struct ggml_tensor * ggml_rwkv_wkv7( + struct ggml_context * ctx, + struct ggml_tensor * r, + struct ggml_tensor * w, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * state); + + /* Solves a specific equation of the form Ax=B, where A is a triangular matrix + * without zeroes on the diagonal (i.e. invertible). + * B can have any number of columns, but must have the same number of rows as A + * If A is [n, n] and B is [n, m], then the result will be [n, m] as well + * Has O(n^3) complexity (unlike most matrix ops out there), so use on cases + * where n > 100 sparingly, pre-chunk if necessary. + * + * If left = false, solves xA=B instead + * If lower = false, assumes upper triangular instead + * If uni = true, assumes diagonal of A to be all ones (will override actual values) + * + * TODO: currently only lower, right, non-unitriangular variant is implemented + */ + GGML_API struct ggml_tensor * ggml_solve_tri( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + bool left, + bool lower, + bool uni); + + // TODO: add ggml_gated_delta_net_set_bcast() to be able to configure Q, K broadcast type: tiled vs interleaved [TAG_GGML_GDN_BCAST] + // ref: https://github.com/ggml-org/llama.cpp/pull/19468#discussion_r2786394306 + GGML_API struct ggml_tensor * ggml_gated_delta_net( + struct ggml_context * ctx, + struct ggml_tensor * q, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * g, + struct ggml_tensor * beta, + struct ggml_tensor * state); + + // custom operators + + typedef void (*ggml_custom1_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, int ith, int nth, void * userdata); + typedef void (*ggml_custom2_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, const struct ggml_tensor * b, int ith, int nth, void * userdata); + typedef void (*ggml_custom3_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, const struct ggml_tensor * b, const struct ggml_tensor * c, int ith, int nth, void * userdata); + +#define GGML_N_TASKS_MAX (-1) + // n_tasks == GGML_N_TASKS_MAX means to use max number of tasks + + GGML_API struct ggml_tensor * ggml_map_custom1( + struct ggml_context * ctx, + struct ggml_tensor * a, + ggml_custom1_op_t fun, + int n_tasks, + void * userdata); + + GGML_API struct ggml_tensor * ggml_map_custom1_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + ggml_custom1_op_t fun, + int n_tasks, + void * userdata); + + GGML_API struct ggml_tensor * ggml_map_custom2( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + ggml_custom2_op_t fun, + int n_tasks, + void * userdata); + + GGML_API struct ggml_tensor * ggml_map_custom2_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + ggml_custom2_op_t fun, + int n_tasks, + void * userdata); + + GGML_API struct ggml_tensor * ggml_map_custom3( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + ggml_custom3_op_t fun, + int n_tasks, + void * userdata); + + GGML_API struct ggml_tensor * ggml_map_custom3_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * b, + struct ggml_tensor * c, + ggml_custom3_op_t fun, + int n_tasks, + void * userdata); + + typedef void (*ggml_custom_op_t)(struct ggml_tensor * dst , int ith, int nth, void * userdata); + + GGML_API struct ggml_tensor * ggml_custom_4d( + struct ggml_context * ctx, + enum ggml_type type, + int64_t ne0, + int64_t ne1, + int64_t ne2, + int64_t ne3, + struct ggml_tensor ** args, + int n_args, + ggml_custom_op_t fun, + int n_tasks, + void * userdata); + + GGML_API struct ggml_tensor * ggml_custom_inplace( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor ** args, + int n_args, + ggml_custom_op_t fun, + int n_tasks, + void * userdata); + + // loss function + + GGML_API struct ggml_tensor * ggml_cross_entropy_loss( + struct ggml_context * ctx, + struct ggml_tensor * a, // logits + struct ggml_tensor * b); // labels + + GGML_API struct ggml_tensor * ggml_cross_entropy_loss_back( + struct ggml_context * ctx, + struct ggml_tensor * a, // logits + struct ggml_tensor * b, // labels + struct ggml_tensor * c); // gradients of cross_entropy_loss result + + // AdamW optimizer step + // Paper: https://arxiv.org/pdf/1711.05101v3.pdf + // PyTorch: https://pytorch.org/docs/stable/generated/torch.optim.AdamW.html + GGML_API struct ggml_tensor * ggml_opt_step_adamw( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * grad, + struct ggml_tensor * m, + struct ggml_tensor * v, + struct ggml_tensor * adamw_params); // parameters such as the learning rate + + // stochastic gradient descent step (with weight decay) + GGML_API struct ggml_tensor * ggml_opt_step_sgd( + struct ggml_context * ctx, + struct ggml_tensor * a, + struct ggml_tensor * grad, + struct ggml_tensor * sgd_params); // alpha, weight decay + + // build forward multiple tensors and select one of them for computing + // this is useful for creating graphs that have constant topology but compute different things based on the input + // ref: https://github.com/ggml-org/llama.cpp/pull/18550 + // + // nodes: + // | - build forward into the graph but do not compute + // c - build forward into the graph and compute + // + // | | ... c ... | + // | | ... c ... | + // | | ... c ... | + // [0 1 ... idx ... n-1] <-- ggml_build_forward_select(..., n, idx) + // c + // c + // + // example: + // struct ggml_tensor * curs[3]; + // + // curs[0] = compute0(...); + // curs[1] = compute1(...); + // curs[2] = compute2(...); + // + // int idx = select_branch(some_input); + // + // struct ggml_tensor * out = ggml_build_forward_select(cgraph, curs, 3, idx); + // + GGML_API struct ggml_tensor * ggml_build_forward_select( + struct ggml_cgraph * cgraph, + struct ggml_tensor ** tensors, + int n_tensors, + int idx); + + GGML_API void ggml_build_forward_expand( + struct ggml_cgraph * cgraph, + struct ggml_tensor * tensor); + + GGML_API void ggml_build_backward_expand( + struct ggml_context * ctx, // context for gradient computation + struct ggml_cgraph * cgraph, + struct ggml_tensor ** grad_accs); + + // graph allocation in a context + GGML_API struct ggml_cgraph * ggml_new_graph (struct ggml_context * ctx); // size = GGML_DEFAULT_GRAPH_SIZE, grads = false + GGML_API struct ggml_cgraph * ggml_new_graph_custom(struct ggml_context * ctx, size_t size, bool grads); + GGML_API struct ggml_cgraph * ggml_graph_dup (struct ggml_context * ctx, struct ggml_cgraph * cgraph, bool force_grads); + GGML_API void ggml_graph_cpy (struct ggml_cgraph * src, struct ggml_cgraph * dst); + GGML_API void ggml_graph_reset (struct ggml_cgraph * cgraph); // set regular grads + optimizer momenta to 0, set loss grad to 1 + GGML_API void ggml_graph_clear (struct ggml_cgraph * cgraph); + + GGML_API int ggml_graph_size (struct ggml_cgraph * cgraph); + GGML_API struct ggml_tensor * ggml_graph_node (struct ggml_cgraph * cgraph, int i); // if i < 0, returns nodes[n_nodes + i] + GGML_API struct ggml_tensor ** ggml_graph_nodes (struct ggml_cgraph * cgraph); + GGML_API int ggml_graph_n_nodes(struct ggml_cgraph * cgraph); + + GGML_API void ggml_graph_add_node(struct ggml_cgraph * cgraph, struct ggml_tensor * tensor); + + GGML_API size_t ggml_graph_overhead(void); + GGML_API size_t ggml_graph_overhead_custom(size_t size, bool grads); + + GGML_API struct ggml_tensor * ggml_graph_get_tensor (const struct ggml_cgraph * cgraph, const char * name); + GGML_API struct ggml_tensor * ggml_graph_get_grad (const struct ggml_cgraph * cgraph, const struct ggml_tensor * node); + GGML_API struct ggml_tensor * ggml_graph_get_grad_acc(const struct ggml_cgraph * cgraph, const struct ggml_tensor * node); + + // print info and performance information for the graph + GGML_API void ggml_graph_print(const struct ggml_cgraph * cgraph); + + // dump the graph into a file using the dot format + GGML_API void ggml_graph_dump_dot(const struct ggml_cgraph * gb, const struct ggml_cgraph * cgraph, const char * filename); + + // TODO these functions were sandwiched in the old optimization interface, is there a better place for them? + typedef void (*ggml_log_callback)(enum ggml_log_level level, const char * text, void * user_data); + + // Set callback for all future logging events. + // If this is not called, or NULL is supplied, everything is output on stderr. + GGML_API void ggml_log_get(ggml_log_callback * log_callback, void ** user_data); + GGML_API void ggml_log_set(ggml_log_callback log_callback, void * user_data); + + GGML_API struct ggml_tensor * ggml_set_zero(struct ggml_tensor * tensor); + + // + // quantization + // + + // - ggml_quantize_init can be called multiple times with the same type + // it will only initialize the quantization tables for the first call or after ggml_quantize_free + // automatically called by ggml_quantize_chunk for convenience + // + // - ggml_quantize_free will free any memory allocated by ggml_quantize_init + // call this at the end of the program to avoid memory leaks + // + // note: these are thread-safe + // + GGML_API void ggml_quantize_init(enum ggml_type type); + GGML_API void ggml_quantize_free(void); + + // some quantization type cannot be used without an importance matrix + GGML_API bool ggml_quantize_requires_imatrix(enum ggml_type type); + + // calls ggml_quantize_init internally (i.e. can allocate memory) + GGML_API size_t ggml_quantize_chunk( + enum ggml_type type, + const float * src, + void * dst, + int64_t start, + int64_t nrows, + int64_t n_per_row, + const float * imatrix); + +#ifdef __cplusplus + // restrict not standard in C++ +# if defined(__GNUC__) +# define GGML_RESTRICT __restrict__ +# elif defined(__clang__) +# define GGML_RESTRICT __restrict +# elif defined(_MSC_VER) +# define GGML_RESTRICT __restrict +# else +# define GGML_RESTRICT +# endif +#else +# if defined (_MSC_VER) && (__STDC_VERSION__ < 201112L) +# define GGML_RESTRICT __restrict +# else +# define GGML_RESTRICT restrict +# endif +#endif + typedef void (*ggml_to_float_t) (const void * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); + typedef void (*ggml_from_float_t)(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k); + + struct ggml_type_traits { + const char * type_name; + int64_t blck_size; + int64_t blck_size_interleave; // interleave elements in blocks + size_t type_size; + bool is_quantized; + ggml_to_float_t to_float; + ggml_from_float_t from_float_ref; + }; + + GGML_API const struct ggml_type_traits * ggml_get_type_traits(enum ggml_type type); + + // ggml threadpool + // TODO: currently, only a few functions are in the base ggml API, while the rest are in the CPU backend + // the goal should be to create an API that other backends can use move everything to the ggml base + + // scheduling priorities + enum ggml_sched_priority { + GGML_SCHED_PRIO_LOW = -1, + GGML_SCHED_PRIO_NORMAL, + GGML_SCHED_PRIO_MEDIUM, + GGML_SCHED_PRIO_HIGH, + GGML_SCHED_PRIO_REALTIME + }; + + // threadpool params + // Use ggml_threadpool_params_default() or ggml_threadpool_params_init() to populate the defaults + struct ggml_threadpool_params { + bool cpumask[GGML_MAX_N_THREADS]; // mask of cpu cores (all-zeros means use default affinity settings) + int n_threads; // number of threads + enum ggml_sched_priority prio; // thread priority + uint32_t poll; // polling level (0 - no polling, 100 - aggressive polling) + bool strict_cpu; // strict cpu placement + bool paused; // start in paused state + }; + + struct ggml_threadpool; // forward declaration, see ggml.c + + typedef struct ggml_threadpool * ggml_threadpool_t; + + GGML_API struct ggml_threadpool_params ggml_threadpool_params_default(int n_threads); + GGML_API void ggml_threadpool_params_init (struct ggml_threadpool_params * p, int n_threads); + GGML_API bool ggml_threadpool_params_match (const struct ggml_threadpool_params * p0, const struct ggml_threadpool_params * p1); + +#ifdef __cplusplus +} +#endif diff --git a/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Versions/A/Headers/gguf.h b/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Versions/A/Headers/gguf.h new file mode 100644 index 0000000..02d5f22 --- /dev/null +++ b/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Versions/A/Headers/gguf.h @@ -0,0 +1,204 @@ +// This file contains functionality related to "GGUF" files, the binary file format used by ggml. +// GGUF files have the following structure: +// +// 1. File magic "GGUF" (4 bytes). +// 2. File version (uint32_t). +// 3. Number of ggml tensors in file (int64_t). +// 4. Number of key-value-pairs in file (int64_t). +// 5. For each KV pair: +// 1. The key (string). +// 2. The value type (gguf_type). +// 3a. If the value type is GGUF_TYPE_ARRAY: +// 1. The type of the array (gguf_type). +// 2. The number of elements in the array (uint64_t). +// 3. The binary representation of each element in the array. +// 3b. Otherwise: +// 1. The binary representation of the value. +// 6. For each ggml tensor: +// 1. The tensor name (string). +// 2. The number of dimensions of the tensor (uint32_t). +// 3. For each dimension: +// 1. The size of the tensor in the dimension (int64_t). +// 4. The tensor data type (ggml_type). +// 5. The tensor data offset in the tensor data binary blob (uint64_t). +// 7. The tensor data binary blob (optional, aligned). +// +// Strings are serialized as the string length (uint64_t) followed by the C string without the null terminator. +// All enums are stored as int32_t. +// All bool values are stored as int8_t. +// If the special key "general.alignment" (uint32_t) is defined it is used for alignment, +// otherwise GGUF_DEFAULT_ALIGNMENT is used. +// +// Module maintainer: Johannes Gäßler (@JohannesGaessler, johannesg@5d6.de) + +#pragma once + +#include "ggml.h" + +#include +#include + +#define GGUF_MAGIC "GGUF" +#define GGUF_VERSION 3 + +#define GGUF_KEY_GENERAL_ALIGNMENT "general.alignment" + +#define GGUF_DEFAULT_ALIGNMENT 32 + +#ifdef __cplusplus +extern "C" { +#endif + + // types that can be stored as GGUF KV data + enum gguf_type { + GGUF_TYPE_UINT8 = 0, + GGUF_TYPE_INT8 = 1, + GGUF_TYPE_UINT16 = 2, + GGUF_TYPE_INT16 = 3, + GGUF_TYPE_UINT32 = 4, + GGUF_TYPE_INT32 = 5, + GGUF_TYPE_FLOAT32 = 6, + GGUF_TYPE_BOOL = 7, + GGUF_TYPE_STRING = 8, + GGUF_TYPE_ARRAY = 9, + GGUF_TYPE_UINT64 = 10, + GGUF_TYPE_INT64 = 11, + GGUF_TYPE_FLOAT64 = 12, + GGUF_TYPE_COUNT, // marks the end of the enum + }; + + struct gguf_context; + + struct gguf_init_params { + bool no_alloc; + + // if not NULL, create a ggml_context and allocate the tensor data in it + struct ggml_context ** ctx; + }; + + GGML_API struct gguf_context * gguf_init_empty(void); + GGML_API struct gguf_context * gguf_init_from_file_ptr(FILE * file, struct gguf_init_params params); + GGML_API struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_params params); + //GGML_API struct gguf_context * gguf_init_from_buffer(..); + + GGML_API void gguf_free(struct gguf_context * ctx); + + GGML_API const char * gguf_type_name(enum gguf_type type); + + GGML_API uint32_t gguf_get_version (const struct gguf_context * ctx); + GGML_API size_t gguf_get_alignment (const struct gguf_context * ctx); + GGML_API size_t gguf_get_data_offset(const struct gguf_context * ctx); + + GGML_API int64_t gguf_get_n_kv(const struct gguf_context * ctx); + GGML_API int64_t gguf_find_key(const struct gguf_context * ctx, const char * key); // returns -1 if key is not found + GGML_API const char * gguf_get_key (const struct gguf_context * ctx, int64_t key_id); + + GGML_API enum gguf_type gguf_get_kv_type (const struct gguf_context * ctx, int64_t key_id); + GGML_API enum gguf_type gguf_get_arr_type(const struct gguf_context * ctx, int64_t key_id); + + // will abort if the wrong type is used for the key + GGML_API uint8_t gguf_get_val_u8 (const struct gguf_context * ctx, int64_t key_id); + GGML_API int8_t gguf_get_val_i8 (const struct gguf_context * ctx, int64_t key_id); + GGML_API uint16_t gguf_get_val_u16 (const struct gguf_context * ctx, int64_t key_id); + GGML_API int16_t gguf_get_val_i16 (const struct gguf_context * ctx, int64_t key_id); + GGML_API uint32_t gguf_get_val_u32 (const struct gguf_context * ctx, int64_t key_id); + GGML_API int32_t gguf_get_val_i32 (const struct gguf_context * ctx, int64_t key_id); + GGML_API float gguf_get_val_f32 (const struct gguf_context * ctx, int64_t key_id); + GGML_API uint64_t gguf_get_val_u64 (const struct gguf_context * ctx, int64_t key_id); + GGML_API int64_t gguf_get_val_i64 (const struct gguf_context * ctx, int64_t key_id); + GGML_API double gguf_get_val_f64 (const struct gguf_context * ctx, int64_t key_id); + GGML_API bool gguf_get_val_bool(const struct gguf_context * ctx, int64_t key_id); + GGML_API const char * gguf_get_val_str (const struct gguf_context * ctx, int64_t key_id); + GGML_API const void * gguf_get_val_data(const struct gguf_context * ctx, int64_t key_id); + GGML_API size_t gguf_get_arr_n (const struct gguf_context * ctx, int64_t key_id); + + // get raw pointer to the first element of the array with the given key_id + // for bool arrays, note that they are always stored as int8 on all platforms (usually this makes no difference) + GGML_API const void * gguf_get_arr_data(const struct gguf_context * ctx, int64_t key_id); + + // get ith C string from array with given key_id + GGML_API const char * gguf_get_arr_str (const struct gguf_context * ctx, int64_t key_id, size_t i); + + GGML_API int64_t gguf_get_n_tensors (const struct gguf_context * ctx); + GGML_API int64_t gguf_find_tensor (const struct gguf_context * ctx, const char * name); // returns -1 if the tensor is not found + GGML_API size_t gguf_get_tensor_offset(const struct gguf_context * ctx, int64_t tensor_id); + GGML_API const char * gguf_get_tensor_name (const struct gguf_context * ctx, int64_t tensor_id); + GGML_API enum ggml_type gguf_get_tensor_type (const struct gguf_context * ctx, int64_t tensor_id); + GGML_API size_t gguf_get_tensor_size (const struct gguf_context * ctx, int64_t tensor_id); + + // removes key if it exists, returns id that the key had prior to removal (-1 if it didn't exist) + GGML_API int64_t gguf_remove_key(struct gguf_context * ctx, const char * key); + + // overrides an existing KV pair or adds a new one, the new KV pair is always at the back + GGML_API void gguf_set_val_u8 (struct gguf_context * ctx, const char * key, uint8_t val); + GGML_API void gguf_set_val_i8 (struct gguf_context * ctx, const char * key, int8_t val); + GGML_API void gguf_set_val_u16 (struct gguf_context * ctx, const char * key, uint16_t val); + GGML_API void gguf_set_val_i16 (struct gguf_context * ctx, const char * key, int16_t val); + GGML_API void gguf_set_val_u32 (struct gguf_context * ctx, const char * key, uint32_t val); + GGML_API void gguf_set_val_i32 (struct gguf_context * ctx, const char * key, int32_t val); + GGML_API void gguf_set_val_f32 (struct gguf_context * ctx, const char * key, float val); + GGML_API void gguf_set_val_u64 (struct gguf_context * ctx, const char * key, uint64_t val); + GGML_API void gguf_set_val_i64 (struct gguf_context * ctx, const char * key, int64_t val); + GGML_API void gguf_set_val_f64 (struct gguf_context * ctx, const char * key, double val); + GGML_API void gguf_set_val_bool(struct gguf_context * ctx, const char * key, bool val); + GGML_API void gguf_set_val_str (struct gguf_context * ctx, const char * key, const char * val); + + // creates a new array with n elements of the given type and copies the corresponding number of bytes from data + GGML_API void gguf_set_arr_data(struct gguf_context * ctx, const char * key, enum gguf_type type, const void * data, size_t n); + + // creates a new array with n strings and copies the corresponding strings from data + GGML_API void gguf_set_arr_str (struct gguf_context * ctx, const char * key, const char ** data, size_t n); + + // set or add KV pairs from another context + GGML_API void gguf_set_kv(struct gguf_context * ctx, const struct gguf_context * src); + + // add tensor to GGUF context, tensor name must be unique + GGML_API void gguf_add_tensor(struct gguf_context * ctx, const struct ggml_tensor * tensor); + + // after changing a tensor's type, the offsets of all tensors with higher indices are immediately recalculated + // in such a way that the tensor data remains as one contiguous block (except for padding) + GGML_API void gguf_set_tensor_type(struct gguf_context * ctx, const char * name, enum ggml_type type); + + // assumes that at least gguf_get_tensor_size bytes can be read from data + GGML_API void gguf_set_tensor_data(struct gguf_context * ctx, const char * name, const void * data); + + // writing gguf files can be done in 3 ways: + // + // - write the entire gguf_context to a binary file in a single pass: + // + // gguf_write_to_file(ctx, fname, /*only_meta =*/ false); + // + // - write only the meta data to a file, then re-open the file and append the tensor data: + // + // gguf_write_to_file(ctx, fname, /*only_meta =*/ true); + // FILE * f = fopen(fname, "ab"); + // fwrite(f, ...); // write tensor data + // fclose(f); + // + // - first prepare a file with a placeholder for the meta data, write the tensor data, then write the meta data: + // + // FILE * f = fopen(fname, "wb"); + // const size_t size_meta = gguf_get_meta_size(ctx); + // fseek(f, size_meta, SEEK_SET); + // fwrite(f, ...); // write tensor data + // void * data = malloc(size_meta); + // gguf_get_meta_data(ctx, data); + // rewind(f); + // fwrite(data, 1, data, f); + // free(data); + // fclose(f); + // + + // write the entire context to a binary file + GGML_API bool gguf_write_to_file_ptr(const struct gguf_context * ctx, FILE * file, bool only_meta); + GGML_API bool gguf_write_to_file(const struct gguf_context * ctx, const char * fname, bool only_meta); + + // get the size in bytes of the meta data (header, kv pairs, tensor info) including padding + GGML_API size_t gguf_get_meta_size(const struct gguf_context * ctx); + + // writes the meta data to pointer "data" + GGML_API void gguf_get_meta_data(const struct gguf_context * ctx, void * data); + +#ifdef __cplusplus +} +#endif diff --git a/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Versions/A/Headers/llama.h b/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Versions/A/Headers/llama.h new file mode 100644 index 0000000..308e8ba --- /dev/null +++ b/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Versions/A/Headers/llama.h @@ -0,0 +1,1570 @@ +#ifndef LLAMA_H +#define LLAMA_H + +#include "ggml.h" +#include "ggml-cpu.h" +#include "ggml-backend.h" +#include "ggml-opt.h" +#include "gguf.h" + +#include +#include +#include +#include + +#ifdef LLAMA_SHARED +# if defined(_WIN32) && !defined(__MINGW32__) +# ifdef LLAMA_BUILD +# define LLAMA_API __declspec(dllexport) +# else +# define LLAMA_API __declspec(dllimport) +# endif +# else +# define LLAMA_API __attribute__ ((visibility ("default"))) +# endif +#else +# define LLAMA_API +#endif + +#ifdef __GNUC__ +# define DEPRECATED(func, hint) func __attribute__((deprecated(hint))) +#elif defined(_MSC_VER) +# define DEPRECATED(func, hint) __declspec(deprecated(hint)) func +#else +# define DEPRECATED(func, hint) func +#endif + +#define LLAMA_DEFAULT_SEED 0xFFFFFFFF + +#define LLAMA_TOKEN_NULL -1 + +#define LLAMA_FILE_MAGIC_GGLA 0x67676c61u // 'ggla' +#define LLAMA_FILE_MAGIC_GGSN 0x6767736eu // 'ggsn' +#define LLAMA_FILE_MAGIC_GGSQ 0x67677371u // 'ggsq' + +#define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN +#define LLAMA_SESSION_VERSION 9 + +#define LLAMA_STATE_SEQ_MAGIC LLAMA_FILE_MAGIC_GGSQ +#define LLAMA_STATE_SEQ_VERSION 2 + +#ifdef __cplusplus +extern "C" { +#endif + + // + // C interface + // + // TODO: show sample usage + // + + struct llama_vocab; + struct llama_model; + struct llama_context; + struct llama_sampler; + + typedef struct llama_memory_i * llama_memory_t; + + typedef int32_t llama_pos; + typedef int32_t llama_token; + typedef int32_t llama_seq_id; + + enum llama_vocab_type { + LLAMA_VOCAB_TYPE_NONE = 0, // For models without vocab + LLAMA_VOCAB_TYPE_SPM = 1, // LLaMA tokenizer based on byte-level BPE with byte fallback + LLAMA_VOCAB_TYPE_BPE = 2, // GPT-2 tokenizer based on byte-level BPE + LLAMA_VOCAB_TYPE_WPM = 3, // BERT tokenizer based on WordPiece + LLAMA_VOCAB_TYPE_UGM = 4, // T5 tokenizer based on Unigram + LLAMA_VOCAB_TYPE_RWKV = 5, // RWKV tokenizer based on greedy tokenization + LLAMA_VOCAB_TYPE_PLAMO2 = 6, // PLaMo-2 tokenizer based on Aho-Corasick with dynamic programming + }; + + enum llama_rope_type { + LLAMA_ROPE_TYPE_NONE = -1, + LLAMA_ROPE_TYPE_NORM = 0, + LLAMA_ROPE_TYPE_NEOX = GGML_ROPE_TYPE_NEOX, + LLAMA_ROPE_TYPE_MROPE = GGML_ROPE_TYPE_MROPE, + LLAMA_ROPE_TYPE_IMROPE = GGML_ROPE_TYPE_IMROPE, + LLAMA_ROPE_TYPE_VISION = GGML_ROPE_TYPE_VISION, + }; + + enum llama_token_type { //TODO: remove, required until per token attributes are available from GGUF file + LLAMA_TOKEN_TYPE_UNDEFINED = 0, + LLAMA_TOKEN_TYPE_NORMAL = 1, + LLAMA_TOKEN_TYPE_UNKNOWN = 2, + LLAMA_TOKEN_TYPE_CONTROL = 3, + LLAMA_TOKEN_TYPE_USER_DEFINED = 4, + LLAMA_TOKEN_TYPE_UNUSED = 5, + LLAMA_TOKEN_TYPE_BYTE = 6, + }; + + enum llama_token_attr { + LLAMA_TOKEN_ATTR_UNDEFINED = 0, + LLAMA_TOKEN_ATTR_UNKNOWN = 1 << 0, + LLAMA_TOKEN_ATTR_UNUSED = 1 << 1, + LLAMA_TOKEN_ATTR_NORMAL = 1 << 2, + LLAMA_TOKEN_ATTR_CONTROL = 1 << 3, // SPECIAL? + LLAMA_TOKEN_ATTR_USER_DEFINED = 1 << 4, + LLAMA_TOKEN_ATTR_BYTE = 1 << 5, + LLAMA_TOKEN_ATTR_NORMALIZED = 1 << 6, + LLAMA_TOKEN_ATTR_LSTRIP = 1 << 7, + LLAMA_TOKEN_ATTR_RSTRIP = 1 << 8, + LLAMA_TOKEN_ATTR_SINGLE_WORD = 1 << 9, + }; + + // model file types + enum llama_ftype { + LLAMA_FTYPE_ALL_F32 = 0, + LLAMA_FTYPE_MOSTLY_F16 = 1, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q4_0 = 2, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q4_1 = 3, // except 1d tensors + // LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16 + // LLAMA_FTYPE_MOSTLY_Q4_2 = 5, // support has been removed + // LLAMA_FTYPE_MOSTLY_Q4_3 = 6, // support has been removed + LLAMA_FTYPE_MOSTLY_Q8_0 = 7, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q5_0 = 8, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q5_1 = 9, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q2_K = 10, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q3_K_S = 11, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q3_K_M = 12, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q3_K_L = 13, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q4_K_S = 14, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q4_K_M = 15, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q5_K_S = 16, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q5_K_M = 17, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q6_K = 18, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ2_XXS = 19, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ2_XS = 20, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q2_K_S = 21, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ3_XS = 22, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ3_XXS = 23, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ1_S = 24, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ4_NL = 25, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ3_S = 26, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ3_M = 27, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ2_S = 28, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ2_M = 29, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ4_XS = 30, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ1_M = 31, // except 1d tensors + LLAMA_FTYPE_MOSTLY_BF16 = 32, // except 1d tensors + //LLAMA_FTYPE_MOSTLY_Q4_0_4_4 = 33, // removed from gguf files, use Q4_0 and runtime repack + //LLAMA_FTYPE_MOSTLY_Q4_0_4_8 = 34, // removed from gguf files, use Q4_0 and runtime repack + //LLAMA_FTYPE_MOSTLY_Q4_0_8_8 = 35, // removed from gguf files, use Q4_0 and runtime repack + LLAMA_FTYPE_MOSTLY_TQ1_0 = 36, // except 1d tensors + LLAMA_FTYPE_MOSTLY_TQ2_0 = 37, // except 1d tensors + LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38, // except 1d tensors + LLAMA_FTYPE_MOSTLY_NVFP4 = 39, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q1_0 = 40, // except 1d tensors + + LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file + }; + + enum llama_rope_scaling_type { + LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED = -1, + LLAMA_ROPE_SCALING_TYPE_NONE = 0, + LLAMA_ROPE_SCALING_TYPE_LINEAR = 1, + LLAMA_ROPE_SCALING_TYPE_YARN = 2, + LLAMA_ROPE_SCALING_TYPE_LONGROPE = 3, + LLAMA_ROPE_SCALING_TYPE_MAX_VALUE = LLAMA_ROPE_SCALING_TYPE_LONGROPE, + }; + + enum llama_pooling_type { + LLAMA_POOLING_TYPE_UNSPECIFIED = -1, + LLAMA_POOLING_TYPE_NONE = 0, + LLAMA_POOLING_TYPE_MEAN = 1, + LLAMA_POOLING_TYPE_CLS = 2, + LLAMA_POOLING_TYPE_LAST = 3, + LLAMA_POOLING_TYPE_RANK = 4, // used by reranking models to attach the classification head to the graph + }; + + enum llama_attention_type { + LLAMA_ATTENTION_TYPE_UNSPECIFIED = -1, + LLAMA_ATTENTION_TYPE_CAUSAL = 0, + LLAMA_ATTENTION_TYPE_NON_CAUSAL = 1, + }; + + enum llama_flash_attn_type { + LLAMA_FLASH_ATTN_TYPE_AUTO = -1, + LLAMA_FLASH_ATTN_TYPE_DISABLED = 0, + LLAMA_FLASH_ATTN_TYPE_ENABLED = 1, + }; + + LLAMA_API const char * llama_flash_attn_type_name(enum llama_flash_attn_type flash_attn_type); + + enum llama_split_mode { + LLAMA_SPLIT_MODE_NONE = 0, // single GPU + LLAMA_SPLIT_MODE_LAYER = 1, // split layers and KV across GPUs + LLAMA_SPLIT_MODE_ROW = 2, // split layers and KV across GPUs, use tensor parallelism if supported + LLAMA_SPLIT_MODE_TENSOR = 3, + }; + + // TODO: simplify (https://github.com/ggml-org/llama.cpp/pull/9294#pullrequestreview-2286561979) + typedef struct llama_token_data { + llama_token id; // token id + float logit; // log-odds of the token + float p; // probability of the token + } llama_token_data; + + typedef struct llama_token_data_array { + // TODO: consider SoA + // NOTE: this pointer can be modified by the samplers + llama_token_data * data; + size_t size; + int64_t selected; // this is the index in the data array (i.e. not the token id) + bool sorted; // note: do not assume the data is sorted - always check this flag + } llama_token_data_array; + + typedef bool (*llama_progress_callback)(float progress, void * user_data); + + // Input data for llama_encode/llama_decode + // A llama_batch object can contain input about one or many sequences + // The provided arrays (i.e. token, embd, pos, etc.) must have size of n_tokens + // + // - token : the token ids of the input (used when embd is NULL) + // - embd : token embeddings (i.e. float vector of size n_embd) (used when token is NULL) + // - pos : the positions of the respective token in the sequence + // (if set to NULL, the token position will be tracked automatically by llama_encode/llama_decode) + // - seq_id : the sequence to which the respective token belongs + // (if set to NULL, the sequence ID will be assumed to be 0) + // - logits : if zero, the logits (and/or the embeddings) for the respective token will not be output + // (if set to NULL: + // - if embeddings: all tokens are output + // - if not: only the last token is output + // ) + // + typedef struct llama_batch { + int32_t n_tokens; + + llama_token * token; + float * embd; + llama_pos * pos; + int32_t * n_seq_id; + llama_seq_id ** seq_id; + int8_t * logits; // TODO: rename this to "output" + } llama_batch; + + enum llama_model_kv_override_type { + LLAMA_KV_OVERRIDE_TYPE_INT, + LLAMA_KV_OVERRIDE_TYPE_FLOAT, + LLAMA_KV_OVERRIDE_TYPE_BOOL, + LLAMA_KV_OVERRIDE_TYPE_STR, + }; + + enum llama_model_meta_key { + LLAMA_MODEL_META_KEY_SAMPLING_SEQUENCE, + LLAMA_MODEL_META_KEY_SAMPLING_TOP_K, + LLAMA_MODEL_META_KEY_SAMPLING_TOP_P, + LLAMA_MODEL_META_KEY_SAMPLING_MIN_P, + LLAMA_MODEL_META_KEY_SAMPLING_XTC_PROBABILITY, + LLAMA_MODEL_META_KEY_SAMPLING_XTC_THRESHOLD, + LLAMA_MODEL_META_KEY_SAMPLING_TEMP, + LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_LAST_N, + LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_REPEAT, + LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT, + LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_TAU, + LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_ETA, + }; + + struct llama_model_kv_override { + enum llama_model_kv_override_type tag; + + char key[128]; + + union { + int64_t val_i64; + double val_f64; + bool val_bool; + char val_str[128]; + }; + }; + + struct llama_model_tensor_buft_override { + const char * pattern; + ggml_backend_buffer_type_t buft; + }; + + struct llama_model_params { + // NULL-terminated list of devices to use for offloading (if NULL, all available devices are used) + ggml_backend_dev_t * devices; + + // NULL-terminated list of buffer types to use for tensors that match a pattern + const struct llama_model_tensor_buft_override * tensor_buft_overrides; + + int32_t n_gpu_layers; // number of layers to store in VRAM, a negative value means all layers + enum llama_split_mode split_mode; // how to split the model across multiple GPUs + + // the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONE + int32_t main_gpu; + + // proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices() + const float * tensor_split; + + // Called with a progress value between 0.0 and 1.0. Pass NULL to disable. + // If the provided progress_callback returns true, model loading continues. + // If it returns false, model loading is immediately aborted. + llama_progress_callback progress_callback; + + // context pointer passed to the progress callback + void * progress_callback_user_data; + + // override key-value pairs of the model meta data + const struct llama_model_kv_override * kv_overrides; + + // Keep the booleans together to avoid misalignment during copy-by-value. + bool vocab_only; // only load the vocabulary, no weights + bool use_mmap; // use mmap if possible + bool use_direct_io; // use direct io, takes precedence over use_mmap when supported + bool use_mlock; // force system to keep model in RAM + bool check_tensors; // validate model tensor data + bool use_extra_bufts; // use extra buffer types (used for weight repacking) + bool no_host; // bypass host buffer allowing extra buffers to be used + bool no_alloc; // only load metadata and simulate memory allocations + }; + + struct llama_sampler_seq_config { + llama_seq_id seq_id; + struct llama_sampler * sampler; + }; + + // NOTE: changing the default values of parameters marked as [EXPERIMENTAL] may cause crashes or incorrect results in certain configurations + // https://github.com/ggml-org/llama.cpp/pull/7544 + struct llama_context_params { + uint32_t n_ctx; // text context, 0 = from model + uint32_t n_batch; // logical maximum batch size that can be submitted to llama_decode + uint32_t n_ubatch; // physical maximum batch size + uint32_t n_seq_max; // max number of sequences (i.e. distinct states for recurrent models) + int32_t n_threads; // number of threads to use for generation + int32_t n_threads_batch; // number of threads to use for batch processing + + enum llama_rope_scaling_type rope_scaling_type; // RoPE scaling type, from `enum llama_rope_scaling_type` + enum llama_pooling_type pooling_type; // whether to pool (sum) embedding results by sequence id + enum llama_attention_type attention_type; // attention type to use for embeddings + enum llama_flash_attn_type flash_attn_type; // when to enable Flash Attention + + // ref: https://github.com/ggml-org/llama.cpp/pull/2054 + float rope_freq_base; // RoPE base frequency, 0 = from model + float rope_freq_scale; // RoPE frequency scaling factor, 0 = from model + float yarn_ext_factor; // YaRN extrapolation mix factor, negative = from model + float yarn_attn_factor; // YaRN magnitude scaling factor + float yarn_beta_fast; // YaRN low correction dim + float yarn_beta_slow; // YaRN high correction dim + uint32_t yarn_orig_ctx; // YaRN original context size + float defrag_thold; // [DEPRECATED] defragment the KV cache if holes/size > thold, <= 0 disabled (default) + + ggml_backend_sched_eval_callback cb_eval; + void * cb_eval_user_data; + + enum ggml_type type_k; // data type for K cache [EXPERIMENTAL] + enum ggml_type type_v; // data type for V cache [EXPERIMENTAL] + + // Abort callback + // if it returns true, execution of llama_decode() will be aborted + // currently works only with CPU execution + ggml_abort_callback abort_callback; + void * abort_callback_data; + + // Keep the booleans together and at the end of the struct to avoid misalignment during copy-by-value. + bool embeddings; // if true, extract embeddings (together with logits) + bool offload_kqv; // offload the KQV ops (including the KV cache) to GPU + bool no_perf; // measure performance timings + bool op_offload; // offload host tensor operations to device + bool swa_full; // use full-size SWA cache (https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) + // NOTE: setting to false when n_seq_max > 1 can cause bad performance in some cases + // ref: https://github.com/ggml-org/llama.cpp/pull/13845#issuecomment-2924800573 + bool kv_unified; // use a unified buffer across the input sequences when computing the attention + // try to disable when n_seq_max > 1 for improved performance when the sequences do not share a large prefix + // ref: https://github.com/ggml-org/llama.cpp/pull/14363 + + // [EXPERIMENTAL] + // backend sampler chain configuration (make sure the caller keeps the sampler chains alive) + // note: the samplers must be sampler chains (i.e. use llama_sampler_chain_init) + struct llama_sampler_seq_config * samplers; + size_t n_samplers; + }; + + struct llama_model_tensor_override { + const char * pattern; + enum ggml_type type; + }; + + struct llama_model_imatrix_data { + const char * name; + const float * data; + size_t size; + }; + + // model quantization parameters + typedef struct llama_model_quantize_params { + int32_t nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency() + enum llama_ftype ftype; // quantize to this llama_ftype + enum ggml_type output_tensor_type; // output tensor type + enum ggml_type token_embedding_type; // token embeddings tensor type + bool allow_requantize; // allow quantizing non-f32/f16 tensors + bool quantize_output_tensor; // quantize output.weight + bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored + bool pure; // quantize all tensors to the default type + bool keep_split; // quantize to the same number of shards + bool dry_run; // calculate and show the final quantization size without performing quantization + const struct llama_model_imatrix_data * imatrix; // pointer to importance matrix data + const struct llama_model_kv_override * kv_overrides; // pointer to kv overrides + const struct llama_model_tensor_override * tt_overrides; // pointer to tensor overrides + const int32_t * prune_layers; // pointer to layer indices to prune + } llama_model_quantize_params; + + typedef struct llama_logit_bias { + llama_token token; + float bias; + } llama_logit_bias; + + typedef struct llama_sampler_chain_params { + bool no_perf; // whether to measure performance timings + } llama_sampler_chain_params; + + // used in chat template + typedef struct llama_chat_message { + const char * role; + const char * content; + } llama_chat_message; + + // lora adapter + struct llama_adapter_lora; + + // Helpers for getting default parameters + // TODO: update API to start accepting pointers to params structs (https://github.com/ggml-org/llama.cpp/discussions/9172) + LLAMA_API struct llama_model_params llama_model_default_params(void); + LLAMA_API struct llama_context_params llama_context_default_params(void); + LLAMA_API struct llama_sampler_chain_params llama_sampler_chain_default_params(void); + LLAMA_API struct llama_model_quantize_params llama_model_quantize_default_params(void); + + // Initialize the llama + ggml backend + // If numa is true, use NUMA optimizations + // Call once at the start of the program + LLAMA_API void llama_backend_init(void); + + // Call once at the end of the program - currently only used for MPI + LLAMA_API void llama_backend_free(void); + + //optional: + LLAMA_API void llama_numa_init(enum ggml_numa_strategy numa); + + // Optional: an auto threadpool gets created in ggml if not passed explicitly + LLAMA_API void llama_attach_threadpool( + struct llama_context * ctx, + ggml_threadpool_t threadpool, + ggml_threadpool_t threadpool_batch); + + LLAMA_API void llama_detach_threadpool(struct llama_context * ctx); + + typedef void (*llama_model_set_tensor_data_t)(struct ggml_tensor * tensor, void * userdata); + + // Create a new model from GGUF metadata as well as a function to set the tensor data + // - tensors are created as GGML_TYPE_F32 by default, + // override by adding a tensor with the same name but a different name to the context + LLAMA_API struct llama_model * llama_model_init_from_user( + struct gguf_context * metadata, + llama_model_set_tensor_data_t set_tensor_data, // function to initialize tensor data with + void * set_tensor_data_ud, // userdata for function + struct llama_model_params params); + + DEPRECATED(LLAMA_API struct llama_model * llama_load_model_from_file( + const char * path_model, + struct llama_model_params params), + "use llama_model_load_from_file instead"); + + // Load a model from a file + // If the file is split into multiple parts, the file name must follow this pattern: -%05d-of-%05d.gguf + // If the split file name does not follow this pattern, use llama_model_load_from_splits + LLAMA_API struct llama_model * llama_model_load_from_file( + const char * path_model, + struct llama_model_params params); + + // Load a model from an open FILE pointer + LLAMA_API struct llama_model * llama_model_load_from_file_ptr( + FILE * file, + struct llama_model_params params); + + // Load a model from multiple splits (support custom naming scheme) + // The paths must be in the correct order + LLAMA_API struct llama_model * llama_model_load_from_splits( + const char ** paths, + size_t n_paths, + struct llama_model_params params); + + LLAMA_API void llama_model_save_to_file( + const struct llama_model * model, + const char * path_model); + + DEPRECATED(LLAMA_API void llama_free_model(struct llama_model * model), + "use llama_model_free instead"); + + LLAMA_API void llama_model_free(struct llama_model * model); + + LLAMA_API struct llama_context * llama_init_from_model( + struct llama_model * model, + struct llama_context_params params); + + DEPRECATED(LLAMA_API struct llama_context * llama_new_context_with_model( + struct llama_model * model, + struct llama_context_params params), + "use llama_init_from_model instead"); + + // Frees all allocated memory + LLAMA_API void llama_free(struct llama_context * ctx); + + LLAMA_API int64_t llama_time_us(void); + + LLAMA_API size_t llama_max_devices(void); + LLAMA_API size_t llama_max_parallel_sequences(void); + LLAMA_API size_t llama_max_tensor_buft_overrides(void); + + LLAMA_API bool llama_supports_mmap (void); + LLAMA_API bool llama_supports_mlock (void); + LLAMA_API bool llama_supports_gpu_offload(void); + LLAMA_API bool llama_supports_rpc (void); + + // NOTE: After creating a llama_context, it is recommended to query the actual values using these functions + // In some cases the requested values via llama_context_params may differ from the actual values used by the context + // ref: https://github.com/ggml-org/llama.cpp/pull/17046#discussion_r2503085732 + LLAMA_API uint32_t llama_n_ctx (const struct llama_context * ctx); + LLAMA_API uint32_t llama_n_ctx_seq (const struct llama_context * ctx); + LLAMA_API uint32_t llama_n_batch (const struct llama_context * ctx); + LLAMA_API uint32_t llama_n_ubatch (const struct llama_context * ctx); + LLAMA_API uint32_t llama_n_seq_max (const struct llama_context * ctx); + + DEPRECATED(LLAMA_API int32_t llama_n_ctx_train(const struct llama_model * model), "use llama_model_n_ctx_train instead"); + DEPRECATED(LLAMA_API int32_t llama_n_embd (const struct llama_model * model), "use llama_model_n_embd instead"); + DEPRECATED(LLAMA_API int32_t llama_n_layer (const struct llama_model * model), "use llama_model_n_layer instead"); + DEPRECATED(LLAMA_API int32_t llama_n_head (const struct llama_model * model), "use llama_model_n_head instead"); + + DEPRECATED(LLAMA_API int32_t llama_n_vocab (const struct llama_vocab * vocab), "use llama_vocab_n_tokens instead"); + + LLAMA_API const struct llama_model * llama_get_model (const struct llama_context * ctx); + LLAMA_API llama_memory_t llama_get_memory (const struct llama_context * ctx); + LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx); // TODO: rename to llama_get_pooling_type + + LLAMA_API const struct llama_vocab * llama_model_get_vocab(const struct llama_model * model); + LLAMA_API enum llama_rope_type llama_model_rope_type(const struct llama_model * model); + + LLAMA_API int32_t llama_model_n_ctx_train(const struct llama_model * model); + LLAMA_API int32_t llama_model_n_embd (const struct llama_model * model); + LLAMA_API int32_t llama_model_n_embd_inp (const struct llama_model * model); + LLAMA_API int32_t llama_model_n_embd_out (const struct llama_model * model); + LLAMA_API int32_t llama_model_n_layer (const struct llama_model * model); + LLAMA_API int32_t llama_model_n_head (const struct llama_model * model); + LLAMA_API int32_t llama_model_n_head_kv (const struct llama_model * model); + LLAMA_API int32_t llama_model_n_swa (const struct llama_model * model); + + // Get the model's RoPE frequency scaling factor + LLAMA_API float llama_model_rope_freq_scale_train(const struct llama_model * model); + + // Returns the number of classifier outputs (only valid for classifier models) + // Undefined behavior for non-classifier models + LLAMA_API uint32_t llama_model_n_cls_out(const struct llama_model * model); + + // Returns label of classifier output by index ( 1` + // p0 < 0 : [0, p1] + // p1 < 0 : [p0, inf) + LLAMA_API void llama_memory_seq_div( + llama_memory_t mem, + llama_seq_id seq_id, + llama_pos p0, + llama_pos p1, + int d); + + // Returns the smallest position present in the memory for the specified sequence + // This is typically non-zero only for SWA caches + // Note that all positions in the range [pos_min, pos_max] are guaranteed to be present in the memory + // Return -1 if the sequence is empty + LLAMA_API llama_pos llama_memory_seq_pos_min( + llama_memory_t mem, + llama_seq_id seq_id); + + // Returns the largest position present in the memory for the specified sequence + // Note that all positions in the range [pos_min, pos_max] are guaranteed to be present in the memory + // Return -1 if the sequence is empty + LLAMA_API llama_pos llama_memory_seq_pos_max( + llama_memory_t mem, + llama_seq_id seq_id); + + // Check if the memory supports shifting + LLAMA_API bool llama_memory_can_shift(llama_memory_t mem); + + // + // State / sessions + // + + // Returns the *actual* size in bytes of the state + // (logits, embedding and memory) + // Only use when saving the state, not when restoring it, otherwise the size may be too small. + LLAMA_API size_t llama_state_get_size(struct llama_context * ctx); + LLAMA_API DEPRECATED(size_t llama_get_state_size(struct llama_context * ctx), + "use llama_state_get_size instead"); + + // Copies the state to the specified destination address. + // Destination needs to have allocated enough memory. + // Returns the number of bytes copied + LLAMA_API size_t llama_state_get_data( + struct llama_context * ctx, + uint8_t * dst, + size_t size); + LLAMA_API DEPRECATED(size_t llama_copy_state_data( + struct llama_context * ctx, + uint8_t * dst), + "use llama_state_get_data instead"); + + // Set the state reading from the specified address + // Returns the number of bytes read + LLAMA_API size_t llama_state_set_data( + struct llama_context * ctx, + const uint8_t * src, + size_t size); + LLAMA_API DEPRECATED(size_t llama_set_state_data( + struct llama_context * ctx, + const uint8_t * src), + "use llama_state_set_data instead"); + + // Save/load session file + LLAMA_API bool llama_state_load_file( + struct llama_context * ctx, + const char * path_session, + llama_token * tokens_out, + size_t n_token_capacity, + size_t * n_token_count_out); + LLAMA_API DEPRECATED(bool llama_load_session_file( + struct llama_context * ctx, + const char * path_session, + llama_token * tokens_out, + size_t n_token_capacity, + size_t * n_token_count_out), + "use llama_state_load_file instead"); + + LLAMA_API bool llama_state_save_file( + struct llama_context * ctx, + const char * path_session, + const llama_token * tokens, + size_t n_token_count); + LLAMA_API DEPRECATED(bool llama_save_session_file( + struct llama_context * ctx, + const char * path_session, + const llama_token * tokens, + size_t n_token_count), + "use llama_state_save_file instead"); + + // Get the exact size needed to copy the state of a single sequence + LLAMA_API size_t llama_state_seq_get_size( + struct llama_context * ctx, + llama_seq_id seq_id); + + // Copy the state of a single sequence into the specified buffer + LLAMA_API size_t llama_state_seq_get_data( + struct llama_context * ctx, + uint8_t * dst, + size_t size, + llama_seq_id seq_id); + + // Copy the sequence data (originally copied with `llama_state_seq_get_data`) into the specified sequence + // Returns: + // - Positive: Ok + // - Zero: Failed to load + LLAMA_API size_t llama_state_seq_set_data( + struct llama_context * ctx, + const uint8_t * src, + size_t size, + llama_seq_id dest_seq_id); + + LLAMA_API size_t llama_state_seq_save_file( + struct llama_context * ctx, + const char * filepath, + llama_seq_id seq_id, + const llama_token * tokens, + size_t n_token_count); + + LLAMA_API size_t llama_state_seq_load_file( + struct llama_context * ctx, + const char * filepath, + llama_seq_id dest_seq_id, + llama_token * tokens_out, + size_t n_token_capacity, + size_t * n_token_count_out); + +#define LLAMA_STATE_SEQ_FLAGS_NONE 0 + +// for backwards-compat +#define LLAMA_STATE_SEQ_FLAGS_SWA_ONLY 1 + +// work only with partial states, such as SWA KV cache or recurrent cache (e.g. Mamba) +#define LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY 1 + +// keeps the tensor data on device buffers (i.e. not accessible in host memory, but faster save/load) +#define LLAMA_STATE_SEQ_FLAGS_ON_DEVICE 2 + + typedef uint32_t llama_state_seq_flags; + + LLAMA_API size_t llama_state_seq_get_size_ext( + struct llama_context * ctx, + llama_seq_id seq_id, + llama_state_seq_flags flags); + + LLAMA_API size_t llama_state_seq_get_data_ext( + struct llama_context * ctx, + uint8_t * dst, + size_t size, + llama_seq_id seq_id, + llama_state_seq_flags flags); + + LLAMA_API size_t llama_state_seq_set_data_ext( + struct llama_context * ctx, + const uint8_t * src, + size_t size, + llama_seq_id dest_seq_id, + llama_state_seq_flags flags); + + // + // Decoding + // + + // Return batch for single sequence of tokens + // The sequence ID will be fixed to 0 + // The position of the tokens will be tracked automatically by llama_decode + // + // NOTE: this is a helper function to facilitate transition to the new batch API - avoid using it + // + LLAMA_API struct llama_batch llama_batch_get_one( + llama_token * tokens, + int32_t n_tokens); + + // Allocates a batch of tokens on the heap that can hold a maximum of n_tokens + // Each token can be assigned up to n_seq_max sequence ids + // The batch has to be freed with llama_batch_free() + // If embd != 0, llama_batch.embd will be allocated with size of n_tokens * embd * sizeof(float) + // Otherwise, llama_batch.token will be allocated to store n_tokens llama_token + // The rest of the llama_batch members are allocated with size n_tokens + // All members are left uninitialized + LLAMA_API struct llama_batch llama_batch_init( + int32_t n_tokens, + int32_t embd, + int32_t n_seq_max); + + // Frees a batch of tokens allocated with llama_batch_init() + LLAMA_API void llama_batch_free(struct llama_batch batch); + + // Process a batch of tokens. + // In contrast to llama_decode() - this call does not use KV cache. + // For encode-decoder contexts, processes the batch using the encoder. + // Can store the encoder output internally for later use by the decoder's cross-attention layers. + // 0 - success + // < 0 - error. the memory state is restored to the state before this call + LLAMA_API int32_t llama_encode( + struct llama_context * ctx, + struct llama_batch batch); + + // Process a batch of tokens. + // Requires the context to have a memory. + // For encode-decoder contexts, processes the batch using the decoder. + // Positive return values does not mean a fatal error, but rather a warning. + // Upon fatal-error or abort, the ubatches that managed to be been processed will remain in the memory state of the context + // To handle this correctly, query the memory state using llama_memory_seq_pos_min() and llama_memory_seq_pos_max() + // Upon other return values, the memory state is restored to the state before this call + // 0 - success + // 1 - could not find a KV slot for the batch (try reducing the size of the batch or increase the context) + // 2 - aborted (processed ubatches will remain in the context's memory) + // -1 - invalid input batch + // < -1 - fatal error (processed ubatches will remain in the context's memory) + LLAMA_API int32_t llama_decode( + struct llama_context * ctx, + struct llama_batch batch); + + // Set the number of threads used for decoding + // n_threads is the number of threads used for generation (single token) + // n_threads_batch is the number of threads used for prompt and batch processing (multiple tokens) + LLAMA_API void llama_set_n_threads(struct llama_context * ctx, int32_t n_threads, int32_t n_threads_batch); + + // Get the number of threads used for generation of a single token. + LLAMA_API int32_t llama_n_threads(struct llama_context * ctx); + + // Get the number of threads used for prompt and batch processing (multiple token). + LLAMA_API int32_t llama_n_threads_batch(struct llama_context * ctx); + + // Set whether the context outputs embeddings or not + // TODO: rename to avoid confusion with llama_get_embeddings() + LLAMA_API void llama_set_embeddings(struct llama_context * ctx, bool embeddings); + + // Set whether to use causal attention or not + // If set to true, the model will only attend to the past tokens + LLAMA_API void llama_set_causal_attn(struct llama_context * ctx, bool causal_attn); + + // Set whether the model is in warmup mode or not + // If true, all model tensors are activated during llama_decode() to load and cache their weights. + LLAMA_API void llama_set_warmup(struct llama_context * ctx, bool warmup); + + // Set abort callback + LLAMA_API void llama_set_abort_callback(struct llama_context * ctx, ggml_abort_callback abort_callback, void * abort_callback_data); + + // Wait until all computations are finished + // This is automatically done when using one of the functions below to obtain the computation results + // and is not necessary to call it explicitly in most cases + LLAMA_API void llama_synchronize(struct llama_context * ctx); + + // Token logits obtained from the last call to llama_decode() + // The logits for which llama_batch.logits[i] != 0 are stored contiguously + // in the order they have appeared in the batch. + // Rows: number of tokens for which llama_batch.logits[i] != 0 + // Cols: n_vocab + // TODO: deprecate in favor of llama_get_logits_ith() (ref: https://github.com/ggml-org/llama.cpp/pull/14853#issuecomment-3113143522) + LLAMA_API float * llama_get_logits(struct llama_context * ctx); + + // Logits for the ith token. For positive indices, Equivalent to: + // llama_get_logits(ctx) + ctx->output_ids[i]*n_vocab + // Negative indices can be used to access logits in reverse order, -1 is the last logit. + // returns NULL for invalid ids. + LLAMA_API float * llama_get_logits_ith(struct llama_context * ctx, int32_t i); + + // Get all output token embeddings. + // when pooling_type == LLAMA_POOLING_TYPE_NONE or when using a generative model, + // the embeddings for which llama_batch.logits[i] != 0 are stored contiguously + // in the order they have appeared in the batch. + // shape: [n_outputs*n_embd] + // Otherwise, returns NULL. + // TODO: deprecate in favor of llama_get_embeddings_ith() (ref: https://github.com/ggml-org/llama.cpp/pull/14853#issuecomment-3113143522) + LLAMA_API float * llama_get_embeddings(struct llama_context * ctx); + + // Get the embeddings for the ith token. For positive indices, Equivalent to: + // llama_get_embeddings(ctx) + ctx->output_ids[i]*n_embd + // Negative indices can be used to access embeddings in reverse order, -1 is the last embedding. + // shape: [n_embd] (1-dimensional) + // returns NULL for invalid ids. + LLAMA_API float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i); + + // Get the embeddings for a sequence id + // Returns NULL if pooling_type is LLAMA_POOLING_TYPE_NONE + // when pooling_type == LLAMA_POOLING_TYPE_RANK, returns float[n_cls_out] with the rank(s) of the sequence + // otherwise: float[n_embd] (1-dimensional) + LLAMA_API float * llama_get_embeddings_seq(struct llama_context * ctx, llama_seq_id seq_id); + + // + // backend sampling API [EXPERIMENTAL] + // note: use only if the llama_context was created with at least one llama_sampler_seq_config + // + + // Get the backend sampled token for the ith token. + // Returns LLAMA_TOKEN_NULL if no token was sampled. + LLAMA_API llama_token llama_get_sampled_token_ith(struct llama_context * ctx, int32_t i); + + // Get the backend sampled probabilities for the ith token + // The index matches llama_get_sampled_token_ith(). + // Returns NULL if no probabilities were generated. + LLAMA_API float * llama_get_sampled_probs_ith (struct llama_context * ctx, int32_t i); + LLAMA_API uint32_t llama_get_sampled_probs_count_ith(struct llama_context * ctx, int32_t i); + + // Get the backend sampled logits for the ith token + // Returns NULL if no logits were sampled. + LLAMA_API float * llama_get_sampled_logits_ith (struct llama_context * ctx, int32_t i); + LLAMA_API uint32_t llama_get_sampled_logits_count_ith(struct llama_context * ctx, int32_t i); + + // Get the backend sampled candidates (token ids) for the ith token + // These are needed to map probability/logit indices to vocab token ids. + // Returns NULL if no candidates were sampled. + LLAMA_API llama_token * llama_get_sampled_candidates_ith (struct llama_context * ctx, int32_t i); + LLAMA_API uint32_t llama_get_sampled_candidates_count_ith(struct llama_context * ctx, int32_t i); + + // + // Vocab + // + + LLAMA_API const char * llama_vocab_get_text(const struct llama_vocab * vocab, llama_token token); + + LLAMA_API float llama_vocab_get_score(const struct llama_vocab * vocab, llama_token token); + + LLAMA_API enum llama_token_attr llama_vocab_get_attr(const struct llama_vocab * vocab, llama_token token); + + // Check if the token is supposed to end generation (end-of-generation, eg. EOS, EOT, etc.) + LLAMA_API bool llama_vocab_is_eog(const struct llama_vocab * vocab, llama_token token); + + // Identify if Token Id is a control token or a render-able token + LLAMA_API bool llama_vocab_is_control(const struct llama_vocab * vocab, llama_token token); + + // Special tokens + LLAMA_API llama_token llama_vocab_bos(const struct llama_vocab * vocab); // beginning-of-sentence + LLAMA_API llama_token llama_vocab_eos(const struct llama_vocab * vocab); // end-of-sentence + LLAMA_API llama_token llama_vocab_eot(const struct llama_vocab * vocab); // end-of-turn + LLAMA_API llama_token llama_vocab_sep(const struct llama_vocab * vocab); // sentence separator + LLAMA_API llama_token llama_vocab_nl (const struct llama_vocab * vocab); // next-line + LLAMA_API llama_token llama_vocab_pad(const struct llama_vocab * vocab); // padding + LLAMA_API llama_token llama_vocab_mask(const struct llama_vocab * vocab); // mask + + LLAMA_API bool llama_vocab_get_add_bos(const struct llama_vocab * vocab); + LLAMA_API bool llama_vocab_get_add_eos(const struct llama_vocab * vocab); + LLAMA_API bool llama_vocab_get_add_sep(const struct llama_vocab * vocab); + + LLAMA_API llama_token llama_vocab_fim_pre(const struct llama_vocab * vocab); + LLAMA_API llama_token llama_vocab_fim_suf(const struct llama_vocab * vocab); + LLAMA_API llama_token llama_vocab_fim_mid(const struct llama_vocab * vocab); + LLAMA_API llama_token llama_vocab_fim_pad(const struct llama_vocab * vocab); + LLAMA_API llama_token llama_vocab_fim_rep(const struct llama_vocab * vocab); + LLAMA_API llama_token llama_vocab_fim_sep(const struct llama_vocab * vocab); + + DEPRECATED(LLAMA_API const char * llama_token_get_text(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_text instead"); + DEPRECATED(LLAMA_API float llama_token_get_score(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_score instead"); + DEPRECATED(LLAMA_API enum llama_token_attr llama_token_get_attr(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_attr instead"); + DEPRECATED(LLAMA_API bool llama_token_is_eog(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_is_eog instead"); + DEPRECATED(LLAMA_API bool llama_token_is_control(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_is_control instead"); + DEPRECATED(LLAMA_API llama_token llama_token_bos(const struct llama_vocab * vocab), "use llama_vocab_bos instead"); + DEPRECATED(LLAMA_API llama_token llama_token_eos(const struct llama_vocab * vocab), "use llama_vocab_eos instead"); + DEPRECATED(LLAMA_API llama_token llama_token_eot(const struct llama_vocab * vocab), "use llama_vocab_eot instead"); + DEPRECATED(LLAMA_API llama_token llama_token_cls(const struct llama_vocab * vocab), "use llama_vocab_cls instead"); + DEPRECATED(LLAMA_API llama_token llama_token_sep(const struct llama_vocab * vocab), "use llama_vocab_sep instead"); + DEPRECATED(LLAMA_API llama_token llama_token_nl (const struct llama_vocab * vocab), "use llama_vocab_nl instead"); + DEPRECATED(LLAMA_API llama_token llama_token_pad(const struct llama_vocab * vocab), "use llama_vocab_pad instead"); + DEPRECATED(LLAMA_API bool llama_add_bos_token(const struct llama_vocab * vocab), "use llama_vocab_get_add_bos instead"); + DEPRECATED(LLAMA_API bool llama_add_eos_token(const struct llama_vocab * vocab), "use llama_vocab_get_add_eos instead"); + DEPRECATED(LLAMA_API llama_token llama_token_fim_pre(const struct llama_vocab * vocab), "use llama_vocab_fim_pre instead"); + DEPRECATED(LLAMA_API llama_token llama_token_fim_suf(const struct llama_vocab * vocab), "use llama_vocab_fim_suf instead"); + DEPRECATED(LLAMA_API llama_token llama_token_fim_mid(const struct llama_vocab * vocab), "use llama_vocab_fim_mid instead"); + DEPRECATED(LLAMA_API llama_token llama_token_fim_pad(const struct llama_vocab * vocab), "use llama_vocab_fim_pad instead"); + DEPRECATED(LLAMA_API llama_token llama_token_fim_rep(const struct llama_vocab * vocab), "use llama_vocab_fim_rep instead"); + DEPRECATED(LLAMA_API llama_token llama_token_fim_sep(const struct llama_vocab * vocab), "use llama_vocab_fim_sep instead"); + + // CLS is equivalent to BOS + DEPRECATED(LLAMA_API llama_token llama_vocab_cls(const struct llama_vocab * vocab), // classification + "use llama_vocab_bos instead"); + + // + // Tokenization + // + // The API is thread-safe. + // + + /// @details Convert the provided text into tokens. + /// @param tokens The tokens pointer must be large enough to hold the resulting tokens. + /// @return Returns the number of tokens on success, no more than n_tokens_max + /// @return Returns a negative number on failure - the number of tokens that would have been returned + /// @return Returns INT32_MIN on overflow (e.g., tokenization result size exceeds int32_t limit) + /// @param add_special Allow to add BOS and EOS tokens if model is configured to do so. + /// @param parse_special Allow tokenizing special and/or control tokens which otherwise are not exposed and treated + /// as plaintext. Does not insert a leading space. + LLAMA_API int32_t llama_tokenize( + const struct llama_vocab * vocab, + const char * text, + int32_t text_len, + llama_token * tokens, + int32_t n_tokens_max, + bool add_special, + bool parse_special); + + // Token Id -> Piece. + // Uses the vocabulary in the provided context. + // Does not write null terminator to the buffer. + // User can skip up to 'lstrip' leading spaces before copying (useful when encoding/decoding multiple tokens with 'add_space_prefix') + // @param special If true, special tokens are rendered in the output. + LLAMA_API int32_t llama_token_to_piece( + const struct llama_vocab * vocab, + llama_token token, + char * buf, + int32_t length, + int32_t lstrip, + bool special); + + /// @details Convert the provided tokens into text (inverse of llama_tokenize()). + /// @param text The char pointer must be large enough to hold the resulting text. + /// @return Returns the number of chars/bytes on success, no more than text_len_max. + /// @return Returns a negative number on failure - the number of chars/bytes that would have been returned. + /// @param remove_special Allow to remove BOS and EOS tokens if model is configured to do so. + /// @param unparse_special If true, special tokens are rendered in the output. + LLAMA_API int32_t llama_detokenize( + const struct llama_vocab * vocab, + const llama_token * tokens, + int32_t n_tokens, + char * text, + int32_t text_len_max, + bool remove_special, + bool unparse_special); + + // + // Chat templates + // + + /// Apply chat template. Inspired by hf apply_chat_template() on python. + /// + /// NOTE: This function does not use a jinja parser. It only support a pre-defined list of template. See more: https://github.com/ggml-org/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template + /// @param tmpl A Jinja template to use for this chat. + /// @param chat Pointer to a list of multiple llama_chat_message + /// @param n_msg Number of llama_chat_message in this chat + /// @param add_ass Whether to end the prompt with the token(s) that indicate the start of an assistant message. + /// @param buf A buffer to hold the output formatted prompt. The recommended alloc size is 2 * (total number of characters of all messages) + /// @param length The size of the allocated buffer + /// @return The total number of bytes of the formatted prompt. If is it larger than the size of buffer, you may need to re-alloc it and then re-apply the template. + LLAMA_API int32_t llama_chat_apply_template( + const char * tmpl, + const struct llama_chat_message * chat, + size_t n_msg, + bool add_ass, + char * buf, + int32_t length); + + // Get list of built-in chat templates + LLAMA_API int32_t llama_chat_builtin_templates(const char ** output, size_t len); + + // + // Sampling API + // + // Sample usage: + // + // // prepare the sampling chain at the start + // auto sparams = llama_sampler_chain_default_params(); + // + // llama_sampler * smpl = llama_sampler_chain_init(sparams); + // + // llama_sampler_chain_add(smpl, llama_sampler_init_top_k(50)); + // llama_sampler_chain_add(smpl, llama_sampler_init_top_p(0.9, 1)); + // llama_sampler_chain_add(smpl, llama_sampler_init_temp (0.8)); + // + // // typically, the chain should end with a sampler such as "greedy", "dist" or "mirostat" + // // this sampler will be responsible to select the actual token + // llama_sampler_chain_add(smpl, llama_sampler_init_dist(seed)); + // + // ... + // + // // decoding loop: + // while (...) { + // ... + // + // llama_decode(ctx, batch); + // + // // sample from the logits of the last token in the batch + // const llama_token id = llama_sampler_sample(smpl, ctx, -1); + // + // ... + // } + // + // llama_sampler_free(smpl); + // + + typedef void * llama_sampler_context_t; + + struct llama_sampler_data { + struct ggml_tensor * logits; + struct ggml_tensor * probs; + struct ggml_tensor * sampled; + struct ggml_tensor * candidates; + }; + + // user code can implement the interface below in order to create custom llama_sampler + struct llama_sampler_i { + const char * (*name) (const struct llama_sampler * smpl); // can be NULL + void (*accept)( struct llama_sampler * smpl, llama_token token); // can be NULL + void (*apply) ( struct llama_sampler * smpl, llama_token_data_array * cur_p); // required + void (*reset) ( struct llama_sampler * smpl); // can be NULL + struct llama_sampler * (*clone) (const struct llama_sampler * smpl); // can be NULL if ctx is NULL + void (*free) ( struct llama_sampler * smpl); // can be NULL if ctx is NULL + + // [EXPERIMENTAL] + // backend sampling interface: + + // return true if the backend supports all ops needed by the sampler + // note: call once per sampler + bool (*backend_init)(struct llama_sampler * smpl, ggml_backend_buffer_type_t buft); + + // call after .backend_apply() + void (*backend_accept)( + struct llama_sampler * smpl, + struct ggml_context * ctx, + struct ggml_cgraph * gf, + struct ggml_tensor * selected_token); + + // call after .backend_init() + void (*backend_apply)( + struct llama_sampler * smpl, + struct ggml_context * ctx, + struct ggml_cgraph * gf, + struct llama_sampler_data * data); + + // called before graph execution to set inputs for the current ubatch + void (*backend_set_input)(struct llama_sampler * smpl); + }; + + struct llama_sampler { + struct llama_sampler_i * iface; + + llama_sampler_context_t ctx; + }; + + // [EXPERIMENTAL] + // attach a sampler to the context + // note: prefer initializing the context with llama_context_params.samplers when possible + LLAMA_API bool llama_set_sampler(struct llama_context * ctx, llama_seq_id seq_id, struct llama_sampler * smpl); + + // mirror of llama_sampler_i: + LLAMA_API struct llama_sampler * llama_sampler_init ( struct llama_sampler_i * iface, llama_sampler_context_t ctx); + LLAMA_API const char * llama_sampler_name (const struct llama_sampler * smpl); + LLAMA_API void llama_sampler_accept( struct llama_sampler * smpl, llama_token token); + LLAMA_API void llama_sampler_apply ( struct llama_sampler * smpl, llama_token_data_array * cur_p); + LLAMA_API void llama_sampler_reset ( struct llama_sampler * smpl); + LLAMA_API struct llama_sampler * llama_sampler_clone (const struct llama_sampler * smpl); + // important: do not free if the sampler has been added to a llama_sampler_chain (via llama_sampler_chain_add) + LLAMA_API void llama_sampler_free ( struct llama_sampler * smpl); + + // llama_sampler_chain + // a type of llama_sampler that can chain multiple samplers one after another + + LLAMA_API struct llama_sampler * llama_sampler_chain_init(struct llama_sampler_chain_params params); + + // important: takes ownership of the sampler object and will free it when llama_sampler_free is called + LLAMA_API void llama_sampler_chain_add( struct llama_sampler * chain, struct llama_sampler * smpl); + + // return NULL if: + // - the sampler is NULL + // - the sampler is not a llama_sampler_chain + // - the index is out of bounds, unless i == -1 + // - if i == -1, returns the chain itself (can be used to check if the sampler is a chain) + LLAMA_API struct llama_sampler * llama_sampler_chain_get( struct llama_sampler * chain, int32_t i); + + // the total number of samplers in the chain + LLAMA_API int llama_sampler_chain_n (const struct llama_sampler * chain); + + // after removing a sampler, the chain will no longer own it, and it will not be freed when the chain is freed + LLAMA_API struct llama_sampler * llama_sampler_chain_remove( struct llama_sampler * chain, int32_t i); + + // available samplers: + + LLAMA_API struct llama_sampler * llama_sampler_init_greedy(void); + + /// seed == LLAMA_DEFAULT_SEED to use a random seed. + LLAMA_API struct llama_sampler * llama_sampler_init_dist(uint32_t seed); + + /// @details Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 + /// Setting k <= 0 makes this a noop + LLAMA_API struct llama_sampler * llama_sampler_init_top_k (int32_t k); + + /// @details Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751 + LLAMA_API struct llama_sampler * llama_sampler_init_top_p (float p, size_t min_keep); + + /// @details Minimum P sampling as described in https://github.com/ggml-org/llama.cpp/pull/3841 + LLAMA_API struct llama_sampler * llama_sampler_init_min_p (float p, size_t min_keep); + + /// @details Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666. + LLAMA_API struct llama_sampler * llama_sampler_init_typical (float p, size_t min_keep); + + /// #details Updates the logits l_i` = l_i/t. When t <= 0.0f, the maximum logit is kept at it's original value, the rest are set to -inf + LLAMA_API struct llama_sampler * llama_sampler_init_temp (float t); + + /// @details Dynamic temperature implementation (a.k.a. entropy) described in the paper https://arxiv.org/abs/2309.02772. + LLAMA_API struct llama_sampler * llama_sampler_init_temp_ext (float t, float delta, float exponent); + + /// @details XTC sampler as described in https://github.com/oobabooga/text-generation-webui/pull/6335 + LLAMA_API struct llama_sampler * llama_sampler_init_xtc (float p, float t, size_t min_keep, uint32_t seed); + + /// @details Top n sigma sampling as described in academic paper "Top-nσ: Not All Logits Are You Need" https://arxiv.org/pdf/2411.07641 + LLAMA_API struct llama_sampler * llama_sampler_init_top_n_sigma(float n); + + /// @details Mirostat 1.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. + /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. + /// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. + /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates. + /// @param m The number of tokens considered in the estimation of `s_hat`. This is an arbitrary value that is used to calculate `s_hat`, which in turn helps to calculate the value of `k`. In the paper, they use `m = 100`, but you can experiment with different values to see how it affects the performance of the algorithm. + /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal. + LLAMA_API struct llama_sampler * llama_sampler_init_mirostat( + int32_t n_vocab, + uint32_t seed, + float tau, + float eta, + int32_t m); + + /// @details Mirostat 2.0 algorithm described in the paper https://arxiv.org/abs/2007.14966. Uses tokens instead of words. + /// @param candidates A vector of `llama_token_data` containing the candidate tokens, their probabilities (p), and log-odds (logit) for the current position in the generated text. + /// @param tau The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text. + /// @param eta The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates. + /// @param mu Maximum cross-entropy. This value is initialized to be twice the target cross-entropy (`2 * tau`) and is updated in the algorithm based on the error between the target and observed surprisal. + LLAMA_API struct llama_sampler * llama_sampler_init_mirostat_v2( + uint32_t seed, + float tau, + float eta); + + /// @details Initializes a GBNF grammar, see grammars/README.md for details. + /// @param vocab The vocabulary that this grammar will be used with. + /// @param grammar_str The production rules for the grammar, encoded as a string. Returns an empty grammar if empty. Returns NULL if parsing of grammar_str fails. + /// @param grammar_root The name of the start symbol for the grammar. + LLAMA_API struct llama_sampler * llama_sampler_init_grammar( + const struct llama_vocab * vocab, + const char * grammar_str, + const char * grammar_root); + + DEPRECATED(LLAMA_API struct llama_sampler * llama_sampler_init_grammar_lazy( + const struct llama_vocab * vocab, + const char * grammar_str, + const char * grammar_root, + const char ** trigger_words, + size_t num_trigger_words, + const llama_token * trigger_tokens, + size_t num_trigger_tokens), + "use llama_sampler_init_grammar_lazy_patterns instead"); + + + /// @details Lazy grammar sampler, introduced in https://github.com/ggml-org/llama.cpp/pull/9639 + /// @param trigger_patterns A list of patterns that will trigger the grammar sampler. Pattern will be matched from the start of the generation output, and grammar sampler will be fed content starting from its first match group. + /// @param trigger_tokens A list of tokens that will trigger the grammar sampler. Grammar sampler will be fed content starting from the trigger token included. + LLAMA_API struct llama_sampler * llama_sampler_init_grammar_lazy_patterns( + const struct llama_vocab * vocab, + const char * grammar_str, + const char * grammar_root, + const char ** trigger_patterns, + size_t num_trigger_patterns, + const llama_token * trigger_tokens, + size_t num_trigger_tokens); + + + /// NOTE: Avoid using on the full vocabulary as searching for repeated tokens can become slow. For example, apply top-k or top-p sampling first. + LLAMA_API struct llama_sampler * llama_sampler_init_penalties( + int32_t penalty_last_n, // last n tokens to penalize (0 = disable penalty, -1 = context size) + float penalty_repeat, // 1.0 = disabled + float penalty_freq, // 0.0 = disabled + float penalty_present); // 0.0 = disabled + + /// @details DRY sampler, designed by p-e-w, as described in: https://github.com/oobabooga/text-generation-webui/pull/5677, porting Koboldcpp implementation authored by pi6am: https://github.com/LostRuins/koboldcpp/pull/982 + LLAMA_API struct llama_sampler * llama_sampler_init_dry( + const struct llama_vocab * vocab, + int32_t n_ctx_train, + float dry_multiplier, + float dry_base, + int32_t dry_allowed_length, + int32_t dry_penalty_last_n, + const char ** seq_breakers, + size_t num_breakers); + + /// adaptive-p: select tokens near a configurable target probability over time. + /// + /// the adaptive-p sampler transforms the token probability distribution to favor tokens + /// that fall near a user-configurable probability target. + /// + /// internally, the sampler maintains an exponential moving average of the *ORIGINAL* + /// probabilities of selected tokens at each sampling step. it uses this EMA to compute an + /// adapted target probability at each sampling step, thus maintaining the desired target + /// probability over time. + /// + /// adaptive-p selects a token ID rather than just mutating candidates, so it must be last + /// in the sampler chain (like mirostat, dist, greedy). + /// + /// only mild truncation before this sampler is recommended. we suggest applying min-p + /// before adaptive-p as the only other active sampler in the chain. + /// + /// @param target select tokens near this probability (valid range 0.0 to 1.0; negative = disabled) + /// @param decay EMA decay for adaptation; history ≈ 1/(1-decay) tokens (valid range 0.0 - 0.99) + /// @param seed RNG seed + /// + /// ref: https://github.com/ggml-org/llama.cpp/pull/17927 + /// + LLAMA_API struct llama_sampler * llama_sampler_init_adaptive_p( + float target, + float decay, + uint32_t seed); + + LLAMA_API struct llama_sampler * llama_sampler_init_logit_bias( + int32_t n_vocab, + int32_t n_logit_bias, + const llama_logit_bias * logit_bias); + + // this sampler is meant to be used for fill-in-the-middle infilling + // it's supposed to be used after top_k + top_p sampling + // + // 1. if the sum of the EOG probs times the number of candidates is higher than the sum of the other probs -> pick EOG + // 2. combine probs of tokens that have the same prefix + // + // example: + // + // - before: + // "hel": 0.5 + // "hell": 0.2 + // "hello": 0.1 + // "dummy": 0.1 + // + // - after: + // "hel": 0.8 + // "dummy": 0.1 + // + // 3. discard non-EOG tokens with low prob + // 4. if no tokens are left -> pick EOT + // + LLAMA_API struct llama_sampler * llama_sampler_init_infill(const struct llama_vocab * vocab); + + // Returns the seed used by the sampler if applicable, LLAMA_DEFAULT_SEED otherwise + LLAMA_API uint32_t llama_sampler_get_seed(const struct llama_sampler * smpl); + + /// @details Sample and accept a token from the idx-th output of the last evaluation + // + // Shorthand for: + // const auto * logits = llama_get_logits_ith(ctx, idx); + // llama_token_data_array cur_p = { ... init from logits ... }; + // llama_sampler_apply(smpl, &cur_p); + // auto token = cur_p.data[cur_p.selected].id; + // llama_sampler_accept(smpl, token); + // return token; + // Returns the sampled token + LLAMA_API llama_token llama_sampler_sample(struct llama_sampler * smpl, struct llama_context * ctx, int32_t idx); + + // TODO: extend in the future + //LLAMA_API void llama_decode_with_sampler(struct llama_context * ctx, struct llama_sampler * smpl, struct llama_batch batch, ...); + + // + // Model split + // + + /// @details Build a split GGUF final path for this chunk. + /// llama_split_path(split_path, sizeof(split_path), "/models/ggml-model-q4_0", 2, 4) => split_path = "/models/ggml-model-q4_0-00002-of-00004.gguf" + // Returns the split_path length. + LLAMA_API int32_t llama_split_path(char * split_path, size_t maxlen, const char * path_prefix, int32_t split_no, int32_t split_count); + + /// @details Extract the path prefix from the split_path if and only if the split_no and split_count match. + /// llama_split_prefix(split_prefix, 64, "/models/ggml-model-q4_0-00002-of-00004.gguf", 2, 4) => split_prefix = "/models/ggml-model-q4_0" + // Returns the split_prefix length. + LLAMA_API int32_t llama_split_prefix(char * split_prefix, size_t maxlen, const char * split_path, int32_t split_no, int32_t split_count); + + // Print system information + LLAMA_API const char * llama_print_system_info(void); + + // Set callback for all future logging events. + // If this is not called, or NULL is supplied, everything is output on stderr. + // The logger state is global so these functions are NOT thread safe. + LLAMA_API void llama_log_get(ggml_log_callback * log_callback, void ** user_data); + LLAMA_API void llama_log_set(ggml_log_callback log_callback, void * user_data); + + // + // Performance utils + // + // NOTE: Used by llama.cpp examples/tools, avoid using in third-party apps. Instead, do your own performance measurements. + // + + struct llama_perf_context_data { + // ms == milliseconds + double t_start_ms; // absolute start time + double t_load_ms; // time needed for loading the model + double t_p_eval_ms; // time needed for processing the prompt + double t_eval_ms; // time needed for generating tokens + + int32_t n_p_eval; // number of prompt tokens + int32_t n_eval; // number of generated tokens + int32_t n_reused; // number of times a ggml compute graph had been reused + }; + + struct llama_perf_sampler_data { + double t_sample_ms; // time needed for sampling in ms + + int32_t n_sample; // number of sampled tokens + }; + + LLAMA_API struct llama_perf_context_data llama_perf_context (const struct llama_context * ctx); + LLAMA_API void llama_perf_context_print(const struct llama_context * ctx); + LLAMA_API void llama_perf_context_reset( struct llama_context * ctx); + + // NOTE: the following work only with samplers constructed via llama_sampler_chain_init + LLAMA_API struct llama_perf_sampler_data llama_perf_sampler (const struct llama_sampler * chain); + LLAMA_API void llama_perf_sampler_print(const struct llama_sampler * chain); + LLAMA_API void llama_perf_sampler_reset( struct llama_sampler * chain); + + // + // training + // + + // function that returns whether or not a given tensor contains trainable parameters + typedef bool (*llama_opt_param_filter)(const struct ggml_tensor * tensor, void * userdata); + + // always returns true + LLAMA_API bool llama_opt_param_filter_all(const struct ggml_tensor * tensor, void * userdata); + + struct llama_opt_params { + uint32_t n_ctx_train; // assumed context size post training, use context size specified in llama_context if 0 + + llama_opt_param_filter param_filter; // callback for determining which tensors contain trainable parameters + void * param_filter_ud; // userdata for determining which tensors contain trainable parameters + + ggml_opt_get_optimizer_params get_opt_pars; // callback for calculating optimizer parameters + void * get_opt_pars_ud; // userdata for calculating optimizer parameters + + enum ggml_opt_optimizer_type optimizer_type; + }; + + LLAMA_API void llama_opt_init(struct llama_context * lctx, struct llama_model * model, struct llama_opt_params lopt_params); + + LLAMA_API void llama_opt_epoch( + struct llama_context * lctx, + ggml_opt_dataset_t dataset, + ggml_opt_result_t result_train, + ggml_opt_result_t result_eval, + int64_t idata_split, + ggml_opt_epoch_callback callback_train, + ggml_opt_epoch_callback callback_eval); + +#ifdef __cplusplus +} +#endif + +#endif // LLAMA_H diff --git a/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Versions/A/Modules/module.modulemap b/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Versions/A/Modules/module.modulemap new file mode 100644 index 0000000..dc2eecd --- /dev/null +++ b/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Versions/A/Modules/module.modulemap @@ -0,0 +1,17 @@ +framework module llama { + header "llama.h" + header "ggml.h" + header "ggml-alloc.h" + header "ggml-backend.h" + header "ggml-metal.h" + header "ggml-cpu.h" + header "ggml-blas.h" + header "gguf.h" + + link "c++" + link framework "Accelerate" + link framework "Metal" + link framework "Foundation" + + export * +} diff --git a/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Versions/A/Resources/Info.plist b/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Versions/A/Resources/Info.plist new file mode 100644 index 0000000..fb9154a --- /dev/null +++ b/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Versions/A/Resources/Info.plist @@ -0,0 +1,32 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleExecutable + llama + CFBundleIdentifier + org.ggml.llama + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + llama + CFBundlePackageType + FMWK + CFBundleShortVersionString + 1.0 + CFBundleVersion + 1 + MinimumOSVersion + 13.3 + CFBundleSupportedPlatforms + + MacOSX + + DTPlatformName + macosx + DTSDKName + macosx13.3 + + diff --git a/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Versions/A/llama b/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Versions/A/llama new file mode 100755 index 0000000..789ba4b Binary files /dev/null and b/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Versions/A/llama differ diff --git a/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Versions/Current b/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Versions/Current new file mode 120000 index 0000000..8c7e5a6 --- /dev/null +++ b/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/Versions/Current @@ -0,0 +1 @@ +A \ No newline at end of file diff --git a/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/llama b/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/llama new file mode 120000 index 0000000..a5ed5e7 --- /dev/null +++ b/Frameworks/llama.xcframework/ios-arm64-maccatalyst/llama.framework/llama @@ -0,0 +1 @@ +Versions/Current/llama \ No newline at end of file diff --git a/Frameworks/llama.xcframework/ios-arm64/llama.framework/Headers/ggml-backend.h b/Frameworks/llama.xcframework/ios-arm64/llama.framework/Headers/ggml-backend.h index 9fd3f7f..b6f7373 100644 --- a/Frameworks/llama.xcframework/ios-arm64/llama.framework/Headers/ggml-backend.h +++ b/Frameworks/llama.xcframework/ios-arm64/llama.framework/Headers/ggml-backend.h @@ -68,7 +68,7 @@ extern "C" { GGML_API void ggml_backend_buffer_reset (ggml_backend_buffer_t buffer); // tensor copy between different backends - GGML_API void ggml_backend_tensor_copy(struct ggml_tensor * src, struct ggml_tensor * dst); + GGML_API void ggml_backend_tensor_copy(const struct ggml_tensor * src, struct ggml_tensor * dst); // // Backend (stream) @@ -83,13 +83,17 @@ extern "C" { GGML_API size_t ggml_backend_get_alignment(ggml_backend_t backend); GGML_API size_t ggml_backend_get_max_size(ggml_backend_t backend); - GGML_API void ggml_backend_tensor_set_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size); - GGML_API void ggml_backend_tensor_get_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_set_async (ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_get_async (ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_set_2d_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); + GGML_API void ggml_backend_tensor_get_2d_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); // "offset" refers to the offset in tensor->data for setting/getting data - GGML_API void ggml_backend_tensor_set( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size); - GGML_API void ggml_backend_tensor_get(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size); - GGML_API void ggml_backend_tensor_memset( struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_set ( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_get (const struct ggml_tensor * tensor, void * data, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_set_2d( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); + GGML_API void ggml_backend_tensor_get_2d(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); + GGML_API void ggml_backend_tensor_memset( struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size); GGML_API void ggml_backend_synchronize(ggml_backend_t backend); @@ -109,7 +113,7 @@ extern "C" { // the copy is performed after all the currently queued operations in backend_src // backend_dst will wait for the copy to complete before performing other operations // automatic fallback to sync copy if async is not supported - GGML_API void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, struct ggml_tensor * src, struct ggml_tensor * dst); + GGML_API void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, const struct ggml_tensor * src, struct ggml_tensor * dst); GGML_API ggml_backend_dev_t ggml_backend_get_device(ggml_backend_t backend); @@ -135,7 +139,9 @@ extern "C" { // integrated GPU device using host memory GGML_BACKEND_DEVICE_TYPE_IGPU, // accelerator devices intended to be used together with the CPU backend (e.g. BLAS or AMX) - GGML_BACKEND_DEVICE_TYPE_ACCEL + GGML_BACKEND_DEVICE_TYPE_ACCEL, + // "meta" device wrapping multiple other devices for tensor parallelism + GGML_BACKEND_DEVICE_TYPE_META, }; // functionality supported by the device @@ -163,7 +169,7 @@ extern "C" { // device type enum ggml_backend_dev_type type; // device id - // for PCI devices, this should be the PCI bus id formatted as "domain:bus:device.function" (e.g. "0000:01:00.0") + // for PCI devices, this should be the lower-case PCI bus id formatted as "domain:bus:device.function" (e.g. "0000:c1:00.0") // if the id is unknown, this should be NULL const char * device_id; // device capabilities @@ -196,7 +202,12 @@ extern "C" { // Common functions that may be obtained using ggml_backend_reg_get_proc_address - // Split buffer type for tensor parallelism + // Context management and operations for faster communication between backends, used for tensor parallelism (meta backend) + typedef void * (*ggml_backend_comm_init_t)(ggml_backend_t * backends, size_t n_backends); + typedef void (*ggml_backend_comm_free_t)(void * comm_ctx); + typedef bool (*ggml_backend_comm_allreduce_tensor_t)(void * comm_ctx, struct ggml_tensor ** tensors); + + // Split buffer type for tensor parallelism (old) typedef ggml_backend_buffer_type_t (*ggml_backend_split_buffer_type_t)(int main_device, const float * tensor_split); // Set the number of threads for the backend typedef void (*ggml_backend_set_n_threads_t)(ggml_backend_t backend, int n_threads); @@ -340,6 +351,53 @@ extern "C" { // Set a callback to be called for each resulting node during graph compute GGML_API void ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backend_sched_eval_callback callback, void * user_data); + // + // Meta backend + // + +#define GGML_BACKEND_META_MAX_DEVICES 16 + + enum ggml_backend_meta_split_axis { + // tensor split by tensor dimensions: + GGML_BACKEND_SPLIT_AXIS_0 = 0, + GGML_BACKEND_SPLIT_AXIS_1 = 1, + GGML_BACKEND_SPLIT_AXIS_2 = 2, + GGML_BACKEND_SPLIT_AXIS_3 = 3, + + GGML_BACKEND_SPLIT_AXIS_MIRRORED = 10, // all values on all backends + GGML_BACKEND_SPLIT_AXIS_PARTIAL = 11, // each backend has a partial sum + + // for internal bookkeeping only: + GGML_BACKEND_SPLIT_AXIS_NONE = 98, + GGML_BACKEND_SPLIT_AXIS_UNKNOWN = 99, + }; + GGML_API const char * ggml_backend_meta_split_axis_name(enum ggml_backend_meta_split_axis split_axis); + + struct ggml_backend_meta_split_state { + enum ggml_backend_meta_split_axis axis; + + // for tensors with axis >= 0 && axis < GGML_MAX_DIMS: + // - each device has a slice of the tensor along the split axis + // - most tensors have n_segments == 1 and a contiguous slice of the tensor data + // - some tensors have an inhomogenenous data layout along the split axis, + // those tensors are divided into segments which are each individually split across devices + // - ne has one entry per segment and device that add up to ggml_tensor::ne for that axis, + // the outer/inner loops are over segments/devices like [seg0_dev0, seg0_dev1, seg1_dev0, seg1_dev1], + // - for example, a transformer may have a fused QKV matrix rather than 3 matrices, those would be 3 separate segments + // that each need to be split individually across devices so that each device gets a slice of Q, K, and V + int64_t ne[16*GGML_BACKEND_META_MAX_DEVICES]; + uint32_t n_segments; + }; + + // function to assign split states for statically allocated tensors, compute tensor split states will be assigned to be compatible: + typedef struct ggml_backend_meta_split_state(*ggml_backend_meta_get_split_state_t)(const struct ggml_tensor * tensor, void * userdata); + + // create a new meta device from "simple" devices, meta buffer type/buffer/backend is then derived from this: + // TODO: this looks a bit strange - a backend API creates a device. I think we should try + // express this as a backend registry functionality instead + GGML_API ggml_backend_dev_t ggml_backend_meta_device( + ggml_backend_dev_t * devs, size_t n_devs, ggml_backend_meta_get_split_state_t get_split_state, void * get_split_state_ud); + // // Utils // diff --git a/Frameworks/llama.xcframework/ios-arm64/llama.framework/Headers/ggml.h b/Frameworks/llama.xcframework/ios-arm64/llama.framework/Headers/ggml.h index 669f66b..3357a0d 100644 --- a/Frameworks/llama.xcframework/ios-arm64/llama.framework/Headers/ggml.h +++ b/Frameworks/llama.xcframework/ios-arm64/llama.framework/Headers/ggml.h @@ -428,7 +428,8 @@ extern "C" { // GGML_TYPE_IQ4_NL_8_8 = 38, GGML_TYPE_MXFP4 = 39, // MXFP4 (1 block) GGML_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale) - GGML_TYPE_COUNT = 41, + GGML_TYPE_Q1_0 = 41, + GGML_TYPE_COUNT = 42, }; // precision @@ -437,6 +438,12 @@ extern "C" { GGML_PREC_F32 = 10, }; + // op hint + enum ggml_op_hint { + GGML_HINT_NONE = 0, + GGML_HINT_SRC0_IS_HADAMARD = 1, + }; + // model file types enum ggml_ftype { GGML_FTYPE_UNKNOWN = -1, @@ -465,6 +472,7 @@ extern "C" { GGML_FTYPE_MOSTLY_BF16 = 24, // except 1d tensors GGML_FTYPE_MOSTLY_MXFP4 = 25, // except 1d tensors GGML_FTYPE_MOSTLY_NVFP4 = 26, // except 1d tensors + GGML_FTYPE_MOSTLY_Q1_0 = 27, // except 1d tensors }; // available tensor operations: @@ -900,15 +908,17 @@ extern "C" { struct ggml_tensor * b, struct ggml_tensor * ids); - GGML_API struct ggml_tensor * ggml_add1( + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_add1( struct ggml_context * ctx, struct ggml_tensor * a, - struct ggml_tensor * b); + struct ggml_tensor * b), + "use ggml_add instead"); - GGML_API struct ggml_tensor * ggml_add1_inplace( + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_add1_inplace( struct ggml_context * ctx, struct ggml_tensor * a, - struct ggml_tensor * b); + struct ggml_tensor * b), + "use ggml_add_inplace instead"); // dst = a // view(dst, nb1, nb2, nb3, offset) += b @@ -1415,6 +1425,11 @@ extern "C" { struct ggml_tensor * a, enum ggml_prec prec); + // change the hint of a matrix multiplication + GGML_API void ggml_mul_mat_set_hint( + struct ggml_tensor * a, + enum ggml_op_hint hint); + // indirect matrix multiplication GGML_API struct ggml_tensor * ggml_mul_mat_id( struct ggml_context * ctx, @@ -1769,8 +1784,32 @@ extern "C" { int n_dims, int mode); - // custom RoPE + // RoPE operations with extended options + // a is the input tensor to apply RoPE to, shape [n_embd, n_head, n_token] + // b is an int32 vector with size n_token // c is freq factors (e.g. phi3-128k), (optional) + // mode can be GGML_ROPE_TYPE_NORMAL or NEOX; for MROPE and VISION mode, use ggml_rope_multi + // + // pseudo-code for computing theta: + // for i in [0, n_dims/2): + // theta[i] = b[i] * powf(freq_base, -2.0 * i / n_dims); + // theta[i] = theta[i] / c[i]; # if c is provided, divide theta by c + // theta[i] = rope_yarn(theta[i], ...); # note: theta = theta * freq_scale is applied here + // + // other params are used by YaRN RoPE scaling, these default values will disable YaRN: + // freq_scale = 1.0f + // ext_factor = 0.0f + // attn_factor = 1.0f + // beta_fast = 0.0f + // beta_slow = 0.0f + // + // example: + // (marking: c = cos, s = sin, 0 = unrotated) + // given a single head with size = 8 --> [00000000] + // GGML_ROPE_TYPE_NORMAL n_dims = 4 --> [cscs0000] + // GGML_ROPE_TYPE_NORMAL n_dims = 8 --> [cscscscs] + // GGML_ROPE_TYPE_NEOX n_dims = 4 --> [ccss0000] + // GGML_ROPE_TYPE_NEOX n_dims = 8 --> [ccccssss] GGML_API struct ggml_tensor * ggml_rope_ext( struct ggml_context * ctx, struct ggml_tensor * a, @@ -1786,6 +1825,36 @@ extern "C" { float beta_fast, float beta_slow); + // multi-dimensional RoPE, for Qwen-VL and similar vision models + // mode can be either VISION, MROPE, IMROPE, cannot be combined with NORMAL or NEOX + // sections specify how many dimensions to rotate in each section: + // section length is equivalent to number of cos/sin pairs, NOT the number of dims + // (i.e. sum of 4 sections are expected to be n_dims/2) + // last sections can be 0, means ignored + // all other options are identical to ggml_rope_ext + // + // important note: + // - NEOX ordering is automatically applied and cannot be disabled for MROPE and VISION + // if you need normal ordering, there are 2 methods: + // (1) split the tensor manually using ggml_view + // (2) permute the weight upon conversion + // - for VISION, n_dims must be head_size/2 + // + // example M-RoPE: + // given sections = [t=4, y=2, x=2, 0] + // given a single head with size = 18 --> [000000000000000000] + // GGML_ROPE_TYPE_MROPE n_dims = 16 --> [ttttyyxxttttyyxx00] (cos/sin are applied in NEOX ordering) + // GGML_ROPE_TYPE_IMROPE n_dims = 16 --> [ttyxttyxttyxttyx00] (interleaved M-RoPE, still NEOX ordering) + // note: the theta for each dim is computed the same way as ggml_rope_ext, no matter the section + // in other words, idx used for theta: [0123456789... until n_dims/2], not reset for each section + // + // example vision RoPE: + // given sections = [y=4, x=4, 0, 0] (last 2 sections are ignored) + // given a single head with size = 8 --> [00000000] + // GGML_ROPE_TYPE_VISION n_dims = 4 --> [yyyyxxxx] + // other values of n_dims are untested and is undefined behavior + // note: unlike MROPE, the theta for each dim is computed differently for each section + // in other words, idx used for theta: [0123] for y section, then [0123] for x section GGML_API struct ggml_tensor * ggml_rope_multi( struct ggml_context * ctx, struct ggml_tensor * a, diff --git a/Frameworks/llama.xcframework/ios-arm64/llama.framework/Headers/gguf.h b/Frameworks/llama.xcframework/ios-arm64/llama.framework/Headers/gguf.h index 79ee202..02d5f22 100644 --- a/Frameworks/llama.xcframework/ios-arm64/llama.framework/Headers/gguf.h +++ b/Frameworks/llama.xcframework/ios-arm64/llama.framework/Headers/gguf.h @@ -77,6 +77,7 @@ extern "C" { }; GGML_API struct gguf_context * gguf_init_empty(void); + GGML_API struct gguf_context * gguf_init_from_file_ptr(FILE * file, struct gguf_init_params params); GGML_API struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_params params); //GGML_API struct gguf_context * gguf_init_from_buffer(..); @@ -189,6 +190,7 @@ extern "C" { // // write the entire context to a binary file + GGML_API bool gguf_write_to_file_ptr(const struct gguf_context * ctx, FILE * file, bool only_meta); GGML_API bool gguf_write_to_file(const struct gguf_context * ctx, const char * fname, bool only_meta); // get the size in bytes of the meta data (header, kv pairs, tensor info) including padding diff --git a/Frameworks/llama.xcframework/ios-arm64/llama.framework/Headers/llama.h b/Frameworks/llama.xcframework/ios-arm64/llama.framework/Headers/llama.h index 6e72db7..308e8ba 100644 --- a/Frameworks/llama.xcframework/ios-arm64/llama.framework/Headers/llama.h +++ b/Frameworks/llama.xcframework/ios-arm64/llama.framework/Headers/llama.h @@ -154,6 +154,7 @@ extern "C" { LLAMA_FTYPE_MOSTLY_TQ2_0 = 37, // except 1d tensors LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38, // except 1d tensors LLAMA_FTYPE_MOSTLY_NVFP4 = 39, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q1_0 = 40, // except 1d tensors LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file }; @@ -191,9 +192,10 @@ extern "C" { LLAMA_API const char * llama_flash_attn_type_name(enum llama_flash_attn_type flash_attn_type); enum llama_split_mode { - LLAMA_SPLIT_MODE_NONE = 0, // single GPU - LLAMA_SPLIT_MODE_LAYER = 1, // split layers and KV across GPUs - LLAMA_SPLIT_MODE_ROW = 2, // split layers and KV across GPUs, use tensor parallelism if supported + LLAMA_SPLIT_MODE_NONE = 0, // single GPU + LLAMA_SPLIT_MODE_LAYER = 1, // split layers and KV across GPUs + LLAMA_SPLIT_MODE_ROW = 2, // split layers and KV across GPUs, use tensor parallelism if supported + LLAMA_SPLIT_MODE_TENSOR = 3, }; // TODO: simplify (https://github.com/ggml-org/llama.cpp/pull/9294#pullrequestreview-2286561979) @@ -380,22 +382,33 @@ extern "C" { size_t n_samplers; }; + struct llama_model_tensor_override { + const char * pattern; + enum ggml_type type; + }; + + struct llama_model_imatrix_data { + const char * name; + const float * data; + size_t size; + }; + // model quantization parameters typedef struct llama_model_quantize_params { - int32_t nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency() - enum llama_ftype ftype; // quantize to this llama_ftype - enum ggml_type output_tensor_type; // output tensor type - enum ggml_type token_embedding_type; // token embeddings tensor type - bool allow_requantize; // allow quantizing non-f32/f16 tensors - bool quantize_output_tensor; // quantize output.weight - bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored - bool pure; // quantize all tensors to the default type - bool keep_split; // quantize to the same number of shards - bool dry_run; // calculate and show the final quantization size without performing quantization - void * imatrix; // pointer to importance matrix data - void * kv_overrides; // pointer to vector containing overrides - void * tensor_types; // pointer to vector containing tensor types - void * prune_layers; // pointer to vector containing layer indices to prune + int32_t nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency() + enum llama_ftype ftype; // quantize to this llama_ftype + enum ggml_type output_tensor_type; // output tensor type + enum ggml_type token_embedding_type; // token embeddings tensor type + bool allow_requantize; // allow quantizing non-f32/f16 tensors + bool quantize_output_tensor; // quantize output.weight + bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored + bool pure; // quantize all tensors to the default type + bool keep_split; // quantize to the same number of shards + bool dry_run; // calculate and show the final quantization size without performing quantization + const struct llama_model_imatrix_data * imatrix; // pointer to importance matrix data + const struct llama_model_kv_override * kv_overrides; // pointer to kv overrides + const struct llama_model_tensor_override * tt_overrides; // pointer to tensor overrides + const int32_t * prune_layers; // pointer to layer indices to prune } llama_model_quantize_params; typedef struct llama_logit_bias { @@ -465,6 +478,11 @@ extern "C" { const char * path_model, struct llama_model_params params); + // Load a model from an open FILE pointer + LLAMA_API struct llama_model * llama_model_load_from_file_ptr( + FILE * file, + struct llama_model_params params); + // Load a model from multiple splits (support custom naming scheme) // The paths must be in the correct order LLAMA_API struct llama_model * llama_model_load_from_splits( @@ -493,27 +511,6 @@ extern "C" { // Frees all allocated memory LLAMA_API void llama_free(struct llama_context * ctx); - enum llama_params_fit_status { - LLAMA_PARAMS_FIT_STATUS_SUCCESS = 0, // found allocations that are projected to fit - LLAMA_PARAMS_FIT_STATUS_FAILURE = 1, // could not find allocations that are projected to fit - LLAMA_PARAMS_FIT_STATUS_ERROR = 2, // a hard error occurred, e.g. because no model could be found at the specified path - }; - - // fits mparams and cparams to free device memory (assumes system memory is unlimited) - // - returns true if the parameters could be successfully modified to fit device memory - // - this function is NOT thread safe because it modifies the global llama logger state - // - only parameters that have the same value as in llama_default_model_params are modified - // with the exception of the context size which is modified if and only if equal to 0 - LLAMA_API enum llama_params_fit_status llama_params_fit( - const char * path_model, - struct llama_model_params * mparams, - struct llama_context_params * cparams, - float * tensor_split, // writable buffer for tensor split, needs at least llama_max_devices elements - struct llama_model_tensor_buft_override * tensor_buft_overrides, // writable buffer for overrides, needs at least llama_max_tensor_buft_overrides elements - size_t * margins, // margins of memory to leave per device in bytes - uint32_t n_ctx_min, // minimum context size to set when trying to reduce memory use - enum ggml_log_level log_level); // minimum log level to print during fitting, lower levels go to debug log - LLAMA_API int64_t llama_time_us(void); LLAMA_API size_t llama_max_devices(void); @@ -861,12 +858,17 @@ extern "C" { size_t n_token_capacity, size_t * n_token_count_out); +#define LLAMA_STATE_SEQ_FLAGS_NONE 0 + // for backwards-compat #define LLAMA_STATE_SEQ_FLAGS_SWA_ONLY 1 // work only with partial states, such as SWA KV cache or recurrent cache (e.g. Mamba) #define LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY 1 +// keeps the tensor data on device buffers (i.e. not accessible in host memory, but faster save/load) +#define LLAMA_STATE_SEQ_FLAGS_ON_DEVICE 2 + typedef uint32_t llama_state_seq_flags; LLAMA_API size_t llama_state_seq_get_size_ext( @@ -1528,9 +1530,6 @@ extern "C" { LLAMA_API void llama_perf_sampler_print(const struct llama_sampler * chain); LLAMA_API void llama_perf_sampler_reset( struct llama_sampler * chain); - // print a breakdown of per-device memory use via LLAMA_LOG: - LLAMA_API void llama_memory_breakdown_print(const struct llama_context * ctx); - // // training // diff --git a/Frameworks/llama.xcframework/ios-arm64/llama.framework/llama b/Frameworks/llama.xcframework/ios-arm64/llama.framework/llama index 9fe48dd..e5bbbbb 100755 Binary files a/Frameworks/llama.xcframework/ios-arm64/llama.framework/llama and b/Frameworks/llama.xcframework/ios-arm64/llama.framework/llama differ diff --git a/Frameworks/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/Headers/ggml-backend.h b/Frameworks/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/Headers/ggml-backend.h index 9fd3f7f..b6f7373 100644 --- a/Frameworks/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/Headers/ggml-backend.h +++ b/Frameworks/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/Headers/ggml-backend.h @@ -68,7 +68,7 @@ extern "C" { GGML_API void ggml_backend_buffer_reset (ggml_backend_buffer_t buffer); // tensor copy between different backends - GGML_API void ggml_backend_tensor_copy(struct ggml_tensor * src, struct ggml_tensor * dst); + GGML_API void ggml_backend_tensor_copy(const struct ggml_tensor * src, struct ggml_tensor * dst); // // Backend (stream) @@ -83,13 +83,17 @@ extern "C" { GGML_API size_t ggml_backend_get_alignment(ggml_backend_t backend); GGML_API size_t ggml_backend_get_max_size(ggml_backend_t backend); - GGML_API void ggml_backend_tensor_set_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size); - GGML_API void ggml_backend_tensor_get_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_set_async (ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_get_async (ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_set_2d_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); + GGML_API void ggml_backend_tensor_get_2d_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); // "offset" refers to the offset in tensor->data for setting/getting data - GGML_API void ggml_backend_tensor_set( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size); - GGML_API void ggml_backend_tensor_get(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size); - GGML_API void ggml_backend_tensor_memset( struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_set ( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_get (const struct ggml_tensor * tensor, void * data, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_set_2d( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); + GGML_API void ggml_backend_tensor_get_2d(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); + GGML_API void ggml_backend_tensor_memset( struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size); GGML_API void ggml_backend_synchronize(ggml_backend_t backend); @@ -109,7 +113,7 @@ extern "C" { // the copy is performed after all the currently queued operations in backend_src // backend_dst will wait for the copy to complete before performing other operations // automatic fallback to sync copy if async is not supported - GGML_API void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, struct ggml_tensor * src, struct ggml_tensor * dst); + GGML_API void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, const struct ggml_tensor * src, struct ggml_tensor * dst); GGML_API ggml_backend_dev_t ggml_backend_get_device(ggml_backend_t backend); @@ -135,7 +139,9 @@ extern "C" { // integrated GPU device using host memory GGML_BACKEND_DEVICE_TYPE_IGPU, // accelerator devices intended to be used together with the CPU backend (e.g. BLAS or AMX) - GGML_BACKEND_DEVICE_TYPE_ACCEL + GGML_BACKEND_DEVICE_TYPE_ACCEL, + // "meta" device wrapping multiple other devices for tensor parallelism + GGML_BACKEND_DEVICE_TYPE_META, }; // functionality supported by the device @@ -163,7 +169,7 @@ extern "C" { // device type enum ggml_backend_dev_type type; // device id - // for PCI devices, this should be the PCI bus id formatted as "domain:bus:device.function" (e.g. "0000:01:00.0") + // for PCI devices, this should be the lower-case PCI bus id formatted as "domain:bus:device.function" (e.g. "0000:c1:00.0") // if the id is unknown, this should be NULL const char * device_id; // device capabilities @@ -196,7 +202,12 @@ extern "C" { // Common functions that may be obtained using ggml_backend_reg_get_proc_address - // Split buffer type for tensor parallelism + // Context management and operations for faster communication between backends, used for tensor parallelism (meta backend) + typedef void * (*ggml_backend_comm_init_t)(ggml_backend_t * backends, size_t n_backends); + typedef void (*ggml_backend_comm_free_t)(void * comm_ctx); + typedef bool (*ggml_backend_comm_allreduce_tensor_t)(void * comm_ctx, struct ggml_tensor ** tensors); + + // Split buffer type for tensor parallelism (old) typedef ggml_backend_buffer_type_t (*ggml_backend_split_buffer_type_t)(int main_device, const float * tensor_split); // Set the number of threads for the backend typedef void (*ggml_backend_set_n_threads_t)(ggml_backend_t backend, int n_threads); @@ -340,6 +351,53 @@ extern "C" { // Set a callback to be called for each resulting node during graph compute GGML_API void ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backend_sched_eval_callback callback, void * user_data); + // + // Meta backend + // + +#define GGML_BACKEND_META_MAX_DEVICES 16 + + enum ggml_backend_meta_split_axis { + // tensor split by tensor dimensions: + GGML_BACKEND_SPLIT_AXIS_0 = 0, + GGML_BACKEND_SPLIT_AXIS_1 = 1, + GGML_BACKEND_SPLIT_AXIS_2 = 2, + GGML_BACKEND_SPLIT_AXIS_3 = 3, + + GGML_BACKEND_SPLIT_AXIS_MIRRORED = 10, // all values on all backends + GGML_BACKEND_SPLIT_AXIS_PARTIAL = 11, // each backend has a partial sum + + // for internal bookkeeping only: + GGML_BACKEND_SPLIT_AXIS_NONE = 98, + GGML_BACKEND_SPLIT_AXIS_UNKNOWN = 99, + }; + GGML_API const char * ggml_backend_meta_split_axis_name(enum ggml_backend_meta_split_axis split_axis); + + struct ggml_backend_meta_split_state { + enum ggml_backend_meta_split_axis axis; + + // for tensors with axis >= 0 && axis < GGML_MAX_DIMS: + // - each device has a slice of the tensor along the split axis + // - most tensors have n_segments == 1 and a contiguous slice of the tensor data + // - some tensors have an inhomogenenous data layout along the split axis, + // those tensors are divided into segments which are each individually split across devices + // - ne has one entry per segment and device that add up to ggml_tensor::ne for that axis, + // the outer/inner loops are over segments/devices like [seg0_dev0, seg0_dev1, seg1_dev0, seg1_dev1], + // - for example, a transformer may have a fused QKV matrix rather than 3 matrices, those would be 3 separate segments + // that each need to be split individually across devices so that each device gets a slice of Q, K, and V + int64_t ne[16*GGML_BACKEND_META_MAX_DEVICES]; + uint32_t n_segments; + }; + + // function to assign split states for statically allocated tensors, compute tensor split states will be assigned to be compatible: + typedef struct ggml_backend_meta_split_state(*ggml_backend_meta_get_split_state_t)(const struct ggml_tensor * tensor, void * userdata); + + // create a new meta device from "simple" devices, meta buffer type/buffer/backend is then derived from this: + // TODO: this looks a bit strange - a backend API creates a device. I think we should try + // express this as a backend registry functionality instead + GGML_API ggml_backend_dev_t ggml_backend_meta_device( + ggml_backend_dev_t * devs, size_t n_devs, ggml_backend_meta_get_split_state_t get_split_state, void * get_split_state_ud); + // // Utils // diff --git a/Frameworks/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/Headers/ggml.h b/Frameworks/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/Headers/ggml.h index 669f66b..3357a0d 100644 --- a/Frameworks/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/Headers/ggml.h +++ b/Frameworks/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/Headers/ggml.h @@ -428,7 +428,8 @@ extern "C" { // GGML_TYPE_IQ4_NL_8_8 = 38, GGML_TYPE_MXFP4 = 39, // MXFP4 (1 block) GGML_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale) - GGML_TYPE_COUNT = 41, + GGML_TYPE_Q1_0 = 41, + GGML_TYPE_COUNT = 42, }; // precision @@ -437,6 +438,12 @@ extern "C" { GGML_PREC_F32 = 10, }; + // op hint + enum ggml_op_hint { + GGML_HINT_NONE = 0, + GGML_HINT_SRC0_IS_HADAMARD = 1, + }; + // model file types enum ggml_ftype { GGML_FTYPE_UNKNOWN = -1, @@ -465,6 +472,7 @@ extern "C" { GGML_FTYPE_MOSTLY_BF16 = 24, // except 1d tensors GGML_FTYPE_MOSTLY_MXFP4 = 25, // except 1d tensors GGML_FTYPE_MOSTLY_NVFP4 = 26, // except 1d tensors + GGML_FTYPE_MOSTLY_Q1_0 = 27, // except 1d tensors }; // available tensor operations: @@ -900,15 +908,17 @@ extern "C" { struct ggml_tensor * b, struct ggml_tensor * ids); - GGML_API struct ggml_tensor * ggml_add1( + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_add1( struct ggml_context * ctx, struct ggml_tensor * a, - struct ggml_tensor * b); + struct ggml_tensor * b), + "use ggml_add instead"); - GGML_API struct ggml_tensor * ggml_add1_inplace( + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_add1_inplace( struct ggml_context * ctx, struct ggml_tensor * a, - struct ggml_tensor * b); + struct ggml_tensor * b), + "use ggml_add_inplace instead"); // dst = a // view(dst, nb1, nb2, nb3, offset) += b @@ -1415,6 +1425,11 @@ extern "C" { struct ggml_tensor * a, enum ggml_prec prec); + // change the hint of a matrix multiplication + GGML_API void ggml_mul_mat_set_hint( + struct ggml_tensor * a, + enum ggml_op_hint hint); + // indirect matrix multiplication GGML_API struct ggml_tensor * ggml_mul_mat_id( struct ggml_context * ctx, @@ -1769,8 +1784,32 @@ extern "C" { int n_dims, int mode); - // custom RoPE + // RoPE operations with extended options + // a is the input tensor to apply RoPE to, shape [n_embd, n_head, n_token] + // b is an int32 vector with size n_token // c is freq factors (e.g. phi3-128k), (optional) + // mode can be GGML_ROPE_TYPE_NORMAL or NEOX; for MROPE and VISION mode, use ggml_rope_multi + // + // pseudo-code for computing theta: + // for i in [0, n_dims/2): + // theta[i] = b[i] * powf(freq_base, -2.0 * i / n_dims); + // theta[i] = theta[i] / c[i]; # if c is provided, divide theta by c + // theta[i] = rope_yarn(theta[i], ...); # note: theta = theta * freq_scale is applied here + // + // other params are used by YaRN RoPE scaling, these default values will disable YaRN: + // freq_scale = 1.0f + // ext_factor = 0.0f + // attn_factor = 1.0f + // beta_fast = 0.0f + // beta_slow = 0.0f + // + // example: + // (marking: c = cos, s = sin, 0 = unrotated) + // given a single head with size = 8 --> [00000000] + // GGML_ROPE_TYPE_NORMAL n_dims = 4 --> [cscs0000] + // GGML_ROPE_TYPE_NORMAL n_dims = 8 --> [cscscscs] + // GGML_ROPE_TYPE_NEOX n_dims = 4 --> [ccss0000] + // GGML_ROPE_TYPE_NEOX n_dims = 8 --> [ccccssss] GGML_API struct ggml_tensor * ggml_rope_ext( struct ggml_context * ctx, struct ggml_tensor * a, @@ -1786,6 +1825,36 @@ extern "C" { float beta_fast, float beta_slow); + // multi-dimensional RoPE, for Qwen-VL and similar vision models + // mode can be either VISION, MROPE, IMROPE, cannot be combined with NORMAL or NEOX + // sections specify how many dimensions to rotate in each section: + // section length is equivalent to number of cos/sin pairs, NOT the number of dims + // (i.e. sum of 4 sections are expected to be n_dims/2) + // last sections can be 0, means ignored + // all other options are identical to ggml_rope_ext + // + // important note: + // - NEOX ordering is automatically applied and cannot be disabled for MROPE and VISION + // if you need normal ordering, there are 2 methods: + // (1) split the tensor manually using ggml_view + // (2) permute the weight upon conversion + // - for VISION, n_dims must be head_size/2 + // + // example M-RoPE: + // given sections = [t=4, y=2, x=2, 0] + // given a single head with size = 18 --> [000000000000000000] + // GGML_ROPE_TYPE_MROPE n_dims = 16 --> [ttttyyxxttttyyxx00] (cos/sin are applied in NEOX ordering) + // GGML_ROPE_TYPE_IMROPE n_dims = 16 --> [ttyxttyxttyxttyx00] (interleaved M-RoPE, still NEOX ordering) + // note: the theta for each dim is computed the same way as ggml_rope_ext, no matter the section + // in other words, idx used for theta: [0123456789... until n_dims/2], not reset for each section + // + // example vision RoPE: + // given sections = [y=4, x=4, 0, 0] (last 2 sections are ignored) + // given a single head with size = 8 --> [00000000] + // GGML_ROPE_TYPE_VISION n_dims = 4 --> [yyyyxxxx] + // other values of n_dims are untested and is undefined behavior + // note: unlike MROPE, the theta for each dim is computed differently for each section + // in other words, idx used for theta: [0123] for y section, then [0123] for x section GGML_API struct ggml_tensor * ggml_rope_multi( struct ggml_context * ctx, struct ggml_tensor * a, diff --git a/Frameworks/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/Headers/gguf.h b/Frameworks/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/Headers/gguf.h index 79ee202..02d5f22 100644 --- a/Frameworks/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/Headers/gguf.h +++ b/Frameworks/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/Headers/gguf.h @@ -77,6 +77,7 @@ extern "C" { }; GGML_API struct gguf_context * gguf_init_empty(void); + GGML_API struct gguf_context * gguf_init_from_file_ptr(FILE * file, struct gguf_init_params params); GGML_API struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_params params); //GGML_API struct gguf_context * gguf_init_from_buffer(..); @@ -189,6 +190,7 @@ extern "C" { // // write the entire context to a binary file + GGML_API bool gguf_write_to_file_ptr(const struct gguf_context * ctx, FILE * file, bool only_meta); GGML_API bool gguf_write_to_file(const struct gguf_context * ctx, const char * fname, bool only_meta); // get the size in bytes of the meta data (header, kv pairs, tensor info) including padding diff --git a/Frameworks/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/Headers/llama.h b/Frameworks/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/Headers/llama.h index 6e72db7..308e8ba 100644 --- a/Frameworks/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/Headers/llama.h +++ b/Frameworks/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/Headers/llama.h @@ -154,6 +154,7 @@ extern "C" { LLAMA_FTYPE_MOSTLY_TQ2_0 = 37, // except 1d tensors LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38, // except 1d tensors LLAMA_FTYPE_MOSTLY_NVFP4 = 39, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q1_0 = 40, // except 1d tensors LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file }; @@ -191,9 +192,10 @@ extern "C" { LLAMA_API const char * llama_flash_attn_type_name(enum llama_flash_attn_type flash_attn_type); enum llama_split_mode { - LLAMA_SPLIT_MODE_NONE = 0, // single GPU - LLAMA_SPLIT_MODE_LAYER = 1, // split layers and KV across GPUs - LLAMA_SPLIT_MODE_ROW = 2, // split layers and KV across GPUs, use tensor parallelism if supported + LLAMA_SPLIT_MODE_NONE = 0, // single GPU + LLAMA_SPLIT_MODE_LAYER = 1, // split layers and KV across GPUs + LLAMA_SPLIT_MODE_ROW = 2, // split layers and KV across GPUs, use tensor parallelism if supported + LLAMA_SPLIT_MODE_TENSOR = 3, }; // TODO: simplify (https://github.com/ggml-org/llama.cpp/pull/9294#pullrequestreview-2286561979) @@ -380,22 +382,33 @@ extern "C" { size_t n_samplers; }; + struct llama_model_tensor_override { + const char * pattern; + enum ggml_type type; + }; + + struct llama_model_imatrix_data { + const char * name; + const float * data; + size_t size; + }; + // model quantization parameters typedef struct llama_model_quantize_params { - int32_t nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency() - enum llama_ftype ftype; // quantize to this llama_ftype - enum ggml_type output_tensor_type; // output tensor type - enum ggml_type token_embedding_type; // token embeddings tensor type - bool allow_requantize; // allow quantizing non-f32/f16 tensors - bool quantize_output_tensor; // quantize output.weight - bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored - bool pure; // quantize all tensors to the default type - bool keep_split; // quantize to the same number of shards - bool dry_run; // calculate and show the final quantization size without performing quantization - void * imatrix; // pointer to importance matrix data - void * kv_overrides; // pointer to vector containing overrides - void * tensor_types; // pointer to vector containing tensor types - void * prune_layers; // pointer to vector containing layer indices to prune + int32_t nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency() + enum llama_ftype ftype; // quantize to this llama_ftype + enum ggml_type output_tensor_type; // output tensor type + enum ggml_type token_embedding_type; // token embeddings tensor type + bool allow_requantize; // allow quantizing non-f32/f16 tensors + bool quantize_output_tensor; // quantize output.weight + bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored + bool pure; // quantize all tensors to the default type + bool keep_split; // quantize to the same number of shards + bool dry_run; // calculate and show the final quantization size without performing quantization + const struct llama_model_imatrix_data * imatrix; // pointer to importance matrix data + const struct llama_model_kv_override * kv_overrides; // pointer to kv overrides + const struct llama_model_tensor_override * tt_overrides; // pointer to tensor overrides + const int32_t * prune_layers; // pointer to layer indices to prune } llama_model_quantize_params; typedef struct llama_logit_bias { @@ -465,6 +478,11 @@ extern "C" { const char * path_model, struct llama_model_params params); + // Load a model from an open FILE pointer + LLAMA_API struct llama_model * llama_model_load_from_file_ptr( + FILE * file, + struct llama_model_params params); + // Load a model from multiple splits (support custom naming scheme) // The paths must be in the correct order LLAMA_API struct llama_model * llama_model_load_from_splits( @@ -493,27 +511,6 @@ extern "C" { // Frees all allocated memory LLAMA_API void llama_free(struct llama_context * ctx); - enum llama_params_fit_status { - LLAMA_PARAMS_FIT_STATUS_SUCCESS = 0, // found allocations that are projected to fit - LLAMA_PARAMS_FIT_STATUS_FAILURE = 1, // could not find allocations that are projected to fit - LLAMA_PARAMS_FIT_STATUS_ERROR = 2, // a hard error occurred, e.g. because no model could be found at the specified path - }; - - // fits mparams and cparams to free device memory (assumes system memory is unlimited) - // - returns true if the parameters could be successfully modified to fit device memory - // - this function is NOT thread safe because it modifies the global llama logger state - // - only parameters that have the same value as in llama_default_model_params are modified - // with the exception of the context size which is modified if and only if equal to 0 - LLAMA_API enum llama_params_fit_status llama_params_fit( - const char * path_model, - struct llama_model_params * mparams, - struct llama_context_params * cparams, - float * tensor_split, // writable buffer for tensor split, needs at least llama_max_devices elements - struct llama_model_tensor_buft_override * tensor_buft_overrides, // writable buffer for overrides, needs at least llama_max_tensor_buft_overrides elements - size_t * margins, // margins of memory to leave per device in bytes - uint32_t n_ctx_min, // minimum context size to set when trying to reduce memory use - enum ggml_log_level log_level); // minimum log level to print during fitting, lower levels go to debug log - LLAMA_API int64_t llama_time_us(void); LLAMA_API size_t llama_max_devices(void); @@ -861,12 +858,17 @@ extern "C" { size_t n_token_capacity, size_t * n_token_count_out); +#define LLAMA_STATE_SEQ_FLAGS_NONE 0 + // for backwards-compat #define LLAMA_STATE_SEQ_FLAGS_SWA_ONLY 1 // work only with partial states, such as SWA KV cache or recurrent cache (e.g. Mamba) #define LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY 1 +// keeps the tensor data on device buffers (i.e. not accessible in host memory, but faster save/load) +#define LLAMA_STATE_SEQ_FLAGS_ON_DEVICE 2 + typedef uint32_t llama_state_seq_flags; LLAMA_API size_t llama_state_seq_get_size_ext( @@ -1528,9 +1530,6 @@ extern "C" { LLAMA_API void llama_perf_sampler_print(const struct llama_sampler * chain); LLAMA_API void llama_perf_sampler_reset( struct llama_sampler * chain); - // print a breakdown of per-device memory use via LLAMA_LOG: - LLAMA_API void llama_memory_breakdown_print(const struct llama_context * ctx); - // // training // diff --git a/Frameworks/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/llama b/Frameworks/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/llama index 3b43165..a32bce8 100755 Binary files a/Frameworks/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/llama and b/Frameworks/llama.xcframework/ios-arm64_x86_64-simulator/llama.framework/llama differ diff --git a/Frameworks/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/ggml-backend.h b/Frameworks/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/ggml-backend.h index 9fd3f7f..b6f7373 100644 --- a/Frameworks/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/ggml-backend.h +++ b/Frameworks/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/ggml-backend.h @@ -68,7 +68,7 @@ extern "C" { GGML_API void ggml_backend_buffer_reset (ggml_backend_buffer_t buffer); // tensor copy between different backends - GGML_API void ggml_backend_tensor_copy(struct ggml_tensor * src, struct ggml_tensor * dst); + GGML_API void ggml_backend_tensor_copy(const struct ggml_tensor * src, struct ggml_tensor * dst); // // Backend (stream) @@ -83,13 +83,17 @@ extern "C" { GGML_API size_t ggml_backend_get_alignment(ggml_backend_t backend); GGML_API size_t ggml_backend_get_max_size(ggml_backend_t backend); - GGML_API void ggml_backend_tensor_set_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size); - GGML_API void ggml_backend_tensor_get_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_set_async (ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_get_async (ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_set_2d_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); + GGML_API void ggml_backend_tensor_get_2d_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); // "offset" refers to the offset in tensor->data for setting/getting data - GGML_API void ggml_backend_tensor_set( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size); - GGML_API void ggml_backend_tensor_get(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size); - GGML_API void ggml_backend_tensor_memset( struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_set ( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_get (const struct ggml_tensor * tensor, void * data, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_set_2d( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); + GGML_API void ggml_backend_tensor_get_2d(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); + GGML_API void ggml_backend_tensor_memset( struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size); GGML_API void ggml_backend_synchronize(ggml_backend_t backend); @@ -109,7 +113,7 @@ extern "C" { // the copy is performed after all the currently queued operations in backend_src // backend_dst will wait for the copy to complete before performing other operations // automatic fallback to sync copy if async is not supported - GGML_API void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, struct ggml_tensor * src, struct ggml_tensor * dst); + GGML_API void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, const struct ggml_tensor * src, struct ggml_tensor * dst); GGML_API ggml_backend_dev_t ggml_backend_get_device(ggml_backend_t backend); @@ -135,7 +139,9 @@ extern "C" { // integrated GPU device using host memory GGML_BACKEND_DEVICE_TYPE_IGPU, // accelerator devices intended to be used together with the CPU backend (e.g. BLAS or AMX) - GGML_BACKEND_DEVICE_TYPE_ACCEL + GGML_BACKEND_DEVICE_TYPE_ACCEL, + // "meta" device wrapping multiple other devices for tensor parallelism + GGML_BACKEND_DEVICE_TYPE_META, }; // functionality supported by the device @@ -163,7 +169,7 @@ extern "C" { // device type enum ggml_backend_dev_type type; // device id - // for PCI devices, this should be the PCI bus id formatted as "domain:bus:device.function" (e.g. "0000:01:00.0") + // for PCI devices, this should be the lower-case PCI bus id formatted as "domain:bus:device.function" (e.g. "0000:c1:00.0") // if the id is unknown, this should be NULL const char * device_id; // device capabilities @@ -196,7 +202,12 @@ extern "C" { // Common functions that may be obtained using ggml_backend_reg_get_proc_address - // Split buffer type for tensor parallelism + // Context management and operations for faster communication between backends, used for tensor parallelism (meta backend) + typedef void * (*ggml_backend_comm_init_t)(ggml_backend_t * backends, size_t n_backends); + typedef void (*ggml_backend_comm_free_t)(void * comm_ctx); + typedef bool (*ggml_backend_comm_allreduce_tensor_t)(void * comm_ctx, struct ggml_tensor ** tensors); + + // Split buffer type for tensor parallelism (old) typedef ggml_backend_buffer_type_t (*ggml_backend_split_buffer_type_t)(int main_device, const float * tensor_split); // Set the number of threads for the backend typedef void (*ggml_backend_set_n_threads_t)(ggml_backend_t backend, int n_threads); @@ -340,6 +351,53 @@ extern "C" { // Set a callback to be called for each resulting node during graph compute GGML_API void ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backend_sched_eval_callback callback, void * user_data); + // + // Meta backend + // + +#define GGML_BACKEND_META_MAX_DEVICES 16 + + enum ggml_backend_meta_split_axis { + // tensor split by tensor dimensions: + GGML_BACKEND_SPLIT_AXIS_0 = 0, + GGML_BACKEND_SPLIT_AXIS_1 = 1, + GGML_BACKEND_SPLIT_AXIS_2 = 2, + GGML_BACKEND_SPLIT_AXIS_3 = 3, + + GGML_BACKEND_SPLIT_AXIS_MIRRORED = 10, // all values on all backends + GGML_BACKEND_SPLIT_AXIS_PARTIAL = 11, // each backend has a partial sum + + // for internal bookkeeping only: + GGML_BACKEND_SPLIT_AXIS_NONE = 98, + GGML_BACKEND_SPLIT_AXIS_UNKNOWN = 99, + }; + GGML_API const char * ggml_backend_meta_split_axis_name(enum ggml_backend_meta_split_axis split_axis); + + struct ggml_backend_meta_split_state { + enum ggml_backend_meta_split_axis axis; + + // for tensors with axis >= 0 && axis < GGML_MAX_DIMS: + // - each device has a slice of the tensor along the split axis + // - most tensors have n_segments == 1 and a contiguous slice of the tensor data + // - some tensors have an inhomogenenous data layout along the split axis, + // those tensors are divided into segments which are each individually split across devices + // - ne has one entry per segment and device that add up to ggml_tensor::ne for that axis, + // the outer/inner loops are over segments/devices like [seg0_dev0, seg0_dev1, seg1_dev0, seg1_dev1], + // - for example, a transformer may have a fused QKV matrix rather than 3 matrices, those would be 3 separate segments + // that each need to be split individually across devices so that each device gets a slice of Q, K, and V + int64_t ne[16*GGML_BACKEND_META_MAX_DEVICES]; + uint32_t n_segments; + }; + + // function to assign split states for statically allocated tensors, compute tensor split states will be assigned to be compatible: + typedef struct ggml_backend_meta_split_state(*ggml_backend_meta_get_split_state_t)(const struct ggml_tensor * tensor, void * userdata); + + // create a new meta device from "simple" devices, meta buffer type/buffer/backend is then derived from this: + // TODO: this looks a bit strange - a backend API creates a device. I think we should try + // express this as a backend registry functionality instead + GGML_API ggml_backend_dev_t ggml_backend_meta_device( + ggml_backend_dev_t * devs, size_t n_devs, ggml_backend_meta_get_split_state_t get_split_state, void * get_split_state_ud); + // // Utils // diff --git a/Frameworks/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/ggml.h b/Frameworks/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/ggml.h index 669f66b..3357a0d 100644 --- a/Frameworks/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/ggml.h +++ b/Frameworks/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/ggml.h @@ -428,7 +428,8 @@ extern "C" { // GGML_TYPE_IQ4_NL_8_8 = 38, GGML_TYPE_MXFP4 = 39, // MXFP4 (1 block) GGML_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale) - GGML_TYPE_COUNT = 41, + GGML_TYPE_Q1_0 = 41, + GGML_TYPE_COUNT = 42, }; // precision @@ -437,6 +438,12 @@ extern "C" { GGML_PREC_F32 = 10, }; + // op hint + enum ggml_op_hint { + GGML_HINT_NONE = 0, + GGML_HINT_SRC0_IS_HADAMARD = 1, + }; + // model file types enum ggml_ftype { GGML_FTYPE_UNKNOWN = -1, @@ -465,6 +472,7 @@ extern "C" { GGML_FTYPE_MOSTLY_BF16 = 24, // except 1d tensors GGML_FTYPE_MOSTLY_MXFP4 = 25, // except 1d tensors GGML_FTYPE_MOSTLY_NVFP4 = 26, // except 1d tensors + GGML_FTYPE_MOSTLY_Q1_0 = 27, // except 1d tensors }; // available tensor operations: @@ -900,15 +908,17 @@ extern "C" { struct ggml_tensor * b, struct ggml_tensor * ids); - GGML_API struct ggml_tensor * ggml_add1( + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_add1( struct ggml_context * ctx, struct ggml_tensor * a, - struct ggml_tensor * b); + struct ggml_tensor * b), + "use ggml_add instead"); - GGML_API struct ggml_tensor * ggml_add1_inplace( + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_add1_inplace( struct ggml_context * ctx, struct ggml_tensor * a, - struct ggml_tensor * b); + struct ggml_tensor * b), + "use ggml_add_inplace instead"); // dst = a // view(dst, nb1, nb2, nb3, offset) += b @@ -1415,6 +1425,11 @@ extern "C" { struct ggml_tensor * a, enum ggml_prec prec); + // change the hint of a matrix multiplication + GGML_API void ggml_mul_mat_set_hint( + struct ggml_tensor * a, + enum ggml_op_hint hint); + // indirect matrix multiplication GGML_API struct ggml_tensor * ggml_mul_mat_id( struct ggml_context * ctx, @@ -1769,8 +1784,32 @@ extern "C" { int n_dims, int mode); - // custom RoPE + // RoPE operations with extended options + // a is the input tensor to apply RoPE to, shape [n_embd, n_head, n_token] + // b is an int32 vector with size n_token // c is freq factors (e.g. phi3-128k), (optional) + // mode can be GGML_ROPE_TYPE_NORMAL or NEOX; for MROPE and VISION mode, use ggml_rope_multi + // + // pseudo-code for computing theta: + // for i in [0, n_dims/2): + // theta[i] = b[i] * powf(freq_base, -2.0 * i / n_dims); + // theta[i] = theta[i] / c[i]; # if c is provided, divide theta by c + // theta[i] = rope_yarn(theta[i], ...); # note: theta = theta * freq_scale is applied here + // + // other params are used by YaRN RoPE scaling, these default values will disable YaRN: + // freq_scale = 1.0f + // ext_factor = 0.0f + // attn_factor = 1.0f + // beta_fast = 0.0f + // beta_slow = 0.0f + // + // example: + // (marking: c = cos, s = sin, 0 = unrotated) + // given a single head with size = 8 --> [00000000] + // GGML_ROPE_TYPE_NORMAL n_dims = 4 --> [cscs0000] + // GGML_ROPE_TYPE_NORMAL n_dims = 8 --> [cscscscs] + // GGML_ROPE_TYPE_NEOX n_dims = 4 --> [ccss0000] + // GGML_ROPE_TYPE_NEOX n_dims = 8 --> [ccccssss] GGML_API struct ggml_tensor * ggml_rope_ext( struct ggml_context * ctx, struct ggml_tensor * a, @@ -1786,6 +1825,36 @@ extern "C" { float beta_fast, float beta_slow); + // multi-dimensional RoPE, for Qwen-VL and similar vision models + // mode can be either VISION, MROPE, IMROPE, cannot be combined with NORMAL or NEOX + // sections specify how many dimensions to rotate in each section: + // section length is equivalent to number of cos/sin pairs, NOT the number of dims + // (i.e. sum of 4 sections are expected to be n_dims/2) + // last sections can be 0, means ignored + // all other options are identical to ggml_rope_ext + // + // important note: + // - NEOX ordering is automatically applied and cannot be disabled for MROPE and VISION + // if you need normal ordering, there are 2 methods: + // (1) split the tensor manually using ggml_view + // (2) permute the weight upon conversion + // - for VISION, n_dims must be head_size/2 + // + // example M-RoPE: + // given sections = [t=4, y=2, x=2, 0] + // given a single head with size = 18 --> [000000000000000000] + // GGML_ROPE_TYPE_MROPE n_dims = 16 --> [ttttyyxxttttyyxx00] (cos/sin are applied in NEOX ordering) + // GGML_ROPE_TYPE_IMROPE n_dims = 16 --> [ttyxttyxttyxttyx00] (interleaved M-RoPE, still NEOX ordering) + // note: the theta for each dim is computed the same way as ggml_rope_ext, no matter the section + // in other words, idx used for theta: [0123456789... until n_dims/2], not reset for each section + // + // example vision RoPE: + // given sections = [y=4, x=4, 0, 0] (last 2 sections are ignored) + // given a single head with size = 8 --> [00000000] + // GGML_ROPE_TYPE_VISION n_dims = 4 --> [yyyyxxxx] + // other values of n_dims are untested and is undefined behavior + // note: unlike MROPE, the theta for each dim is computed differently for each section + // in other words, idx used for theta: [0123] for y section, then [0123] for x section GGML_API struct ggml_tensor * ggml_rope_multi( struct ggml_context * ctx, struct ggml_tensor * a, diff --git a/Frameworks/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/gguf.h b/Frameworks/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/gguf.h index 79ee202..02d5f22 100644 --- a/Frameworks/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/gguf.h +++ b/Frameworks/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/gguf.h @@ -77,6 +77,7 @@ extern "C" { }; GGML_API struct gguf_context * gguf_init_empty(void); + GGML_API struct gguf_context * gguf_init_from_file_ptr(FILE * file, struct gguf_init_params params); GGML_API struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_params params); //GGML_API struct gguf_context * gguf_init_from_buffer(..); @@ -189,6 +190,7 @@ extern "C" { // // write the entire context to a binary file + GGML_API bool gguf_write_to_file_ptr(const struct gguf_context * ctx, FILE * file, bool only_meta); GGML_API bool gguf_write_to_file(const struct gguf_context * ctx, const char * fname, bool only_meta); // get the size in bytes of the meta data (header, kv pairs, tensor info) including padding diff --git a/Frameworks/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/llama.h b/Frameworks/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/llama.h index 6e72db7..308e8ba 100644 --- a/Frameworks/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/llama.h +++ b/Frameworks/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/Headers/llama.h @@ -154,6 +154,7 @@ extern "C" { LLAMA_FTYPE_MOSTLY_TQ2_0 = 37, // except 1d tensors LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38, // except 1d tensors LLAMA_FTYPE_MOSTLY_NVFP4 = 39, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q1_0 = 40, // except 1d tensors LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file }; @@ -191,9 +192,10 @@ extern "C" { LLAMA_API const char * llama_flash_attn_type_name(enum llama_flash_attn_type flash_attn_type); enum llama_split_mode { - LLAMA_SPLIT_MODE_NONE = 0, // single GPU - LLAMA_SPLIT_MODE_LAYER = 1, // split layers and KV across GPUs - LLAMA_SPLIT_MODE_ROW = 2, // split layers and KV across GPUs, use tensor parallelism if supported + LLAMA_SPLIT_MODE_NONE = 0, // single GPU + LLAMA_SPLIT_MODE_LAYER = 1, // split layers and KV across GPUs + LLAMA_SPLIT_MODE_ROW = 2, // split layers and KV across GPUs, use tensor parallelism if supported + LLAMA_SPLIT_MODE_TENSOR = 3, }; // TODO: simplify (https://github.com/ggml-org/llama.cpp/pull/9294#pullrequestreview-2286561979) @@ -380,22 +382,33 @@ extern "C" { size_t n_samplers; }; + struct llama_model_tensor_override { + const char * pattern; + enum ggml_type type; + }; + + struct llama_model_imatrix_data { + const char * name; + const float * data; + size_t size; + }; + // model quantization parameters typedef struct llama_model_quantize_params { - int32_t nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency() - enum llama_ftype ftype; // quantize to this llama_ftype - enum ggml_type output_tensor_type; // output tensor type - enum ggml_type token_embedding_type; // token embeddings tensor type - bool allow_requantize; // allow quantizing non-f32/f16 tensors - bool quantize_output_tensor; // quantize output.weight - bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored - bool pure; // quantize all tensors to the default type - bool keep_split; // quantize to the same number of shards - bool dry_run; // calculate and show the final quantization size without performing quantization - void * imatrix; // pointer to importance matrix data - void * kv_overrides; // pointer to vector containing overrides - void * tensor_types; // pointer to vector containing tensor types - void * prune_layers; // pointer to vector containing layer indices to prune + int32_t nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency() + enum llama_ftype ftype; // quantize to this llama_ftype + enum ggml_type output_tensor_type; // output tensor type + enum ggml_type token_embedding_type; // token embeddings tensor type + bool allow_requantize; // allow quantizing non-f32/f16 tensors + bool quantize_output_tensor; // quantize output.weight + bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored + bool pure; // quantize all tensors to the default type + bool keep_split; // quantize to the same number of shards + bool dry_run; // calculate and show the final quantization size without performing quantization + const struct llama_model_imatrix_data * imatrix; // pointer to importance matrix data + const struct llama_model_kv_override * kv_overrides; // pointer to kv overrides + const struct llama_model_tensor_override * tt_overrides; // pointer to tensor overrides + const int32_t * prune_layers; // pointer to layer indices to prune } llama_model_quantize_params; typedef struct llama_logit_bias { @@ -465,6 +478,11 @@ extern "C" { const char * path_model, struct llama_model_params params); + // Load a model from an open FILE pointer + LLAMA_API struct llama_model * llama_model_load_from_file_ptr( + FILE * file, + struct llama_model_params params); + // Load a model from multiple splits (support custom naming scheme) // The paths must be in the correct order LLAMA_API struct llama_model * llama_model_load_from_splits( @@ -493,27 +511,6 @@ extern "C" { // Frees all allocated memory LLAMA_API void llama_free(struct llama_context * ctx); - enum llama_params_fit_status { - LLAMA_PARAMS_FIT_STATUS_SUCCESS = 0, // found allocations that are projected to fit - LLAMA_PARAMS_FIT_STATUS_FAILURE = 1, // could not find allocations that are projected to fit - LLAMA_PARAMS_FIT_STATUS_ERROR = 2, // a hard error occurred, e.g. because no model could be found at the specified path - }; - - // fits mparams and cparams to free device memory (assumes system memory is unlimited) - // - returns true if the parameters could be successfully modified to fit device memory - // - this function is NOT thread safe because it modifies the global llama logger state - // - only parameters that have the same value as in llama_default_model_params are modified - // with the exception of the context size which is modified if and only if equal to 0 - LLAMA_API enum llama_params_fit_status llama_params_fit( - const char * path_model, - struct llama_model_params * mparams, - struct llama_context_params * cparams, - float * tensor_split, // writable buffer for tensor split, needs at least llama_max_devices elements - struct llama_model_tensor_buft_override * tensor_buft_overrides, // writable buffer for overrides, needs at least llama_max_tensor_buft_overrides elements - size_t * margins, // margins of memory to leave per device in bytes - uint32_t n_ctx_min, // minimum context size to set when trying to reduce memory use - enum ggml_log_level log_level); // minimum log level to print during fitting, lower levels go to debug log - LLAMA_API int64_t llama_time_us(void); LLAMA_API size_t llama_max_devices(void); @@ -861,12 +858,17 @@ extern "C" { size_t n_token_capacity, size_t * n_token_count_out); +#define LLAMA_STATE_SEQ_FLAGS_NONE 0 + // for backwards-compat #define LLAMA_STATE_SEQ_FLAGS_SWA_ONLY 1 // work only with partial states, such as SWA KV cache or recurrent cache (e.g. Mamba) #define LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY 1 +// keeps the tensor data on device buffers (i.e. not accessible in host memory, but faster save/load) +#define LLAMA_STATE_SEQ_FLAGS_ON_DEVICE 2 + typedef uint32_t llama_state_seq_flags; LLAMA_API size_t llama_state_seq_get_size_ext( @@ -1528,9 +1530,6 @@ extern "C" { LLAMA_API void llama_perf_sampler_print(const struct llama_sampler * chain); LLAMA_API void llama_perf_sampler_reset( struct llama_sampler * chain); - // print a breakdown of per-device memory use via LLAMA_LOG: - LLAMA_API void llama_memory_breakdown_print(const struct llama_context * ctx); - // // training // diff --git a/Frameworks/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/llama b/Frameworks/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/llama index 92a3617..94fe9b4 100755 Binary files a/Frameworks/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/llama and b/Frameworks/llama.xcframework/macos-arm64_x86_64/llama.framework/Versions/A/llama differ diff --git a/README.md b/README.md index 3e217b1..38954dd 100644 --- a/README.md +++ b/README.md @@ -1,32 +1,37 @@ # BisonNotes AI -SwiftUI iOS + watchOS app for recording audio, transcribing it with local or cloud engines, and generating summaries, tasks, and reminders. Core Data powers persistence; background jobs handle long/complex processing; WatchConnectivity syncs state between watch and phone. +SwiftUI app for recording audio, transcribing it with local or cloud engines, and generating summaries, tasks, and reminders. Ships on **iOS, iPadOS, watchOS, and macOS (Mac Catalyst)**. Core Data powers persistence; background jobs handle long/complex processing; WatchConnectivity syncs state between watch and phone. AVAILABLE ON THE APP STORE: https://apps.apple.com/us/app/bisonnotes-ai-voice-notes/id6749189425 Quick links: [Full User Guide](docs/bisonnotes-ai-guide.html) • [Mistral AI Free Setup](docs/mistral-free-setup.md) • [Build & Test](#build-and-test) • [Architecture](#architecture) ## Architecture -- Data: Core Data model at `BisonNotes AI/BisonNotes_AI.xcdatamodeld` stores recordings, transcripts, summaries, and jobs. -- Engines: Pluggable services for On Device transcription, OpenAI, OpenAI-compatible APIs, Mistral AI, Google AI Studio, AWS Bedrock/Transcribe, Whisper (REST), Wyoming streaming, Ollama, and On-Device AI. Each engine pairs a service with a settings view. +- Data: Core Data model at `BisonNotes AI/BisonNotes_AI.xcdatamodeld` stores recordings, transcripts, summaries, and jobs. Sensitive credentials (API keys, AWS access keys, Bedrock session tokens) live in the iOS Keychain, never on disk in plaintext. +- Engines: Pluggable services for On Device transcription, OpenAI, OpenAI-compatible APIs, Mistral AI, Google AI Studio, AWS Bedrock/Transcribe, Whisper (REST), Wyoming streaming, Ollama, On-Device AI (llama.cpp), MLX Swift (experimental), and Apple Native (Foundation Models). Each engine pairs a service with a settings view. - Background: `BackgroundProcessingManager` coordinates queued work with retries, timeouts, and recovery. Large files are chunked and processed streaming‑first. -- Watch Sync: `WatchConnectivityManager` (on iOS and watch targets) manages reachability, queued transfers, and state recovery. +- Recording: A platform-aware audio pipeline — `AVAudioRecorder` on iOS/iPadOS, `AVAudioEngine` on Mac Catalyst (`AudioRecorderViewModel+CatalystEngine.swift`) — with shared Pause/Resume support and crash-safe interruption handling. +- Watch Sync: `WatchConnectivityManager` (on iOS and watch targets) manages reachability, queued transfers, and state recovery. Watch complications and a Control Center recording widget are bundled as separate targets. - UI: SwiftUI views under `Views/` implement recording, summaries, transcripts, and settings. AI-generated content uses MarkdownUI for professional formatting. View models isolate state and side effects. ## Project Structure -- `BisonNotes AI/`: iOS app source +- `BisonNotes AI/`: iOS / iPadOS / Mac Catalyst app source - Notable folders: `Models/`, `Views/`, `ViewModels/`, `OpenAI/`, `AWS/`, `Wyoming/`, `WatchConnectivity/`, `OnDeviceLLM/`, `FluidAudio/`, `Services/` - Assets: `Assets.xcassets`; config: `Info.plist`, `.entitlements` - Uses Xcode's file-system synchronized groups, so dropping new Swift files into these folders automatically adds them to the project—no manual `.xcodeproj` edits are necessary. -- `BisonNotes Share/`: Share Extension target for importing audio from other apps -- `BisonNotes AI Watch App/`: watchOS companion app +- `BisonNotes Share/`: Share Extension target for importing audio from other apps (excluded from Mac Catalyst embed phase) +- `BisonNotes AI Watch App/`: watchOS companion app (excluded from Mac Catalyst embed phase) +- `BisonNotes Watch Widget/`: Watch complications surface for live recording state +- `BisonNotes AI Controls/`: Control Center recording widget (Recording Control Widget) - Tests: `BisonNotes AITests/` (unit), `BisonNotes AIUITests/` (UI), plus watch tests ## Build and Test - Open in Xcode: `open "BisonNotes AI/BisonNotes AI.xcodeproj"` - Build (iOS): `xcodebuild -project "BisonNotes AI/BisonNotes AI.xcodeproj" -scheme "BisonNotes AI" -configuration Debug build` - Test (iOS): `xcodebuild test -project "BisonNotes AI/BisonNotes AI.xcodeproj" -scheme "BisonNotes AI" -destination 'platform=iOS Simulator,name=iPhone 15'` +- Build (Mac Catalyst): `xcodebuild -project "BisonNotes AI/BisonNotes AI.xcodeproj" -scheme "BisonNotes AI" -destination 'platform=macOS,variant=Mac Catalyst' -configuration Debug build` - Use the watch app scheme to run the watch target. SwiftPM resolves automatically in Xcode. +- See `CLAUDE.md` for the manual `llama.xcframework` Mac Catalyst slice and `bisonbet/textual` Catalyst guards if you rebuild dependencies. ## Dependencies @@ -45,6 +50,7 @@ The project uses Swift Package Manager for dependency management. Major dependen - Supports GGUF model format with Q4_K_M quantization (optimal for mobile) - Available models: Gemma 3n E4B/E2B, Granite 4.0 H Tiny/Micro, Ministral 3B, LFM 2.5 1.2B, Qwen3.5 2B/4B - Models filtered by device RAM (6GB+ for most, 8GB+ for larger models) + - **Mac Catalyst note**: The upstream xcframework has no `maccatalyst` slice. The `ios-arm64-maccatalyst` slice in this repo was manually created from the macOS arm64 binary (`lipo -thin arm64`) and patched with `vtool -set-build-version maccatalyst 14.0 15.5`. If you rebuild or update the xcframework, repeat these steps and update `Frameworks/llama.xcframework/Info.plist` accordingly. Full instructions are in `CLAUDE.md` under "Mac Catalyst Build Notes". ### **UI & Formatting** - **MarkdownUI**: Professional markdown rendering for AI-generated summaries, headers, lists, and formatted text @@ -69,36 +75,50 @@ All external dependencies are resolved automatically via Swift Package Manager w - Branch/PR: create a feature branch in your fork, push changes, and open a PR. Include build/test results and screenshots for UI changes. ## Key Features +- **Now on Mac (v1.11)**: Mac Catalyst build runs natively on macOS — record, transcribe, and summarize on your Mac with the same Core Data store and engine catalog as iPhone/iPad. +- **Pause and Resume Recording (v1.11)**: Pause mid-meeting without stopping the file. Resume seamlessly across iOS, iPadOS, and Mac Catalyst (separate `AVAudioEngine` path on Catalyst). +- **Hardened Credential Storage (v1.11)**: API keys, AWS credentials, and Bedrock session tokens stored in the iOS Keychain. Legacy values are migrated automatically and kept out of iCloud settings backups. File protection is applied to recordings, transcripts, notes, attachments, and the Core Data SQLite files. +- **Endpoint Safety (v1.11)**: User-configurable OpenAI, OpenAI-compatible, Ollama, and Whisper endpoints are validated — public cleartext (HTTP/WS) destinations are blocked by default; local/private endpoints stay allowed, with a Development Mode toggle for power users. +- **Source-Centric Workflow (v1.11)**: "Generate Transcript" lives on the recording row; "Generate Summary" lives on the transcript. Buttons only appear where they apply and disappear once the artifact exists — regeneration happens from the existing detail view. - **iPhone Action Button Support**: Quick-start recording from the Action Button on iPhone 15 Pro/Pro Max, iPhone 16 Pro/Pro Max, and future Pro models. Press the Action Button to launch the app and start recording instantly, even when your phone is locked. -- **Watch App**: Full recording control from Apple Watch with automatic sync via WatchConnectivity -- **Multiple AI Engines**: Support for OpenAI, AWS Bedrock, Google AI Studio, Mistral AI, Ollama, On-Device AI, and Apple Native (Apple Intelligence) +- **Watch App & Complications**: Full recording control from Apple Watch with automatic sync via WatchConnectivity. Watch complications surface live recording state on the watch face. +- **Control Center Recording Widget**: Start/stop recordings from Control Center on iOS 18+ via the bundled Controls widget. +- **Multiple AI Engines**: Support for OpenAI, AWS Bedrock, Google AI Studio, Mistral AI, OpenAI-compatible endpoints, Ollama, On-Device AI (llama.cpp), Apple Native (Apple Intelligence), and **MLX Swift (experimental)** for local summarization. - **Apple Native AI Engine**: On-device summarization using Apple's Foundation Models framework (iOS 26+, iPhone 15 Pro+). No data leaves the device. +- **MLX Swift Engine (Experimental)**: Apple Silicon-native local summarization built on MLX Swift. Enable in Settings → Experimental summary models & MLX AI engine. - **Mistral AI (Free & Paid Tiers)**: Guided in-app setup wizard for Mistral's free tier -- transcription and summarization with no credit card required. Paid tiers available for higher rate limits. Cloud transcription via Voxtral Mini with speaker diarization support. - **On-Device Processing**: Complete privacy with FluidAudio Parakeet transcription and On-Device AI summarization (default for new installs) +- **Comedy Mode**: Optional summarization tone (snarky and other styles) applied across engines that support custom prompts. +- **Google Calendar Integration**: Send tasks or reminders into Google Calendar (app or web fallback) in addition to Apple Reminders/Calendar. +- **Summary Attachments**: Attach text, PDF, or other documents to a summary and preview them inline (Quick Look fallback for unknown types). +- **Recording Title Editing**: Edit recording titles directly from the audio player or transcript editor; AI-generated alternative titles are still available from the summary view. - **Audio Export**: Share any recording as an audio file via the iOS share sheet - **Audio Archive to iCloud Drive**: Offload selected recordings, or recordings older than a chosen age, while keeping transcripts, summaries, and a saved restore pointer in the app. Third-party file providers are disabled for archive targets for now. - **Video Import**: Import video files; audio is automatically extracted to M4A - **Audio Cleanup**: Optional pre-transcription DSP processing — high-pass filter, noise gate, dynamic normalization, and peak limiting - **Live Transcription**: On-device live speech-to-text via SFSpeechRecognizer during recording; transcript auto-saved on stop -- **Share Extension**: Import audio files directly from Voice Memos, Files, and other apps via the iOS share sheet +- **Share Extension**: Import audio files directly from Voice Memos, Files, and other apps via the iOS share sheet. Token-based authorization prevents the main app from scanning the shared container without an explicit handoff. - **Combine Recordings**: Merge two separate recordings into a single continuous audio file - **PDF Export**: Professional PDF reports with three-pane header (metadata, local map, regional map), pagination, and dedicated tasks/reminders sections - **Background Processing**: Long recordings and complex processing handled automatically in the background with intelligent stale job detection and automatic recovery -- **iCloud Backup & Sync**: Automatic backup on recording creation, CloudKit summary sync with paginated queries and schema-safe fallback, deferred auto-backup +- **iCloud Backup & Sync**: Automatic backup on recording creation, CloudKit summary sync with paginated queries and schema-safe fallback, deferred auto-backup. Sensitive settings (API keys, AWS credentials) are excluded from iCloud settings backups by default. - **Search Functionality**: Powerful search across recordings, transcripts, and summaries. Search by recording name, transcript text, summary content, tasks, reminders, and titles. - **Date Filters**: Filter recordings, transcripts, and summaries by date range. Select start and end dates to quickly find content from specific time periods. ## Key Modules -- Recording: `EnhancedAudioSessionManager`, `AudioFileChunkingService`, `AudioRecorderViewModel`, `RecordingCombiner` +- Recording: `EnhancedAudioSessionManager`, `AudioFileChunkingService`, `AudioRecorderViewModel` (+ `+CatalystEngine`, `+Interruptions`, `+Background`, `+CallIntelligence`, `+Warnings`), `RecordingCombiner`, `TranscriptionStarter` - Transcription: `FluidAudioManager` (Parakeet), `OpenAITranscribeService`, `MistralTranscribeService`, `WhisperService`, `WyomingWhisperClient`, `AWSTranscribeService`, `LiveTranscriptionService` -- Summarization: `OpenAISummarizationService`, `MistralAISummarizationService`, `GoogleAIStudioService`, `AWSBedrockService`, `OnDeviceLLMService`, `AppleNativeEngine` +- Summarization: `OpenAISummarizationService`, `MistralAISummarizationService`, `GoogleAIStudioService`, `AWSBedrockService`, `OnDeviceLLMService`, `MLXSwiftEngine`, `AppleNativeEngine` +- Security: `KeychainSecretStore`, `AWSCredentialsManager`, `AWSClientCredentialResolver`, `EndpointSecurityPolicy`, `AppFileProtection` - Export: `PDFExportService`, `SummaryExportFormatter`, `RecordingArchiveService` - UI: `SummariesView`, `SummaryDetailView`, `TranscriptionProgressView`, `AITextView` (with MarkdownUI), `CombineRecordingsView` - Persistence: `Persistence`, `CoreDataManager`, models under `Models/` - Background: `BackgroundProcessingManager` -- Watch: `WatchConnectivityManager` (both targets) +- Watch: `WatchConnectivityManager` (both targets), `BisonNotesComplications` (Watch Widget target) +- Controls: `RecordingControlWidget` (Control Center recording widget) - Share Extension: `ShareViewController` (imports audio from other apps via share sheet) - Action Button: `StartRecordingIntent`, `ActionButtonLaunchManager`, `AppShortcuts` +- Integrations: `SystemIntegrationManager` (Reminders, Apple Calendar, Google Calendar), `IntegrationSelectionView` ## Audio Archive @@ -164,7 +184,8 @@ The app supports multiple AI engines for summarization and content analysis: | **Google AI Studio** | Gemini 3 Flash Preview (default), Gemini 3.1 Flash Lite Preview | API key, internet | | **AWS Bedrock** | Claude 4.5 Haiku, Claude Sonnet 4.5, Llama 4 Maverick 17B Instruct | AWS credentials | | **Ollama** | Local LLM server (recommended: qwen3:30b, gpt-oss:20b, mistral-small3.2) | Ollama server running | -| **On-Device AI** | Fully offline, privacy-focused — llama.cpp with GGUF models | iPhone 15 Pro+, model (2-4.5 GB) | +| **On-Device AI** | Fully offline, privacy-focused — llama.cpp with GGUF models | 6 GB+ RAM, model (731 MB – 4.5 GB) | +| **MLX Swift** *(experimental)* | Apple Silicon-native local summarization on MLX Swift | Enable experimental engines toggle in Settings | ### OpenAI Models @@ -226,10 +247,12 @@ The on-device AI feature enables completely private, offline AI processing: - **Downloads**: WiFi by default with optional cellular download support ## Configuration -- Secrets are entered in‑app via setup views (OpenAI, Mistral AI, Google, AWS, Ollama, Whisper). Do not commit API keys. -- Enable required capabilities in Xcode (Microphone, Background Modes, iCloud if used). Keep `Info.plist` and `.entitlements` aligned with features. +- Secrets are entered in‑app via setup views (OpenAI, Mistral AI, Google, AWS, Ollama, Whisper). All keys/tokens are persisted to the iOS Keychain through `KeychainSecretStore`; legacy `UserDefaults` values are migrated automatically on first launch of v1.11. Do not commit API keys. +- AWS process-environment credentials (`AWS_ACCESS_KEY_ID` etc.) are cleared at launch; Bedrock, Transcribe, and background jobs use explicit credential resolvers from `AWSCredentialsManager`. +- User-configurable AI endpoints (OpenAI/OpenAI-Compatible/Ollama/Whisper) are validated via `EndpointSecurityPolicy` — public cleartext destinations are blocked unless the per-service Development Mode override is enabled. +- Enable required capabilities in Xcode (Microphone, Background Modes, iCloud if used). Keep `Info.plist` and `.entitlements` aligned with features. `APS_ENVIRONMENT` is set per-configuration so Debug uses `development` and Release uses `production`. - For On Device transcription, Parakeet is the only on-device engine (WhisperKit was removed in v1.8). Download the model in Setup → Transcription Settings → On Device. -- For on-device AI, device capability checks ensure your device meets requirements (iPhone 15 Pro+ for llama.cpp models; iOS 26+ for Apple Native) before allowing downloads. +- For on-device AI, device capability checks ensure your device meets requirements (6 GB+ RAM for llama.cpp models; iOS 26+ and an Apple Intelligence–capable device for Apple Native) before allowing downloads. ## iPhone Action Button Setup If you have an iPhone 15 Pro, iPhone 15 Pro Max, iPhone 16 Pro, iPhone 16 Pro Max, or future iPhone Pro models with an Action Button, you can configure it to start recording instantly: diff --git a/docs/bisonnotes-ai-guide.html b/docs/bisonnotes-ai-guide.html index 77f3bba..f345ac5 100644 --- a/docs/bisonnotes-ai-guide.html +++ b/docs/bisonnotes-ai-guide.html @@ -131,8 +131,8 @@

BisonNotes AI

-

Complete User Guide

- v1.8 +

Complete User Guide — iPhone, iPad, Apple Watch, and Mac

+ v1.11
@@ -147,6 +147,8 @@

BisonNotes AI

  • Recording Features
      +
    • Mac Catalyst
    • +
    • Pause & Resume
    • Action Button
    • Share Extension
    • Combining Recordings
    • @@ -157,6 +159,7 @@

      BisonNotes AI

    • Working with Summaries
    • Audio Playback
    • Setup & Configuration
    • +
    • Privacy & Security
    • Troubleshooting
    • Advanced Features
    • Best Practices
    • @@ -164,6 +167,20 @@

      BisonNotes AI

      + + + +
      + What’s new in v1.11: +
        +
      • Now on Mac. BisonNotes AI runs on macOS via Mac Catalyst — record, transcribe, and summarize on your Mac with the same data as iPhone and iPad.
      • +
      • Pause & Resume recording. Step away from a meeting without losing your spot.
      • +
      • Major security hardening. API keys, AWS credentials, and tokens live in the iOS Keychain. Recordings, transcripts, notes, and attachments get explicit file protection, and user-configured AI endpoints are validated to block insecure cleartext destinations.
      • +
      • Cleaner workflow. “Generate Transcript” lives on the recording, “Generate Summary” lives on the transcript — the right action shows up in the right place.
      • +
      • Refreshed local AI engine. Updated llama.cpp build with a dedicated Mac Catalyst slice; experimental MLX Swift summarization engine for Apple Silicon.
      • +
      +
      + @@ -196,23 +213,26 @@

      Initial Setup Options

      Your First Recording

        -
      1. Start Recording — Tap the large microphone button on the main screen.
      2. -
      3. Microphone Permission — On your first recording, iOS will ask for microphone access. Tap "OK". If denied, re-enable in Settings → Privacy & Security → Microphone.
      4. -
      5. Recording Status — You'll see a red recording indicator, live timer, and location indicator (if enabled).
      6. -
      7. Stop Recording — Tap the stop button to end.
      8. -
      9. Background Recording — The app continues recording when minimized or the phone is locked.
      10. +
      11. Start Recording — Tap the large microphone button on the main screen.
      12. +
      13. Microphone Permission — On your first recording, the system will ask for microphone access. Tap “OK”. If denied, re-enable in Settings → Privacy & Security → Microphone (or System Settings → Privacy & Security → Microphone on Mac).
      14. +
      15. Recording Status — You’ll see a red recording indicator, live timer, and location indicator (if enabled).
      16. +
      17. Pause & Resume (v1.11) — Tap the Pause button to pause without ending the file; tap Resume to continue. Works on iPhone, iPad, and Mac.
      18. +
      19. Stop Recording — Tap the stop button to end.
      20. +
      21. Background Recording — The app continues recording when minimized or the phone is locked. On Mac, the app keeps recording when in the background or behind other windows.

      First Transcript & Summary

      +
      v1.11 workflow: The “Generate Transcript” action lives on the recording; the “Generate Summary” action lives on the transcript. Each button only appears where it makes sense and disappears once the artifact exists — if you want to redo something, open the existing transcript or summary and regenerate from there.
      +
      Generate Your First Transcript
        -
      1. Access Recording — After stopping, you'll see it in the recordings list.
      2. -
      3. Start Transcription — Tap the recording, then tap "Generate Transcript". The app processes your audio using your selected engine.
      4. -
      5. Progress — You'll see processing status and time remaining. You can continue using the app while it processes.
      6. -
      7. View Results — Full transcript with editable text, timestamps (if supported), and confidence indicators.
      8. +
      9. Access Recording — After stopping, you’ll see it in the Recordings list.
      10. +
      11. Start Transcription — Tap “Generate Transcript” on the recording row, or open the audio player and tap it there. The app processes your audio using your selected transcription engine.
      12. +
      13. Progress — You’ll see processing status and an estimate. You can continue using the app while it processes.
      14. +
      15. View Results — Open the Transcripts tab to read the full transcript with editable text and confidence indicators.
      @@ -221,9 +241,9 @@

      First Transcript & Summary

      Generate Your First Summary
        -
      1. Prerequisites — You need a transcript first.
      2. -
      3. Start Summary — In the recording detail view, tap "Generate Summary".
      4. -
      5. AI Processing — The app analyzes your transcript and creates: +
      6. Prerequisites — You need a transcript first.
      7. +
      8. Start Summary — Open the transcript (Transcripts tab) and tap “Generate Summary”. A second “Generate Summary” button is also available directly on the transcript row.
      9. +
      10. AI Processing — The app analyzes your transcript and creates:
        • Enhanced Summary: Main content overview
        • Action Items: Tasks with priority levels
        • @@ -231,7 +251,7 @@

          First Transcript & Summary

        • Alternative Titles: AI-generated recording names
      11. -
      12. Review Results — Expandable sections, priority indicators, interactive maps, and integration options.
      13. +
      14. Review Results — Expandable sections, priority indicators, interactive maps, attachments, and integration options (Apple Reminders, Apple Calendar, Google Calendar).
      @@ -267,6 +287,24 @@

      First Transcript & Summary

      Recording Features

      +

      Mac Catalyst (v1.11)

      +

      BisonNotes AI runs natively on macOS as a Mac Catalyst app. The Mac build uses an AVAudioEngine-based recording pipeline that is tuned for desktop audio sessions:

      +
        +
      • Same Core Data store — Recordings, transcripts, and summaries sync with iPhone/iPad through iCloud (if enabled).
      • +
      • Pause & Resume — Full pause/resume support, the same as on iOS.
      • +
      • Microphone selection — Choose your built-in mic, USB interface, or Bluetooth device from Transcription Settings.
      • +
      • Local AI — The on-device llama.cpp engine ships with a dedicated Mac Catalyst slice; experimental MLX Swift summarization is available on Apple Silicon Macs.
      • +
      • Settings sheets — Settings forms use a Mac-friendly layout for reliable scrolling and button hit targets.
      • +
      + +

      Pause & Resume Recording (v1.11)

      +

      Pause and resume the current recording without stopping the file:

      +
        +
      • Tap the Pause button while recording. The timer freezes; audio capture stops; the file stays open.
      • +
      • Tap Resume to keep adding to the same audio file from where you left off.
      • +
      • Works on iPhone, iPad, and Mac. On Mac the underlying pipeline uses AVAudioEngine for a clean pause boundary.
      • +
      +

      iPhone Action Button Integration

      Available on: iPhone 15 Pro, iPhone 15 Pro Max, iPhone 16 Pro, iPhone 16 Pro Max, and future iPhone Pro models with Action Button.
      @@ -632,6 +670,28 @@

      Setup

      + +
      + 9. MLX Swift — Experimental On-Device Summarization +
      + On-Device + Free +

      Type: On-device, Apple Silicon • Cost: Free • Privacy: 100% local • Internet: Only for initial model download

      + +
      Experimental: The MLX Swift engine is gated behind the experimental toggle in Settings. It runs on Apple’s MLX framework and is best treated as a preview while we tune memory pressure on lower-RAM devices.
      + +

      Setup

      +
        +
      1. Go to Setup → Additional Settings.
      2. +
      3. Turn on “Experimental summary models & MLX AI engine”.
      4. +
      5. Return to AI Settings and pick MLX Swift from the engine library.
      6. +
      7. Download an MLX model from the MLX Swift settings screen.
      8. +
      + +

      Best for: Apple Silicon devices (especially Mac and iPad with M-series chips) where you want fully local summarization with a different model lineup than llama.cpp.

      +
      +
      + @@ -742,11 +802,13 @@

      Viewing Summaries

      Summary Features

        -
      • Expandable Sections — Tap to expand/collapse
      • -
      • Task Extraction — AI-identified actionable items
      • -
      • Reminder Detection — Time-sensitive reminders
      • -
      • Priority Indicators — Color-coded priorities
      • -
      • Location Maps — Interactive maps showing recording location
      • +
      • Expandable Sections — Tap to expand/collapse
      • +
      • Task Extraction — AI-identified actionable items
      • +
      • Reminder Detection — Time-sensitive reminders
      • +
      • Priority Indicators — Color-coded priorities
      • +
      • Location Maps — Interactive maps showing recording location
      • +
      • Attachments — Attach text, PDF, or other documents to a summary. Inline previews are available for text and PDF; other formats open with Quick Look. Note exports save your notes for sharing.
      • +
      • Send to Reminders / Calendar — Push extracted tasks and reminders into Apple Reminders, Apple Calendar, or Google Calendar (uses the Google Calendar app if installed, falls back to the web).

      Search & Filtering

      @@ -766,13 +828,18 @@

      Editing Metadata

      Change Title, Date, or Location

      Recording Title

      -

      Open summary → "Titles" section → "Edit" → enter new title or select from AI alternatives → "Use This Title".

      +

      You can rename a recording from three places:

      +
        +
      • The audio player view (tap the title to edit)
      • +
      • The transcript editor (tap the title at the top)
      • +
      • The summary view — “Titles” section → “Edit” → enter a new title or pick one of the AI-generated alternatives → “Use This Title”.
      • +

      Custom Date & Time

      -

      Open summary → "Recording Date & Time" → "Set Custom Date & Time" → use pickers → "Save".

      +

      Open summary → “Recording Date & Time” → “Set Custom Date & Time” → use pickers → “Save”. BisonNotes AI also persists the explicit recording start timestamp so date sorting matches when the audio was captured, not when it was imported.

      Location

      -

      Open summary → location section → "Add Location" or "Edit Location". Choose: Current Location (GPS), Map Selection, or Manual Entry → "Save".

      +

      Open summary → location section → “Add Location” or “Edit Location”. Choose: Current Location (GPS), Map Selection, or Manual Entry → “Save”.

      @@ -816,10 +883,10 @@

      Simple Setup

      Additional Settings

      Full control over all options.

        -
      • All 8 AI engines
      • +
      • All 8 AI engines plus the experimental MLX Swift engine
      • All transcription engines
      • Microphone selection
      • -
      • Preferences, iCloud, background processing
      • +
      • Preferences, iCloud, background processing, experimental toggles
      @@ -828,19 +895,60 @@

      Additional Settings

      Audio Settings

        -
      • Quality: Whisper Optimized (22 kHz, 64 kbps AAC) — optimized for voice
      • -
      • Microphone Selection: Choose from available mics (built-in, Bluetooth, USB). Saved across recordings. Auto-fallback if unavailable.
      • -
      • Mixed Audio: Record without interrupting system audio
      • -
      • Background Recording: Continue when app is minimized
      • +
      • Quality: Whisper Optimized (22 kHz, 64 kbps AAC) — optimized for voice
      • +
      • Microphone Selection: Choose from available mics (built-in, Bluetooth, USB, audio interface). Saved across recordings. Auto-fallback if unavailable. Available on iPhone, iPad, and Mac.
      • +
      • Mixed Audio: Record without interrupting system audio (iOS / iPadOS)
      • +
      • Background Recording: Continue when the app is minimized (iOS / iPadOS) or behind other windows (Mac)
      • +
      • Pause & Resume (v1.11): Pause without ending the file; resume to keep adding to the same recording

      AI Settings

        -
      • Engine Selection — Choose from all 7 AI engines
      • -
      • Model Configuration — Temperature, max tokens, etc.
      • -
      • Connection Testing — Verify API connectivity
      • -
      • Batch Regeneration — Update all summaries with new engine
      • +
      • Engine Selection — Choose from all 8 AI engines (plus MLX Swift if experimental engines are enabled)
      • +
      • Model Configuration — Temperature, max tokens, comedy mode tone, etc.
      • +
      • Connection Testing — Verify API connectivity
      • +
      • Batch Regeneration — Update all summaries with new engine
      • +
      + +

      Comedy Mode

      +

      Comedy Mode adds an optional tone to AI-generated summaries. Turn it on in Settings, pick a style (e.g. snarky), and supported engines will rewrite summaries in that voice. Disable it to return to the standard, neutral summary style.

      + + + + +

      Privacy & Security

      + +
      v1.11 security hardening: BisonNotes AI now stores all sensitive credentials in the iOS Keychain, applies file protection to recordings and notes, and validates user-configurable AI endpoints to block insecure cleartext destinations.
      + +

      How credentials are stored

      +
        +
      • Keychain-backed storage — OpenAI keys, Mistral keys, Google AI keys, AWS access keys/secret keys/session tokens, OpenAI-Compatible keys, and Whisper auth tokens all live in the iOS Keychain.
      • +
      • Automatic migration — Existing values stored in legacy locations are migrated into the Keychain on first launch of v1.11 and then removed from their old locations.
      • +
      • Process environment is cleared — Any inherited AWS_* environment variables are scrubbed at launch so AWS SDKs only use the credentials you configured in-app.
      • +
      • iCloud settings backup — Sensitive settings (API keys, AWS credentials) are excluded from iCloud settings backups by default. If a previous backup contained sensitive values, they are restored into the Keychain rather than re-saved to settings storage.
      • +
      + +

      File protection

      +

      BisonNotes AI applies explicit iOS file protection to:

      +
        +
      • Audio recordings, imported audio, restored archives, and watch backups
      • +
      • Live transcription placeholder audio
      • +
      • Summary attachments and their metadata
      • +
      • Persistent error logs
      • +
      • The Core Data SQLite store and related files
      • +
      + +

      Endpoint safety

      +

      User-configurable AI services (OpenAI, OpenAI-Compatible, Ollama, Whisper) are checked against an endpoint policy:

      +
        +
      • Local/private endpoints (loopback, link-local, RFC1918 networks) are allowed by default.
      • +
      • Public cleartext endpoints (plain HTTP or WS pointing at the open internet) are blocked unless you explicitly enable Development Mode for that service.
      • +
      • HTTPS / WSS public endpoints (Groq, Nebius, OpenAI, etc.) work as expected.
      +
      Development Mode: Enable it only on networks you control. It exists so you can connect to a self-hosted server on a public address while you set it up — it does not weaken transport security for any other engine.
      + +

      Share Extension hardening

      +

      Imports from the Share Extension are now authorized with one-time tokens before the main app scans the shared container, and the imported files themselves are file-protected. Full file paths are no longer written to logs.

      @@ -908,14 +1016,15 @@

      Advanced Features

      Background Processing
        -
      • Job Queue — Multiple operations queued and processed in order
      • -
      • Progress Tracking — Real-time updates with accurate status display
      • -
      • Error Recovery — Automatic retry for failures
      • -
      • Stale Job Detection (v1.7) — Intelligent detection and recovery of stuck jobs, with timeouts based on actual processing start time
      • -
      • Job Cancellation (v1.7) — Properly cancel and interrupt running jobs when the app is backgrounded or closed
      • -
      • On-Device AI Handling — On-device AI jobs are gracefully interrupted when the app is backgrounded, since GPU processing requires the app to stay open
      • -
      • Live Transcription (v1.8) — On-device speech-to-text streamed in real time during recording via SFSpeechRecognizer; transcript auto-saved when recording stops
      • -
      • Duplicate Summary Cleanup (v1.8) — Automatically detects and removes duplicate summaries; manual cleanup available in Database Tools
      • +
      • Job Queue — Multiple operations queued and processed in order
      • +
      • Progress Tracking — Real-time updates with accurate status display
      • +
      • Error Recovery — Automatic retry for failures
      • +
      • Stale Job Detection (v1.7) — Intelligent detection and recovery of stuck jobs, with timeouts based on actual processing start time
      • +
      • Job Cancellation (v1.7) — Properly cancel and interrupt running jobs when the app is backgrounded or closed
      • +
      • On-Device AI Handling — On-device AI jobs are gracefully interrupted when the app is backgrounded, since GPU processing requires the app to stay open
      • +
      • Live Transcription (v1.8) — On-device speech-to-text streamed in real time during recording via SFSpeechRecognizer; transcript auto-saved when recording stops
      • +
      • Duplicate Summary Cleanup (v1.8) — Automatically detects and removes duplicate summaries; manual cleanup available in Database Tools
      • +
      • Explicit credential resolvers (v1.11) — Background AWS jobs always pull fresh credentials from the Keychain so stale process-environment values can’t leak in
      @@ -924,17 +1033,30 @@

      Advanced Features

      File Management
        -
      • Import/Export — M4A, MP3, WAV, CAF, AIFF, AIF
      • -
      • Audio Export (v1.8) — Share any recording as an audio file via the iOS share sheet
      • -
      • Audio Archive — Offload selected or older audio files to iCloud Drive, keep an in-app restore pointer, and remove the archived cloud copy after restore
      • -
      • Video Import (v1.8) — Import video files; audio is automatically extracted to M4A
      • -
      • Audio Cleanup (v1.8) — Optional pre-transcription DSP: high-pass filter, noise gate, dynamic normalization, peak limiting
      • -
      • Share Extension — Import from Voice Memos, Files, etc.
      • -
      • Combining Recordings — Merge two into one
      • -
      • PDF Export — Professional reports with metadata, maps, page numbers
      • -
      • File Relationships — Audio/transcript/summary links maintained
      • -
      • Orphaned File Detection — Find and clean disconnected files
      • -
      • Selective Deletion — Choose what to keep
      • +
      • Import/Export — M4A, MP3, WAV, CAF, AIFF, AIF
      • +
      • Audio Export (v1.8) — Share any recording as an audio file via the iOS share sheet
      • +
      • Clean Audio Export (v1.10) — Exports use a clean, user-friendly filename derived from the recording title
      • +
      • Audio Archive — Offload selected or older audio files to iCloud Drive, keep an in-app restore pointer, and remove the archived cloud copy after restore
      • +
      • Video Import (v1.8) — Import video files; audio is automatically extracted to M4A
      • +
      • Audio Cleanup (v1.8) — Optional pre-transcription DSP: high-pass filter, noise gate, dynamic normalization, peak limiting
      • +
      • Share Extension — Import from Voice Memos, Files, etc., with one-time token authorization (v1.11)
      • +
      • Combining Recordings — Merge two into one
      • +
      • PDF Export — Professional reports with metadata, maps, page numbers
      • +
      • File Relationships — Audio/transcript/summary links maintained
      • +
      • Orphaned File Detection — Find and clean disconnected files
      • +
      • Selective Deletion — Choose what to keep
      • +
      • File Protection (v1.11) — Recordings, attachments, notes, and the Core Data store are stored with explicit iOS file protection
      • +
      +
      + + +
      + Watch Complications & Control Center Widget +
      +
        +
      • Apple Watch Complications (v1.9) — Surface live recording state on watch faces. Tap the complication to open the watch app.
      • +
      • Control Center Recording Widget (iOS 18+) — Add the BisonNotes recording control to Control Center to start/stop recordings without opening the app.
      • +
      • Watch App — Full standalone recording on Apple Watch with automatic sync back to iPhone via WatchConnectivity.
      @@ -957,10 +1079,11 @@

      Advanced Features

      iCloud Backup & Sync
        -
      • iCloud Backup — Full backup of all data to iCloud Drive
      • -
      • Audio Archive to iCloud Drive — Space-saving audio offload with tracked restore locations. Third-party Files providers are paused for archive targets for now.
      • -
      • Auto-Backup (v1.7) — Automatically backs up when new recordings are created
      • -
      • CloudKit Sync — Optional summary synchronization across devices with paginated queries and schema-safe fallback for reliability
      • +
      • iCloud Backup — Full backup of all data to iCloud Drive
      • +
      • Audio Archive to iCloud Drive — Space-saving audio offload with tracked restore locations. Third-party Files providers are paused for archive targets for now.
      • +
      • Auto-Backup (v1.7) — Automatically backs up when new recordings are created
      • +
      • CloudKit Sync — Optional summary synchronization across devices with paginated queries and schema-safe fallback for reliability
      • +
      • Sensitive data excluded (v1.11) — API keys and AWS credentials are never written into iCloud settings backups; if a previous backup contained them, they are restored into the Keychain instead.
      • Prompted after generating your first summary