From da7a535c392f9a007a14d4db36e0ec89aaae3e6c Mon Sep 17 00:00:00 2001 From: MarcusJRLee <7527115+MarcusJRLee@users.noreply.github.com> Date: Fri, 28 Aug 2026 00:03:18 -0400 Subject: [PATCH] Add deterministic casing and spoken list semantics --- README.md | 8 +- .../application_preferences.swift | 4 +- .../local_ai_dictation.swift | 74 ++++---- .../personal_dictionary.swift | 37 ++++ .../HardwareControllerCore/voice_casing.swift | 153 ++++++++++++++++ .../voice_list_intent.swift | 61 ++++++ .../voice_spoken_edit.swift | 4 +- .../voice_spoken_edit_engine.swift | 173 +++++++++++++++--- .../voice_spoken_edit_replayer.swift | 62 ++++++- .../local_ai_dictation_controller.swift | 58 ++++-- .../voice_history_reformatter.swift | 16 +- .../application_preferences_test.swift | 21 +++ .../local_ai_dictation_tests.swift | 21 +++ .../voice_casing_policy_test.swift | 45 +++++ .../voice_list_intent_test.swift | 14 ++ .../voice_spoken_edit_engine_test.swift | 32 ++++ .../voice_spoken_edit_replayer_test.swift | 16 ++ .../local_ai_dictation_controller_test.swift | 36 ++++ .../voice_history_reformatter_test.swift | 19 ++ .../VoiceInput.xcodeproj/project.pbxproj | 12 ++ .../app/voice_input_document_pipeline.swift | 13 +- apps/ios/voice_input/project.yml | 6 + .../voice_input_document_pipeline_test.swift | 18 ++ docs/architecture.md | 22 ++- .../0043_ios_local_formatting_and_history.md | 6 +- ..._voice_casing_and_spoken_list_semantics.md | 62 +++++++ docs/game_plan.md | 2 +- docs/product_brief.md | 10 +- docs/voice_cujs.md | 12 +- 29 files changed, 906 insertions(+), 111 deletions(-) create mode 100644 Sources/HardwareControllerCore/personal_dictionary.swift create mode 100644 Sources/HardwareControllerCore/voice_casing.swift create mode 100644 Sources/HardwareControllerCore/voice_list_intent.swift create mode 100644 Tests/HardwareControllerCoreTests/voice_casing_policy_test.swift create mode 100644 Tests/HardwareControllerCoreTests/voice_list_intent_test.swift create mode 100644 docs/decisions/0053_voice_casing_and_spoken_list_semantics.md diff --git a/README.md b/README.md index d94b42d..8ca4df1 100644 --- a/README.md +++ b/README.md @@ -201,8 +201,14 @@ supported recognition errors, and applies the selected Natural, Casual Message, Formal, Technical, or Verbatim Style. It creates validated paragraph and list blocks, then preserves or flattens structure for the target. It also applies exact spoken commands such as **scratch that**, **delete that -sentence**, **new paragraph**, and numbered-list boundaries before formatting; +sentence**, **new paragraph**, **start a bullet list**, **bullet**, **next +item**, and numbered-list boundaries before formatting; say **literal** immediately before a command phrase to keep the phrase. It +normalizes conservative grocery, shopping, packing, task, explicit-marker, +and sequential-ordinal list cues before formatting. Typed casing policy can +retain the selected Style, lowercase prose while preserving source-signaled +names, or enforce strict lowercase while protecting operational tokens. The +same casing and spoken-list rules run on macOS and iOS. It validates protected numbers, URLs, email addresses, paths, code-like tokens, quotations, and dictionary terms. A provider error, invalid output, or three-second deadline delivers the deterministic Edited transcript once when diff --git a/Sources/HardwareControllerApp/application_preferences.swift b/Sources/HardwareControllerApp/application_preferences.swift index 54ac12e..6e62525 100644 --- a/Sources/HardwareControllerApp/application_preferences.swift +++ b/Sources/HardwareControllerApp/application_preferences.swift @@ -42,7 +42,7 @@ struct PreferredMicrophone: Codable, Equatable, Identifiable, Sendable { /// Stores versioned application presentation preferences. struct ApplicationPreferences: Codable, Equatable, Sendable { - static let currentSchemaVersion = 6 + static let currentSchemaVersion = 7 var appearance: ApplicationAppearance var sidebarVisibility: SidebarVisibilityPreference @@ -306,7 +306,7 @@ struct ApplicationPreferencesStore: _ preferences: ApplicationPreferences ) throws -> ApplicationPreferences { switch preferences.schemaVersion { - case 1, 2, 3, 4, 5: + case 1, 2, 3, 4, 5, 6: var migrated = preferences migrated.schemaVersion = ApplicationPreferences.currentSchemaVersion if preferences.schemaVersion == 1 { diff --git a/Sources/HardwareControllerCore/local_ai_dictation.swift b/Sources/HardwareControllerCore/local_ai_dictation.swift index bd63b9f..d9f4ec7 100644 --- a/Sources/HardwareControllerCore/local_ai_dictation.swift +++ b/Sources/HardwareControllerCore/local_ai_dictation.swift @@ -45,42 +45,6 @@ public struct LocalAIModelSelection: Codable, Equatable, Sendable { } } -public struct PersonalDictionaryReplacement: - Codable, - Equatable, - Identifiable, - Sendable -{ - public let id: UUID - public var spokenForm: String - public var replacement: String - - public init( - id: UUID = UUID(), - spokenForm: String, - replacement: String - ) { - self.id = id - self.spokenForm = spokenForm - self.replacement = replacement - } -} - -public struct PersonalDictionary: Codable, Equatable, Sendable { - public var vocabulary: [String] - public var replacements: [PersonalDictionaryReplacement] - - public init( - vocabulary: [String] = [], - replacements: [PersonalDictionaryReplacement] = [] - ) { - self.vocabulary = vocabulary - self.replacements = replacements - } - - public static let empty = PersonalDictionary() -} - public struct LocalAISettings: Codable, Equatable, Sendable { public static let defaultRecommendedModelName = "qwen3.5:4b" @@ -91,6 +55,7 @@ public struct LocalAISettings: Codable, Equatable, Sendable { public var dictionary: PersonalDictionary public var additionalInstructions: String public var style: VoiceStyle + public var casingPolicy: VoiceCasingPolicy public init( provider: LocalAIProviderKind = .appleOnDevice, @@ -101,7 +66,8 @@ public struct LocalAISettings: Codable, Equatable, Sendable { includeNearbyText: Bool = false, dictionary: PersonalDictionary = .empty, additionalInstructions: String = "", - style: VoiceStyle = .natural + style: VoiceStyle = .natural, + casingPolicy: VoiceCasingPolicy = .styleDefault ) { self.provider = provider self.ollamaModel = ollamaModel @@ -110,6 +76,7 @@ public struct LocalAISettings: Codable, Equatable, Sendable { self.dictionary = dictionary self.additionalInstructions = additionalInstructions self.style = style + self.casingPolicy = casingPolicy } private enum CodingKeys: String, CodingKey { @@ -120,6 +87,7 @@ public struct LocalAISettings: Codable, Equatable, Sendable { case dictionary case additionalInstructions case style + case casingPolicy } public init(from decoder: any Decoder) throws { @@ -140,6 +108,11 @@ public struct LocalAISettings: Codable, Equatable, Sendable { forKey: .additionalInstructions ) style = try container.decodeIfPresent(VoiceStyle.self, forKey: .style) ?? .natural + casingPolicy = + try container.decodeIfPresent( + VoiceCasingPolicy.self, + forKey: .casingPolicy + ) ?? .styleDefault } public func encode(to encoder: any Encoder) throws { @@ -151,6 +124,7 @@ public struct LocalAISettings: Codable, Equatable, Sendable { try container.encode(dictionary, forKey: .dictionary) try container.encode(additionalInstructions, forKey: .additionalInstructions) try container.encode(style, forKey: .style) + try container.encode(casingPolicy, forKey: .casingPolicy) } public static let `default` = LocalAISettings() @@ -170,6 +144,23 @@ public enum LocalAISettingsValidationError: Error, Equatable, Sendable { } extension LocalAISettings { + public var effectiveCasingPolicy: VoiceCasingPolicy { + guard casingPolicy == .styleDefault else { + return casingPolicy + } + let instruction = + additionalInstructions + .split(whereSeparator: { $0.isWhitespace }) + .joined(separator: " ") + .lowercased() + let requestsOnlyLowercase = + instruction.contains("only provide text in lowercase") + || instruction.contains("only use lowercase") + || instruction.contains("all lowercase") + || instruction.contains("lowercase only") + return requestsOnlyLowercase ? .strictLowercase : .styleDefault + } + public func validate() throws { guard style.revision == VoiceStyle.currentRevision else { throw LocalAISettingsValidationError.unsupportedStyleRevision( @@ -257,6 +248,8 @@ public struct LocalAIRefinementRequest: Equatable, Sendable { public let dictionary: PersonalDictionary public let additionalInstructions: String public let style: VoiceStyle + public let casingPolicy: VoiceCasingPolicy + public let listIntent: VoiceListIntent public init( sessionID: UUID, @@ -264,7 +257,9 @@ public struct LocalAIRefinementRequest: Equatable, Sendable { context: LocalAITargetContext, dictionary: PersonalDictionary, additionalInstructions: String, - style: VoiceStyle = .natural + style: VoiceStyle = .natural, + casingPolicy: VoiceCasingPolicy = .styleDefault, + listIntent: VoiceListIntent? = nil ) { self.sessionID = sessionID self.transcript = transcript @@ -272,6 +267,9 @@ public struct LocalAIRefinementRequest: Equatable, Sendable { self.dictionary = dictionary self.additionalInstructions = additionalInstructions self.style = style + self.casingPolicy = casingPolicy + self.listIntent = + listIntent ?? VoiceListIntentDetector().detect(in: transcript) } } diff --git a/Sources/HardwareControllerCore/personal_dictionary.swift b/Sources/HardwareControllerCore/personal_dictionary.swift new file mode 100644 index 0000000..82d8848 --- /dev/null +++ b/Sources/HardwareControllerCore/personal_dictionary.swift @@ -0,0 +1,37 @@ +import Foundation + +public struct PersonalDictionaryReplacement: + Codable, + Equatable, + Identifiable, + Sendable +{ + public let id: UUID + public var spokenForm: String + public var replacement: String + + public init( + id: UUID = UUID(), + spokenForm: String, + replacement: String + ) { + self.id = id + self.spokenForm = spokenForm + self.replacement = replacement + } +} + +public struct PersonalDictionary: Codable, Equatable, Sendable { + public var vocabulary: [String] + public var replacements: [PersonalDictionaryReplacement] + + public init( + vocabulary: [String] = [], + replacements: [PersonalDictionaryReplacement] = [] + ) { + self.vocabulary = vocabulary + self.replacements = replacements + } + + public static let empty = PersonalDictionary() +} diff --git a/Sources/HardwareControllerCore/voice_casing.swift b/Sources/HardwareControllerCore/voice_casing.swift new file mode 100644 index 0000000..7a171e8 --- /dev/null +++ b/Sources/HardwareControllerCore/voice_casing.swift @@ -0,0 +1,153 @@ +import Foundation + +public enum VoiceCasingPolicy: + String, + CaseIterable, + Codable, + Equatable, + Sendable +{ + case styleDefault + case lowercaseProse + case strictLowercase +} + +public struct VoiceCasingTransformer: Sendable { + public init() {} + + public func apply( + _ policy: VoiceCasingPolicy, + to text: String, + preserving source: String, + dictionary: PersonalDictionary + ) -> String { + guard policy != .styleDefault else { + return text + } + let protectedTokens = intentionalTokens( + in: source, + dictionary: dictionary, + preserveProseCasing: policy == .lowercaseProse + ) + let ranges = nonoverlappingRanges( + of: protectedTokens, + in: text + ) + var result = "" + var cursor = text.startIndex + for range in ranges { + result += text[cursor.. [String] { + let patterns = [ + #"https?://[^\s]+"#, + #"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b"#, + #"(?:/[^\s/]+){2,}"#, + #"\b[A-Za-z][A-Za-z0-9]*_[A-Za-z0-9_]+\b"#, + #"\b[a-z][A-Za-z0-9]*[A-Z][A-Za-z0-9]*\b"#, + #"\b[A-Z][a-z]+(?:[A-Z][A-Za-z0-9]*)+\b"#, + #"[\"“][^\"”]+[\"”]"#, + ] + var tokens = Set( + patterns.flatMap { matches(pattern: $0, in: source) } + ) + tokens.formUnion(dictionary.vocabulary) + tokens.formUnion(dictionary.replacements.map(\.replacement)) + if preserveProseCasing { + tokens.formUnion(matches(pattern: #"\b[A-Z]{2,}\b"#, in: source)) + tokens.formUnion(sourceSignaledNames(in: source)) + } + return tokens.filter { !$0.isEmpty }.sorted { + if $0.count != $1.count { + return $0.count > $1.count + } + return $0 < $1 + } + } + + private func sourceSignaledNames(in source: String) -> [String] { + guard + let expression = try? NSRegularExpression( + pattern: #"\b[A-Z][a-z]+\b"# + ) + else { + return [] + } + let range = NSRange(source.startIndex..., in: source) + return expression.matches(in: source, range: range).compactMap { match in + guard let wordRange = Range(match.range, in: source) else { + return nil + } + let prefix = source[.. [String] { + guard let expression = try? NSRegularExpression(pattern: pattern) else { + return [] + } + let range = NSRange(source.startIndex..., in: source) + return expression.matches(in: source, range: range).compactMap { match in + Range(match.range, in: source).map { String(source[$0]) } + } + } + + private func nonoverlappingRanges( + of tokens: [String], + in text: String + ) -> [Range] { + let candidates = tokens.flatMap { token in + ranges(of: token, in: text) + }.sorted { + if $0.lowerBound != $1.lowerBound { + return $0.lowerBound < $1.lowerBound + } + return text.distance(from: $0.lowerBound, to: $0.upperBound) + > text.distance(from: $1.lowerBound, to: $1.upperBound) + } + var selected: [Range] = [] + for candidate in candidates + where selected.last?.upperBound ?? text.startIndex <= candidate.lowerBound { + selected.append(candidate) + } + return selected + } + + private func ranges( + of token: String, + in text: String + ) -> [Range] { + var ranges: [Range] = [] + var searchStart = text.startIndex + while searchStart < text.endIndex, + let range = text.range( + of: token, + range: searchStart.. VoiceListIntent { + guard !text.isEmpty else { + return .none + } + if contains(#"(?m)^\s*\d+\.\s+\S"#, in: text) + || containsSequentialOrdinals(in: text) + { + return .ordered + } + if contains(#"(?m)^\s*[-*•]\s+\S"#, in: text) { + return .unordered + } + let hasListCue = contains( + #"(?i)\b(?:(?:grocery|shopping|packing|task|to-do)\s+)?list\b"#, + in: text + ) + let hasDelimitedItems = + text.contains(";") + || text.filter({ $0 == "," }).count >= 2 + || (text.contains(":") && text.contains(",")) + return hasListCue && hasDelimitedItems ? .unordered : .none + } + + private func containsSequentialOrdinals(in text: String) -> Bool { + guard + let first = text.range( + of: #"\bfirst\b"#, + options: [.regularExpression, .caseInsensitive] + ), + let second = text.range( + of: #"\bsecond\b"#, + options: [.regularExpression, .caseInsensitive], + range: first.upperBound.. first.upperBound + } + + private func contains(_ pattern: String, in text: String) -> Bool { + guard let expression = try? NSRegularExpression(pattern: pattern) else { + return false + } + return expression.firstMatch( + in: text, + range: NSRange(text.startIndex..., in: text) + ) != nil + } +} diff --git a/Sources/HardwareControllerCore/voice_spoken_edit.swift b/Sources/HardwareControllerCore/voice_spoken_edit.swift index 5204fc1..17602bd 100644 --- a/Sources/HardwareControllerCore/voice_spoken_edit.swift +++ b/Sources/HardwareControllerCore/voice_spoken_edit.swift @@ -9,6 +9,8 @@ public enum VoiceSpokenEditOperationKind: case insertParagraphBreak case beginOrderedList case beginOrderedListItem + case beginUnorderedList + case beginUnorderedListItem case endList case preserveLiteralCommand } @@ -39,7 +41,7 @@ public struct VoiceSpokenEditOperation: Codable, Equatable, Sendable { } public struct VoiceSpokenEditResult: Codable, Equatable, Sendable { - public static let currentRevision = 1 + public static let currentRevision = 2 public let revision: Int public let sourceText: String diff --git a/Sources/HardwareControllerCore/voice_spoken_edit_engine.swift b/Sources/HardwareControllerCore/voice_spoken_edit_engine.swift index 20057f1..a493b0c 100644 --- a/Sources/HardwareControllerCore/voice_spoken_edit_engine.swift +++ b/Sources/HardwareControllerCore/voice_spoken_edit_engine.swift @@ -1,7 +1,13 @@ import Foundation public struct VoiceSpokenEditEngine: Sendable { - private struct OrderedListState { + private enum ListKind { + case ordered + case unordered + } + + private struct ListState { + let kind: ListKind let itemNumber: Int let itemContentStartUTF8Offset: Int } @@ -10,7 +16,7 @@ public struct VoiceSpokenEditEngine: Sendable { let kind: VoiceSpokenEditOperationKind let affectedStart: String.Index let replacementText: String - let nextListState: OrderedListState? + let nextListState: ListState? let changesListState: Bool } @@ -19,13 +25,25 @@ public struct VoiceSpokenEditEngine: Sendable { case deleteThatSentence = "delete that sentence" case newParagraph = "new paragraph" case startNumberedList = "start a numbered list" + case startBulletList = "start a bullet list" + case startBulletedList = "start a bulleted list" + case bullet = "bullet" + case nextItem = "next item" case endList = "end list" } - public init() {} + private let revision: Int + + public init() { + revision = VoiceSpokenEditResult.currentRevision + } + + init(revision: Int) { + self.revision = revision + } public func apply(to sourceText: String) -> VoiceSpokenEditResult { - let matches = Self.commandExpression.matches( + let matches = commandExpression.matches( in: sourceText, range: NSRange(sourceText.startIndex..., in: sourceText) ) @@ -33,7 +51,7 @@ public struct VoiceSpokenEditEngine: Sendable { var sourceCursorUTF8Offset = 0 var editedText = "" var operations: [VoiceSpokenEditOperation] = [] - var listState: OrderedListState? + var listState: ListState? for match in matches { guard let range = Range(match.range, in: sourceText) else { @@ -118,6 +136,7 @@ public struct VoiceSpokenEditEngine: Sendable { ) } return VoiceSpokenEditResult( + revision: revision, sourceText: sourceText, editedText: editedText, operations: operations @@ -127,7 +146,7 @@ public struct VoiceSpokenEditEngine: Sendable { private func edit( for command: Command, editedText: String, - listState: OrderedListState? + listState: ListState? ) -> Edit? { switch command { case .scratchThat: @@ -151,6 +170,42 @@ public struct VoiceSpokenEditEngine: Sendable { ) case .startNumberedList: return beginListEdit( + kind: .ordered, + editedText: editedText, + listState: listState + ) + case .startBulletList, .startBulletedList: + return beginListEdit( + kind: .unordered, + editedText: editedText, + listState: listState + ) + case .bullet: + guard let listState else { + guard + !hasMeaningfulText(editedText) + || hasTrailingListCue(editedText) + else { + return nil + } + return beginListEdit( + kind: .unordered, + editedText: editedText, + listState: nil + ) + } + guard listState.kind == .unordered else { + return nil + } + return nextListItemEdit( + editedText: editedText, + listState: listState + ) + case .nextItem: + guard let listState else { + return nil + } + return nextListItemEdit( editedText: editedText, listState: listState ) @@ -166,7 +221,7 @@ public struct VoiceSpokenEditEngine: Sendable { kind: VoiceSpokenEditOperationKind, boundaries: Set, editedText: String, - listState: OrderedListState? + listState: ListState? ) -> Edit? { guard var start = deletionStart( @@ -206,7 +261,7 @@ public struct VoiceSpokenEditEngine: Sendable { private func paragraphEdit( editedText: String, - listState: OrderedListState? + listState: ListState? ) -> Edit? { let affectedStart = trailingWhitespaceStart(in: editedText) if let listState { @@ -219,19 +274,9 @@ public struct VoiceSpokenEditEngine: Sendable { else { return nil } - let nextNumber = listState.itemNumber + 1 - let replacement = "\n\(nextNumber). " - return Edit( - kind: .beginOrderedListItem, - affectedStart: affectedStart, - replacementText: replacement, - nextListState: OrderedListState( - itemNumber: nextNumber, - itemContentStartUTF8Offset: - editedText.voiceUTF8Offset(of: affectedStart) - + replacement.utf8.count - ), - changesListState: true + return nextListItemEdit( + editedText: editedText, + listState: listState ) } guard @@ -250,21 +295,24 @@ public struct VoiceSpokenEditEngine: Sendable { } private func beginListEdit( + kind: ListKind, editedText: String, - listState: OrderedListState? + listState: ListState? ) -> Edit? { guard listState == nil else { return nil } let affectedStart = trailingWhitespaceStart(in: editedText) + let marker = kind == .ordered ? "1. " : "- " let replacement = hasMeaningfulText(editedText[.. Edit? { + let affectedStart = trailingWhitespaceStart(in: editedText) + guard + let itemStart = editedText.voiceIndex( + atUTF8Offset: listState.itemContentStartUTF8Offset + ), + itemStart <= affectedStart, + hasMeaningfulText(editedText[itemStart.. Edit? { guard let listState, let itemStart = editedText.voiceIndex( @@ -346,6 +427,14 @@ public struct VoiceSpokenEditEngine: Sendable { } } + private func hasTrailingListCue(_ text: String) -> Bool { + text.range( + of: + #"(?i)\b(?:(?:grocery|shopping|packing|task|to-do)\s+)?list\s*[:,-]?\s*$"#, + options: .regularExpression + ) != nil + } + private func literalCommandText(_ text: String) -> String? { let components = text.split( maxSplits: 1, @@ -364,7 +453,13 @@ public struct VoiceSpokenEditEngine: Sendable { let normalized = text.split(whereSeparator: { $0.isWhitespace }) .joined(separator: " ") .lowercased() - return Command(rawValue: normalized) + guard let command = Command(rawValue: normalized) else { + return nil + } + guard revision >= 2 || Self.revisionOneCommands.contains(command) else { + return nil + } + return command } private func extendedCommandEnd( @@ -385,7 +480,12 @@ public struct VoiceSpokenEditEngine: Sendable { return cursor } - private static let commandExpression: NSRegularExpression = { + private var commandExpression: NSRegularExpression { + revision >= 2 + ? Self.revisionTwoCommandExpression + : Self.revisionOneCommandExpression + } + private static let revisionOneCommandExpression: NSRegularExpression = { do { return try NSRegularExpression( pattern: @@ -395,6 +495,23 @@ public struct VoiceSpokenEditEngine: Sendable { preconditionFailure("The fixed spoken-edit command pattern is invalid: \(error)") } }() + private static let revisionTwoCommandExpression: NSRegularExpression = { + do { + return try NSRegularExpression( + pattern: + "(?i)\\b(?:literal[ \\t]+)?(?:scratch[ \\t]+that|delete[ \\t]+that[ \\t]+sentence|new[ \\t]+paragraph|start[ \\t]+a[ \\t]+numbered[ \\t]+list|start[ \\t]+a[ \\t]+bullet(?:ed)?[ \\t]+list|next[ \\t]+item|bullet|end[ \\t]+list)\\b" + ) + } catch { + preconditionFailure("The fixed spoken-edit command pattern is invalid: \(error)") + } + }() + private static let revisionOneCommands: Set = [ + .scratchThat, + .deleteThatSentence, + .newParagraph, + .startNumberedList, + .endList, + ] private static let clauseBoundaries: Set = [ ".", "?", "!", ";", ":", ",", "\n", ] diff --git a/Sources/HardwareControllerCore/voice_spoken_edit_replayer.swift b/Sources/HardwareControllerCore/voice_spoken_edit_replayer.swift index 436bef6..288e099 100644 --- a/Sources/HardwareControllerCore/voice_spoken_edit_replayer.swift +++ b/Sources/HardwareControllerCore/voice_spoken_edit_replayer.swift @@ -6,12 +6,13 @@ public struct VoiceSpokenEditReplayer: Sendable { public func replay( _ result: VoiceSpokenEditResult ) throws -> String { - guard result.revision == VoiceSpokenEditResult.currentRevision else { + guard Self.supportedRevisions.contains(result.revision) else { throw VoiceSpokenEditError.unsupportedRevision(result.revision) } return try replay( sourceText: result.sourceText, - operations: result.operations + operations: result.operations, + revision: result.revision ) } @@ -19,7 +20,9 @@ public struct VoiceSpokenEditReplayer: Sendable { guard try replay(result) == result.editedText else { throw VoiceSpokenEditError.resultMismatch } - let canonical = VoiceSpokenEditEngine().apply(to: result.sourceText) + let canonical = VoiceSpokenEditEngine(revision: result.revision).apply( + to: result.sourceText + ) guard canonical.operations == result.operations else { throw VoiceSpokenEditError.nonCanonicalOperations } @@ -28,6 +31,18 @@ public struct VoiceSpokenEditReplayer: Sendable { public func replay( sourceText: String, operations: [VoiceSpokenEditOperation] + ) throws -> String { + try replay( + sourceText: sourceText, + operations: operations, + revision: VoiceSpokenEditResult.currentRevision + ) + } + + private func replay( + sourceText: String, + operations: [VoiceSpokenEditOperation], + revision: Int ) throws -> String { var sourceCursor = sourceText.startIndex var sourceCursorOffset = 0 @@ -51,7 +66,8 @@ public struct VoiceSpokenEditReplayer: Sendable { } try validateCommandEvidence( sourceText[sourceStart..= 2 && normalized == "next item") case .beginOrderedList: valid = normalized == "start a numbered list" + case .beginUnorderedList: + valid = + revision >= 2 + && ["start a bullet list", "start a bulleted list", "bullet"] + .contains(normalized) + case .beginUnorderedListItem: + valid = + revision >= 2 + && ["new paragraph", "next item", "bullet"].contains(normalized) case .endList: valid = normalized == "end list" case .preserveLiteralCommand: @@ -139,7 +173,7 @@ public struct VoiceSpokenEditReplayer: Sendable { valid = literalWords.count == 2 && literalWords[0].lowercased() == "literal" - && Self.commandPhrases.contains( + && commandPhrases(revision: revision).contains( literalWords[1].split(whereSeparator: { $0.isWhitespace }) .joined(separator: " ").lowercased() ) @@ -175,4 +209,18 @@ public struct VoiceSpokenEditReplayer: Sendable { "start a numbered list", "end list", ] + + private func commandPhrases(revision: Int) -> Set { + guard revision >= 2 else { + return Self.commandPhrases + } + return Self.commandPhrases.union([ + "start a bullet list", + "start a bulleted list", + "bullet", + "next item", + ]) + } + + private static let supportedRevisions: Set = [1, 2] } diff --git a/Sources/HardwareControllerMac/local_ai_dictation_controller.swift b/Sources/HardwareControllerMac/local_ai_dictation_controller.swift index 6fee3ad..d57061a 100644 --- a/Sources/HardwareControllerMac/local_ai_dictation_controller.swift +++ b/Sources/HardwareControllerMac/local_ai_dictation_controller.swift @@ -45,6 +45,7 @@ public actor LocalAIDictationController { private let refiner: any LocalAIRefinementRouting private let validator: RefinedTranscriptValidator private let polisher: DeterministicTranscriptPolisher + private let casingTransformer: VoiceCasingTransformer private let replacementApplier: PersonalDictionaryReplacementApplier private let spokenEditEngine: VoiceSpokenEditEngine private let formattedDocumentBuilder: VoiceFormattedDocumentBuilder @@ -117,6 +118,7 @@ public actor LocalAIDictationController { self.refiner = refiner validator = RefinedTranscriptValidator() polisher = DeterministicTranscriptPolisher() + casingTransformer = VoiceCasingTransformer() replacementApplier = PersonalDictionaryReplacementApplier() spokenEditEngine = VoiceSpokenEditEngine() formattedDocumentBuilder = VoiceFormattedDocumentBuilder() @@ -211,17 +213,22 @@ public actor LocalAIDictationController { dictionary: .empty, additionalInstructions: currentSettings.additionalInstructions, - style: currentSettings.style + style: currentSettings.style, + casingPolicy: currentSettings.effectiveCasingPolicy ) let response = try await responseBeforeTimeout( request, settings: currentSettings, preparationTask: preparationTask ) - let polished = polishedText( - response.text, + let polished = casedText( + polishedText( + response.text, + preserving: transcript, + style: currentSettings.style + ), preserving: transcript, - style: currentSettings.style + settings: currentSettings ) _ = try validator.validate( polished, @@ -514,7 +521,8 @@ public actor LocalAIDictationController { dictionary: currentSettings.dictionary, additionalInstructions: currentSettings.additionalInstructions, - style: currentSettings.style + style: currentSettings.style, + casingPolicy: currentSettings.effectiveCasingPolicy ) let start = MonotonicClock.nowNanoseconds() @@ -523,7 +531,11 @@ public actor LocalAIDictationController { let candidate: String if currentSettings.style.kind == .verbatim { response = nil - candidate = normalizedTranscript + candidate = casedText( + normalizedTranscript, + preserving: normalizedTranscript, + settings: currentSettings + ) } else { let modelResponse = try await responseBeforeTimeout( request, @@ -531,10 +543,14 @@ public actor LocalAIDictationController { preparationTask: preparationTask ) response = modelResponse - candidate = polishedText( - modelResponse.text, + candidate = casedText( + polishedText( + modelResponse.text, + preserving: normalizedTranscript, + style: currentSettings.style + ), preserving: normalizedTranscript, - style: currentSettings.style + settings: currentSettings ) } guard !Task.isCancelled, state.sessionID == sessionID else { @@ -738,14 +754,19 @@ public actor LocalAIDictationController { } do { replace(phase: .delivering, refinedText: "") + let casedFallbackText = casedText( + fallbackText, + preserving: fallbackText, + settings: settings + ) let fallbackDocument = try? formattedDocumentBuilder.build( - formattedText: fallbackText, + formattedText: casedFallbackText, rawText: rawText, style: settings.style, validationStatus: .sourceFallback ) let deliveredFallback = deterministicFallbackText( - fallbackText, + casedFallbackText, supportsMultiline: targetContext?.supportsMultilineText ?? target.supportsMultilineText ) @@ -754,7 +775,7 @@ public actor LocalAIDictationController { sessionID: sessionID, rawText: rawText, editedText: fallbackText, - formattedText: fallbackText, + formattedText: casedFallbackText, deliveredText: deliveredFallback, targetApplicationName: target.applicationName, deliveryOutcome: .inserted, @@ -991,6 +1012,19 @@ public actor LocalAIDictationController { } } + private func casedText( + _ text: String, + preserving source: String, + settings: LocalAISettings + ) -> String { + casingTransformer.apply( + settings.effectiveCasingPolicy, + to: text, + preserving: source, + dictionary: settings.dictionary + ) + } + private func deterministicFallbackText( _ text: String, supportsMultiline: Bool diff --git a/Sources/HardwareControllerMac/voice_history_reformatter.swift b/Sources/HardwareControllerMac/voice_history_reformatter.swift index 6131f28..e1370f2 100644 --- a/Sources/HardwareControllerMac/voice_history_reformatter.swift +++ b/Sources/HardwareControllerMac/voice_history_reformatter.swift @@ -9,6 +9,7 @@ public actor LocalAIVoiceHistoryReformatter: private let validator = RefinedTranscriptValidator() private let builder = VoiceFormattedDocumentBuilder() private let renderer = VoiceFormattedTextRenderer() + private let casingTransformer = VoiceCasingTransformer() private var settings: LocalAISettings private var operationInProgress = false private var operationWaiters: [CheckedContinuation] = [] @@ -58,10 +59,10 @@ public actor LocalAIVoiceHistoryReformatter: nearbyText: nil ) let response: LocalAIRefinementResponse? - let candidate: String + let rawCandidate: String if style.kind == .verbatim { response = nil - candidate = text + rawCandidate = text } else { try await refiner.prepare(settings: selectedSettings) let generated = try await refiner.refine( @@ -72,13 +73,20 @@ public actor LocalAIVoiceHistoryReformatter: dictionary: selectedSettings.dictionary, additionalInstructions: selectedSettings.additionalInstructions, - style: style + style: style, + casingPolicy: selectedSettings.effectiveCasingPolicy ), settings: selectedSettings ) response = generated - candidate = generated.text + rawCandidate = generated.text } + let candidate = casingTransformer.apply( + selectedSettings.effectiveCasingPolicy, + to: rawCandidate, + preserving: text, + dictionary: selectedSettings.dictionary + ) let validated = try validator.validate( candidate, preserving: text, diff --git a/Tests/HardwareControllerAppTests/application_preferences_test.swift b/Tests/HardwareControllerAppTests/application_preferences_test.swift index fcc1823..1c2e241 100644 --- a/Tests/HardwareControllerAppTests/application_preferences_test.swift +++ b/Tests/HardwareControllerAppTests/application_preferences_test.swift @@ -177,6 +177,27 @@ struct ApplicationPreferencesStoreTests { ) } + @Test + func schemaSixDefaultsCasingPolicyAndMigrates() throws { + let files = PreferenceFileAccess() + let encoded = try JSONEncoder().encode( + ApplicationPreferences(schemaVersion: 6) + ) + var object = try #require( + JSONSerialization.jsonObject(with: encoded) as? [String: Any] + ) + var localAI = try #require(object["localAI"] as? [String: Any]) + localAI.removeValue(forKey: "casingPolicy") + object["localAI"] = localAI + files.data = try JSONSerialization.data(withJSONObject: object) + + let result = makeStore(files: files).load() + + #expect(result.issue == nil) + #expect(result.preferences.localAI.casingPolicy == .styleDefault) + #expect(result.preferences.schemaVersion == 7) + } + @Test func invalidVoiceHistoryRetentionIsPreservedForRecovery() throws { let files = PreferenceFileAccess() diff --git a/Tests/HardwareControllerCoreTests/local_ai_dictation_tests.swift b/Tests/HardwareControllerCoreTests/local_ai_dictation_tests.swift index ee4b2e4..338af34 100644 --- a/Tests/HardwareControllerCoreTests/local_ai_dictation_tests.swift +++ b/Tests/HardwareControllerCoreTests/local_ai_dictation_tests.swift @@ -107,4 +107,25 @@ struct LocalAIDictationTests { !LocalAIProviderLocality.remoteCapable.permitsContentInLocalOnlyMode ) } + + @Test + func refinementRequestNormalizesListIntent() { + let request = LocalAIRefinementRequest( + sessionID: UUID(), + transcript: "grocery list: apples, bananas, and coffee", + context: LocalAITargetContext( + localeIdentifier: "en_US", + profileName: "Default", + applicationName: "Notes", + applicationBundleIdentifier: nil, + targetRole: nil, + supportsMultilineText: true, + nearbyText: nil + ), + dictionary: .empty, + additionalInstructions: "" + ) + + #expect(request.listIntent == .unordered) + } } diff --git a/Tests/HardwareControllerCoreTests/voice_casing_policy_test.swift b/Tests/HardwareControllerCoreTests/voice_casing_policy_test.swift new file mode 100644 index 0000000..be53fae --- /dev/null +++ b/Tests/HardwareControllerCoreTests/voice_casing_policy_test.swift @@ -0,0 +1,45 @@ +import Testing + +@testable import HardwareControllerCore + +struct VoiceCasingPolicyTest { + @Test + func lowercaseInstructionResolvesToStrictPolicy() { + var settings = LocalAISettings.default + settings.additionalInstructions = "only provide text in lowercase" + + #expect(settings.effectiveCasingPolicy == .strictLowercase) + } + + @Test + func strictLowercaseProtectsIntentionalTokens() { + let text = + "Send NASA status to Ops@Example.com from /Users/Demo/Input and call parseJSON." + + let result = VoiceCasingTransformer().apply( + .strictLowercase, + to: text, + preserving: text, + dictionary: .empty + ) + + #expect( + result + == "send nasa status to Ops@Example.com from /Users/Demo/Input and call parseJSON." + ) + } + + @Test + func lowercaseProsePreservesSourceNamesAndAcronyms() { + let text = "Meet Sarah from NASA and call parseJSON." + + let result = VoiceCasingTransformer().apply( + .lowercaseProse, + to: text, + preserving: text, + dictionary: .empty + ) + + #expect(result == "meet Sarah from NASA and call parseJSON.") + } +} diff --git a/Tests/HardwareControllerCoreTests/voice_list_intent_test.swift b/Tests/HardwareControllerCoreTests/voice_list_intent_test.swift new file mode 100644 index 0000000..f8e8471 --- /dev/null +++ b/Tests/HardwareControllerCoreTests/voice_list_intent_test.swift @@ -0,0 +1,14 @@ +import Testing + +@testable import HardwareControllerCore + +struct VoiceListIntentDetectorTest { + @Test + func groceryListWithDelimitedItemsIsUnordered() { + let intent = VoiceListIntentDetector().detect( + in: "grocery list: apples, bananas, and coffee" + ) + + #expect(intent == .unordered) + } +} diff --git a/Tests/HardwareControllerCoreTests/voice_spoken_edit_engine_test.swift b/Tests/HardwareControllerCoreTests/voice_spoken_edit_engine_test.swift index 9113d9b..5191179 100644 --- a/Tests/HardwareControllerCoreTests/voice_spoken_edit_engine_test.swift +++ b/Tests/HardwareControllerCoreTests/voice_spoken_edit_engine_test.swift @@ -54,6 +54,28 @@ struct VoiceSpokenEditEngineTest { #expect(try replayer.replay(result) == result.editedText) } + @Test + func bulletAndNextItemCommandsProduceAnUnorderedList() throws { + let source = + "Groceries start a bullet list apples next item bananas bullet coffee end list Done" + + let result = engine.apply(to: source) + + #expect( + result.editedText + == "Groceries\n\n- apples\n- bananas\n- coffee\n\nDone" + ) + #expect( + result.operations.map(\.kind) == [ + .beginUnorderedList, + .beginUnorderedListItem, + .beginUnorderedListItem, + .endList, + ] + ) + #expect(try replayer.replay(result) == result.editedText) + } + @Test func literalPreservesOnlyAnExactFollowingCommand() throws { let source = @@ -82,6 +104,16 @@ struct VoiceSpokenEditEngineTest { #expect(result.operations.isEmpty) } + @Test + func bulletUsedAsAnOrdinaryNounRemainsLiteral() { + let source = "The bullet reached the target." + + let result = engine.apply(to: source) + + #expect(result.editedText == source) + #expect(result.operations.isEmpty) + } + @Test func destructiveEditCannotRemoveTheActiveListMarker() throws { let source = diff --git a/Tests/HardwareControllerCoreTests/voice_spoken_edit_replayer_test.swift b/Tests/HardwareControllerCoreTests/voice_spoken_edit_replayer_test.swift index 8f4d4f8..9076338 100644 --- a/Tests/HardwareControllerCoreTests/voice_spoken_edit_replayer_test.swift +++ b/Tests/HardwareControllerCoreTests/voice_spoken_edit_replayer_test.swift @@ -22,6 +22,22 @@ struct VoiceSpokenEditReplayerTest { #expect(try replayer.replay(decoded) == decoded.editedText) } + @Test + func revisionOneTraceRemainsCanonicalAfterAddingListCommands() throws { + let current = VoiceSpokenEditEngine().apply( + to: "Intro new paragraph Outro" + ) + let legacy = VoiceSpokenEditResult( + revision: 1, + sourceText: current.sourceText, + editedText: current.editedText, + operations: current.operations + ) + + try replayer.validate(legacy) + #expect(try replayer.replay(legacy) == "Intro\n\nOutro") + } + @Test func rejectsOverlappingSourceEvidence() { let operations = [ diff --git a/Tests/HardwareControllerMacTests/local_ai_dictation_controller_test.swift b/Tests/HardwareControllerMacTests/local_ai_dictation_controller_test.swift index 2610293..a5b3fe7 100644 --- a/Tests/HardwareControllerMacTests/local_ai_dictation_controller_test.swift +++ b/Tests/HardwareControllerMacTests/local_ai_dictation_controller_test.swift @@ -7,6 +7,42 @@ import Testing @testable import HardwareControllerMac struct LocalAIDictationControllerTest { + @Test + func strictLowercaseOverridesNaturalStyleCapitalization() async throws { + let fixture = LocalAIControllerFixture( + refinement: .output("Buy Milk for Ops@Example.com") + ) + var settings = LocalAISettings.default + settings.additionalInstructions = "only provide text in lowercase" + let controller = fixture.makeController(settings: settings) + + await controller.handle(.begin) + try await fixture.waitUntilListening(controller) + fixture.session.emit(.committed("buy milk for Ops@Example.com")) + await controller.handle(.finish) + try await fixture.waitUntilCompleted(controller) + + #expect(fixture.writer.inserted == ["buy milk for Ops@Example.com."]) + } + + @Test + func strictLowercaseAlsoAppliesToFormattingFallback() async throws { + let fixture = LocalAIControllerFixture( + refinement: .failure(.providerUnavailable("No formatter.")) + ) + var settings = LocalAISettings.default + settings.casingPolicy = .strictLowercase + let controller = fixture.makeController(settings: settings) + + await controller.handle(.begin) + try await fixture.waitUntilListening(controller) + fixture.session.emit(.committed("Buy Milk for Ops@Example.com")) + await controller.handle(.finish) + try await fixture.waitUntilCompleted(controller) + + #expect(fixture.writer.inserted == ["buy milk for Ops@Example.com"]) + } + @Test func storesDeliveredDictationWithPlayableAudio() async throws { let rootDirectory = FileManager.default.temporaryDirectory diff --git a/Tests/HardwareControllerMacTests/voice_history_reformatter_test.swift b/Tests/HardwareControllerMacTests/voice_history_reformatter_test.swift index c7aee3f..11d9334 100644 --- a/Tests/HardwareControllerMacTests/voice_history_reformatter_test.swift +++ b/Tests/HardwareControllerMacTests/voice_history_reformatter_test.swift @@ -30,6 +30,25 @@ struct VoiceHistoryReformatterTest { #expect(await router.requestedStyles == [.formal]) } + @Test + func reformatUsesTheConfiguredCasingPolicy() async throws { + let router = HistoryRefinementRouter() + var settings = LocalAISettings.default + settings.casingPolicy = .strictLowercase + let reformatter = LocalAIVoiceHistoryReformatter( + settings: settings, + refiner: router + ) + + let result = try await reformatter.reformat( + text: "send the plan", + sessionID: UUID(), + style: .natural + ) + + #expect(result.text == "send the plan.") + } + @Test func verbatimSkipsGenerationAndStillBuildsValidatedEvidence() async throws diff --git a/apps/ios/voice_input/VoiceInput.xcodeproj/project.pbxproj b/apps/ios/voice_input/VoiceInput.xcodeproj/project.pbxproj index 40b59c2..56d297a 100644 --- a/apps/ios/voice_input/VoiceInput.xcodeproj/project.pbxproj +++ b/apps/ios/voice_input/VoiceInput.xcodeproj/project.pbxproj @@ -57,6 +57,7 @@ 7208598ECEC8F62F81605D1B /* voice_input_history_retention_preferences.swift in Sources */ = {isa = PBXBuildFile; fileRef = E831DBA41333B43F6903CD37 /* voice_input_history_retention_preferences.swift */; }; 727ECEAB4825A9602485A6E9 /* voice_input_keychain_store.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5D74E1DEDF0ABA766350BF40 /* voice_input_keychain_store.swift */; }; 73A5C53DFB572A01EBE14294 /* voice_input_document_pipeline_test.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1CCC0C83D88B8C773AAF1690 /* voice_input_document_pipeline_test.swift */; }; + 74D9D762E406146BB8D05F80 /* voice_list_intent.swift in Sources */ = {isa = PBXBuildFile; fileRef = 34139F6C135D8EB64229FA38 /* voice_list_intent.swift */; }; 75E70A2A8B3AF50404F4A49F /* voice_input_onboarding_view.swift in Sources */ = {isa = PBXBuildFile; fileRef = A657531BE5596A20AB1D1FAE /* voice_input_onboarding_view.swift */; }; 7A3270432E6092B9B99EACC4 /* HardwareControllerVoiceCore.framework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = 29B7722AD489E6A643C8C88F /* HardwareControllerVoiceCore.framework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; }; 7B763C2403C0C8BBBCE35BDC /* voice_input_history_model_test.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7BC7E40456DA743C56A2E637 /* voice_input_history_model_test.swift */; }; @@ -73,6 +74,7 @@ 9E6B5EB0F96B87BCF2991CA5 /* voice_input_asr_workflow_test.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8710132CC58D01E271258655 /* voice_input_asr_workflow_test.swift */; }; 9FBC75AA92E1CB05DBF073D1 /* voice_input_history_audio_player_model_test.swift in Sources */ = {isa = PBXBuildFile; fileRef = A7123485BE89DAC67B2C3451 /* voice_input_history_audio_player_model_test.swift */; }; A644DD8B9766C5E6137FC1F9 /* voice_input_asr_workflow.swift in Sources */ = {isa = PBXBuildFile; fileRef = C5768072590C38554047CC0F /* voice_input_asr_workflow.swift */; }; + A6F44CCF8D1D897699621821 /* voice_casing.swift in Sources */ = {isa = PBXBuildFile; fileRef = 86752435C202AF492298C9A4 /* voice_casing.swift */; }; A8E07CC63450BBCC75FE97F6 /* voice_formatted_document_builder.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7FC7415074EA515761E3F82D /* voice_formatted_document_builder.swift */; }; A9885ACF09D692124FA1B7D3 /* voice_spoken_edit_engine.swift in Sources */ = {isa = PBXBuildFile; fileRef = 87C14BD07CFF04C81A0DCF4B /* voice_spoken_edit_engine.swift */; }; ABE23293CA06AEE1FB74ECCE /* VoiceInputShared.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 0123BE2D32E45C9A1A736155 /* VoiceInputShared.framework */; }; @@ -105,6 +107,7 @@ ED030D03B1A4E4246F8CFE50 /* voice_input_system_capture.swift in Sources */ = {isa = PBXBuildFile; fileRef = 91E71B1E4A8DBE0DB21EEF8E /* voice_input_system_capture.swift */; }; F0B221E9C8D6539465ACB680 /* voice_input_model_library_view.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C4D086A77C159F47E36FA8B /* voice_input_model_library_view.swift */; }; F6791D02D2B8E619A2B94C7C /* voice_input_widgets.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9B9EDA822613D13F8604F58F /* voice_input_widgets.swift */; }; + F6A5A60CF6EC4AE67BDCB407 /* personal_dictionary.swift in Sources */ = {isa = PBXBuildFile; fileRef = 725F5BB88E6E3AC39D387DB6 /* personal_dictionary.swift */; }; F6B022DE0F836E72C99FC1EF /* voice_input_history_retention_preferences_test.swift in Sources */ = {isa = PBXBuildFile; fileRef = 2E50A5416BA4B684E5EE13EE /* voice_input_history_retention_preferences_test.swift */; }; F6C56F90353A839C7C830400 /* voice_input_host_field_policy_test.swift in Sources */ = {isa = PBXBuildFile; fileRef = E59D05A6E89992CBAB7EB9E0 /* voice_input_host_field_policy_test.swift */; }; FA0DB71BF6F5068E9994C35E /* valid in Resources */ = {isa = PBXBuildFile; fileRef = 70AD4B72B184FCE0C2B0075E /* valid */; }; @@ -274,6 +277,7 @@ 2E50A5416BA4B684E5EE13EE /* voice_input_history_retention_preferences_test.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = voice_input_history_retention_preferences_test.swift; sourceTree = ""; }; 2E55D9A1E5484487CECB7774 /* voice_input_insertion_recovery.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = voice_input_insertion_recovery.swift; sourceTree = ""; }; 3364171653512AF3B1CB5BBF /* VoiceInputTests.xctest */ = {isa = PBXFileReference; includeInIndex = 0; lastKnownFileType = wrapper.cfbundle; path = VoiceInputTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; + 34139F6C135D8EB64229FA38 /* voice_list_intent.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; name = voice_list_intent.swift; path = ../../../Sources/HardwareControllerCore/voice_list_intent.swift; sourceTree = ""; }; 3AC71BC68D1A1B8B71EDD8A8 /* voice_input_environment.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = voice_input_environment.swift; sourceTree = ""; }; 3B91A570D429EEB46217A80C /* voice_input_capture_service.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = voice_input_capture_service.swift; sourceTree = ""; }; 40E06E252FCFBB5CBDEDA1BD /* voice_input_system_capture_intents.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = voice_input_system_capture_intents.swift; sourceTree = ""; }; @@ -288,11 +292,13 @@ 66894E9CB12ADD64305532DB /* voice_input_app.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = voice_input_app.swift; sourceTree = ""; }; 6A193C9D9D6CEADAB85C8ED1 /* voice_input_model_package_fixture.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = voice_input_model_package_fixture.swift; sourceTree = ""; }; 70AD4B72B184FCE0C2B0075E /* valid */ = {isa = PBXFileReference; lastKnownFileType = folder; name = valid; path = ../../../Tests/cuj/voice_model_package_v1/valid; sourceTree = SOURCE_ROOT; }; + 725F5BB88E6E3AC39D387DB6 /* personal_dictionary.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; name = personal_dictionary.swift; path = ../../../Sources/HardwareControllerCore/personal_dictionary.swift; sourceTree = ""; }; 76B49DD809855BED87F883A1 /* VoiceFFI.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; name = VoiceFFI.xcframework; path = ../../../.build/ios_voice_ffi/VoiceFFI.xcframework; sourceTree = ""; }; 7BC7E40456DA743C56A2E637 /* voice_input_history_model_test.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = voice_input_history_model_test.swift; sourceTree = ""; }; 7DFDF7EBFC4891F604797237 /* voice_input_style_test.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = voice_input_style_test.swift; sourceTree = ""; }; 7FC7415074EA515761E3F82D /* voice_formatted_document_builder.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; name = voice_formatted_document_builder.swift; path = ../../../Sources/HardwareControllerCore/voice_formatted_document_builder.swift; sourceTree = ""; }; 83300748E2DEF79D228F4A66 /* voice_input_model_library_model_test.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = voice_input_model_library_model_test.swift; sourceTree = ""; }; + 86752435C202AF492298C9A4 /* voice_casing.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; name = voice_casing.swift; path = ../../../Sources/HardwareControllerCore/voice_casing.swift; sourceTree = ""; }; 870E8658F0A5428F729940C7 /* VoiceInputKeyboard.appex */ = {isa = PBXFileReference; includeInIndex = 0; lastKnownFileType = "wrapper.app-extension"; path = VoiceInputKeyboard.appex; sourceTree = BUILT_PRODUCTS_DIR; }; 8710132CC58D01E271258655 /* voice_input_asr_workflow_test.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = voice_input_asr_workflow_test.swift; sourceTree = ""; }; 87C14BD07CFF04C81A0DCF4B /* voice_spoken_edit_engine.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; name = voice_spoken_edit_engine.swift; path = ../../../Sources/HardwareControllerCore/voice_spoken_edit_engine.swift; sourceTree = ""; }; @@ -564,10 +570,13 @@ isa = PBXGroup; children = ( 661DBA058E96E22130B681B0 /* local_ai_provider_kind.swift */, + 725F5BB88E6E3AC39D387DB6 /* personal_dictionary.swift */, + 86752435C202AF492298C9A4 /* voice_casing.swift */, 7FC7415074EA515761E3F82D /* voice_formatted_document_builder.swift */, 8E74E0CA5543420625918CB3 /* voice_formatted_text_renderer.swift */, DDC5BAFE6FF8BFED876860BE /* voice_formatting.swift */, 0D025266467687E8B7FC7E78 /* voice_history_retention.swift */, + 34139F6C135D8EB64229FA38 /* voice_list_intent.swift */, 87C14BD07CFF04C81A0DCF4B /* voice_spoken_edit_engine.swift */, FF4E6158219BA1EF466A8CDD /* voice_spoken_edit_replayer.swift */, E1FB32FBAE2A5D044818C786 /* voice_spoken_edit.swift */, @@ -999,10 +1008,13 @@ buildActionMask = 2147483647; files = ( 3A6162C568500D54ADAD3C08 /* local_ai_provider_kind.swift in Sources */, + F6A5A60CF6EC4AE67BDCB407 /* personal_dictionary.swift in Sources */, + A6F44CCF8D1D897699621821 /* voice_casing.swift in Sources */, A8E07CC63450BBCC75FE97F6 /* voice_formatted_document_builder.swift in Sources */, B85D3BCD785A3F423A125B79 /* voice_formatted_text_renderer.swift in Sources */, 57B8F8C872E3D7F6DE8E32EF /* voice_formatting.swift in Sources */, 48535CE8D39C0F799E3A1980 /* voice_history_retention.swift in Sources */, + 74D9D762E406146BB8D05F80 /* voice_list_intent.swift in Sources */, 400E6F4BBCAC9FD9D06CFC74 /* voice_spoken_edit.swift in Sources */, A9885ACF09D692124FA1B7D3 /* voice_spoken_edit_engine.swift in Sources */, 227702CC2D64B2A40F1BEF48 /* voice_spoken_edit_replayer.swift in Sources */, diff --git a/apps/ios/voice_input/app/voice_input_document_pipeline.swift b/apps/ios/voice_input/app/voice_input_document_pipeline.swift index 2247c93..44529be 100644 --- a/apps/ios/voice_input/app/voice_input_document_pipeline.swift +++ b/apps/ios/voice_input/app/voice_input_document_pipeline.swift @@ -11,12 +11,15 @@ struct VoiceInputProcessedTranscript: Codable, Equatable, Sendable { struct VoiceInputDocumentPipeline: Sendable { private let spokenEditEngine = VoiceSpokenEditEngine() + private let casingTransformer = VoiceCasingTransformer() private let documentBuilder = VoiceFormattedDocumentBuilder() private let renderer = VoiceFormattedTextRenderer() func process( _ rawTranscript: VoiceInputRawTranscript, - style: VoiceStyle + style: VoiceStyle, + casingPolicy: VoiceCasingPolicy = .styleDefault, + dictionary: PersonalDictionary = .empty ) throws -> VoiceInputProcessedTranscript { let spokenEdits = style.kind == .verbatim @@ -26,8 +29,14 @@ struct VoiceInputDocumentPipeline: Sendable { operations: [] ) : spokenEditEngine.apply(to: rawTranscript.text) + let casedText = casingTransformer.apply( + casingPolicy, + to: spokenEdits.editedText, + preserving: spokenEdits.editedText, + dictionary: dictionary + ) let document = try documentBuilder.build( - formattedText: spokenEdits.editedText, + formattedText: casedText, rawText: rawTranscript.text, style: style ) diff --git a/apps/ios/voice_input/project.yml b/apps/ios/voice_input/project.yml index 75ccbb3..5e977aa 100644 --- a/apps/ios/voice_input/project.yml +++ b/apps/ios/voice_input/project.yml @@ -73,6 +73,8 @@ targets: sources: - path: ../../../Sources/HardwareControllerCore/local_ai_provider_kind.swift group: portable_voice_core + - path: ../../../Sources/HardwareControllerCore/personal_dictionary.swift + group: portable_voice_core - path: ../../../Sources/HardwareControllerCore/voice_formatted_document_builder.swift group: portable_voice_core - path: ../../../Sources/HardwareControllerCore/voice_formatted_text_renderer.swift @@ -81,6 +83,10 @@ targets: group: portable_voice_core - path: ../../../Sources/HardwareControllerCore/voice_history_retention.swift group: portable_voice_core + - path: ../../../Sources/HardwareControllerCore/voice_casing.swift + group: portable_voice_core + - path: ../../../Sources/HardwareControllerCore/voice_list_intent.swift + group: portable_voice_core - path: ../../../Sources/HardwareControllerCore/voice_spoken_edit.swift group: portable_voice_core - path: ../../../Sources/HardwareControllerCore/voice_spoken_edit_engine.swift diff --git a/apps/ios/voice_input/tests/voice_input_document_pipeline_test.swift b/apps/ios/voice_input/tests/voice_input_document_pipeline_test.swift index bdcaddc..f7784b9 100644 --- a/apps/ios/voice_input/tests/voice_input_document_pipeline_test.swift +++ b/apps/ios/voice_input/tests/voice_input_document_pipeline_test.swift @@ -75,4 +75,22 @@ final class VoiceInputDocumentPipelineTest: XCTestCase { XCTAssertEqual(result.spokenEdits.operations, []) XCTAssertEqual(result.formattedDocument.blocks.map(\.kind), [.verbatim]) } + + func testStrictLowercaseAppliesAfterSpokenEdits() throws { + let raw = VoiceInputRawTranscript( + text: "Buy Milk new paragraph Call parseJSON", + segments: [], + modelPackageID: "model", + modelVersion: "1" + ) + + let result = try VoiceInputDocumentPipeline().process( + raw, + style: .natural, + casingPolicy: .strictLowercase + ) + + XCTAssertEqual(result.editedText, "Buy Milk\n\nCall parseJSON") + XCTAssertEqual(result.formattedText, "buy milk\n\ncall parseJSON") + } } diff --git a/docs/architecture.md b/docs/architecture.md index f4e8a55..0c072d8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -394,7 +394,8 @@ Current executors: microphone and Apple recognition controller in final-only mode. It warms the selected provider while speech continues, applies typed spoken-edit operations to Raw before deterministic Dictionary replacements, sends immutable typed - context to one local refiner, validates protected content and semantic bounds, parses + context including normalized list intent and casing policy to one local + refiner, validates protected content and semantic bounds, parses evidence-backed paragraph and list blocks, then renders target-safe refined or Edited fallback text exactly once. Verbatim Style bypasses provider preparation and generation. @@ -463,17 +464,21 @@ to an ordered-list block and validation runs again on the canonical rendering. The sanitized provider test shares the three-second preparation-plus-generation deadline; settings changes and shutdown cancel it and suppress stale results. -The revision-1 Swift spoken-edit engine recognizes only exact, case-insensitive +The revision-2 Swift spoken-edit engine recognizes only exact, case-insensitive command phrases in immutable Raw text, so a Dictionary replacement cannot synthesize a destructive command. Each accepted command records its source UTF-8 range, affected pre-Dictionary suffix, typed operation, and replacement. Replay rejects unsupported revisions, noncanonical command evidence, overlapping source evidence, non-suffix destructive ranges, invalid structure replacements, and mismatched stored results. Clause and sentence deletion stop at explicit -stable punctuation or a list-item marker. In ordered-list mode, `new paragraph` -begins the next numbered item; `literal` preserves one immediately following -exact command. An inapplicable destructive/list command and every near-match -remain ordinary transcript text. The model receives only the resulting Edited +stable punctuation or a list-item marker. Ordered and unordered modes accept +`new paragraph` or `next item`; unordered mode also accepts `bullet`. Exact +`start a bullet list` and `start a bulleted list` phrases enter unordered mode. +A bare `bullet` starts a list only at the start of text or after a list cue, so +ordinary noun use remains text. `literal` preserves one immediately following +exact command. Revision-1 traces retain their original command vocabulary and +remain replayable. An inapplicable destructive/list command and every +near-match remain ordinary transcript text. The model receives only the resulting Edited text. If all Edited text is removed, the session completes without generation or insertion while retaining its Raw evidence. Dictionary replacements then produce the final Edited text. @@ -571,12 +576,13 @@ Schema 5 adds the Local AI Dictation Action identity. Schema-4 Profiles migrate without changing any Action, Binding, interaction mode, or fallback. Application appearance, sidebar visibility, microphone identity, Local AI -settings, and the Voice chord use a separate schema-5 `preferences.json` file. +settings, and the Voice chord use a separate schema-7 `preferences.json` file. Earlier schemas migrate with System Default microphone and conservative Local AI defaults: Apple On-Device, recommended Ollama model identity, five-minute retention, nearby context off, empty dictionary, no additional instructions, and the Voice -chord disabled. A valid +chord disabled. Schema 7 adds typed casing; schema 6 defaults it to Style +Default without changing prior output. A valid future preference schema is preserved and never overwritten. This store uses the same atomic-write and corruption-preservation policy because application preferences are not work-mode data. diff --git a/docs/decisions/0043_ios_local_formatting_and_history.md b/docs/decisions/0043_ios_local_formatting_and_history.md index b70ab40..5d6afb7 100644 --- a/docs/decisions/0043_ios_local_formatting_and_history.md +++ b/docs/decisions/0043_ios_local_formatting_and_history.md @@ -25,8 +25,10 @@ macOS History implementation into the mobile target. playback, Keychain, SQLite, and filesystem adapters in the containing app. - Treat whisper output as immutable Raw evidence. Apply explicit spoken edits to create Edited, then build and validate semantic paragraph/list blocks and - render Formatted. Verbatim bypasses spoken commands and preserves literal - text. + render Formatted. Compile the same typed casing transformer, list-intent + detector, and revisioned spoken-list engine into the portable target so iOS + and macOS share those semantics. Verbatim bypasses spoken commands and + preserves literal text unless an explicit casing policy is applied. - Copy each completed CAF through a protected partial file, synchronize it, atomically finalize it, calculate SHA-256 and bytes, and commit a versioned SQLite session containing Raw, Edited, Formatted, Style, spoken operations, diff --git a/docs/decisions/0053_voice_casing_and_spoken_list_semantics.md b/docs/decisions/0053_voice_casing_and_spoken_list_semantics.md new file mode 100644 index 0000000..20af4cd --- /dev/null +++ b/docs/decisions/0053_voice_casing_and_spoken_list_semantics.md @@ -0,0 +1,62 @@ +# Decision 0053: Voice casing and spoken-list semantics + +**Status:** Accepted + +## Context + +Natural Style capitalization and deterministic polishing could override a +lowercase formatting instruction. A provider returned one text field, so it +could not reliably distinguish paragraphs from spoken grocery-list items. +macOS and iOS also needed one semantic contract before their model adapters +converge. + +| Criterion | Typed deterministic policy | Prompt-only instruction | Platform-specific rules | +| --- | --- | --- | --- | +| Provider and fallback agreement | Exact | Model-dependent | Adapter-dependent | +| Protected operational tokens | Explicit | Best effort | Drift-prone | +| macOS/iOS parity | Shared source | Requires equal models | Duplicated | +| Stored spoken-edit compatibility | Revisioned | Not applicable | Duplicated | +| Decision | Selected | Rejected | Rejected | + +## Decision + +- Model casing as Style Default, Lowercase Prose, or Strict Lowercase. An + explicit policy overrides Style capitalization. The legacy instruction + “only provide text in lowercase” normalizes to Strict Lowercase. +- Apply casing after deterministic polishing on accepted provider output and to + fallback output. Reformatting History and iOS deterministic formatting use + the same transformer. +- Preserve URLs, email addresses, paths, code identifiers, quoted phrases, and + Dictionary values under both lowercase policies. Lowercase Prose also + preserves source-signaled names and acronyms; Strict Lowercase does not. +- Keep privacy, fidelity, protected-content, and data-handling invariants above + user formatting preferences. A casing preference cannot authorize semantic + additions or protected-token mutation. +- Carry typed list intent on each refinement request. Infer ordered intent from + explicit numeric markers or sequential ordinals. Infer unordered intent from + explicit bullet markers or a list cue with conservative delimiters. Do not + guess boundaries from an undelimited word sequence. +- Add exact unordered-list and item-boundary commands in spoken-edit revision 2: + `start a bullet list`, `start a bulleted list`, `bullet`, and `next item`. + Retain revision-1 replay with its original command vocabulary. +- Compile casing, list intent, and spoken-edit sources into the portable iOS + core. Platform ASR and generative formatting may differ; these semantics do + not. +- Store casing in application-preference schema 7. Schema 6 defaults to Style + Default, preserving existing output behavior. Older apps reject schema 7 + rather than silently erasing the policy. + +## Verification + +Focused tests cover policy normalization, protected tokens, provider and +fallback delivery, History reformatting, schema migration, conservative list +intent, explicit unordered-list commands, ordinary noun ambiguity, replay, and +revision-1 compatibility. The iOS simulator runs the same transformer after +spoken edits. + +## Implications + +Prompt wording is no longer the only enforcement point for casing. Formatting +providers can use list intent to emit typed blocks, while deterministic +fallback remains conservative. ASR model selection remains a separate platform +adapter decision. diff --git a/docs/game_plan.md b/docs/game_plan.md index b7ed6d4..fc4e0e4 100644 --- a/docs/game_plan.md +++ b/docs/game_plan.md @@ -17,7 +17,7 @@ evidence is retained in [`release_validation.md`](release_validation.md). | Voice M1 tracer | Local AI Dictation tees immutable audio off the capture path, inserts once, and atomically stores one CAF plus separate final text stages in SQLite. | [`voice_cujs.md`](voice_cujs.md#m1--hold-to-dictate-and-recover) | | Voice M2 trigger | One opt-in machine-wide exact chord starts Local AI Dictation immediately, supports hold or double-press latch, finishes once, cancels on interruption, and reports reservation conflicts. | [`voice_cujs.md`](voice_cujs.md#m2--latch-a-long-prompt) | | Voice M3 formatting | Five versioned Styles produce validated evidence-backed paragraph/list blocks; one renderer preserves multiline structure or safely flattens it, and Verbatim skips the model. | [`voice_cujs.md`](voice_cujs.md#m3--format-for-purpose) | -| Voice M4 spoken edits | Exact backtrack, paragraph, numbered-list, and literal commands produce persisted replayable operations before formatting; ambiguous or inapplicable phrases remain text. | [`voice_cujs.md`](voice_cujs.md#m4--backtrack-explicitly) | +| Voice M4 spoken edits | Exact backtrack, paragraph, numbered-list, bullet-list, item-boundary, and literal commands produce persisted replayable operations before formatting; ambiguous or inapplicable phrases remain text. | [`voice_cujs.md`](voice_cujs.md#m4--backtrack-explicitly) | | Voice M5 ownership guard | Local AI preserves its captured route, rejects nonempty or changed carets, distinguishes process/secure/focus/caret invalidation, withholds later mutations, and stores a typed reason. | [`voice_cujs.md`](voice_cujs.md#m5--preserve-ownership-when-the-target-changes) | | Voice M6 History | A fourth native destination searches every text stage, exposes immutable provenance and timed audio, and appends corrections, retranscriptions, reformats, and explicit re-delivery outcomes without rewriting earlier results. Export, pin, and transactional delete are available. | [`voice_cujs.md`](voice_cujs.md#m6--browse-and-reuse-history) | | Voice M7 retention | Versioned age, byte, count, and low-disk rules expire only eligible audio, retain searchable transcript evidence, protect active/pinned/recovery artifacts, and disclose typed reasons. | [`decisions/0031_bounded_voice_history_audio.md`](decisions/0031_bounded_voice_history_audio.md) | diff --git a/docs/product_brief.md b/docs/product_brief.md index 7b0c14a..61cf72e 100644 --- a/docs/product_brief.md +++ b/docs/product_brief.md @@ -109,9 +109,15 @@ failure path. generative refinement. - Accept machine-wide recognition vocabulary, deterministic exact replacements, and optional formatting instructions. +- Apply a typed Style Default, Lowercase Prose, or Strict Lowercase policy after + formatting and on deterministic fallback. Preserve intentional operational + tokens; Lowercase Prose also preserves source-signaled names and acronyms. - Apply exact `scratch that`, `delete that sentence`, `new paragraph`, `start a - numbered list`, and `end list` commands before formatting. `literal` preserves - the immediately following exact command phrase; near-misses remain text. + numbered list`, `start a bullet list`, `bullet`, `next item`, and `end list` + commands before formatting. `literal` preserves the immediately following + exact command phrase; near-misses and ordinary uses remain text. +- Normalize explicit markers, sequential ordinals, and conservatively delimited + grocery, shopping, packing, task, or to-do list cues into typed list intent. - Optionally use a bounded caret window from the current nonsecure multiline target. Never read browser URLs, terminal contents, whole documents, screenshots, the pasteboard, or secure fields. diff --git a/docs/voice_cujs.md b/docs/voice_cujs.md index f019bd0..acf8a5f 100644 --- a/docs/voice_cujs.md +++ b/docs/voice_cujs.md @@ -119,6 +119,9 @@ Natural, or Verbatim without retranscription. Dictated ordinal structure becomes validated list blocks. A single-line target receives a safe plain-text rendering; a multiline target preserves list and paragraph structure. Protected names, commands, URLs, code tokens, and Dictionary spellings remain unchanged. +Typed casing is applied after formatting and on fallback: Strict Lowercase +lowers prose while preserving operational tokens; Lowercase Prose additionally +preserves source-signaled names and acronyms. **Current evidence:** General stores one versioned Natural, Casual Message, Formal, Technical, or Verbatim Style. The prompt carries the selected Style as @@ -139,11 +142,14 @@ databases without rewriting their rows. literal text. A destructive command cannot remove stable text outside its defined range. The Raw transcript remains inspectable. -**Current evidence:** the deterministic Swift engine recognizes only the five -exact command phrases and one exact `literal` prefix. Operations record source +**Current evidence:** the deterministic Swift engine recognizes exact +backtrack, paragraph, ordered-list, unordered-list, item-boundary, end-list, +and `literal` phrases. Operations record source UTF-8 evidence, the affected Edited suffix, and a typed replacement. Clause and sentence deletion stop at stable punctuation or an active list-item marker; -`new paragraph` begins the next item while a numbered list is active. Revision, +`new paragraph` and `next item` begin the next active list item; `bullet` begins +or advances an unordered list only in an explicit list context. Revision-1 +stored traces retain their original command vocabulary. Revision, command evidence, canonical ranges, ordering, replay, and stored-result corruption are rejected during SQLite write and read. Commands run against Raw before Dictionary output can synthesize one. The formatter receives Edited text, provider fallback delivers