diff --git a/Packages/LiveTranscribeKit/Sources/Bench/main.swift b/Packages/LiveTranscribeKit/Sources/Bench/main.swift index de3601a..8d8c2d9 100644 --- a/Packages/LiveTranscribeKit/Sources/Bench/main.swift +++ b/Packages/LiveTranscribeKit/Sources/Bench/main.swift @@ -33,8 +33,8 @@ struct BenchOptions { var verbose = false /// A speech-to-text model to measure instead of the default, to compare models on the same clips. var sttModel: String? - /// The Language setting, for a model that is told the language to write (Cohere Transcribe): a - /// code, such as `de`. + /// The Language setting, for a model that can be told the language to write (Qwen3-ASR, + /// Whisper, Cohere Transcribe): a code, such as `de`. Absent is Automatic. var sttLanguage: String? /// Export untouched ASR inputs for synthetic cleanup-data preparation, without cleanup. var asrOutput: URL? diff --git a/Packages/LiveTranscribeKit/Sources/DictationUI/MenuBarContent.swift b/Packages/LiveTranscribeKit/Sources/DictationUI/MenuBarContent.swift index ec38c94..843d312 100644 --- a/Packages/LiveTranscribeKit/Sources/DictationUI/MenuBarContent.swift +++ b/Packages/LiveTranscribeKit/Sources/DictationUI/MenuBarContent.swift @@ -3,11 +3,13 @@ import Dictation import Permissions import Shared import SwiftUI +import Transcription import TranscriptUI import Updates /// The menu bar menu: dictation status and controls, stopping the live transcript, the cleanup -/// level, the microphone, the app's windows, About and, in a release, updates. +/// level, the speech model's language, the microphone, the app's windows, About and, in a +/// release, updates. /// /// Made for `MenuBarExtra { MenuBarContent(context:) } label: { MenuBarLabel(context:) }` with /// `.menuBarExtraStyle(.menu)`, so it uses only views a menu can show. What it says and allows @@ -17,6 +19,8 @@ public struct MenuBarContent: View { private static let d = AppSettings.defaults @AppStorage(AppSettingsKey.cleanupLevel.rawValue) private var cleanupLevel = d.cleanupLevel + @AppStorage(AppSettingsKey.sttModel.rawValue) private var speechModel = d.sttModel + @AppStorage(AppSettingsKey.sttLanguage.rawValue) private var speechLanguage: String? @AppStorage(AppSettingsKey.showVirtualInputDevices.rawValue) private var showOtherDevices = d.dictation.showVirtualInputDevices @AppStorage(AppSettingsKey.dictationHotkey.rawValue) private var dictationHotkey = d.dictation.hotkey @@ -76,6 +80,7 @@ public struct MenuBarContent: View { .tag(level) } } + languagePicker if transcript.supportsInputSelection { MenuMicrophoneSubmenu(transcript: transcript, showOtherDevices: $showOtherDevices) } @@ -100,6 +105,32 @@ public struct MenuBarContent: View { .keyboardShortcut("q") } + /// The Language setting, with the languages the chosen speech model can be told. A model + /// that can't be told one has Automatic alone, and the menu is greyed out. + private var languagePicker: some View { + let model = context.speechModels.catalog.model(forSetting: speechModel) + let menu = SpeechLanguageMenu(model: model) + let selection = Binding( + get: { menu.tag(forSetting: speechLanguage) }, + set: { (tag: String) in + let code = SpeechLanguageMenu.setting(forTag: tag) + context.settingsStore.setSpeechLanguage(code) + Log.ui.info("Speech language chosen from the menu bar: \(code ?? "automatic", privacy: .public)") + } + ) + return Picker("Language", selection: selection) { + Text(menu.automaticTitle).tag(SpeechLanguageMenu.automatic) + if menu.canChoose { + Divider() + } + ForEach(menu.choices) { choice in + Text(choice.name).tag(choice.code) + } + } + .disabled(!menu.canChoose) + .help(menu.canChoose ? "The language \(model?.name ?? "the speech model") writes" : "This speech model finds the language itself") + } + /// *Check for Updates…*, which a running check disables, or the release a scheduled check /// found, which can always be brought forward. private func updatesItem(_ updates: any SoftwareUpdating) -> some View { diff --git a/Packages/LiveTranscribeKit/Sources/DictationUI/SpeechLanguageMenu.swift b/Packages/LiveTranscribeKit/Sources/DictationUI/SpeechLanguageMenu.swift new file mode 100644 index 0000000..b210d46 --- /dev/null +++ b/Packages/LiveTranscribeKit/Sources/DictationUI/SpeechLanguageMenu.swift @@ -0,0 +1,37 @@ +import Shared +import Transcription + +/// The Language setting as a menu offers it for one speech model, in the menu bar and on the +/// model's row in Settings › Models: Automatic first, then the languages the model can be told. +/// +/// The setting is one for every model (``AppSettingsKey/sttLanguage``), so a language the model +/// in use doesn't have shows as Automatic, and is kept for a model that has it. +struct SpeechLanguageMenu: Equatable { + /// Automatic's tag in a picker: the setting is absent. + static let automatic = "" + + /// What Automatic does: the model finds the language, or, for one that can't, writes the one + /// in brackets, "Automatic (English)". + let automaticTitle: String + /// The languages the model can be told, in the catalog's order; empty for a model that + /// can't be told one, or one outside the catalog, whose menu is Automatic alone. + let choices: [SpeechModelCatalog.LanguageChoice] + + init(model: SpeechModelCatalog.Model?) { + choices = model?.languageChoices ?? [] + automaticTitle = model?.automaticChoice.map { "Automatic (\($0.name))" } ?? "Automatic" + } + + /// Whether there is anything to choose. + var canChoose: Bool { !choices.isEmpty } + + /// The picker's tag for the setting: its code if the model has it, and otherwise Automatic's. + func tag(forSetting setting: String?) -> String { + choices.contains { $0.code == setting } ? setting ?? Self.automatic : Self.automatic + } + + /// The setting for a tag the picker gives: `nil` for Automatic. + static func setting(forTag tag: String) -> String? { + tag == automatic ? nil : tag + } +} diff --git a/Packages/LiveTranscribeKit/Sources/DictationUI/SpeechModelsSettingsView.swift b/Packages/LiveTranscribeKit/Sources/DictationUI/SpeechModelsSettingsView.swift index 49e3470..4cfa3f0 100644 --- a/Packages/LiveTranscribeKit/Sources/DictationUI/SpeechModelsSettingsView.swift +++ b/Packages/LiveTranscribeKit/Sources/DictationUI/SpeechModelsSettingsView.swift @@ -30,6 +30,7 @@ struct SpeechModelsSettingsView: View { status: status(of: model.mac.repository.rawValue, download: library.state(of: model)), canRemove: canRemove(model), language: languageChoice(for: model), + languages: SpeechLanguageMenu(model: model), actions: actions(for: model) ) } @@ -115,15 +116,17 @@ struct SpeechModelsSettingsView: View { ) } - /// The language a model that is told one writes: the Language setting's, if the model has - /// it, and otherwise its first. `nil` for a model that finds the language itself. + /// The Language setting, for a model that can be told a language: its tag in the model's + /// menu (``SpeechLanguageMenu``). `nil` for a model that can't be told one. private func languageChoice(for model: SpeechModelCatalog.Model) -> Binding? { - guard let first = model.languageChoices.first else { return nil } + let menu = SpeechLanguageMenu(model: model) + guard menu.canChoose else { return nil } return Binding( - get: { model.language(forSetting: language)?.code ?? first.code }, - set: { code in + get: { menu.tag(forSetting: language) }, + set: { tag in + let code = SpeechLanguageMenu.setting(forTag: tag) context.settingsStore.setSpeechLanguage(code) - Log.ui.info("Speech language chosen in Settings: \(code, privacy: .public)") + Log.ui.info("Speech language chosen in Settings: \(code ?? "automatic", privacy: .public)") } ) } @@ -145,15 +148,16 @@ struct SpeechModelsSettingsView: View { } } -/// One catalog model: what it is for, the language it writes if it's told one, and its status or -/// what can be done with it. +/// One catalog model: what it is for, the language it writes if it can be told one, and its +/// status or what can be done with it. private struct SpeechModelRow: View { let model: SpeechModelCatalog.Model let isDefault: Bool let status: SpeechModelRowStatus let canRemove: Bool - /// The Language setting, for a model that is told one. + /// The Language setting, for a model that can be told one. let language: Binding? + let languages: SpeechLanguageMenu let actions: SpeechModelStatusView.Actions var body: some View { @@ -183,7 +187,9 @@ private struct SpeechModelRow: View { .accessibilityElement(children: .combine) if let language { Picker("Language", selection: language) { - ForEach(model.languageChoices) { choice in + Text(languages.automaticTitle).tag(SpeechLanguageMenu.automatic) + Divider() + ForEach(languages.choices) { choice in Text(choice.name).tag(choice.code) } } diff --git a/Packages/LiveTranscribeKit/Sources/Shared/AppSettingsStore.swift b/Packages/LiveTranscribeKit/Sources/Shared/AppSettingsStore.swift index 8a98e83..d426bb9 100644 --- a/Packages/LiveTranscribeKit/Sources/Shared/AppSettingsStore.swift +++ b/Packages/LiveTranscribeKit/Sources/Shared/AppSettingsStore.swift @@ -48,9 +48,10 @@ public enum AppSettingsKey: String, CaseIterable, Sendable { /// The chosen microphone's Core Audio UID; absent means the system default input. /// Not part of ``AppSettings``: it is read at every Start rather than once at launch. case inputDeviceUID - /// The Language setting: the code of the language a speech model that is told one (Cohere - /// Transcribe) writes; absent means the model's first, English. The other models find the - /// language themselves. Not part of ``AppSettings``: it is read at every transcription. + /// The Language setting: the code of the language a speech model that can be told one + /// (Qwen3-ASR, Whisper, Cohere Transcribe) writes; absent means Automatic: the model finds + /// the language, or, for one that can't (Cohere Transcribe), writes English. Not part of + /// ``AppSettings``: it is read at every transcription. case sttLanguage } @@ -213,8 +214,7 @@ public struct AppSettingsStore: Sendable { } } - /// The Language setting's code, or `nil` for each model's first - /// (``AppSettingsKey/sttLanguage``). + /// The Language setting's code, or `nil` for Automatic (``AppSettingsKey/sttLanguage``). public var speechLanguage: String? { guard let code = defaults.string(forKey: AppSettingsKey.sttLanguage.rawValue), !code.isEmpty else { return nil @@ -222,8 +222,8 @@ public struct AppSettingsStore: Sendable { return code } - /// Stores the language a speech model that is told one writes, by its code; `nil` goes back - /// to each model's first. + /// Stores the language a speech model that can be told one writes, by its code; `nil` goes + /// back to Automatic. public func setSpeechLanguage(_ code: String?) { if let code, !code.isEmpty { defaults.set(code, forKey: AppSettingsKey.sttLanguage.rawValue) diff --git a/Packages/LiveTranscribeKit/Sources/Transcription/LockedVocabulary.swift b/Packages/LiveTranscribeKit/Sources/Transcription/LockedVocabulary.swift new file mode 100644 index 0000000..4d4d08b --- /dev/null +++ b/Packages/LiveTranscribeKit/Sources/Transcription/LockedVocabulary.swift @@ -0,0 +1,106 @@ +import Foundation + +/// Qwen3-ASR's vocabulary as the bytes each token writes, from the model's `vocab.json`, to find +/// the tokens a ``ScriptLock`` lets the model write next. +/// +/// The vocabulary holds the byte-level BPE tokens only. Qwen's added tokens (`<|im_end|>`, +/// ``, ...) come after them, and the model's logits after those, padded; none of them +/// writes text, so the lock allows none but the ends of a reply. +final class LockedVocabulary { + enum LoadError: Error, Equatable { + case unreadable(String) + /// A token id is missing, so the ids don't run from 0 without a gap. + case missingToken(Int) + } + + /// Each token's bytes, one token after another: token `i`'s run from `starts[i]` up to + /// `starts[i + 1]`. + private let bytes: [UInt8] + private let starts: [Int] + /// The tokens that end the reply, which may follow a whole character only. + private let endOfReply: Set + + /// How many tokens the vocabulary holds. + var count: Int { starts.count - 1 } + + /// Reads `vocab.json`, which spells each token's bytes in GPT-2's byte-level alphabet. + init(vocabularyJSON data: Data, endOfReply: Set) throws(LoadError) { + let ids: [String: Int] + do { + ids = try JSONDecoder().decode([String: Int].self, from: data) + } catch { + throw .unreadable(String(describing: error)) + } + var tokens = [String?](repeating: nil, count: ids.count) + for (token, id) in ids where tokens.indices.contains(id) { + tokens[id] = token + } + var bytes: [UInt8] = [] + var starts = [0] + for (id, token) in tokens.enumerated() { + guard let token else { throw .missingToken(id) } + // A character outside the alphabet, which no byte-level vocabulary has, writes nothing, + // and the lock never allows writing nothing but the end. + if let written = Self.bytes(spelt: token) { + bytes += written + } + starts.append(bytes.count) + } + self.bytes = bytes + self.starts = starts + self.endOfReply = endOfReply + } + + /// The bytes `token` writes; none past the vocabulary. + func bytes(of token: Int) -> ArraySlice { + guard token >= 0, token < count else { return [] } + return bytes[starts[token].. [Bool] { + var allowed = [Bool](repeating: false, count: logitCount) + for token in 0.. [UInt8]? { + var bytes: [UInt8] = [] + bytes.reserveCapacity(token.unicodeScalars.count) + for scalar in token.unicodeScalars { + guard let byte = byteOf[scalar] else { return nil } + bytes.append(byte) + } + return bytes + } + + /// GPT-2's byte-level alphabet, as Qwen's vocabulary spells bytes: the printable Latin-1 + /// bytes stand for themselves, and the other 68 for the characters from U+0100 on, in byte + /// order. + private static let byteOf: [Unicode.Scalar: UInt8] = { + var byteOf: [Unicode.Scalar: UInt8] = [:] + var next: UInt32 = 0x100 + for byte in UInt8.min...UInt8.max { + let printable = (0x21...0x7E).contains(byte) || (0xA1...0xAC).contains(byte) || (0xAE...0xFF).contains(byte) + let scalar: UInt32 + if printable { + scalar = UInt32(byte) + } else { + scalar = next + next += 1 + } + byteOf[Unicode.Scalar(scalar)!] = byte + } + return byteOf + }() +} diff --git a/Packages/LiveTranscribeKit/Sources/Transcription/MLXTranscriber.swift b/Packages/LiveTranscribeKit/Sources/Transcription/MLXTranscriber.swift index c1608f5..99856ce 100644 --- a/Packages/LiveTranscribeKit/Sources/Transcription/MLXTranscriber.swift +++ b/Packages/LiveTranscribeKit/Sources/Transcription/MLXTranscriber.swift @@ -12,9 +12,13 @@ import Shared /// /// The model is loaded once and warmed up, and replaced when ``switchModel(to:progress:)`` names /// another. It never leaves this actor, which runs on its own serial queue so the blocking MLX -/// inference does not occupy the cooperative thread pool. A catalog model that is told which -/// language to write (Cohere Transcribe) is told the Language setting's at each transcription, so -/// a change applies to the next without loading the model again. +/// inference does not occupy the cooperative thread pool. A catalog model that can be told which +/// language to write (Qwen3-ASR, Whisper, Cohere Transcribe) is told the Language setting's at +/// each transcription, so a change applies to the next without loading the model again. +/// Automatic tells it nothing, and it finds the language, except Cohere Transcribe, which can't +/// and is told English. Qwen3-ASR takes the language it's told as no more than a hint, so it's +/// also kept to that language's letters (``ScriptLock``), and decoded here rather than by +/// mlx-audio-swift. public actor MLXTranscriber: Transcriber { /// Shorter clips are returned as empty text rather than sent to the model: they hold no word, /// and some models' subsampling (Parakeet's conformer, say) needs a minimum number of frames. @@ -23,8 +27,12 @@ public actor MLXTranscriber: Transcriber { /// The Speech-to-text setting to load: the one the transcriber was made with, or the last one /// it was switched to. private var modelID: String - /// The loaded model, and the setting it was loaded for. - private var loaded: (modelID: String, model: any STTGenerationModel)? + /// The loaded model, the setting it was loaded for and its folder. + private var loaded: (modelID: String, model: any STTGenerationModel, folder: URL)? + /// The masks that lock the loaded Qwen3-ASR's letters to the language it's told, made from + /// its vocabulary when a language is first told, and the setting they were made for; `nil` + /// masks if its vocabulary couldn't be read. + private var lockedLogits: (modelID: String, logits: LockedLogits?)? /// A model is taking the place of the one before, which is already gone. private var isReplacingModel = false /// Transcriptions asked for meanwhile, which wait for the new model. @@ -32,7 +40,7 @@ public actor MLXTranscriber: Transcriber { private let catalog: SpeechModelCatalog private let downloads: SpeechModelDownloads - /// The Language setting, a code, read at each transcription; `nil` for each model's first. + /// The Language setting, a code, read at each transcription; `nil` for Automatic. private let language: @Sendable () -> String? private let queue = DispatchSerialQueue(label: "LiveTranscribe.MLXTranscriber", qos: .userInitiated) @@ -67,6 +75,7 @@ public actor MLXTranscriber: Transcriber { defer { finishReplacingModel() } if loaded != nil { loaded = nil + lockedLogits = nil Memory.clearCache() } let model = try await kind.load(folder) @@ -78,7 +87,7 @@ public actor MLXTranscriber: Transcriber { audio: MLXArray.zeros([AudioFormat.sampleRate]), generationParameters: OutputLimit.capping(model.defaultGenerationParameters, sampleCount: AudioFormat.sampleRate) ) - loaded = (modelID, model) + loaded = (modelID, model, folder) progress(ModelLoadProgress(modelID: modelID, stage: .ready, fractionCompleted: 1)) Log.transcription.info( "STT model ready: \(location.description, privacy: .public), \(kind.modelType, privacy: .public) (warm-up \(started.duration(to: .now).wholeMilliseconds) ms)" @@ -105,31 +114,63 @@ public actor MLXTranscriber: Transcriber { let interval = signposter.beginInterval("STT", id: signposter.makeSignpostID()) defer { signposter.endInterval("STT", interval) } - let told = Self.parameters( - model.defaultGenerationParameters, - for: catalog.model(forSetting: loaded.modelID), - language: language() - ) + let catalogModel = catalog.model(forSetting: loaded.modelID) + let setting = language() + let told = Self.parameters(model.defaultGenerationParameters, for: catalogModel, language: setting) let parameters = OutputLimit.capping(told, sampleCount: samples.count) - let output = model.generate(audio: MLXArray(samples), generationParameters: parameters) - if parameters.maxTokens > 0, output.generationTokens >= parameters.maxTokens { + let text: String + let generationTokens: Int + if let qwen = model as? Qwen3ASRModel, let lock = Self.lock(for: catalogModel, language: setting), + let logits = lockedLogits(for: loaded) { + (text, generationTokens) = qwen.generate(audio: MLXArray(samples), parameters: parameters, locking: lock, with: logits) + } else { + let output = model.generate(audio: MLXArray(samples), generationParameters: parameters) + (text, generationTokens) = (output.text, output.generationTokens) + } + if parameters.maxTokens > 0, generationTokens >= parameters.maxTokens { Log.transcription.warning( "STT stopped at its limit of \(parameters.maxTokens, privacy: .public) tokens for \(samples.count / (AudioFormat.sampleRate / 1000), privacy: .public) ms of audio; the model was probably repeating itself" ) } - return output.text.trimmingCharacters(in: .whitespacesAndNewlines) + return text.trimmingCharacters(in: .whitespacesAndNewlines) + } + + /// The lock on a catalog model's letters for the Language setting: the language's letters, + /// for a language of the model's that names them (Qwen3-ASR's, ``ScriptLock``). `nil` for + /// Automatic, which tells Qwen3-ASR nothing, and for a model whose letters aren't locked. + static func lock(for model: SpeechModelCatalog.Model?, language setting: String?) -> ScriptLock? { + model?.language(forSetting: setting)?.script.map(ScriptLock.init) + } + + /// The masks for the loaded model, made from its `vocab.json` the first time a language is + /// told. If the vocabulary can't be read, the model is told the language unlocked, as + /// Automatic's would be guessed. + private func lockedLogits(for loaded: (modelID: String, model: any STTGenerationModel, folder: URL)) -> LockedLogits? { + if let made = lockedLogits, made.modelID == loaded.modelID { return made.logits } + var logits: LockedLogits? + do { + let json = try Data(contentsOf: loaded.folder.appending(path: "vocab.json")) + logits = LockedLogits(vocabulary: try LockedVocabulary(vocabularyJSON: json, endOfReply: Qwen3ASRModel.endOfReply)) + } catch { + Log.transcription.error( + "STT can't lock its letters to the language it's told, so it's told the language alone: \(String(describing: error), privacy: .public)" + ) + } + lockedLogits = (loaded.modelID, logits) + return logits } - /// `parameters`, with the language to write for a catalog model that is told one: the - /// Language setting's if the model has it, and otherwise its first - /// (``SpeechModelCatalog/Model/language(forSetting:)``). Any other model keeps its own. + /// `parameters`, with the language to write for a catalog model that can be told one: the + /// Language setting's if the model has it, and otherwise Automatic's + /// (``SpeechModelCatalog/Model/language(forSetting:)``). A model that finds the language + /// itself, and any model outside the catalog, keeps its own parameters, which tell it none. static func parameters( _ parameters: STTGenerateParameters, for model: SpeechModelCatalog.Model?, language setting: String? ) -> STTGenerateParameters { - guard let told = model?.language(forSetting: setting) else { return parameters } - return parameters.replacing(language: told.code) + guard let model, let choice = model.language(forSetting: setting) else { return parameters } + return parameters.replacing(language: model.told(choice)) } /// The model's folder, downloaded first when it's a repository. diff --git a/Packages/LiveTranscribeKit/Sources/Transcription/Qwen3ASR+ScriptLock.swift b/Packages/LiveTranscribeKit/Sources/Transcription/Qwen3ASR+ScriptLock.swift new file mode 100644 index 0000000..e19118e --- /dev/null +++ b/Packages/LiveTranscribeKit/Sources/Transcription/Qwen3ASR+ScriptLock.swift @@ -0,0 +1,113 @@ +import Foundation +@preconcurrency import MLX +import MLXAudioSTT + +/// The masks a ``ScriptLock`` puts on Qwen3-ASR's logits: 0 on each token it allows and −∞ on the +/// rest. Each is made the first time a language and an unfinished character need it, which takes +/// a few milliseconds over the 151,643 tokens, and kept while the model is loaded. +final class LockedLogits { + let vocabulary: LockedVocabulary + private var masks: [Key: MLXArray] = [:] + + private struct Key: Hashable { + let lock: ScriptLock + let partial: PartialCharacter + let logitCount: Int + } + + init(vocabulary: LockedVocabulary) { + self.vocabulary = vocabulary + } + + /// `logits` (the last token's, `[1, vocabulary]`) with what `lock` forbids after `partial` + /// taken out. + func locking(_ logits: MLXArray, by lock: ScriptLock, after partial: PartialCharacter) -> MLXArray { + let key = Key(lock: lock, partial: partial, logitCount: logits.dim(-1)) + let mask: MLXArray + if let made = masks[key] { + mask = made + } else { + let allowed = vocabulary.allowed(by: lock, after: partial, logitCount: key.logitCount) + mask = MLXArray(allowed.map { $0 ? Float(0) : -Float.infinity }) + masks[key] = mask + } + return logits.asType(.float32) + mask + } +} + +extension Qwen3ASRModel { + /// The tokens that end a reply: `<|im_end|>` and `<|endoftext|>`. + static let endOfReply: Set = [151_645, 151_643] + + /// Transcribes `audio` told the language `parameters` names, writing only what `lock` + /// allows: mlx-audio-swift's `generate(audio:generationParameters:)` (as at 01dec7c), step for + /// step, with the lock's mask on the logits before each greedy choice. + /// + /// As there: the audio is split at quiet points into chunks of `chunkDuration` at most, and + /// each chunk's reply ends at an end-of-reply token, at the tokens left of `maxTokens`, or, with + /// no repetition penalty, once its last 24 tokens hold 3 or fewer different tokens. + func generate( + audio: MLXArray, + parameters: STTGenerateParameters, + locking lock: ScriptLock, + with logits: LockedLogits + ) -> (text: String, generationTokens: Int) { + guard let tokenizer else { return ("", 0) } + let chunks = splitAudioIntoChunks( + audio, + sampleRate: sampleRate, + chunkDuration: parameters.chunkDuration, + minChunkDuration: parameters.minChunkDuration + ) + var texts: [String] = [] + var generationTokens = 0 + for (chunk, _) in chunks { + let remaining = parameters.maxTokens - generationTokens + guard remaining > 0 else { break } + let tokens = reply(to: chunk, parameters: parameters, maxTokens: remaining, locking: lock, with: logits) + texts.append(tokenizer.decode(tokens: tokens).trimmingCharacters(in: .whitespacesAndNewlines)) + generationTokens += tokens.count + Memory.clearCache() + } + return (texts.joined(separator: " ").trimmingCharacters(in: .whitespacesAndNewlines), generationTokens) + } + + /// One chunk's reply, as mlx-audio-swift's `generateSingleChunk` decodes it, locked. + private func reply( + to audio: MLXArray, + parameters: STTGenerateParameters, + maxTokens: Int, + locking lock: ScriptLock, + with lockedLogits: LockedLogits + ) -> [Int] { + let (features, featureMask, audioTokenCount) = preprocessAudio(audio) + let prompt = buildPrompt(numAudioTokens: audioTokenCount, context: "", language: parameters.language) + let cache = makeCache() + // The prompt in one pass, which merges the audio's features in; mlx-audio-swift takes it + // 2,048 tokens at a time, which a dictation's prompt never reaches. + var last = self(inputIds: prompt, inputFeatures: features, featureAttentionMask: featureMask, cache: cache)[0..., -1, 0...] + var tokens: [Int] = [] + var partial = PartialCharacter.nothing + while tokens.count < maxTokens { + let token = lockedLogits.locking(last, by: lock, after: partial).argMax(axis: -1).item(Int.self) + if Self.endOfReply.contains(token) { break } + tokens.append(token) + // The mask allowed the token, so the lock reads it. + partial = lock.partial(after: partial, reading: lockedLogits.vocabulary.bytes(of: token)) ?? .nothing + if parameters.repetitionPenalty == 1, tokens.count >= 24, Set(tokens.suffix(24)).count <= 3 { break } + if tokens.count == maxTokens { break } + + last = self(inputIds: MLXArray([Int32(token)]).expandedDimensions(axis: 0), cache: cache)[0..., -1, 0...] + if parameters.repetitionPenalty != 1 { + let recent = MLXArray(tokens.suffix(max(1, parameters.repetitionContextSize)).map { Int32($0) }) + let recentLogits = last[0..., recent] + let penalty = MLXArray(parameters.repetitionPenalty) + last[0..., recent] = MLX.where(recentLogits .> 0, recentLogits / penalty, recentLogits * penalty) + } + if tokens.count % 256 == 0 { + Memory.clearCache() + } + } + return tokens + } +} diff --git a/Packages/LiveTranscribeKit/Sources/Transcription/Resources/speech-models.json b/Packages/LiveTranscribeKit/Sources/Transcription/Resources/speech-models.json index 6cfbd10..f9c13ef 100644 --- a/Packages/LiveTranscribeKit/Sources/Transcription/Resources/speech-models.json +++ b/Packages/LiveTranscribeKit/Sources/Transcription/Resources/speech-models.json @@ -8,6 +8,39 @@ "languages": "30 languages and Sinhala", "licence": "CC-BY-SA-4.0", "credit": "Qwen3-ASR-0.6B by the Qwen team, Alibaba Cloud, fine-tuned for Sinhala by Nerdstorm on OpenSLR 52 (Google, CC BY-SA 4.0)", + "language_choices": [ + {"code": "en", "name": "English", "script": "Latn"}, + {"code": "ar", "name": "Arabic", "script": "Arab"}, + {"code": "yue", "name": "Cantonese", "script": "Hani"}, + {"code": "zh", "name": "Chinese", "script": "Hani"}, + {"code": "cs", "name": "Czech", "script": "Latn"}, + {"code": "da", "name": "Danish", "script": "Latn"}, + {"code": "nl", "name": "Dutch", "script": "Latn"}, + {"code": "fil", "name": "Filipino", "script": "Latn"}, + {"code": "fi", "name": "Finnish", "script": "Latn"}, + {"code": "fr", "name": "French", "script": "Latn"}, + {"code": "de", "name": "German", "script": "Latn"}, + {"code": "el", "name": "Greek", "script": "Grek"}, + {"code": "hi", "name": "Hindi", "script": "Deva"}, + {"code": "hu", "name": "Hungarian", "script": "Latn"}, + {"code": "id", "name": "Indonesian", "script": "Latn"}, + {"code": "it", "name": "Italian", "script": "Latn"}, + {"code": "ja", "name": "Japanese", "script": "Jpan"}, + {"code": "ko", "name": "Korean", "script": "Kore"}, + {"code": "mk", "name": "Macedonian", "script": "Cyrl"}, + {"code": "ms", "name": "Malay", "script": "Latn"}, + {"code": "fa", "name": "Persian", "script": "Arab"}, + {"code": "pl", "name": "Polish", "script": "Latn"}, + {"code": "pt", "name": "Portuguese", "script": "Latn"}, + {"code": "ro", "name": "Romanian", "script": "Latn"}, + {"code": "ru", "name": "Russian", "script": "Cyrl"}, + {"code": "si", "name": "Sinhala", "script": "Sinh"}, + {"code": "es", "name": "Spanish", "script": "Latn"}, + {"code": "sv", "name": "Swedish", "script": "Latn"}, + {"code": "th", "name": "Thai", "script": "Thai"}, + {"code": "tr", "name": "Turkish", "script": "Latn"}, + {"code": "vi", "name": "Vietnamese", "script": "Latn"} + ], "mac": { "repository": "Nerdstorm/Qwen3-ASR-0.6B-Sinhala-8bit", "revision": "047217cb611ad5f51cb2284e4a3617a5eaa9b688", @@ -62,6 +95,38 @@ "languages": "30 languages", "licence": "Apache-2.0", "credit": "Qwen3-ASR-0.6B by the Qwen team, Alibaba Cloud", + "language_choices": [ + {"code": "en", "name": "English", "script": "Latn"}, + {"code": "ar", "name": "Arabic", "script": "Arab"}, + {"code": "yue", "name": "Cantonese", "script": "Hani"}, + {"code": "zh", "name": "Chinese", "script": "Hani"}, + {"code": "cs", "name": "Czech", "script": "Latn"}, + {"code": "da", "name": "Danish", "script": "Latn"}, + {"code": "nl", "name": "Dutch", "script": "Latn"}, + {"code": "fil", "name": "Filipino", "script": "Latn"}, + {"code": "fi", "name": "Finnish", "script": "Latn"}, + {"code": "fr", "name": "French", "script": "Latn"}, + {"code": "de", "name": "German", "script": "Latn"}, + {"code": "el", "name": "Greek", "script": "Grek"}, + {"code": "hi", "name": "Hindi", "script": "Deva"}, + {"code": "hu", "name": "Hungarian", "script": "Latn"}, + {"code": "id", "name": "Indonesian", "script": "Latn"}, + {"code": "it", "name": "Italian", "script": "Latn"}, + {"code": "ja", "name": "Japanese", "script": "Jpan"}, + {"code": "ko", "name": "Korean", "script": "Kore"}, + {"code": "mk", "name": "Macedonian", "script": "Cyrl"}, + {"code": "ms", "name": "Malay", "script": "Latn"}, + {"code": "fa", "name": "Persian", "script": "Arab"}, + {"code": "pl", "name": "Polish", "script": "Latn"}, + {"code": "pt", "name": "Portuguese", "script": "Latn"}, + {"code": "ro", "name": "Romanian", "script": "Latn"}, + {"code": "ru", "name": "Russian", "script": "Cyrl"}, + {"code": "es", "name": "Spanish", "script": "Latn"}, + {"code": "sv", "name": "Swedish", "script": "Latn"}, + {"code": "th", "name": "Thai", "script": "Thai"}, + {"code": "tr", "name": "Turkish", "script": "Latn"}, + {"code": "vi", "name": "Vietnamese", "script": "Latn"} + ], "mac": { "repository": "mlx-community/Qwen3-ASR-0.6B-8bit", "revision": "89e96d92ba34aca20b3e29fb10cc284097d1219f", @@ -76,6 +141,38 @@ "languages": "30 languages", "licence": "Apache-2.0", "credit": "Qwen3-ASR-1.7B by the Qwen team, Alibaba Cloud", + "language_choices": [ + {"code": "en", "name": "English", "script": "Latn"}, + {"code": "ar", "name": "Arabic", "script": "Arab"}, + {"code": "yue", "name": "Cantonese", "script": "Hani"}, + {"code": "zh", "name": "Chinese", "script": "Hani"}, + {"code": "cs", "name": "Czech", "script": "Latn"}, + {"code": "da", "name": "Danish", "script": "Latn"}, + {"code": "nl", "name": "Dutch", "script": "Latn"}, + {"code": "fil", "name": "Filipino", "script": "Latn"}, + {"code": "fi", "name": "Finnish", "script": "Latn"}, + {"code": "fr", "name": "French", "script": "Latn"}, + {"code": "de", "name": "German", "script": "Latn"}, + {"code": "el", "name": "Greek", "script": "Grek"}, + {"code": "hi", "name": "Hindi", "script": "Deva"}, + {"code": "hu", "name": "Hungarian", "script": "Latn"}, + {"code": "id", "name": "Indonesian", "script": "Latn"}, + {"code": "it", "name": "Italian", "script": "Latn"}, + {"code": "ja", "name": "Japanese", "script": "Jpan"}, + {"code": "ko", "name": "Korean", "script": "Kore"}, + {"code": "mk", "name": "Macedonian", "script": "Cyrl"}, + {"code": "ms", "name": "Malay", "script": "Latn"}, + {"code": "fa", "name": "Persian", "script": "Arab"}, + {"code": "pl", "name": "Polish", "script": "Latn"}, + {"code": "pt", "name": "Portuguese", "script": "Latn"}, + {"code": "ro", "name": "Romanian", "script": "Latn"}, + {"code": "ru", "name": "Russian", "script": "Cyrl"}, + {"code": "es", "name": "Spanish", "script": "Latn"}, + {"code": "sv", "name": "Swedish", "script": "Latn"}, + {"code": "th", "name": "Thai", "script": "Thai"}, + {"code": "tr", "name": "Turkish", "script": "Latn"}, + {"code": "vi", "name": "Vietnamese", "script": "Latn"} + ], "mac": { "repository": "mlx-community/Qwen3-ASR-1.7B-8bit", "revision": "a8379a2e2f9e313c9292cdf1af4055ab56d50d55", @@ -179,9 +276,111 @@ "id": "whisper-large-v3-turbo", "name": "Whisper large-v3-turbo", "summary": "The most languages.", - "languages": "99 languages", + "languages": "100 languages", "licence": "MIT", "credit": "Whisper large-v3-turbo by OpenAI", + "language_choices": [ + {"code": "en", "name": "English"}, + {"code": "af", "name": "Afrikaans"}, + {"code": "sq", "name": "Albanian"}, + {"code": "am", "name": "Amharic"}, + {"code": "ar", "name": "Arabic"}, + {"code": "hy", "name": "Armenian"}, + {"code": "as", "name": "Assamese"}, + {"code": "az", "name": "Azerbaijani"}, + {"code": "ba", "name": "Bashkir"}, + {"code": "eu", "name": "Basque"}, + {"code": "be", "name": "Belarusian"}, + {"code": "bn", "name": "Bengali"}, + {"code": "bs", "name": "Bosnian"}, + {"code": "br", "name": "Breton"}, + {"code": "bg", "name": "Bulgarian"}, + {"code": "my", "name": "Burmese"}, + {"code": "yue", "name": "Cantonese"}, + {"code": "ca", "name": "Catalan"}, + {"code": "zh", "name": "Chinese"}, + {"code": "hr", "name": "Croatian"}, + {"code": "cs", "name": "Czech"}, + {"code": "da", "name": "Danish"}, + {"code": "nl", "name": "Dutch"}, + {"code": "et", "name": "Estonian"}, + {"code": "fo", "name": "Faroese"}, + {"code": "fi", "name": "Finnish"}, + {"code": "fr", "name": "French"}, + {"code": "gl", "name": "Galician"}, + {"code": "ka", "name": "Georgian"}, + {"code": "de", "name": "German"}, + {"code": "el", "name": "Greek"}, + {"code": "gu", "name": "Gujarati"}, + {"code": "ht", "name": "Haitian Creole"}, + {"code": "ha", "name": "Hausa"}, + {"code": "haw", "name": "Hawaiian"}, + {"code": "he", "name": "Hebrew"}, + {"code": "hi", "name": "Hindi"}, + {"code": "hu", "name": "Hungarian"}, + {"code": "is", "name": "Icelandic"}, + {"code": "id", "name": "Indonesian"}, + {"code": "it", "name": "Italian"}, + {"code": "ja", "name": "Japanese"}, + {"code": "jw", "name": "Javanese"}, + {"code": "kn", "name": "Kannada"}, + {"code": "kk", "name": "Kazakh"}, + {"code": "km", "name": "Khmer"}, + {"code": "ko", "name": "Korean"}, + {"code": "lo", "name": "Lao"}, + {"code": "la", "name": "Latin"}, + {"code": "lv", "name": "Latvian"}, + {"code": "ln", "name": "Lingala"}, + {"code": "lt", "name": "Lithuanian"}, + {"code": "lb", "name": "Luxembourgish"}, + {"code": "mk", "name": "Macedonian"}, + {"code": "mg", "name": "Malagasy"}, + {"code": "ms", "name": "Malay"}, + {"code": "ml", "name": "Malayalam"}, + {"code": "mt", "name": "Maltese"}, + {"code": "mi", "name": "Maori"}, + {"code": "mr", "name": "Marathi"}, + {"code": "mn", "name": "Mongolian"}, + {"code": "ne", "name": "Nepali"}, + {"code": "no", "name": "Norwegian"}, + {"code": "nn", "name": "Norwegian Nynorsk"}, + {"code": "oc", "name": "Occitan"}, + {"code": "ps", "name": "Pashto"}, + {"code": "fa", "name": "Persian"}, + {"code": "pl", "name": "Polish"}, + {"code": "pt", "name": "Portuguese"}, + {"code": "pa", "name": "Punjabi"}, + {"code": "ro", "name": "Romanian"}, + {"code": "ru", "name": "Russian"}, + {"code": "sa", "name": "Sanskrit"}, + {"code": "sr", "name": "Serbian"}, + {"code": "sn", "name": "Shona"}, + {"code": "sd", "name": "Sindhi"}, + {"code": "si", "name": "Sinhala"}, + {"code": "sk", "name": "Slovak"}, + {"code": "sl", "name": "Slovenian"}, + {"code": "so", "name": "Somali"}, + {"code": "es", "name": "Spanish"}, + {"code": "su", "name": "Sundanese"}, + {"code": "sw", "name": "Swahili"}, + {"code": "sv", "name": "Swedish"}, + {"code": "tl", "name": "Tagalog"}, + {"code": "tg", "name": "Tajik"}, + {"code": "ta", "name": "Tamil"}, + {"code": "tt", "name": "Tatar"}, + {"code": "te", "name": "Telugu"}, + {"code": "th", "name": "Thai"}, + {"code": "bo", "name": "Tibetan"}, + {"code": "tr", "name": "Turkish"}, + {"code": "tk", "name": "Turkmen"}, + {"code": "uk", "name": "Ukrainian"}, + {"code": "ur", "name": "Urdu"}, + {"code": "uz", "name": "Uzbek"}, + {"code": "vi", "name": "Vietnamese"}, + {"code": "cy", "name": "Welsh"}, + {"code": "yi", "name": "Yiddish"}, + {"code": "yo", "name": "Yoruba"} + ], "mac": { "repository": "mlx-community/whisper-large-v3-turbo-8bit", "revision": "62103fc276a35fdc76e318f314d8ff47987fba89", @@ -196,6 +395,7 @@ "languages": "14 languages", "licence": "Apache-2.0", "credit": "Cohere Transcribe by Cohere Labs, converted to MLX by beshkenadze on Hugging Face", + "automatic_language": "en", "language_choices": [ {"code": "en", "name": "English"}, {"code": "ar", "name": "Arabic"}, diff --git a/Packages/LiveTranscribeKit/Sources/Transcription/ScriptLock.swift b/Packages/LiveTranscribeKit/Sources/Transcription/ScriptLock.swift new file mode 100644 index 0000000..3802fe4 --- /dev/null +++ b/Packages/LiveTranscribeKit/Sources/Transcription/ScriptLock.swift @@ -0,0 +1,108 @@ +/// What Qwen3-ASR may write once it's told a language: the language's letters (its +/// ``WritingScript``), Latin letters for words in English (as the Sinhala fine-tune writes them), +/// and the digits, punctuation and spaces every language shares. +/// +/// Telling Qwen3-ASR a language only stands in for its own guess at it, so a language spoken +/// clearly still came out in its own letters (LiveTranscribe-0415). Locked, a model told English +/// can't write Sinhala or Chinese, whatever it hears. It can still write German, which has the +/// same letters. +/// +/// The model writes UTF-8 a token at a time, and a token can hold part of a character, so the +/// lock reads a token's bytes after whatever character the tokens before it left unfinished +/// (``PartialCharacter``). +struct ScriptLock: Hashable, Sendable { + /// What every language may write: ASCII; Latin-1's symbols and punctuation (« » ¿ ° × ÷); the + /// combining diacritics; general punctuation (– — “ ” …, and the zero-width joiners Sinhala's + /// conjuncts are written with); currency; letterlike symbols and number forms (℃ ½). + static let shared: [ClosedRange] = [ + 0x0000...0x007F, 0x00A0...0x00BF, 0x00D7...0x00D7, 0x00F7...0x00F7, 0x0300...0x036F, + 0x2000...0x206F, 0x20A0...0x20CF, 0x2100...0x218F, + ] + + let script: WritingScript + /// The code points it allows, in order and apart. + private let allowed: [ClosedRange] + + init(script: WritingScript) { + self.script = script + let english = script == .latin ? [] : WritingScript.latin.blocks + allowed = Self.merged(Self.shared + english + script.blocks) + } + + func allows(_ scalar: UInt32) -> Bool { + allowsAny(from: scalar, through: scalar) + } + + /// Whether it allows any code point from `low` through `high`. + func allowsAny(from low: UInt32, through high: UInt32) -> Bool { + allowed.contains { $0.lowerBound <= high && low <= $0.upperBound } + } + + /// The character left unfinished once `bytes` follow `partial`, the one the tokens before + /// them left unfinished: ``PartialCharacter/nothing`` if they end on a whole character. + /// `nil` if they write a character the lock doesn't allow, begin one no allowed character + /// begins with, or aren't UTF-8. + func partial(after partial: PartialCharacter, reading bytes: some Sequence) -> PartialCharacter? { + var partial = partial + for byte in bytes { + guard let next = partial.appending(byte) else { return nil } + if next.isWhole { + guard allows(next.value) else { return nil } + partial = .nothing + } else { + guard allowsAny(from: next.lowest, through: next.highest) else { return nil } + partial = next + } + } + return partial + } + + private static func merged(_ ranges: [ClosedRange]) -> [ClosedRange] { + var merged: [ClosedRange] = [] + for range in ranges.sorted(by: { $0.lowerBound < $1.lowerBound }) { + if let last = merged.last, range.lowerBound <= last.upperBound + 1 { + merged[merged.count - 1] = last.lowerBound...max(last.upperBound, range.upperBound) + } else { + merged.append(range) + } + } + return merged + } +} + +/// A character whose first UTF-8 bytes are written and whose last aren't yet. +struct PartialCharacter: Hashable, Sendable { + /// No character is unfinished. + static let nothing = PartialCharacter(length: 0, count: 0, value: 0) + + /// How many bytes the character takes, which its first byte says. + private let length: Int + /// How many of them are written. + private let count: Int + /// The bits of the code point the written bytes hold. + fileprivate let value: UInt32 + + /// Its bytes are all written, or none is unfinished. + var isWhole: Bool { count == length } + + /// The lowest and highest code points the character can still turn out to be. The lowest + /// leaves out the overlong encodings, which write a code point in more bytes than it takes. + fileprivate var lowest: UInt32 { max(value << (6 * UInt32(length - count)), [0, 0, 0x80, 0x800, 0x10000][length]) } + fileprivate var highest: UInt32 { value << (6 * UInt32(length - count)) | ((1 << (6 * UInt32(length - count))) - 1) } + + /// The character with `byte` written next, or the one `byte` begins once this one is whole; + /// `nil` if UTF-8 can't have `byte` next. + fileprivate func appending(_ byte: UInt8) -> PartialCharacter? { + guard isWhole else { + guard byte & 0xC0 == 0x80 else { return nil } + return PartialCharacter(length: length, count: count + 1, value: value << 6 | UInt32(byte & 0x3F)) + } + switch byte { + case 0x00...0x7F: return PartialCharacter(length: 1, count: 1, value: UInt32(byte)) + case 0xC2...0xDF: return PartialCharacter(length: 2, count: 1, value: UInt32(byte & 0x1F)) + case 0xE0...0xEF: return PartialCharacter(length: 3, count: 1, value: UInt32(byte & 0x0F)) + case 0xF0...0xF4: return PartialCharacter(length: 4, count: 1, value: UInt32(byte & 0x07)) + default: return nil + } + } +} diff --git a/Packages/LiveTranscribeKit/Sources/Transcription/SpeechModelCatalog.swift b/Packages/LiveTranscribeKit/Sources/Transcription/SpeechModelCatalog.swift index fce25cf..38e3142 100644 --- a/Packages/LiveTranscribeKit/Sources/Transcription/SpeechModelCatalog.swift +++ b/Packages/LiveTranscribeKit/Sources/Transcription/SpeechModelCatalog.swift @@ -8,9 +8,11 @@ import Shared /// of its Hugging Face repository: every copy of a release downloads the files it was tested /// with, and moving a pin is a change like any other. The Linux and Windows app reads the same /// file, so a model is named, described and credited alike everywhere; each platform has its own -/// download in it, and a model without the Mac's is left out here. A model that is told which -/// language to write, where others find it, lists the languages it can be told -/// (`language_choices`), for the Language setting. +/// download in it, and a model without the Mac's is left out here. A model that can be told which +/// language to write lists the languages it takes (`language_choices`), for the Language +/// setting; one that can't find the language itself names the one Automatic tells it +/// (`automatic_language`). Qwen3-ASR's languages name their letters too (`script`), which its +/// transcription is locked to (``ScriptLock``). public struct SpeechModelCatalog: Sendable { /// One model: what Settings shows about it, and where the Mac downloads it from. public struct Model: Decodable, Identifiable, Equatable, Sendable { @@ -24,26 +26,44 @@ public struct SpeechModelCatalog: Sendable { public let licence: String /// Who made it, credited as its licence asks. public let credit: String - /// The languages it can be told to write, its default first; empty for a model that - /// finds the language itself. + /// The languages it can be told to write, in the order Settings lists them; empty for a + /// model that can't be told one (Parakeet), which finds the language itself. public let languageChoices: [LanguageChoice] + /// The code of the language Automatic tells it, for a model that can't find the language + /// itself (Cohere Transcribe); `nil` for one that finds it. + public let automaticLanguage: String? /// The Mac's download. public let mac: Download - /// The language to tell it to write, for the Language setting (a code): the setting's, - /// if it's one of the model's, and otherwise its first. `nil` for a model that isn't - /// told one. + /// The language Automatic tells it, if it can't find the language itself. + public var automaticChoice: LanguageChoice? { + languageChoices.first { $0.code == automaticLanguage } + } + + /// The language to tell it to write, for the Language setting (a code, `nil` for + /// Automatic): the setting's, if it's one of the model's, and otherwise Automatic's, + /// which is `nil` for a model that finds the language itself. public func language(forSetting setting: String?) -> LanguageChoice? { - languageChoices.first { $0.code == setting } ?? languageChoices.first + languageChoices.first { $0.code == setting } ?? automaticChoice + } + + /// How the model is told `choice`: Qwen3-ASR by the name its prompt takes ("language + /// English"), and the others by the code. + public func told(_ choice: LanguageChoice) -> String { + mac.kind == "qwen3_asr" ? choice.name : choice.code } } /// A language a model can be told to write. public struct LanguageChoice: Decodable, Hashable, Identifiable, Sendable { - /// Its ISO 639-1 code, as the model is told it: `en`, `de`. + /// Its ISO 639 code: `en`, `de`, `fil`. One setting holds it for every model, so a + /// language has the same code in each. public let code: String /// Its name in English, as Settings shows it. public let name: String + /// The letters it's written in, for a model whose letters are locked to the language it's + /// told (Qwen3-ASR, ``ScriptLock``); `nil` for one whose aren't. + public let script: WritingScript? public var id: String { code } } @@ -130,18 +150,20 @@ public struct SpeechModelCatalog: Sendable { let licence: String let credit: String let languageChoices: [LanguageChoice]? + let automaticLanguage: String? let mac: Download? private enum CodingKeys: String, CodingKey { case id, name, summary, languages, licence, credit, mac case languageChoices = "language_choices" + case automaticLanguage = "automatic_language" } var model: Model? { mac.map { Model( id: id, name: name, summary: summary, languages: languages, licence: licence, credit: credit, - languageChoices: languageChoices ?? [], mac: $0 + languageChoices: languageChoices ?? [], automaticLanguage: automaticLanguage, mac: $0 ) } } diff --git a/Packages/LiveTranscribeKit/Sources/Transcription/WritingScript.swift b/Packages/LiveTranscribeKit/Sources/Transcription/WritingScript.swift new file mode 100644 index 0000000..3772969 --- /dev/null +++ b/Packages/LiveTranscribeKit/Sources/Transcription/WritingScript.swift @@ -0,0 +1,52 @@ +/// The letters a language is written in, by its ISO 15924 code, as the catalog's language +/// choices name them (`"script": "Sinh"`): what a ``ScriptLock`` keeps a told Qwen3-ASR to. +public enum WritingScript: String, CaseIterable, Decodable, Hashable, Sendable { + case latin = "Latn" + case cyrillic = "Cyrl" + case greek = "Grek" + case arabic = "Arab" + case devanagari = "Deva" + case sinhala = "Sinh" + case thai = "Thai" + /// Chinese characters, for Chinese and Cantonese. + case han = "Hani" + /// Chinese characters, hiragana and katakana. + case japanese = "Jpan" + /// Hangul. + case korean = "Kore" + + /// Its letters and its own punctuation, as Unicode blocks. + var blocks: [ClosedRange] { + switch self { + case .latin: + // Latin-1's letters, Extended-A and -B, the modifier letters (the ʻokina) and Extended + // Additional (Vietnamese). + [0x00C0...0x024F, 0x02B0...0x02FF, 0x1E00...0x1EFF] + case .cyrillic: + [0x0400...0x052F] + case .greek: + [0x0370...0x03FF, 0x1F00...0x1FFF] + case .arabic: + [0x0600...0x06FF, 0x0750...0x077F, 0x08A0...0x08FF, 0xFB50...0xFDFF, 0xFE70...0xFEFF] + case .devanagari: + [0x0900...0x097F, 0xA8E0...0xA8FF] + case .sinhala: + [0x0D80...0x0DFF] + case .thai: + [0x0E00...0x0E7F] + case .han: + Self.cjkPunctuation + Self.hanCharacters + case .japanese: + Self.cjkPunctuation + Self.hanCharacters + [0x3040...0x30FF, 0x31F0...0x31FF] + case .korean: + Self.cjkPunctuation + [0x1100...0x11FF, 0x3130...0x318F, 0xAC00...0xD7AF] + } + } + + /// CJK symbols and punctuation (。、「」), and the full-width forms (，！？). + private static let cjkPunctuation: [ClosedRange] = [0x3000...0x303F, 0xFF00...0xFFEF] + /// The radicals, the unified ideographs and their extensions, and the compatibility ideographs. + private static let hanCharacters: [ClosedRange] = [ + 0x2E80...0x2FDF, 0x3400...0x4DBF, 0x4E00...0x9FFF, 0xF900...0xFAFF, 0x20000...0x2FA1F, + ] +} diff --git a/Packages/LiveTranscribeKit/Tests/DictationUITests/SpeechLanguageMenuTests.swift b/Packages/LiveTranscribeKit/Tests/DictationUITests/SpeechLanguageMenuTests.swift new file mode 100644 index 0000000..d4b5c6a --- /dev/null +++ b/Packages/LiveTranscribeKit/Tests/DictationUITests/SpeechLanguageMenuTests.swift @@ -0,0 +1,42 @@ +@testable import DictationUI +import Testing +import Transcription + +/// The Language menu, in the menu bar and on a model's row in Settings › Models. +@Suite("Speech language menu") +struct SpeechLanguageMenuTests { + private let catalog = SpeechModelCatalog.bundled + + private func menu(_ id: String) throws -> SpeechLanguageMenu { + SpeechLanguageMenu(model: try #require(catalog.models.first { $0.id == id })) + } + + @Test func automaticComesFirstAndOnlyTheModelsLanguagesFollow() throws { + let qwen = try menu("qwen3-asr-0.6b-sinhala") + #expect(qwen.automaticTitle == "Automatic") + #expect(qwen.choices.first?.name == "English") + #expect(qwen.choices.contains { $0.code == "si" }) + #expect(try !menu("qwen3-asr-0.6b").choices.contains { $0.code == "si" }, "only the fine-tune knows Sinhala") + } + + @Test func automaticSaysWhatAModelThatCantFindTheLanguageWrites() throws { + #expect(try menu("cohere-transcribe").automaticTitle == "Automatic (English)") + } + + @Test func aModelThatCantBeToldALanguageHasAutomaticAlone() throws { + let parakeet = try menu("parakeet-tdt-0.6b-v3") + #expect(!parakeet.canChoose) + #expect(parakeet.tag(forSetting: "de") == SpeechLanguageMenu.automatic) + #expect(!SpeechLanguageMenu(model: nil).canChoose, "a model outside the catalog") + } + + /// The setting is one for every model: a language this one lacks shows as Automatic. + @Test func theSettingShowsAsItsLanguageOrAutomatic() throws { + let qwen = try menu("qwen3-asr-0.6b") + #expect(qwen.tag(forSetting: "en") == "en") + #expect(qwen.tag(forSetting: nil) == SpeechLanguageMenu.automatic) + #expect(qwen.tag(forSetting: "ca") == SpeechLanguageMenu.automatic, "Catalan is Whisper's") + #expect(SpeechLanguageMenu.setting(forTag: SpeechLanguageMenu.automatic) == nil) + #expect(SpeechLanguageMenu.setting(forTag: "en") == "en") + } +} diff --git a/Packages/LiveTranscribeKit/Tests/TranscriptionTests/ScriptLockTests.swift b/Packages/LiveTranscribeKit/Tests/TranscriptionTests/ScriptLockTests.swift new file mode 100644 index 0000000..30f3224 --- /dev/null +++ b/Packages/LiveTranscribeKit/Tests/TranscriptionTests/ScriptLockTests.swift @@ -0,0 +1,97 @@ +import Foundation +import Testing +@testable import Transcription + +/// What a language told to Qwen3-ASR lets it write, read as the UTF-8 bytes its tokens write. +@Suite("Script lock") +struct ScriptLockTests { + private let english = ScriptLock(script: .latin) + private let sinhala = ScriptLock(script: .sinhala) + + /// Reads `text`'s bytes from a whole character: what's left unfinished, or `nil` if refused. + private func read(_ text: String, _ lock: ScriptLock) -> PartialCharacter? { + lock.partial(after: .nothing, reading: Array(text.utf8)) + } + + private func read(_ bytes: [UInt8], _ lock: ScriptLock, after partial: PartialCharacter = .nothing) -> PartialCharacter? { + lock.partial(after: partial, reading: bytes) + } + + @Test func englishWritesLatinLettersDigitsAndPunctuation() { + #expect(read("Send the café’s report — 10% off, “naïve” résumé.\n- Item 2", english) == .nothing) + #expect(read("Việt Nam, Łódź, ʻŌlelo", english) == .nothing, "Latin letters of every language") + #expect(read("ක", english) == nil) + #expect(read("我们", english) == nil) + #expect(read("。", english) == nil, "CJK punctuation is Chinese, Japanese and Korean's") + #expect(read("Привет", english) == nil) + } + + /// The Sinhala fine-tune writes English words in English letters, and every lock but Latin's + /// allows them. + @Test func sinhalaWritesSinhalaAndEnglishWords() { + #expect(read("මම Zoom meeting එකට යනවා.", sinhala) == .nothing) + #expect(read("ප්‍රශ්නය", sinhala) == .nothing, "with the zero-width joiner of its conjuncts") + #expect(read("我们", sinhala) == nil) + #expect(read("我们开会。", ScriptLock(script: .han)) == .nothing) + #expect(read("Hello", ScriptLock(script: .han)) == .nothing) + #expect(read("ひらがなとカタカナと漢字", ScriptLock(script: .japanese)) == .nothing) + #expect(read("한국어", ScriptLock(script: .korean)) == .nothing) + #expect(read("한국어", ScriptLock(script: .japanese)) == nil) + } + + /// "ක" is E0 B6 9A, and a token can end after its first bytes. + @Test func aCharacterSplitAcrossTokensIsReadAcrossThem() throws { + let started = try #require(read([0xE0, 0xB6], sinhala)) + #expect(!started.isWhole) + #expect(read([0x9A], sinhala, after: started) == .nothing) + #expect(read([0x9A, 0x61], sinhala, after: started) == .nothing, "then a whole character") + #expect(read([0x61], sinhala, after: started) == nil, "it must be finished first") + // Told English, the first byte already can't begin anything it allows. + #expect(read([0xE0], english) == nil) + // "—" (U+2014, E2 80 94) is allowed, so its first byte is. + #expect(read([0xE2], english) != nil) + } + + @Test func bytesThatArentUTF8AreRefused() { + #expect(read([0x80], english) == nil, "a continuation byte with nothing to continue") + #expect(read([0xC0, 0x80], english) == nil, "an overlong lead byte") + #expect(read([0xFF], english) == nil) + } + + // MARK: The vocabulary + + /// Qwen's vocabulary spells bytes in GPT-2's byte-level alphabet. + @Test func tokensAreReadAsTheBytesTheirSpellingStandsFor() { + #expect(LockedVocabulary.bytes(spelt: "Hello") == Array("Hello".utf8)) + #expect(LockedVocabulary.bytes(spelt: "Ġworld") == Array(" world".utf8)) + #expect(LockedVocabulary.bytes(spelt: "Ċ") == [0x0A]) + #expect(LockedVocabulary.bytes(spelt: "à¶") == [0xE0, 0xB6]) + #expect(LockedVocabulary.bytes(spelt: "ļ") == [0x9A]) + #expect(LockedVocabulary.bytes(spelt: "ක") == nil, "not in the alphabet") + } + + private func vocabulary() throws -> LockedVocabulary { + // "Hello", " world", "à¶" (E0 B6), "ļ" (9A), "æĪij" (我: E6 88 91) and "Ā" (00). + let json = #"{"Hello": 0, "Ġworld": 1, "à¶": 2, "ļ": 3, "æĪij": 4, "Ā": 5}"# + return try LockedVocabulary(vocabularyJSON: Data(json.utf8), endOfReply: [7]) + } + + @Test func theLockAllowsTheTokensThatWriteWhatItAllowsAndEndsAfterAWholeCharacter() throws { + let vocabulary = try vocabulary() + #expect(vocabulary.count == 6) + // Tokens 6 and 8 are past the vocabulary, as Qwen's added tokens are; 7 ends the reply. + #expect(vocabulary.allowed(by: english, after: .nothing, logitCount: 9) == [true, true, false, false, false, true, false, true, false]) + #expect(vocabulary.allowed(by: ScriptLock(script: .han), after: .nothing, logitCount: 9)[4]) + let started = try #require(sinhala.partial(after: .nothing, reading: vocabulary.bytes(of: 2))) + #expect( + vocabulary.allowed(by: sinhala, after: started, logitCount: 9) == [false, false, false, true, false, false, false, false, false], + "only the end of the character, and no end of the reply" + ) + } + + @Test func aVocabularyWithAGapInItsIdsIsRefused() { + #expect(throws: LockedVocabulary.LoadError.missingToken(1)) { + try LockedVocabulary(vocabularyJSON: Data(#"{"a": 0, "b": 2}"#.utf8), endOfReply: []) + } + } +} diff --git a/Packages/LiveTranscribeKit/Tests/TranscriptionTests/SpeechModelCatalogTests.swift b/Packages/LiveTranscribeKit/Tests/TranscriptionTests/SpeechModelCatalogTests.swift index 6b9f9a9..c07477f 100644 --- a/Packages/LiveTranscribeKit/Tests/TranscriptionTests/SpeechModelCatalogTests.swift +++ b/Packages/LiveTranscribeKit/Tests/TranscriptionTests/SpeechModelCatalogTests.swift @@ -39,39 +39,120 @@ struct SpeechModelCatalogTests { } /// mlx-audio-swift's Cohere tokenizer knows these codes, and writes English for any other. + /// It can't find the language itself, so Automatic tells it English. @Test func cohereIsToldOneOfTheLanguagesItKnows() throws { let cohere = try #require(catalog.models.first { $0.id == "cohere-transcribe" }) - #expect(cohere.languageChoices.first?.code == "en", "English is the default") #expect(cohere.languageChoices.map(\.code).sorted() == ["ar", "de", "el", "en", "es", "fr", "it", "ja", "ko", "nl", "pl", "pt", "vi", "zh"]) - #expect(cohere.languageChoices.allSatisfy { !$0.name.isEmpty }) - for model in catalog.models where model.id != "cohere-transcribe" { - #expect(model.languageChoices.isEmpty, "\(model.id) finds the language itself") + #expect(cohere.automaticChoice?.name == "English") + } + + /// Qwen3-ASR's prompt takes a language by the name its config.json lists + /// (`support_languages`, at the pinned revisions), and only Sinhala's fine-tune knows Sinhala. + @Test func qwenIsToldTheLanguagesItsConfigLists() throws { + let qwen = ["Chinese", "English", "Cantonese", "Arabic", "German", "French", "Spanish", "Portuguese", "Indonesian", + "Italian", "Korean", "Russian", "Thai", "Vietnamese", "Japanese", "Turkish", "Hindi", "Malay", "Dutch", + "Swedish", "Danish", "Finnish", "Polish", "Czech", "Filipino", "Persian", "Greek", "Romanian", + "Hungarian", "Macedonian"] + for model in catalog.models where model.mac.kind == "qwen3_asr" { + let expected = model.id.hasSuffix("sinhala") ? qwen + ["Sinhala"] : qwen + #expect(Set(model.languageChoices.map(\.name)) == Set(expected), "\(model.id)") + #expect(model.automaticLanguage == nil, "\(model.id) finds the language itself") } } - @Test func theLanguageToldIsTheSettingsIfTheModelHasItAndElseItsFirst() throws { + /// Whisper large-v3's tokenizer has a token for each of these, by its code. + @Test func whisperIsToldItsHundredLanguages() throws { + let whisper = try #require(catalog.models.first { $0.mac.kind == "whisper" }) + #expect(whisper.languageChoices.count == 100) + #expect(whisper.languageChoices.contains { $0.code == "yue" && $0.name == "Cantonese" }) + #expect(whisper.automaticLanguage == nil) + } + + /// One setting holds the language for every model, so a language is the same code and name + /// in each, and Settings lists English first, then the rest by name. + @Test func everyModelsLanguagesAreListedAlike() throws { + var names: [String: String] = [:] + for model in catalog.models where !model.languageChoices.isEmpty { + let codes = model.languageChoices.map(\.code) + #expect(Set(codes).count == codes.count, "\(model.id) lists a language twice") + #expect(codes.allSatisfy { (2...3).contains($0.count) && $0.allSatisfy { $0.isLowercase && $0.isASCII } }, "\(model.id)") + #expect(codes.first == "en", "\(model.id)") + let rest = model.languageChoices.dropFirst().map(\.name) + #expect(rest == rest.sorted(), "\(model.id)") + for choice in model.languageChoices { + #expect(names[choice.code, default: choice.name] == choice.name, "\(choice.code) in \(model.id)") + names[choice.code] = choice.name + } + if let automatic = model.automaticLanguage { + #expect(codes.contains(automatic), "\(model.id)") + } + } + for model in catalog.models where model.mac.kind == "parakeet" { + #expect(model.languageChoices.isEmpty, "\(model.id) can't be told a language") + } + } + + @Test func theLanguageToldIsTheSettingsIfTheModelHasItAndElseAutomatics() throws { let cohere = try #require(catalog.models.first { $0.id == "cohere-transcribe" }) #expect(cohere.language(forSetting: "de")?.name == "German") - #expect(cohere.language(forSetting: nil)?.code == "en") + #expect(cohere.language(forSetting: nil)?.code == "en", "it can't find the language") #expect(cohere.language(forSetting: "si")?.code == "en", "Sinhala isn't one of its languages") + let qwen = try #require(catalog.models.first { $0.id == "qwen3-asr-0.6b-sinhala" }) + #expect(qwen.language(forSetting: "en")?.name == "English") + #expect(qwen.language(forSetting: nil) == nil, "it finds the language") + #expect(qwen.language(forSetting: "ca") == nil, "Catalan isn't one of its languages") let parakeet = try #require(catalog.models.first { $0.id == "parakeet-tdt-0.6b-v3" }) #expect(parakeet.language(forSetting: "de") == nil) } - /// The transcriber tells Cohere the language at each transcription, and leaves the others' - /// parameters as they are. - @Test func onlyAModelThatIsToldALanguageGetsTheSettings() throws { + /// The transcriber tells a model the setting's language at each transcription: Qwen3-ASR by + /// name, the others by code. Automatic leaves a model that finds the language as it is. + @Test func aModelIsToldTheSettingsLanguageAsItTakesIt() throws { let cohere = try #require(catalog.models.first { $0.id == "cohere-transcribe" }) + let qwen = try #require(catalog.models.first { $0.id == "qwen3-asr-0.6b-sinhala" }) + let whisper = try #require(catalog.models.first { $0.mac.kind == "whisper" }) let parakeet = try #require(catalog.models.first { $0.id == "parakeet-tdt-0.6b-v3" }) let english = STTGenerateParameters(maxTokens: 512, temperature: 0.2, language: "en") let told = MLXTranscriber.parameters(english, for: cohere, language: "de") #expect(told.language == "de") #expect(told.maxTokens == 512 && told.temperature == 0.2, "the rest are kept") #expect(MLXTranscriber.parameters(english, for: cohere, language: "si").language == "en") + #expect(MLXTranscriber.parameters(STTGenerateParameters(), for: qwen, language: "en").language == "English") + #expect(MLXTranscriber.parameters(STTGenerateParameters(), for: qwen, language: "si").language == "Sinhala") + #expect(MLXTranscriber.parameters(STTGenerateParameters(), for: qwen, language: nil).language == nil) + #expect(MLXTranscriber.parameters(STTGenerateParameters(), for: whisper, language: "en").language == "en") + #expect(MLXTranscriber.parameters(STTGenerateParameters(), for: whisper, language: "fil").language == nil, "Whisper calls it Tagalog") #expect(MLXTranscriber.parameters(STTGenerateParameters(), for: parakeet, language: "de").language == nil) #expect(MLXTranscriber.parameters(english, for: nil, language: "de").language == "en", "a model outside the catalog") } + /// Qwen3-ASR takes a language as a hint, so its letters are locked to the one it's told, and + /// each of its languages names them. No other model's are locked. + @Test func everyQwenLanguageNamesItsLettersAndNoOtherModelsDo() { + for model in catalog.models { + for choice in model.languageChoices { + #expect((choice.script != nil) == (model.mac.kind == "qwen3_asr"), "\(model.id): \(choice.code)") + } + } + let qwen = catalog.models.first { $0.id == "qwen3-asr-0.6b-sinhala" } + #expect(qwen?.language(forSetting: "si")?.script == .sinhala) + #expect(qwen?.language(forSetting: "en")?.script == .latin) + #expect(qwen?.language(forSetting: "ja")?.script == .japanese) + } + + @Test func qwenIsLockedToTheLettersOfTheLanguageItsToldAndAutomaticLocksNothing() throws { + let qwen = try #require(catalog.models.first { $0.id == "qwen3-asr-0.6b-sinhala" }) + let whisper = try #require(catalog.models.first { $0.mac.kind == "whisper" }) + let cohere = try #require(catalog.models.first { $0.id == "cohere-transcribe" }) + #expect(MLXTranscriber.lock(for: qwen, language: "si")?.script == .sinhala) + #expect(MLXTranscriber.lock(for: qwen, language: "en")?.script == .latin) + #expect(MLXTranscriber.lock(for: qwen, language: nil) == nil) + #expect(MLXTranscriber.lock(for: qwen, language: "ca") == nil, "Catalan isn't one of its languages") + #expect(MLXTranscriber.lock(for: whisper, language: "en") == nil) + #expect(MLXTranscriber.lock(for: cohere, language: nil) == nil, "Automatic's English is told, not locked") + #expect(MLXTranscriber.lock(for: nil, language: "en") == nil, "a model outside the catalog") + } + /// Parakeet's config.json names no kind, so the app reads it from the repository's name. @Test func aKindItsConfigDoesntNameIsInTheRepositorysName() throws { for model in catalog.models where model.mac.kind == "parakeet" { @@ -94,9 +175,12 @@ struct SpeechModelCatalogTests { @Test func aModelsLanguageChoicesAreReadAndOptional() throws { var told = Self.entry(id: "told", mac: true) told["language_choices"] = [["code": "en", "name": "English"], ["code": "de", "name": "German"]] + told["automatic_language"] = "de" let catalog = try SpeechModelCatalog(json: Self.json(models: [told, Self.entry(id: "finds", mac: true)])) #expect(catalog.models[0].languageChoices.map(\.code) == ["en", "de"]) + #expect(catalog.models[0].automaticChoice?.name == "German") #expect(catalog.models[1].languageChoices.isEmpty) + #expect(catalog.models[1].automaticLanguage == nil) } @Test func anotherFormatIsRefused() { @@ -182,7 +266,7 @@ struct SpeechModelCatalogTests { private static func model(repository: Repo.ID = "owner/a-model") -> SpeechModelCatalog.Model { SpeechModelCatalog.Model( id: "a-model", name: "A Model", summary: "For tests.", languages: "English", licence: "MIT", credit: "Nobody", - languageChoices: [], + languageChoices: [], automaticLanguage: nil, mac: .init(repository: repository, revision: String(repeating: "a", count: 40), bytes: 1, kind: "qwen3_asr") ) } diff --git a/Packages/LiveTranscribeKit/Tests/TranscriptionTests/SpeechModelLibraryTests.swift b/Packages/LiveTranscribeKit/Tests/TranscriptionTests/SpeechModelLibraryTests.swift index 1e5bfa1..c39c796 100644 --- a/Packages/LiveTranscribeKit/Tests/TranscriptionTests/SpeechModelLibraryTests.swift +++ b/Packages/LiveTranscribeKit/Tests/TranscriptionTests/SpeechModelLibraryTests.swift @@ -100,7 +100,7 @@ struct SpeechModelLibraryTests { private func model(_ repository: Repo.ID) -> SpeechModelCatalog.Model { SpeechModelCatalog.Model( id: repository.name, name: repository.name, summary: "", languages: "", licence: "MIT", credit: "", - languageChoices: [], + languageChoices: [], automaticLanguage: nil, mac: .init(repository: repository, revision: String(repeating: "c", count: 40), bytes: 1, kind: "whisper") ) } diff --git a/README.md b/README.md index c5d6fce..c8b7b6d 100644 --- a/README.md +++ b/README.md @@ -299,9 +299,11 @@ restart, each credited there with its licence. The Mac offers seven: the default 1.7B, Parakeet TDT v2 (English) and v3 (European languages), Whisper large-v3-turbo and Cohere Transcribe. Linux and Windows offer four: the default, Parakeet v2 and v3 and Cohere Transcribe; the last three are builds that [sherpa-onnx](https://github.com/k2-fsa/sherpa-onnx) (k2-fsa, Apache-2.0) -quantised to 8 bits for ONNX. Cohere Transcribe can't tell which language it hears, so its row also -chooses which of its 14 it writes (English at first); the others find the language themselves. They -are listed in +quantised to 8 bits for ONNX. Qwen3-ASR, Whisper and Cohere Transcribe can be told which language to +write, from the menu bar or tray, or from the model's row: **Automatic**, the default, lets the model +work it out, or, for Cohere Transcribe, which can't, writes English. Parakeet always works it out. +Qwen3-ASR takes the language as a hint, so it's also kept to that language's letters: with English +chosen, it never writes Sinhala or Chinese. They are listed in [`speech-models.json`](Packages/LiveTranscribeKit/Sources/Transcription/Resources/speech-models.json), each pinned to a tested commit or, for sherpa-onnx archives, a SHA-256, and both apps read that list. **Another model** there takes, on the Mac, any Hugging Face repository that mlx-audio-swift diff --git a/docs/using.md b/docs/using.md index 4a7b9c8..69180cb 100644 --- a/docs/using.md +++ b/docs/using.md @@ -153,6 +153,25 @@ switches back when it reconnects. between dictations, so dictation starts faster and keeps the 300 ms before you pressed the key. macOS's microphone indicator then stays on while the app runs. +### Speech language + +Choose the language the speech model writes from the menu bar's **Language** menu, or from the +**Language** menu on the model's row in **Settings › Models**. Both list only the languages the +chosen model can be told, after **Automatic**, the default. Automatic lets the model work out the +language from each recording; a short or noisy one can come out in another language. The choice is +one for every model: one that doesn't have the language uses Automatic, and the choice comes back +with a model that has it. Qwen3-ASR (30 languages, and Sinhala in the default model) and Whisper +(100) can be told a language. Cohere Transcribe can't work out the language, so its Automatic is +English. Parakeet can't be told one, and its **Language** menu is greyed out. + +Qwen3-ASR takes the language it's told as no more than a hint, so Live Transcribe also keeps it +to that language's letters. With English chosen it never writes Sinhala or Chinese, and something +said in another language comes out in English, translated or misheard. A language written in the +same letters, such as German with English chosen, can still come through as it was said. Every +language not written in Latin letters keeps them as well, for English words, as the Sinhala model +writes them. Whisper and Cohere Transcribe are told the language without being kept to its +letters. + ### Updates A release downloaded from GitHub updates itself with [Sparkle](https://sparkle-project.org). On @@ -181,7 +200,7 @@ of the open-source packages the app is built with. | **Apps** | per app, how text goes in (Accessibility or paste) and whether it takes line breaks | | **History** | dictation history on or off, retention, clearing | | **Permissions** | microphone and Accessibility status, with buttons that open the right System Settings pane, and **Reopen Live Transcribe** when macOS won't let it paste until it reopens | -| **Models** | the speech-to-text model: models to download, switch to and remove, each with its languages, size and licence, and **Another model**, for any Hugging Face repository or a model folder on your Mac | +| **Models** | the speech-to-text model: models to download, switch to and remove, each with its languages, size and licence, a **Language** menu on each model that can be told one ([Speech language](#speech-language)), and **Another model**, for any Hugging Face repository or a model folder on your Mac | | **Advanced** | for dictation and the live transcript: the cleanup model, **Clean up transcripts with the LLM**, **Resolve spoken self-corrections** (both adapters: Standard's and Deep's), the cleanup **Timeout**, the GPU cache and capture restarts; for the live transcript only: the voice-activity model, silence, speech threshold, pre-roll and minimum speech, maximum segment length, live partials, context segments and queue capacity | Settings on every tab but **Advanced** apply immediately, to the next dictation. A speech model @@ -281,8 +300,9 @@ hotkey isn't seen while an app running as administrator has the keyboard. Left-click the tray icon. The menu shows the status ("Hold Right Ctrl to dictate", "Listening…", "Transcribing…", "Downloading speech models… 42%", or what stops it), then **Start Dictation** (it reads **Stop Dictation** while recording), **Cancel Dictation**, **Copy Last Dictation**, -**Cleanup** (None, Standard and Deep; the choice is saved), **Settings…** and **Quit -Live Transcribe**. The icon shows the state: a waveform when ready, a microphone when recording, +**Cleanup** (None, Standard and Deep; the choice is saved), **Language** (Automatic, then the +languages the speech model can be told, as on the Mac: [Speech language](#speech-language)), +**Settings…** and **Quit Live Transcribe**. The icon shows the state: a waveform when ready, a microphone when recording, a warning triangle when something needs attention. ### Settings @@ -293,7 +313,7 @@ dictation. | Tab | What it holds | |---|---| | **General** | **Dictate with the hotkey** (off, and the tray's **Start Dictation** still works), the hotkey, **Double-tap for hands-free**, the cleanup level, the microphone to **Record from** (System default, or one listed), and **Timing**: longest tap 300 ms, double-tap window 300 ms, shortest recording 300 ms, longest recording 300 s, message duration 2.5 s and the clipboard restore delay 250 ms (which does nothing on Windows) | -| **Models** | the speech models, each with its languages, size and licence: **Download**, **Use** and **Remove** them, and **Another model** for a converted model in the models folder. Cohere Transcribe has a **Language** menu (14 languages) | +| **Models** | the speech models, each with its languages, size and licence: **Download**, **Use** and **Remove** them, and **Another model** for a converted model in the models folder. Qwen3-ASR (31 languages) and Cohere Transcribe (14) each have a **Language** menu, the same choice as the tray's | | **Advanced** | **Runs on**: Auto (the NPU if there is one, else the CPU), NPU only or CPU only, for Qwen3-ASR only; GPU only is offered but the packages can't run it; **Clean up transcripts with the LLM**; the cleanup **Timeout** (3 s; at least 8 s at Deep); and the state of both models with **Try Again**. **Restore Defaults** resets these three | The speech models are Qwen3-ASR 0.6B fine-tuned for Sinhala (the default), Parakeet TDT 0.6B v2 diff --git a/linux-windows/README.md b/linux-windows/README.md index 28708ad..4a1266b 100644 --- a/linux-windows/README.md +++ b/linux-windows/README.md @@ -202,8 +202,8 @@ The models it downloads itself, at its first start. and `--device CPU` keeps speech-to-text on the CPU. `livetranscribe keys` names the keys you press, `livetranscribe models` lists the catalog, and `livetranscribe transcribe --model parakeet-tdt-0.6b-v2 clip.wav` transcribes a WAV file with - one of its models, downloading it first (`--language de` tells Cohere Transcribe which - language to write). + one of its models, downloading it first (`--language de` tells Qwen3-ASR or Cohere + Transcribe which language to write; `run --language automatic` sets a saved one aside). On Windows, from source, in Git Bash: `packaging/windows/fetch-openvino.sh`, `cargo build --release -p lt-app`, then put OpenVINO where the installed app has it, beside the diff --git a/linux-windows/crates/app/src/dictation.rs b/linux-windows/crates/app/src/dictation.rs index a25b8c4..46f64a5 100644 --- a/linux-windows/crates/app/src/dictation.rs +++ b/linux-windows/crates/app/src/dictation.rs @@ -51,8 +51,8 @@ pub struct Options { /// can't run), or only on one OpenVINO device: CPU, GPU or NPU [default: Settings'] #[arg(long, value_name = "DEVICE")] device: Option, - /// The language Cohere Transcribe writes, by its code or name, such as de or German; the - /// other models find the language themselves [default: Settings', at first English] + /// The language the speech model writes, by its code or name, such as en or English, or + /// automatic, for Qwen3-ASR and Cohere Transcribe [default: Settings', at first automatic] #[arg(long, value_name = "LANGUAGE")] language: Option, /// The key to hold, as linux/input-event-codes.h names it (`livetranscribe keys` shows the @@ -79,7 +79,12 @@ impl Options { overrides.insert("sttDevice".to_owned(), Value::from(device.as_str())); } if let Some(language) = &self.language { - overrides.insert("sttLanguage".to_owned(), Value::from(language.as_str())); + let language = if language.trim().eq_ignore_ascii_case("automatic") { + Value::Null + } else { + Value::from(language.as_str()) + }; + overrides.insert("sttLanguage".to_owned(), language); } if let Some(key) = &self.key { overrides.insert("dictationHotkey".to_owned(), Value::from(key.as_str())); @@ -221,6 +226,7 @@ fn blocked_status(settings: &Settings, blocker: &Blocker) -> DictationStatus { hotkey: settings.dictation_enabled.then_some(hotkey), has_last_dictation: false, cleanup: settings.cleanup_level, + language: settings.stt_language.clone(), cleanup_model: None, blocker: Some(blocker.clone()), } @@ -297,6 +303,11 @@ mod tests { assert_eq!(given["sttModel"], FOLDER); // The settings keep it by its code (Settings::changed). assert_eq!(given["sttLanguage"], "German"); + assert_eq!( + overrides(&["--language", "Automatic"])["sttLanguage"], + Value::Null, + "a saved language is set aside for the run" + ); } #[test] diff --git a/linux-windows/crates/app/src/dictation/engine.rs b/linux-windows/crates/app/src/dictation/engine.rs index e30db0e..3b74bb9 100644 --- a/linux-windows/crates/app/src/dictation/engine.rs +++ b/linux-windows/crates/app/src/dictation/engine.rs @@ -98,6 +98,8 @@ pub(crate) struct DictationStatus { pub(crate) hotkey: Option, pub(crate) has_last_dictation: bool, pub(crate) cleanup: CleanupLevel, + /// The Language setting, a code; `None` for Automatic. + pub(crate) language: Option, /// Cleanup's language model: `None` while it's turned off in Settings › Advanced. pub(crate) cleanup_model: Option, /// Why dictation can't start at all; the engine never runs then. @@ -400,7 +402,7 @@ impl Running { .and_then(|code| catalog.language_named(code)) { Some(choice) => eprintln!("Language: {}", choice.name), - None => eprintln!("Language: the speech model's default"), + None => eprintln!("Language: Automatic"), } } if cancel && self.controller.is_recording() { @@ -562,6 +564,7 @@ impl Running { hotkey: self.settings.dictation_enabled.then(|| self.hotkey_name.clone()), has_last_dictation: self.controller.last_text().is_some(), cleanup: self.controller.cleanup_level(), + language: self.settings.stt_language.clone(), cleanup_model: self.cleanup.status(), blocker: None, }; diff --git a/linux-windows/crates/app/src/dictation/tray.rs b/linux-windows/crates/app/src/dictation/tray.rs index c5261e1..4db401e 100644 --- a/linux-windows/crates/app/src/dictation/tray.rs +++ b/linux-windows/crates/app/src/dictation/tray.rs @@ -1,17 +1,18 @@ //! The tray icon and its menu, as the Mac app's menu bar extra (MenuBarContent) has them: the //! status line, Start or Stop Dictation, Cancel Dictation, Copy Last Dictation, the Cleanup level, -//! Settings and Quit. Tauri owns the main thread, the menu and the Settings window; the engine's +//! the speech model's Language, Settings and Quit. Tauri owns the main thread, the menu and the Settings window; the engine's //! status arrives from its thread, and the menu's choices go back to it as messages, or as changed //! settings. The icon is drawn for the taskbar's light or dark mode (the desktop's, on Linux) each //! time it changes. use std::cell::Cell; -use std::sync::Arc; use std::sync::mpsc::Sender; +use std::sync::{Arc, Mutex, PoisonError}; use anyhow::Context; use lt_dictation_ui::{Blocker, MenuBarIcon, MenuBarStatus, ModelState, Theme, draw_icon}; use lt_shared::CleanupLevel; +use lt_transcription::catalog::SpeechModel; use serde_json::{Map, Value}; use tauri::image::Image; use tauri::ipc::Invoke; @@ -45,7 +46,7 @@ pub(crate) fn run( .invoke_handler(commands()) .setup(move |app| { settings::follow_changes(app.handle(), &settings, &library); - let tray = Tray::build(app.handle(), messages, settings)?; + let tray = Tray::build(app.handle(), messages, settings, library)?; let handle = app.handle().clone(); start( app.handle(), @@ -177,12 +178,18 @@ struct Tray { cancel: MenuItem, copy: MenuItem, levels: Vec<(CleanupLevel, CheckMenuItem)>, + languages: Arc>, /// The icon showing, and the mode it was drawn for. drawn: Cell>, } impl Tray { - fn build(app: &AppHandle, messages: Sender, settings: Arc) -> tauri::Result { + fn build( + app: &AppHandle, + messages: Sender, + settings: Arc, + library: Arc, + ) -> tauri::Result { let cleanup = settings.current().0.cleanup_level; let status = MenuItem::with_id(app, "status", "Loading speech models…", false, None::<&str>)?; let toggle = MenuItem::with_id(app, "toggle", "Start Dictation", false, None::<&str>)?; @@ -205,6 +212,8 @@ impl Tray { let level_items: Vec<&dyn IsMenuItem> = levels.iter().map(|(_, item)| item as &dyn IsMenuItem).collect(); let cleanup_menu = Submenu::with_id_and_items(app, "cleanup", "Cleanup", true, &level_items)?; + let languages = Arc::new(Mutex::new(LanguageMenu::new(app, library)?)); + let language_menu = lock(&languages).submenu.clone(); let open_settings = MenuItem::with_id(app, "settings", "Settings…", true, None::<&str>)?; let quit = MenuItem::with_id(app, "quit", "Quit Live Transcribe", true, None::<&str>)?; let menu = Menu::with_items( @@ -217,12 +226,14 @@ impl Tray { ©, &PredefinedMenuItem::separator(app)?, &cleanup_menu, + &language_menu, &open_settings, &PredefinedMenuItem::separator(app)?, &quit, ], )?; let choices = levels.clone(); + let chosen_languages = Arc::clone(&languages); let theme = Theme::detect_taskbar(); let icon = TrayIconBuilder::with_id("live-transcribe") .icon(icon_image(MenuBarIcon::Loading, theme)) @@ -247,6 +258,8 @@ impl Tray { id => { if let Some((level, _)) = choices.iter().find(|(level, _)| id == cleanup_id(*level)) { choose_cleanup(&settings, &choices, *level); + } else if let Some(language) = LanguageMenu::setting_of(id) { + choose_language(&settings, &chosen_languages, language); } None } @@ -263,6 +276,7 @@ impl Tray { cancel, copy, levels, + languages, drawn: Cell::new(Some((MenuBarIcon::Loading, theme))), }) } @@ -295,6 +309,7 @@ impl Tray { .iter() .map(|(level, item)| item.set_checked(*level == status.cleanup)), ); + results.push(lock(&self.languages).show(status.model_id.as_deref(), status.language.as_deref())); for error in results.into_iter().filter_map(Result::err) { tracing::warn!("Couldn't update the tray: {error}"); } @@ -324,8 +339,148 @@ fn cleanup_id(level: CleanupLevel) -> String { format!("cleanup-{}", level.as_str()) } +/// The Language submenu, as the Mac's menu bar has it: Automatic, then the languages the speech +/// model in use can be told. It's drawn again when the model changes, and greyed out with +/// Automatic alone for a model that can't be told one (Parakeet) or isn't in the catalog. +struct LanguageMenu { + app: AppHandle, + library: Arc, + submenu: Submenu, + separator: PredefinedMenuItem, + /// The catalog model it was drawn for (`None` for no model, or one outside the catalog); + /// `None` until it's drawn. + drawn_for: Option>, + /// Automatic (`None`), then each language by its code, with its item, as the menu lists them. + items: Vec<(Option, CheckMenuItem)>, +} + +impl LanguageMenu { + const AUTOMATIC: &'static str = "language-automatic"; + + fn new(app: &AppHandle, library: Arc) -> tauri::Result { + Ok(Self { + app: app.clone(), + library, + submenu: Submenu::with_id(app, "language", "Language", false)?, + separator: PredefinedMenuItem::separator(app)?, + drawn_for: None, + items: Vec::new(), + }) + } + + /// The Language setting an item's id stands for: `Some(None)` for Automatic, and `None` for + /// an id that isn't a language's. + fn setting_of(id: &str) -> Option> { + if id == Self::AUTOMATIC { + return Some(None); + } + id.strip_prefix("language-").map(|code| Some(code.to_owned())) + } + + fn id(code: &str) -> String { + format!("language-{code}") + } + + /// Lists the languages of the model `model_id` names, drawing the menu again if it was + /// drawn for another, and ticks the setting's, or Automatic. + fn show(&mut self, model_id: Option<&str>, setting: Option<&str>) -> tauri::Result<()> { + if self.drawn_for.as_ref().map(Option::as_deref) != Some(model_id) { + self.draw(model_id)?; + } + self.tick(setting) + } + + /// Ticks the setting's language if the menu has it, and otherwise Automatic. + fn tick(&self, setting: Option<&str>) -> tauri::Result<()> { + let listed = setting.filter(|code| self.items.iter().any(|(item, _)| item.as_deref() == Some(*code))); + for (code, item) in &self.items { + item.set_checked(code.as_deref() == listed)?; + } + Ok(()) + } + + fn draw(&mut self, model_id: Option<&str>) -> tauri::Result<()> { + let had_languages = self.items.len() > 1; + for (_, item) in self.items.drain(..) { + self.submenu.remove(&item)?; + } + if had_languages { + self.submenu.remove(&self.separator)?; + } + let model = model_id.and_then(|id| self.library.catalog().model(id)); + let choices = model.map(|model| model.language_choices.as_slice()).unwrap_or_default(); + let automatic = match model.and_then(SpeechModel::automatic_choice) { + Some(choice) => format!("Automatic ({})", choice.name), + None => "Automatic".to_owned(), + }; + let mut items = vec![( + None, + CheckMenuItem::with_id(&self.app, Self::AUTOMATIC, automatic, true, false, None::<&str>)?, + )]; + for choice in choices { + let item = CheckMenuItem::with_id( + &self.app, + Self::id(&choice.code), + &choice.name, + true, + false, + None::<&str>, + )?; + items.push((Some(choice.code.clone()), item)); + } + for (index, (_, item)) in items.iter().enumerate() { + self.submenu.append(item)?; + if index == 0 && !choices.is_empty() { + self.submenu.append(&self.separator)?; + } + } + self.submenu.set_enabled(!choices.is_empty())?; + self.items = items; + self.drawn_for = Some(model_id.map(str::to_owned)); + Ok(()) + } +} + +/// Saves the language chosen in the menu (`None` for Automatic), which tells dictation and the +/// Settings window. +fn choose_language(settings: &SettingsService, menu: &Mutex, chosen: Option) { + let mut change = Map::new(); + change.insert("sttLanguage".to_owned(), chosen.map_or(Value::Null, Value::from)); + let language = match settings.change(&change) { + Ok(settings) => settings.stt_language, + Err(error) => { + tracing::error!("Couldn't change the language: {error}"); + settings.current().0.stt_language + } + }; + // Each item ticks itself when chosen, and unticks when chosen again; the menu reads as one + // choice once the rest follow. + if let Err(error) = lock(menu).tick(language.as_deref()) { + tracing::warn!("Couldn't update the Language menu: {error}"); + } +} + +fn lock(menu: &Mutex) -> std::sync::MutexGuard<'_, LanguageMenu> { + menu.lock().unwrap_or_else(PoisonError::into_inner) +} + /// `icon` for the taskbar's light or dark mode. fn icon_image(icon: MenuBarIcon, theme: Theme) -> Image<'static> { let image = draw_icon(icon, theme, ICON_SIZE); Image::new_owned(image.rgba, image.width, image.height) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_language_items_id_is_the_setting_it_stands_for() { + assert_eq!(LanguageMenu::setting_of(LanguageMenu::AUTOMATIC), Some(None)); + assert_eq!( + LanguageMenu::setting_of(&LanguageMenu::id("en")), + Some(Some("en".to_owned())) + ); + assert_eq!(LanguageMenu::setting_of("cleanup-standard"), None); + } +} diff --git a/linux-windows/crates/app/src/main.rs b/linux-windows/crates/app/src/main.rs index da1cf4b..3faa764 100644 --- a/linux-windows/crates/app/src/main.rs +++ b/linux-windows/crates/app/src/main.rs @@ -46,8 +46,9 @@ struct ModelOptions { /// can't run), or only on one OpenVINO device: CPU, GPU or NPU. The other models run on the CPU #[arg(long, default_value = "auto", value_name = "DEVICE")] device: DeviceChoice, - /// The language Cohere Transcribe writes, by its code or name, such as de or German; the - /// other models find the language themselves [default: English] + /// The language the speech model writes, by its code or name, such as en or English, for + /// Qwen3-ASR and Cohere Transcribe [default: Automatic: the model finds it, and Cohere + /// Transcribe writes English] #[arg(long, value_name = "LANGUAGE")] language: Option, } diff --git a/linux-windows/crates/app/src/settings/model.rs b/linux-windows/crates/app/src/settings/model.rs index 8107e31..496f658 100644 --- a/linux-windows/crates/app/src/settings/model.rs +++ b/linux-windows/crates/app/src/settings/model.rs @@ -37,8 +37,9 @@ pub struct Settings { /// The speech model: a catalog model's id, or a folder the setup kit converted a model into, /// in the app's models folder or a path; `None` for the default model. pub stt_model: Option, - /// The language a speech model that is told one (Cohere Transcribe) writes, by its code; - /// `None` for the model's first, English. The other models find the language themselves. + /// The language a speech model that can be told one (Qwen3-ASR, Cohere Transcribe) writes, + /// by its code; `None` for Automatic: the model finds the language, or, for one that can't + /// (Cohere Transcribe), writes English. Parakeet always finds it. pub stt_language: Option, /// Where a Qwen3-ASR model runs: one of [`DEVICES`]. The catalog's other models run on the CPU. pub stt_device: String, @@ -290,6 +291,7 @@ mod tests { ("German", Some("de")), ("DE", Some("de")), (" ja ", Some("ja")), + ("Sinhala", Some("si")), (" ", None), ] { let (settings, problems) = Settings::read(&json!({"sttLanguage": given}).to_string()).unwrap(); @@ -297,10 +299,11 @@ mod tests { assert!(problems.is_empty(), "{problems:?}"); } - let (settings, problems) = Settings::read(r#"{"sttLanguage": "si"}"#).unwrap(); + // Catalan is only Whisper's, which runs on the Mac. + let (settings, problems) = Settings::read(r#"{"sttLanguage": "ca"}"#).unwrap(); assert_eq!(settings.stt_language, None); assert_eq!(problems.len(), 1); - assert!(problems[0].contains("si isn't a language"), "{problems:?}"); + assert!(problems[0].contains("ca isn't a language"), "{problems:?}"); assert!(problems[0].contains("de, el, en"), "{problems:?}"); let refused = Settings::default().changed(&changes(json!({"sttLanguage": "Klingon"}))); diff --git a/linux-windows/crates/app/src/settings/window.rs b/linux-windows/crates/app/src/settings/window.rs index 3790df5..4fa2d38 100644 --- a/linux-windows/crates/app/src/settings/window.rs +++ b/linux-windows/crates/app/src/settings/window.rs @@ -291,9 +291,12 @@ struct CatalogModelView { source: &'static str, /// Whether it runs on the CPU whatever the device setting says. cpu_only: bool, - /// The languages it can be told to write, its default first; empty for a model that finds - /// the language itself. + /// The languages it can be told to write, as its Language menu lists them after Automatic; + /// empty for a model that can't be told one. language_choices: Vec, + /// The name of the language Automatic tells it, for a model that can't find the language + /// itself (Cohere Transcribe). + automatic_language: Option, download: DownloadView, /// Some of its files are here and it isn't downloading. can_remove: bool, @@ -312,6 +315,7 @@ impl CatalogModelView { source: model.source(), cpu_only: !matches!(model.engine, Engine::OpenVino { .. }), language_choices: model.language_choices.clone(), + automatic_language: model.automatic_choice().map(|choice| choice.name.clone()), download: DownloadView::new(state), can_remove: library.can_remove(model), } diff --git a/linux-windows/crates/app/src/speech_models/downloads.rs b/linux-windows/crates/app/src/speech_models/downloads.rs index 9217a09..7d9e977 100644 --- a/linux-windows/crates/app/src/speech_models/downloads.rs +++ b/linux-windows/crates/app/src/speech_models/downloads.rs @@ -340,7 +340,9 @@ pub(crate) mod tests { language_choices: vec![LanguageChoice { code: "en".to_owned(), name: "English".to_owned(), + script: None, }], + automatic_language: Some("en".to_owned()), engine: Engine::SherpaOnnx { family: Family::CohereTranscribe, archive: Archive { diff --git a/linux-windows/crates/app/src/transcribe.rs b/linux-windows/crates/app/src/transcribe.rs index 5458423..00d7ae1 100644 --- a/linux-windows/crates/app/src/transcribe.rs +++ b/linux-windows/crates/app/src/transcribe.rs @@ -116,8 +116,8 @@ fn chosen(option: Option<&str>, library: &SpeechModelLibrary) -> anyhow::Result< } /// The code of the language `--language` names, for the model chosen: `None` without the option, -/// and for a model that finds the language itself, which is said on standard error. A language -/// the model can't be told is refused. +/// Automatic, and for a model that can't be told a language, which is said on standard error. A +/// language the model can't be told is refused. fn told_language<'a>(chosen: &'a ChosenModel, option: Option<&str>) -> anyhow::Result> { let Some(text) = option else { return Ok(None) }; let choices = match chosen { @@ -126,7 +126,7 @@ fn told_language<'a>(chosen: &'a ChosenModel, option: Option<&str>) -> anyhow::R }; if choices.is_empty() { eprintln!( - "⚠ {} finds the language itself: --language is for Cohere Transcribe", + "⚠ {} can't be told a language, and finds it itself: --language is for Qwen3-ASR and Cohere Transcribe", chosen.name() ); return Ok(None); @@ -221,15 +221,19 @@ mod tests { let cohere = ChosenModel::Catalog(catalog.model("cohere-transcribe").unwrap()); assert_eq!(told_language(&cohere, Some("German")).unwrap(), Some("de")); assert_eq!(told_language(&cohere, Some("ja")).unwrap(), Some("ja")); - assert_eq!(told_language(&cohere, None).unwrap(), None, "the model's first"); + assert_eq!(told_language(&cohere, None).unwrap(), None, "Automatic"); let error = told_language(&cohere, Some("Sinhala")).unwrap_err().to_string(); assert!(error.contains("de (German)"), "{error}"); + let qwen = ChosenModel::Catalog(catalog.model("qwen3-asr-0.6b-sinhala").unwrap()); + assert_eq!(told_language(&qwen, Some("English")).unwrap(), Some("en")); + assert_eq!(told_language(&qwen, Some("si")).unwrap(), Some("si")); + let parakeet = ChosenModel::Catalog(catalog.model("parakeet-tdt-0.6b-v3").unwrap()); assert_eq!( told_language(¶keet, Some("de")).unwrap(), None, - "it finds the language" + "it can't be told one" ); } diff --git a/linux-windows/crates/app/ui/settings.js b/linux-windows/crates/app/ui/settings.js index 1b70aed..579df08 100644 --- a/linux-windows/crates/app/ui/settings.js +++ b/linux-windows/crates/app/ui/settings.js @@ -435,8 +435,10 @@ function renderCatalog({ force = false } = {}) { } /** - * The language a model told one writes: the Language setting's, if the model has it, and - * otherwise its first. The setting is one for every such model. + * The language a model that can be told one writes: Automatic, then its languages, with the + * Language setting's chosen if the model has it, and otherwise Automatic. The setting is one for + * every such model. Automatic lets the model find the language, or, for one that can't, says in + * brackets what it writes: "Automatic (English)". */ function languageMenu(model) { const choices = model.languageChoices; @@ -444,10 +446,14 @@ function languageMenu(model) { const select = document.createElement("select"); select.dataset.model = model.id; select.setAttribute("aria-label", `${model.name}’s language`); - select.replaceChildren(...choices.map((choice) => option(choice.code, choice.name))); - select.value = choices.some((choice) => choice.code === setting) ? setting : choices[0].code; + const automatic = model.automaticLanguage ? `Automatic (${model.automaticLanguage})` : "Automatic"; + const languages = document.createElement("optgroup"); + languages.label = "Languages"; + languages.replaceChildren(...choices.map((choice) => option(choice.code, choice.name))); + select.replaceChildren(option("", automatic), languages); + select.value = choices.some((choice) => choice.code === setting) ? setting : ""; select.addEventListener("change", async () => { - await change({ sttLanguage: select.value }); + await change({ sttLanguage: select.value || null }); // The menu has closed, so the rows can be drawn again, keeping the focus on it. if (catalogStale) { renderCatalog({ force: true }); diff --git a/linux-windows/crates/language-model/src/tokenizer.rs b/linux-windows/crates/language-model/src/tokenizer.rs index e361f00..da97129 100644 --- a/linux-windows/crates/language-model/src/tokenizer.rs +++ b/linux-windows/crates/language-model/src/tokenizer.rs @@ -218,6 +218,15 @@ impl Tokenizer { }) } + /// The bytes a vocabulary token writes; `None` for an added token (`<|im_end|>`) or an id + /// the tokenizer doesn't know. + pub fn bytes_of(&self, id: u32) -> Option<&[u8]> { + match self.pieces.get(&id) { + Some(Piece::Bytes(bytes)) => Some(bytes), + _ => None, + } + } + /// The text of `ids`. Ids the tokenizer doesn't know are skipped. pub fn decode(&self, ids: &[u32]) -> String { let mut bytes = Vec::new(); @@ -360,6 +369,15 @@ mod tests { assert_eq!(tokenizer.decode(&ids), "hello world"); } + #[test] + fn a_vocabulary_token_writes_its_bytes_and_an_added_token_none() { + let tokenizer = tokenizer(); + assert_eq!(tokenizer.bytes_of(id(&tokenizer, "Ġwo")), Some(&b" wo"[..])); + assert_eq!(tokenizer.bytes_of(id(&tokenizer, "Ċ")), Some(&b"\n"[..])); + assert_eq!(tokenizer.bytes_of(id(&tokenizer, "<|im_end|>")), None); + assert_eq!(tokenizer.bytes_of(9_999), None); + } + #[test] fn cuts_added_tokens_out_the_longest_first() { let tokenizer = tokenizer(); diff --git a/linux-windows/crates/transcription/src/catalog.rs b/linux-windows/crates/transcription/src/catalog.rs index 2019d8d..004f907 100644 --- a/linux-windows/crates/transcription/src/catalog.rs +++ b/linux-windows/crates/transcription/src/catalog.rs @@ -6,8 +6,10 @@ //! reads its own platform's and leaves out a model without one, so each app has its own list, and //! a model is named, described and credited alike everywhere. Every file is pinned: to a Hugging //! Face commit and each file's SHA-256, or to an archive's SHA-256 and each file's in it. A model -//! that is told which language to write, where others find it, lists the languages it can be told -//! (`language_choices`), for the Language setting. +//! that can be told which language to write lists the languages it takes (`language_choices`), +//! for the Language setting; one that can't find the language itself names the one Automatic +//! tells it (`automatic_language`). Qwen3-ASR's languages name their letters too (`script`), which +//! its transcription is locked to ([`crate::script_lock`]). //! //! [`SpeechModel::verify`] checks a model's folder against those before the model is opened: //! sherpa-onnx aborts the process on a file ONNX Runtime can't read ([`crate::sherpa`]), so only @@ -24,6 +26,7 @@ use std::time::{Instant, UNIX_EPOCH}; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; +use crate::script_lock::WritingScript; use crate::sherpa::Family; /// The version of `speech-models.json` this build reads, as SpeechModelCatalog.swift's `format`. @@ -74,9 +77,12 @@ pub struct SpeechModel { pub licence: String, /// Who made it, and converted it for this platform, credited as its licence asks. pub credit: String, - /// The languages it can be told to write, its default first; empty for a model that finds - /// the language itself, or that this platform's engine can't tell one. + /// The languages it can be told to write, in the order Settings lists them; empty for a model + /// this platform's engine can't tell one (Parakeet), which finds the language itself. pub language_choices: Vec, + /// The code of the language Automatic tells it, for a model that can't find the language + /// itself (Cohere Transcribe); `None` for one that finds it. + pub automatic_language: Option, pub engine: Engine, /// Its files, as they are in its folder. pub files: Vec, @@ -86,10 +92,15 @@ pub struct SpeechModel { #[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct LanguageChoice { - /// Its ISO 639-1 code, as the model is told it: `en`, `de`. + /// Its ISO 639 code: `en`, `de`, `fil`. One setting holds it for every model, so a language + /// has the same code in each. pub code: String, /// Its name in English, as Settings shows it. pub name: String, + /// The letters it's written in, for a model whose letters are locked to the language it's + /// told (our Qwen3-ASR runtime, [`crate::script_lock`]); `None` for one whose aren't. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub script: Option, } impl LanguageChoice { @@ -100,13 +111,17 @@ impl LanguageChoice { } } -/// The language of `choices` to tell a model, for the Language setting (`sttLanguage`, a code): -/// the setting's, if it's one of them, and otherwise the first, the default. `None` when there -/// are no choices. -pub fn chosen_language<'a>(choices: &'a [LanguageChoice], setting: Option<&str>) -> Option<&'a LanguageChoice> { - setting - .and_then(|code| choices.iter().find(|choice| choice.code == code)) - .or_else(|| choices.first()) +/// The language of `choices` to tell a model, for the Language setting (`sttLanguage`, a code, +/// `None` for Automatic): the setting's, if it's one of them, and otherwise Automatic's, +/// `automatic`, for a model that can't find the language itself. `None` tells the model nothing: +/// it finds the language. +pub fn chosen_language<'a>( + choices: &'a [LanguageChoice], + automatic: Option<&str>, + setting: Option<&str>, +) -> Option<&'a LanguageChoice> { + let find = |code: &str| choices.iter().find(|choice| choice.code == code); + setting.and_then(find).or_else(|| automatic.and_then(find)) } /// What runs a model on this platform, and where its files come from. @@ -121,10 +136,28 @@ pub enum Engine { } impl Engine { - /// Whether it tells the model which language to write. Our Qwen3-ASR runtime lets the model - /// find it. + /// Whether it can tell the model which language to write: our Qwen3-ASR runtime, in its + /// prompt, and sherpa-onnx's Cohere Transcribe. NeMo transducers can't be told one. pub fn takes_language(&self) -> bool { - matches!(self, Self::SherpaOnnx { family, .. } if family.takes_language()) + match self { + Self::OpenVino { .. } => true, + Self::SherpaOnnx { family, .. } => family.takes_language(), + } + } + + /// Whether the model finds the language itself, when it isn't told one. Cohere Transcribe + /// can't. + pub fn finds_language(&self) -> bool { + match self { + Self::OpenVino { .. } => true, + Self::SherpaOnnx { family, .. } => family.finds_language(), + } + } + + /// Whether the model's letters are locked to the language it's told: our Qwen3-ASR runtime's, + /// since Qwen3-ASR takes a language as no more than a hint ([`crate::script_lock`]). + pub fn locks_script(&self) -> bool { + matches!(self, Self::OpenVino { .. }) } } @@ -163,9 +196,14 @@ pub enum Role { impl SpeechModel { /// The language to tell it to write, for the Language setting ([`chosen_language`]); `None` - /// for a model that isn't told one. + /// tells it none, and it finds the language. pub fn language(&self, setting: Option<&str>) -> Option<&LanguageChoice> { - chosen_language(&self.language_choices, setting) + chosen_language(&self.language_choices, self.automatic_language.as_deref(), setting) + } + + /// The language Automatic tells it, if it can't find the language itself. + pub fn automatic_choice(&self) -> Option<&LanguageChoice> { + self.language(None) } /// How much a download fetches. @@ -471,6 +509,7 @@ impl SpeechModelCatalog { languages: entry.languages, licence: entry.licence, language_choices: entry.language_choices, + automatic_language: entry.automatic_language, engine, files, }; @@ -478,6 +517,12 @@ impl SpeechModelCatalog { // Another platform's engine may tell the model; this one lets it find the language. if !model.engine.takes_language() { model.language_choices.clear(); + model.automatic_language = None; + } + if !model.engine.locks_script() { + for choice in &mut model.language_choices { + choice.script = None; + } } models.push(model); } @@ -557,9 +602,28 @@ fn check(model: &SpeechModel) -> Result<(), String> { if !codes.insert(choice.code.as_str()) { return Err(format!("the language {} is listed twice", choice.code)); } + if model.engine.locks_script() && choice.script.is_none() { + return Err(format!( + "the language {} names no script, which its letters are locked to", + choice.code + )); + } } - if model.engine.takes_language() && model.language_choices.is_empty() { - return Err("it's told which language to write, and has no language_choices".to_owned()); + // A model that can't find the language must be told one, so it needs both. + let must_be_told = model.engine.takes_language() && !model.engine.finds_language(); + if must_be_told && model.language_choices.is_empty() { + return Err("it can't find the language itself, and has no language_choices".to_owned()); + } + match &model.automatic_language { + Some(code) if !codes.contains(code.as_str()) => { + return Err(format!( + "its automatic_language, {code}, isn't one of its language_choices" + )); + } + None if must_be_told => { + return Err("it can't find the language itself, and has no automatic_language".to_owned()); + } + _ => {} } match &model.engine { Engine::OpenVino { repository, revision } => { @@ -622,6 +686,8 @@ struct Entry { credit: String, #[serde(default)] language_choices: Vec, + #[serde(default)] + automatic_language: Option, linux: Option
, windows: Option
, } @@ -687,19 +753,20 @@ mod tests { "models": [{ "id": "test-model", "name": "Test", "summary": "For tests.", "languages": "English", "licence": "MIT", "credit": "Us", "language_choices": [{"code": "en", "name": "English"}], - "linux": section, "windows": section, + "automatic_language": "en", "linux": section, "windows": section, }], }); SpeechModelCatalog::parse(&text.to_string(), Platform::Linux).unwrap() } - /// A catalog of one model with `language_choices` and the section `linux`. + /// A catalog of one model with `language_choices`, English as Automatic's, and the section + /// `linux`. fn parse_with_languages(language_choices: Value, linux: Value) -> Result { let text = json!({ "format": 1, "models": [{ "id": "told", "name": "Told", "summary": "", "languages": "", "licence": "MIT", "credit": "", - "language_choices": language_choices, "linux": linux, + "language_choices": language_choices, "automatic_language": "en", "linux": linux, }], }); SpeechModelCatalog::parse(&text.to_string(), Platform::Linux) @@ -790,7 +857,11 @@ mod tests { .iter() .map(|choice| choice.code.as_str()) .collect(); - assert_eq!(codes[0], "en", "English is the default"); + assert_eq!( + cohere.automatic_choice().map(|choice| choice.name.as_str()), + Some("English"), + "it can't find the language, so Automatic tells it English" + ); codes.sort_unstable(); // IsValidCohereTranscribeLanguage, in sherpa-onnx's offline-recognizer-cohere-transcribe-impl.h: // with any other, it decodes nothing. @@ -800,26 +871,110 @@ mod tests { "ar", "de", "el", "en", "es", "fr", "it", "ja", "ko", "nl", "pl", "pt", "vi", "zh" ] ); - for id in ["qwen3-asr-0.6b-sinhala", "parakeet-tdt-0.6b-v2", "parakeet-tdt-0.6b-v3"] { + for id in ["parakeet-tdt-0.6b-v2", "parakeet-tdt-0.6b-v3"] { let model = catalog.model(id).unwrap(); - assert!(model.language_choices.is_empty(), "{id} finds the language itself"); + assert!(model.language_choices.is_empty(), "{id} can't be told a language"); } } + /// Our runtime puts the language in Qwen3-ASR's prompt, by the name its config.json lists. #[test] - fn the_language_told_is_the_settings_if_the_model_has_it_and_else_its_first() { + fn qwen3_asr_is_told_the_languages_its_config_lists() { let catalog = SpeechModelCatalog::parse(BUNDLED, Platform::Linux).unwrap(); - let cohere = catalog.model("cohere-transcribe").unwrap(); - let told = |setting| cohere.language(setting).map(|choice| choice.code.as_str()); - assert_eq!(told(Some("de")), Some("de")); - assert_eq!(told(None), Some("en")); - assert_eq!(told(Some("si")), Some("en"), "Sinhala isn't one of its languages"); + let qwen = catalog.model("qwen3-asr-0.6b-sinhala").unwrap(); + assert_eq!(qwen.language_choices.len(), 31); + assert_eq!(qwen.language_choices[0].name, "English"); + assert!( + qwen.language_choices + .iter() + .any(|choice| choice.code == "si" && choice.name == "Sinhala") + ); + assert_eq!(qwen.automatic_choice(), None, "it finds the language itself"); + } + + /// Qwen3-ASR takes a language as a hint, so our runtime locks its letters to the one it's + /// told, and each of its languages names them. sherpa-onnx's models aren't locked. + #[test] + fn qwen3_asrs_languages_name_their_letters_and_no_other_models_keep_them() { + let catalog = SpeechModelCatalog::parse(BUNDLED, Platform::Linux).unwrap(); + let qwen = catalog.model("qwen3-asr-0.6b-sinhala").unwrap(); + let script = |code: &str| { + qwen.language_choices + .iter() + .find(|choice| choice.code == code) + .and_then(|choice| choice.script) + }; + assert_eq!(script("si"), Some(WritingScript::Sinhala)); + assert_eq!(script("en"), Some(WritingScript::Latin)); + assert_eq!(script("ja"), Some(WritingScript::Japanese)); + for model in catalog.models() { + for choice in &model.language_choices { + assert_eq!( + choice.script.is_some(), + model.engine.locks_script(), + "{}: {}", + model.id, + choice.code + ); + } + } + } + + #[test] + fn a_locked_models_language_needs_its_script_and_another_models_isnt_kept() { + let openvino = json!({ + "engine": "openvino", + "repository": "a/b", + "revision": "8298d9b2d532965800b2c0c64b81965ededb03a3", + "files": [{"name": "model.bin", "bytes": 1, "sha256": sha256(b"x")}], + }); + let error = parse_with_languages(json!([{"code": "en", "name": "English"}]), openvino.clone()).unwrap_err(); + assert!(error.to_string().contains("names no script"), "{error}"); + let locked = + parse_with_languages(json!([{"code": "en", "name": "English", "script": "Latn"}]), openvino).unwrap(); assert_eq!( - catalog.model("parakeet-tdt-0.6b-v3").unwrap().language(Some("de")), - None + locked.models()[0].language_choices[0].script, + Some(WritingScript::Latin) + ); + let unlocked = parse_with_languages( + json!([{"code": "en", "name": "English", "script": "Latn"}]), + cohere_section(), + ) + .unwrap(); + assert_eq!(unlocked.models()[0].language_choices[0].script, None); + assert!( + parse_with_languages( + json!([{"code": "en", "name": "English", "script": "Latin"}]), + cohere_section() + ) + .is_err(), + "scripts go by their ISO 15924 codes" ); } + #[test] + fn the_language_told_is_the_settings_if_the_model_has_it_and_else_automatics() { + let catalog = SpeechModelCatalog::parse(BUNDLED, Platform::Linux).unwrap(); + let told = |id, setting| { + catalog + .model(id) + .unwrap() + .language(setting) + .map(|choice| choice.code.as_str()) + }; + assert_eq!(told("cohere-transcribe", Some("de")), Some("de")); + assert_eq!(told("cohere-transcribe", None), Some("en")); + assert_eq!( + told("cohere-transcribe", Some("si")), + Some("en"), + "Sinhala isn't one of its languages" + ); + assert_eq!(told("qwen3-asr-0.6b-sinhala", Some("si")), Some("si")); + assert_eq!(told("qwen3-asr-0.6b-sinhala", None), None, "it finds the language"); + assert_eq!(told("qwen3-asr-0.6b-sinhala", Some("vi")), Some("vi")); + assert_eq!(told("parakeet-tdt-0.6b-v3", Some("de")), None); + } + #[test] fn a_language_is_named_by_its_code_or_its_name() { let catalog = SpeechModelCatalog::parse(BUNDLED, Platform::Linux).unwrap(); @@ -827,7 +982,8 @@ mod tests { assert_eq!(named("de"), Some("de")); assert_eq!(named(" German "), Some("de")); assert_eq!(named("JA"), Some("ja")); - assert_eq!(named("Sinhala"), None); + assert_eq!(named("Sinhala"), Some("si"), "Qwen3-ASR's Sinhala fine-tune knows it"); + assert_eq!(named("Catalan"), None, "only Whisper on the Mac knows it"); } #[test] @@ -841,23 +997,65 @@ mod tests { "listed twice", ), (json!([]), "has no language_choices"), + ( + json!([{"code": "de", "name": "German"}]), + "automatic_language, en, isn't one of its language_choices", + ), ] { let error = parse_with_languages(choices, cohere_section()).unwrap_err(); assert!(error.to_string().contains(problem), "{error}"); } } + #[test] + fn a_model_that_cant_find_the_language_needs_automatics() { + let text = json!({ + "format": 1, + "models": [{ + "id": "told", "name": "Told", "summary": "", "languages": "", "licence": "MIT", "credit": "", + "language_choices": [{"code": "en", "name": "English"}], "linux": cohere_section(), + }], + }); + let error = SpeechModelCatalog::parse(&text.to_string(), Platform::Linux).unwrap_err(); + assert!(error.to_string().contains("has no automatic_language"), "{error}"); + } + #[test] fn a_model_this_platform_cant_tell_keeps_no_language_choices() { + let nemo = json!({ + "engine": "sherpa-onnx", + "family": "nemo-transducer", + "archive": {"url": "https://example.com/p.tar.bz2", "bytes": 1, "sha256": sha256(b"p")}, + "files": [ + {"name": "encoder.onnx", "bytes": 1, "sha256": sha256(b"e"), "role": "encoder"}, + {"name": "decoder.onnx", "bytes": 1, "sha256": sha256(b"d"), "role": "decoder"}, + {"name": "joiner.onnx", "bytes": 1, "sha256": sha256(b"j"), "role": "joiner"}, + {"name": "tokens.txt", "bytes": 1, "sha256": sha256(b"t"), "role": "tokens"}, + ], + }); + let catalog = parse_with_languages(json!([{"code": "en", "name": "English"}]), nemo).unwrap(); + assert!(catalog.models()[0].language_choices.is_empty()); + assert_eq!(catalog.models()[0].automatic_language, None); + assert_eq!(catalog.language_named("English"), None); + } + + #[test] + fn a_model_that_finds_the_language_may_list_none() { let openvino = json!({ "engine": "openvino", "repository": "a/b", "revision": "8298d9b2d532965800b2c0c64b81965ededb03a3", "files": [{"name": "model.bin", "bytes": 1, "sha256": sha256(b"x")}], }); - let catalog = parse_with_languages(json!([{"code": "en", "name": "English"}]), openvino).unwrap(); + let text = json!({ + "format": 1, + "models": [{ + "id": "finds", "name": "F", "summary": "", "languages": "", "licence": "MIT", "credit": "", + "linux": openvino, + }], + }); + let catalog = SpeechModelCatalog::parse(&text.to_string(), Platform::Linux).unwrap(); assert!(catalog.models()[0].language_choices.is_empty()); - assert_eq!(catalog.language_named("English"), None); } #[test] diff --git a/linux-windows/crates/transcription/src/lib.rs b/linux-windows/crates/transcription/src/lib.rs index 2e42e1f..15f61f3 100644 --- a/linux-windows/crates/transcription/src/lib.rs +++ b/linux-windows/crates/transcription/src/lib.rs @@ -14,11 +14,14 @@ //! - [`catalog`] mirrors SpeechModelCatalog.swift: the models the app offers to download, read //! from the catalog the Mac app has too, and the check of a model's files before it opens. //! - [`speech_to_text`] opens a model with whichever of the two runs it. +//! - [`script_lock`] mirrors ScriptLock.swift: the letters Qwen3-ASR may write once it's told a +//! language. //! //! Transcripts are never logged: log lines carry counts only. pub mod catalog; pub mod output_limit; pub mod qwen3_asr; +pub mod script_lock; pub mod sherpa; pub mod speech_to_text; diff --git a/linux-windows/crates/transcription/src/qwen3_asr/generation.rs b/linux-windows/crates/transcription/src/qwen3_asr/generation.rs index 285456e..bebcdb9 100644 --- a/linux-windows/crates/transcription/src/qwen3_asr/generation.rs +++ b/linux-windows/crates/transcription/src/qwen3_asr/generation.rs @@ -1,5 +1,7 @@ use std::collections::HashSet; +use super::locked_tokens::Locked; + /// Tokens that end a reply, and are not part of it: `<|im_end|>` and `<|endoftext|>`. pub const END_OF_REPLY: [u32; 2] = [151_645, 151_643]; @@ -34,13 +36,17 @@ pub struct Decoded { /// - an end-of-reply token ends the reply and is not part of it; /// - once the reply has 24 tokens, it ends as soon as its last 24 hold 3 or fewer different tokens; /// - it ends at `max_tokens` tokens. +/// +/// `locked`, if given, keeps the reply to a language's letters, as the Mac app's own decoding +/// does when it's told one (Qwen3ASR+ScriptLock.swift): each token is the likeliest it allows. pub fn decode_greedily( first_logits: &[f32], max_tokens: usize, + mut locked: Option>, mut next_logits: impl FnMut(u32) -> Result, E>, ) -> Result { let mut tokens = Vec::new(); - let mut next = likeliest(first_logits); + let mut next = choose(first_logits, locked.as_mut()); for index in 0..max_tokens { if END_OF_REPLY.contains(&next) { return Ok(Decoded { @@ -49,6 +55,9 @@ pub fn decode_greedily( }); } tokens.push(next); + if let Some(locked) = locked.as_mut() { + locked.write(next); + } if is_looping(&tokens) { return Ok(Decoded { tokens, @@ -58,7 +67,7 @@ pub fn decode_greedily( if index == max_tokens - 1 { break; } - next = likeliest(&next_logits(next)?); + next = choose(&next_logits(next)?, locked.as_mut()); } Ok(Decoded { tokens, @@ -66,6 +75,24 @@ pub fn decode_greedily( }) } +/// The likeliest token, or, locked, the likeliest the lock allows; the end of the reply if it +/// allows none, which a lock never does. +fn choose(logits: &[f32], locked: Option<&mut Locked<'_>>) -> u32 { + let Some(locked) = locked else { + return likeliest(logits); + }; + let allowed = locked.allowed(); + let mut best: Option = None; + for (index, (&logit, &allowed)) in logits.iter().zip(allowed).enumerate() { + if allowed && best.is_none_or(|best| logit > logits[best]) { + best = Some(index); + } + } + best.map_or(END_OF_REPLY[0], |best| { + u32::try_from(best).expect("a vocabulary fits in u32") + }) +} + /// The index of the largest logit, the first of equals. fn likeliest(logits: &[f32]) -> u32 { let mut best = 0; @@ -100,7 +127,7 @@ mod tests { /// Decodes a scripted reply: each call returns the logits for the next scripted token. fn decode(script: &[u32], max_tokens: usize) -> (Decoded, usize) { let mut steps = 0; - let decoded = decode_greedily::<()>(&picking(script[0]), max_tokens, |_| { + let decoded = decode_greedily::<()>(&picking(script[0]), max_tokens, None, |_| { steps += 1; Ok(picking(script[steps])) }) @@ -108,6 +135,27 @@ mod tests { (decoded, steps) } + /// Locked to Latin letters, the reply takes "Hello" over the likelier 我. + #[test] + fn a_locked_reply_takes_the_likeliest_token_the_lock_allows() { + use crate::qwen3_asr::locked_tokens::LockedTokens; + use crate::qwen3_asr::tokenizer::Tokenizer; + use crate::script_lock::{ScriptLock, WritingScript}; + + let tokenizer = Tokenizer::from_json(r#"{"Hello": 0, "æĪij": 1}"#, r#"{"added_tokens_decoder": {}}"#).unwrap(); + let mut tokens = LockedTokens::new(&tokenizer, 151_936); + let english = ScriptLock::new(WritingScript::Latin); + let mut first = vec![0.0; 151_936]; + first[1] = 2.0; + first[0] = 1.0; + let decoded = decode_greedily::<()>(&first, 10, Some(Locked::new(&english, &mut tokens)), |_| { + Ok(picking(END_OF_REPLY[0])) + }) + .unwrap(); + assert_eq!(decoded.tokens, [0]); + assert_eq!(decoded.stop, Stop::EndOfReply); + } + #[test] fn ends_at_the_end_of_the_reply() { let (decoded, steps) = decode(&[10, 11, 151_645, 12], 100); diff --git a/linux-windows/crates/transcription/src/qwen3_asr/locked_tokens.rs b/linux-windows/crates/transcription/src/qwen3_asr/locked_tokens.rs new file mode 100644 index 0000000..90770bd --- /dev/null +++ b/linux-windows/crates/transcription/src/qwen3_asr/locked_tokens.rs @@ -0,0 +1,137 @@ +//! The tokens a [`ScriptLock`] lets Qwen3-ASR write next, as LockedVocabulary.swift and +//! LockedLogits in Qwen3ASR+ScriptLock.swift find them on the Mac: each vocabulary token whose +//! bytes the lock allows after the character the reply has left unfinished, and the end of the +//! reply after a whole character. Qwen's added tokens write no text, so they never are. + +use std::collections::HashMap; + +use super::generation::END_OF_REPLY; +use super::tokenizer::Tokenizer; +use crate::script_lock::{PartialCharacter, ScriptLock, WritingScript}; + +/// The model's tokens as the bytes each writes, and the tokens each lock allows after each +/// unfinished character, kept once found: finding them reads all 151,936 tokens. +#[derive(Debug)] +pub struct LockedTokens { + /// Each token's bytes, by id, for as many tokens as the model has logits; `None` for an added + /// token or the logits' padding. + bytes: Vec>>, + allowed: HashMap<(WritingScript, PartialCharacter), Vec>, +} + +impl LockedTokens { + /// The tokens of a model with `count` logits. + pub fn new(tokenizer: &Tokenizer, count: usize) -> Self { + let bytes = (0..count) + .map(|id| { + u32::try_from(id) + .ok() + .and_then(|id| tokenizer.bytes_of(id)) + .map(<[u8]>::to_vec) + }) + .collect(); + Self { + bytes, + allowed: HashMap::new(), + } + } + + /// Which tokens `lock` allows after `partial`, by id. + fn allowed(&mut self, lock: &ScriptLock, partial: PartialCharacter) -> &[bool] { + let bytes = &self.bytes; + self.allowed.entry((lock.script(), partial)).or_insert_with(|| { + let mut allowed: Vec = bytes + .iter() + .map(|written| { + written + .as_deref() + .is_some_and(|written| !written.is_empty() && lock.partial_after(partial, written).is_some()) + }) + .collect(); + if partial.is_whole() { + for end in END_OF_REPLY { + if let Some(allowed) = allowed.get_mut(end as usize) { + *allowed = true; + } + } + } + allowed + }) + } + + fn bytes_of(&self, token: u32) -> &[u8] { + self.bytes + .get(token as usize) + .and_then(Option::as_deref) + .unwrap_or_default() + } +} + +/// A reply locked to a language's letters while it's decoded: the lock, the tokens, and the +/// character the reply has left unfinished so far. +pub struct Locked<'a> { + lock: &'a ScriptLock, + tokens: &'a mut LockedTokens, + partial: PartialCharacter, +} + +impl<'a> Locked<'a> { + pub fn new(lock: &'a ScriptLock, tokens: &'a mut LockedTokens) -> Self { + Self { + lock, + tokens, + partial: PartialCharacter::NOTHING, + } + } + + /// Which tokens may come next, by id. + pub fn allowed(&mut self) -> &[bool] { + self.tokens.allowed(self.lock, self.partial) + } + + /// Reads `token`, which came next. + pub fn write(&mut self, token: u32) { + self.partial = self + .lock + .partial_after(self.partial, self.tokens.bytes_of(token)) + .unwrap_or(PartialCharacter::NOTHING); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// "Hello", " world", E0 B6 and 9A ("ක" is E0 B6 9A), 我 (E6 88 91), with an added token at 10. + fn tokenizer() -> Tokenizer { + let vocab = r#"{"Hello": 0, "Ġworld": 1, "à¶": 2, "ļ": 3, "æĪij": 4}"#; + let config = r#"{"added_tokens_decoder": {"10": {"content": "<|im_end|>", "special": true}}}"#; + Tokenizer::from_json(vocab, config).unwrap() + } + + #[test] + fn a_lock_allows_the_tokens_that_write_what_it_allows() { + let mut tokens = LockedTokens::new(&tokenizer(), 12); + let english = ScriptLock::new(WritingScript::Latin); + let allowed = tokens.allowed(&english, PartialCharacter::NOTHING).to_vec(); + assert_eq!(&allowed[..5], [true, true, false, false, false]); + assert!(!allowed[10], "an added token writes no text"); + let han = ScriptLock::new(WritingScript::Han); + assert!(tokens.allowed(&han, PartialCharacter::NOTHING)[4]); + } + + /// Only the rest of an unfinished character may follow it, and no end of the reply. + #[test] + fn an_unfinished_character_is_finished_first() { + let mut tokens = LockedTokens::new(&tokenizer(), 151_936); + let sinhala = ScriptLock::new(WritingScript::Sinhala); + let mut locked = Locked::new(&sinhala, &mut tokens); + assert!(locked.allowed()[151_645], "a reply may end after a whole character"); + locked.write(2); + let allowed = locked.allowed(); + assert_eq!(&allowed[..5], [false, false, false, true, false]); + assert!(!allowed[151_645] && !allowed[151_643]); + locked.write(3); + assert!(locked.allowed()[151_645]); + } +} diff --git a/linux-windows/crates/transcription/src/qwen3_asr/mod.rs b/linux-windows/crates/transcription/src/qwen3_asr/mod.rs index 083d130..7700bea 100644 --- a/linux-windows/crates/transcription/src/qwen3_asr/mod.rs +++ b/linux-windows/crates/transcription/src/qwen3_asr/mod.rs @@ -5,11 +5,15 @@ //! clips under 100 ms give no text; the token limit follows the clip's length //! ([`crate::output_limit`]); a clip under a second is padded with silence to one; its log-mel //! features are encoded in chunks and windows ([`EncoderLayout`]); the audio rows replace the -//! prompt's first placeholders; the reply is decoded greedily ([`generation`]) and read as a -//! language and a transcript ([`Languages::read`]). +//! prompt's first placeholders, and a language the clip is told starts the reply +//! ([`prompt::told_language`]); the reply is decoded greedily ([`generation`]) and read as a +//! language and a transcript ([`Languages::read`]), or, told one, as the transcript. Told a +//! language that names its letters, the reply is kept to them ([`crate::script_lock`]), as the +//! Mac app's own decoding keeps it. mod encoder_layout; pub mod generation; +mod locked_tokens; mod log_mel; mod openvino_model; mod openvino_runtime; @@ -17,10 +21,14 @@ mod prompt; mod tokenizer; mod transcript; +use std::collections::HashMap; use std::fmt; use lt_shared::audio_format::{SAMPLE_RATE, samples_for_milliseconds}; +use crate::script_lock::{ScriptLock, WritingScript}; +pub use locked_tokens::{Locked, LockedTokens}; + pub use encoder_layout::{CHUNK_FRAMES, CHUNKS_PER_WINDOW, EncoderLayout, placeholders}; pub use generation::{Decoded, Stop}; pub use log_mel::{HOP_LENGTH, LogMel, LogMelError, LogMelExtractor, MEL_BINS, N_FFT}; @@ -150,7 +158,7 @@ impl ChunkFrames { #[derive(Clone, Debug, PartialEq, Eq)] pub struct Transcription { pub text: String, - /// The language the model named, as its supported languages spell it. + /// The language the model named, or was told, as its supported languages spell it. pub language: Option, /// Tokens in the reply, and why it ended. pub tokens: usize, @@ -219,6 +227,13 @@ pub struct Transcriber { tokenizer: Tokenizer, prompt: PromptFormat, languages: Languages, + /// The ids that start the reply for each language a clip has been told, by its spelling. + told: HashMap>, + /// A language couldn't be spelled for the prompt, which has been said. + said_untold: bool, + /// The tokens each language's letters allow, read from the tokenizer when a clip is first + /// told a language that names them. + locked_tokens: Option, log_mel: LogMelExtractor, } @@ -232,6 +247,9 @@ impl Transcriber { prompt: PromptFormat::new(&tokenizer)?, tokenizer, languages, + told: HashMap::new(), + said_untold: false, + locked_tokens: None, log_mel: LogMelExtractor::new(), }) } @@ -241,8 +259,16 @@ impl Transcriber { &self.model } - /// The transcript of `samples`, 16 kHz mono. - pub fn transcribe(&mut self, samples: &[f32]) -> Result> { + /// The transcript of `samples`, 16 kHz mono, in `language` (its name, such as "English") if + /// it's given; otherwise the model finds the language. A tokenizer read without its merges + /// can't spell the language for the prompt, so the model finds it then too. Told a language, + /// the transcript is written in `script`'s letters if it's given ([`ScriptLock`]). + pub fn transcribe( + &mut self, + samples: &[f32], + language: Option<&str>, + script: Option, + ) -> Result> { if samples.len() < MINIMUM_SAMPLES { return Ok(Transcription::empty()); } @@ -266,13 +292,28 @@ impl Transcriber { if audio.count() > placeholders { audio = audio.range(0..placeholders); } - let ids = self.prompt.ids(placeholders); + let told = language.and_then(|name| self.told_language(name)); + let mut ids = self.prompt.ids(placeholders); + if let Some((_, answer)) = &told { + ids.extend_from_slice(answer); + } let first = self .model .prefill(&ids, self.prompt.audio_start(), &audio) .map_err(TranscribeError::Model)?; + let lock = script.filter(|_| told.is_some()).map(ScriptLock::new); + let locked = match &lock { + Some(lock) => { + let tokenizer = &self.tokenizer; + let tokens = self + .locked_tokens + .get_or_insert_with(|| LockedTokens::new(tokenizer, first.len())); + Some(Locked::new(lock, tokens)) + } + None => None, + }; let model = &mut self.model; - let decoded = generation::decode_greedily(&first, max_tokens, |token| model.step(token)) + let decoded = generation::decode_greedily(&first, max_tokens, locked, |token| model.step(token)) .map_err(TranscribeError::Model)?; if max_tokens > 0 && decoded.tokens.len() >= max_tokens { tracing::warn!( @@ -281,7 +322,11 @@ impl Transcriber { ); } - let reply = self.languages.read(&self.tokenizer.decode(&decoded.tokens)); + let decoded_text = self.tokenizer.decode(&decoded.tokens); + let reply = match &told { + Some((name, _)) => self.languages.read_told(&decoded_text, name), + None => self.languages.read(&decoded_text), + }; tracing::debug!( "Transcribed {} ms of audio: {} placeholders, {} audio rows, {} tokens ({:?})", samples.len() * 1_000 / SAMPLE_RATE, @@ -297,6 +342,28 @@ impl Transcriber { stop: Some(decoded.stop), }) } + + /// `name` as the model spells it, and the ids that start its reply in that language; `None` + /// when the tokenizer can't encode them, which is said once. + fn told_language(&mut self, name: &str) -> Option<(String, Vec)> { + let spelled = self.languages.spelled(name); + if let Some(ids) = self.told.get(&spelled) { + return Some((spelled, ids.clone())); + } + match prompt::told_language(&self.tokenizer, &spelled) { + Ok(ids) => { + self.told.insert(spelled.clone(), ids.clone()); + Some((spelled, ids)) + } + Err(error) => { + if !self.said_untold { + tracing::warn!("Speech to text can't be told {spelled}, so it finds the language: {error}"); + self.said_untold = true; + } + None + } + } + } } /// The audio encoder over a clip, as mlx-audio-swift runs it: the convolutions over every chunk, @@ -416,18 +483,48 @@ mod tests { } } + const VOCAB: &str = r#"{"system": 8948, "user": 872, "assistant": 77091, "Ċ": 198, "language": 11528, + "ĠEnglish": 6364, "Ship": 29170, "Ġit": 432, ".": 13}"#; + const TOKENIZER_CONFIG: &str = r#"{"added_tokens_decoder": { + "151643": {"content": "<|endoftext|>"}, "151644": {"content": "<|im_start|>"}, + "151645": {"content": "<|im_end|>"}, "151669": {"content": "<|audio_start|>"}, + "151670": {"content": "<|audio_end|>"}, "151676": {"content": "<|audio_pad|>"}, + "151704": {"content": ""} + }}"#; + fn transcriber_replying(reply: &[u32]) -> Transcriber { - let vocab = r#"{"system": 8948, "user": 872, "assistant": 77091, "Ċ": 198, "language": 11528, - "ĠEnglish": 6364, "Ship": 29170, "Ġit": 432, ".": 13}"#; - let config = r#"{"added_tokens_decoder": { - "151643": {"content": "<|endoftext|>"}, "151644": {"content": "<|im_start|>"}, - "151645": {"content": "<|im_end|>"}, "151669": {"content": "<|audio_start|>"}, - "151670": {"content": "<|audio_end|>"}, "151676": {"content": "<|audio_pad|>"}, - "151704": {"content": ""} - }}"#; + transcriber(reply, Tokenizer::from_json(VOCAB, TOKENIZER_CONFIG).unwrap()) + } + + /// With a token for every byte and no merges, the tokenizer encodes, a byte a token. + fn transcriber_that_encodes(reply: &[u32]) -> Transcriber { + let mut vocab: serde_json::Map = serde_json::from_str(VOCAB).unwrap(); + // GPT-2's byte-level alphabet: printable bytes stand for themselves, the rest from U+0100. + let mut shifted = 0; + for byte in 0..=255_u32 { + let printable = + (0x21..=0x7E).contains(&byte) || (0xA1..=0xAC).contains(&byte) || (0xAE..=0xFF).contains(&byte); + let character = if printable { + byte + } else { + shifted += 1; + 255 + shifted + }; + vocab + .entry(char::from_u32(character).unwrap().to_string()) + .or_insert_with(|| (1_000 + byte).into()); + } + let tokenizer = Tokenizer::from_json(&serde_json::Value::Object(vocab).to_string(), TOKENIZER_CONFIG) + .unwrap() + .with_merges("#version: 0.2\n") + .unwrap(); + transcriber(reply, tokenizer) + } + + fn transcriber(reply: &[u32], tokenizer: Tokenizer) -> Transcriber { Transcriber::new( FakeModel::replying(reply), - Tokenizer::from_json(vocab, config).unwrap(), + tokenizer, Languages::new(["English".to_owned()]), ) .unwrap() @@ -436,7 +533,7 @@ mod tests { #[test] fn transcribes_a_clip() { let mut transcriber = transcriber_replying(&[11528, 6364, 151_704, 29170, 432, 13, 151_645]); - let transcription = transcriber.transcribe(&[0.01; 94_560]).unwrap(); + let transcription = transcriber.transcribe(&[0.01; 94_560], None, None).unwrap(); assert_eq!(transcription.text, "Ship it."); assert_eq!(transcription.language.as_deref(), Some("English")); assert_eq!(transcription.tokens, 6); @@ -457,15 +554,70 @@ mod tests { assert_eq!(audio[77], 512.0); } + /// Told a language, the prompt starts the reply with it, as mlx-audio-swift's does, and the + /// whole reply is the transcript. + #[test] + fn a_language_told_starts_the_reply() { + let reply = [29170, 432, 13, 151_645]; + let mut told = transcriber_that_encodes(&reply); + let transcription = told.transcribe(&[0.01; 16_000], Some("english"), None).unwrap(); + assert_eq!(transcription.text, "Ship it."); + assert_eq!( + transcription.language.as_deref(), + Some("English"), + "as the model spells it" + ); + + let mut untold = transcriber_that_encodes(&reply); + untold.transcribe(&[0.01; 16_000], None, None).unwrap(); + let (told_ids, ..) = told.model.prefilled.clone().unwrap(); + let (untold_ids, ..) = untold.model.prefilled.clone().unwrap(); + let answer = prompt::told_language(&told.tokenizer, "English").unwrap(); + assert_eq!(told_ids, [untold_ids, answer.clone()].concat()); + assert_eq!(told.tokenizer.decode(&answer), "language English"); + } + + /// Told a language with its letters, the reply keeps to them: told English, Qwen3-ASR can't + /// write Sinhala, and writes the likeliest token the lock allows instead. + #[test] + fn a_language_told_with_its_letters_keeps_the_reply_to_them() { + // "ක" (E0 B6 9A), a byte a token. + let reply = [1_224, 1_182, 1_154, 151_645]; + let mut unlocked = transcriber_that_encodes(&reply); + let transcription = unlocked.transcribe(&[0.01; 16_000], Some("English"), None).unwrap(); + assert_eq!(transcription.text, "ක"); + let mut locked = transcriber_that_encodes(&reply); + let transcription = locked + .transcribe(&[0.01; 16_000], Some("English"), Some(WritingScript::Latin)) + .unwrap(); + assert_eq!( + transcription.text, "...", + "the scripted tokens aren't allowed, and the likeliest that is, of equals the first, is ." + ); + } + + /// A tokenizer read without its merges can't spell the language, so the model finds it. + #[test] + fn a_language_that_cant_be_spelled_is_left_to_the_model() { + let mut transcriber = transcriber_replying(&[11528, 6364, 151_704, 29170, 432, 13, 151_645]); + let transcription = transcriber.transcribe(&[0.01; 16_000], Some("English"), None).unwrap(); + assert_eq!(transcription.text, "Ship it."); + let (ids, ..) = transcriber.model.prefilled.clone().unwrap(); + assert_eq!(ids.last(), Some(&198), "the prompt ends with the assistant's line feed"); + } + #[test] fn pads_a_short_clip_to_a_second_and_gives_nothing_for_a_click() { let mut transcriber = transcriber_replying(&[151_645]); - let transcription = transcriber.transcribe(&[0.01; 3_200]).unwrap(); + let transcription = transcriber.transcribe(&[0.01; 3_200], None, None).unwrap(); assert_eq!(transcription.text, ""); assert_eq!(transcriber.model.convolved_chunks, [(2, 100)]); let mut transcriber = transcriber_replying(&[151_645]); - assert_eq!(transcriber.transcribe(&[0.01; 1_599]).unwrap(), Transcription::empty()); + assert_eq!( + transcriber.transcribe(&[0.01; 1_599], None, None).unwrap(), + Transcription::empty() + ); assert!(transcriber.model.convolved_chunks.is_empty()); } @@ -473,11 +625,11 @@ mod tests { fn limits_the_reply_by_the_clip_length() { // A second of audio allows 94 tokens. let mut transcriber = transcriber_replying(&vec![432; 200]); - let transcription = transcriber.transcribe(&[0.01; 16_000]).unwrap(); + let transcription = transcriber.transcribe(&[0.01; 16_000], None, None).unwrap(); assert_eq!(transcription.stop, Some(Stop::Loop)); let looping: Vec = (0..200).map(|n| [432, 13, 29170, 6364][n % 4]).collect(); let mut transcriber = transcriber_replying(&looping); - let transcription = transcriber.transcribe(&[0.01; 16_000]).unwrap(); + let transcription = transcriber.transcribe(&[0.01; 16_000], None, None).unwrap(); assert_eq!(transcription.tokens, 94); assert_eq!(transcription.stop, Some(Stop::Limit)); } @@ -485,7 +637,9 @@ mod tests { #[test] fn refuses_a_clip_over_twenty_minutes() { let mut transcriber = transcriber_replying(&[151_645]); - let error = transcriber.transcribe(&vec![0.0; MAX_CLIP_SAMPLES + 1]).unwrap_err(); + let error = transcriber + .transcribe(&vec![0.0; MAX_CLIP_SAMPLES + 1], None, None) + .unwrap_err(); assert!(matches!(error, TranscribeError::TooLong { .. })); } } diff --git a/linux-windows/crates/transcription/src/qwen3_asr/openvino_model.rs b/linux-windows/crates/transcription/src/qwen3_asr/openvino_model.rs index 6617c06..b444e2a 100644 --- a/linux-windows/crates/transcription/src/qwen3_asr/openvino_model.rs +++ b/linux-windows/crates/transcription/src/qwen3_asr/openvino_model.rs @@ -202,7 +202,14 @@ pub fn open_transcriber( cache: Option<&Path>, ) -> Result, OpenError> { let (model, manifest) = OpenVinoModel::load(folder, device, cache).map_err(OpenError::Model)?; - let tokenizer = Tokenizer::load(folder).map_err(OpenError::Tokenizer)?; + // The merges encode the language a clip is told; a converted model outside the catalog may + // lack them, and is never told one. + let tokenizer = if folder.join("merges.txt").exists() { + Tokenizer::load_with_merges(folder) + } else { + Tokenizer::load(folder) + } + .map_err(OpenError::Tokenizer)?; let pad = tokenizer.id("<|audio_pad|>").map_err(OpenError::Tokenizer)?; if pad != manifest.text.audio_token_id { return Err(OpenError::Model(OpenVinoError::Folder { diff --git a/linux-windows/crates/transcription/src/qwen3_asr/prompt.rs b/linux-windows/crates/transcription/src/qwen3_asr/prompt.rs index b7c9140..7b38724 100644 --- a/linux-windows/crates/transcription/src/qwen3_asr/prompt.rs +++ b/linux-windows/crates/transcription/src/qwen3_asr/prompt.rs @@ -1,14 +1,17 @@ use super::tokenizer::{Tokenizer, TokenizerError}; -/// The prompt the Mac app gives Qwen3-ASR: no context and no language, so the model names the -/// language itself before the transcript ("language English…"), as mlx-audio-swift's -/// `buildPromptText` writes it when no language is given: +/// The prompt the Mac app gives Qwen3-ASR, as mlx-audio-swift's `buildPromptText` writes it: +/// no context, and with no language the model names the language itself before the transcript +/// ("language English…"): /// /// ```text /// <|im_start|>system\n<|im_end|>\n<|im_start|>user\n<|audio_start|> /// <|audio_pad|> × the clip's placeholders /// <|audio_end|><|im_end|>\n<|im_start|>assistant\n /// ``` +/// +/// Told a language, the prompt goes on with the start of the reply, "language English" +/// ([`told_language`]), and the model writes the transcript after it. #[derive(Clone, Debug, PartialEq, Eq)] pub struct PromptFormat { prefix: Vec, @@ -63,6 +66,12 @@ impl PromptFormat { } } +/// The ids that start the reply of a model told to write `language`, as its supported languages +/// spell it: "language English". The tokenizer must have its merges to encode them. +pub fn told_language(tokenizer: &Tokenizer, language: &str) -> Result, TokenizerError> { + tokenizer.encode(&format!("language {language}")) +} + #[cfg(test)] mod tests { use super::*; diff --git a/linux-windows/crates/transcription/src/qwen3_asr/transcript.rs b/linux-windows/crates/transcription/src/qwen3_asr/transcript.rs index 7504e8d..1af2523 100644 --- a/linux-windows/crates/transcription/src/qwen3_asr/transcript.rs +++ b/linux-windows/crates/transcription/src/qwen3_asr/transcript.rs @@ -91,6 +91,22 @@ impl Languages { } } + /// The reply of a model told to write `told` ([`Self::spelled`]): all of it is the transcript, + /// in that language, as mlx-audio-swift's `parseGeneratedChunk` reads it with a forced + /// language. + pub fn read_told(&self, decoded: &str, told: &str) -> Reply { + Reply { + language: self.normalized(told), + text: s::trimming(decoded, CharacterSet::WhitespacesAndNewlines).to_owned(), + } + } + + /// `name` as the model's supported languages spell it, as its prompt takes it; a name it + /// doesn't know stays as written. + pub fn spelled(&self, name: &str) -> String { + self.normalized(name).unwrap_or_else(|| name.to_owned()) + } + /// `name` as the model's supported languages spell it, through the aliases; a name it doesn't /// know stays as written. fn normalized(&self, name: &str) -> Option { diff --git a/linux-windows/crates/transcription/src/script_lock.rs b/linux-windows/crates/transcription/src/script_lock.rs new file mode 100644 index 0000000..5493fd3 --- /dev/null +++ b/linux-windows/crates/transcription/src/script_lock.rs @@ -0,0 +1,322 @@ +//! What Qwen3-ASR may write once it's told a language, as ScriptLock.swift and +//! WritingScript.swift decide it on the Mac: the language's letters ([`WritingScript`]), Latin +//! letters for words in English (as the Sinhala fine-tune writes them), and the digits, +//! punctuation and spaces every language shares. +//! +//! Telling Qwen3-ASR a language only stands in for its own guess at it, so a language spoken +//! clearly still came out in its own letters (LiveTranscribe-0415). Locked, a model told English +//! can't write Sinhala or Chinese, whatever it hears. It can still write German, which has the +//! same letters. +//! +//! The model writes UTF-8 a token at a time, and a token can hold part of a character, so the +//! lock reads a token's bytes after whatever character the tokens before it left unfinished +//! ([`PartialCharacter`]). + +use std::ops::RangeInclusive; + +use serde::{Deserialize, Serialize}; + +/// The letters a language is written in, by its ISO 15924 code, as the catalog's language +/// choices name them (`"script": "Sinh"`). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Deserialize, Serialize)] +pub enum WritingScript { + #[serde(rename = "Latn")] + Latin, + #[serde(rename = "Cyrl")] + Cyrillic, + #[serde(rename = "Grek")] + Greek, + #[serde(rename = "Arab")] + Arabic, + #[serde(rename = "Deva")] + Devanagari, + #[serde(rename = "Sinh")] + Sinhala, + #[serde(rename = "Thai")] + Thai, + /// Chinese characters, for Chinese and Cantonese. + #[serde(rename = "Hani")] + Han, + /// Chinese characters, hiragana and katakana. + #[serde(rename = "Jpan")] + Japanese, + /// Hangul. + #[serde(rename = "Kore")] + Korean, +} + +/// CJK symbols and punctuation (。、「」), and the full-width forms (，！？). +const CJK_PUNCTUATION: [RangeInclusive; 2] = [0x3000..=0x303F, 0xFF00..=0xFFEF]; +/// The radicals, the unified ideographs and their extensions, and the compatibility ideographs. +const HAN_CHARACTERS: [RangeInclusive; 5] = [ + 0x2E80..=0x2FDF, + 0x3400..=0x4DBF, + 0x4E00..=0x9FFF, + 0xF900..=0xFAFF, + 0x20000..=0x2FA1F, +]; + +impl WritingScript { + /// Its letters and its own punctuation, as Unicode blocks. + fn blocks(self) -> Vec> { + match self { + // Latin-1's letters, Extended-A and -B, the modifier letters (the ʻokina) and Extended + // Additional (Vietnamese). + Self::Latin => vec![0x00C0..=0x024F, 0x02B0..=0x02FF, 0x1E00..=0x1EFF], + Self::Cyrillic => vec![RangeInclusive::new(0x0400, 0x052F)], + Self::Greek => vec![0x0370..=0x03FF, 0x1F00..=0x1FFF], + Self::Arabic => vec![ + 0x0600..=0x06FF, + 0x0750..=0x077F, + 0x08A0..=0x08FF, + 0xFB50..=0xFDFF, + 0xFE70..=0xFEFF, + ], + Self::Devanagari => vec![0x0900..=0x097F, 0xA8E0..=0xA8FF], + Self::Sinhala => vec![RangeInclusive::new(0x0D80, 0x0DFF)], + Self::Thai => vec![RangeInclusive::new(0x0E00, 0x0E7F)], + Self::Han => [CJK_PUNCTUATION.as_slice(), &HAN_CHARACTERS].concat(), + Self::Japanese => [ + CJK_PUNCTUATION.as_slice(), + &HAN_CHARACTERS, + &[0x3040..=0x30FF, 0x31F0..=0x31FF], + ] + .concat(), + Self::Korean => [ + CJK_PUNCTUATION.as_slice(), + &[0x1100..=0x11FF, 0x3130..=0x318F, 0xAC00..=0xD7AF], + ] + .concat(), + } + } +} + +/// What every language may write: ASCII; Latin-1's symbols and punctuation (« » ¿ ° × ÷); the +/// combining diacritics; general punctuation (– — “ ” …, and the zero-width joiners Sinhala's +/// conjuncts are written with); currency; letterlike symbols and number forms (℃ ½). +const SHARED: [RangeInclusive; 8] = [ + 0x0000..=0x007F, + 0x00A0..=0x00BF, + 0x00D7..=0x00D7, + 0x00F7..=0x00F7, + 0x0300..=0x036F, + 0x2000..=0x206F, + 0x20A0..=0x20CF, + 0x2100..=0x218F, +]; + +/// What a told language lets Qwen3-ASR write. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct ScriptLock { + script: WritingScript, + /// The code points it allows, in order and apart. + allowed: Vec>, +} + +impl ScriptLock { + pub fn new(script: WritingScript) -> Self { + let mut ranges = SHARED.to_vec(); + if script != WritingScript::Latin { + ranges.extend(WritingScript::Latin.blocks()); + } + ranges.extend(script.blocks()); + ranges.sort_by_key(|range| *range.start()); + let mut allowed: Vec> = Vec::new(); + for range in ranges { + match allowed.last_mut() { + Some(last) if *range.start() <= last.end() + 1 => { + *last = *last.start()..=(*last.end()).max(*range.end()); + } + _ => allowed.push(range), + } + } + Self { script, allowed } + } + + pub fn script(&self) -> WritingScript { + self.script + } + + pub fn allows(&self, scalar: u32) -> bool { + self.allows_any(scalar, scalar) + } + + /// Whether it allows any code point from `low` through `high`. + pub fn allows_any(&self, low: u32, high: u32) -> bool { + self.allowed + .iter() + .any(|range| *range.start() <= high && low <= *range.end()) + } + + /// The character left unfinished once `bytes` follow `partial`, the one the tokens before + /// them left unfinished: [`PartialCharacter::NOTHING`] if they end on a whole character. + /// `None` if they write a character the lock doesn't allow, begin one no allowed character + /// begins with, or aren't UTF-8. + pub fn partial_after(&self, partial: PartialCharacter, bytes: &[u8]) -> Option { + let mut partial = partial; + for &byte in bytes { + let next = partial.appending(byte)?; + if next.is_whole() { + if !self.allows(next.value) { + return None; + } + partial = PartialCharacter::NOTHING; + } else { + if !self.allows_any(next.lowest(), next.highest()) { + return None; + } + partial = next; + } + } + Some(partial) + } +} + +/// A character whose first UTF-8 bytes are written and whose last aren't yet. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct PartialCharacter { + /// How many bytes the character takes, which its first byte says. + length: u32, + /// How many of them are written. + count: u32, + /// The bits of the code point the written bytes hold. + value: u32, +} + +impl PartialCharacter { + /// No character is unfinished. + pub const NOTHING: Self = Self { + length: 0, + count: 0, + value: 0, + }; + + /// Its bytes are all written, or none is unfinished. + pub fn is_whole(self) -> bool { + self.count == self.length + } + + /// The lowest code point the character can still turn out to be, leaving out the overlong + /// encodings, which write a code point in more bytes than it takes. + fn lowest(self) -> u32 { + let minimum = [0, 0, 0x80, 0x800, 0x1_0000][self.length as usize]; + (self.value << (6 * (self.length - self.count))).max(minimum) + } + + /// The highest code point the character can still turn out to be. + fn highest(self) -> u32 { + let missing = 6 * (self.length - self.count); + (self.value << missing) | ((1 << missing) - 1) + } + + /// The character with `byte` written next, or the one `byte` begins once this one is whole; + /// `None` if UTF-8 can't have `byte` next. + fn appending(self, byte: u8) -> Option { + if !self.is_whole() { + if byte & 0xC0 != 0x80 { + return None; + } + return Some(Self { + count: self.count + 1, + value: (self.value << 6) | u32::from(byte & 0x3F), + ..self + }); + } + let (length, value) = match byte { + 0x00..=0x7F => (1, u32::from(byte)), + 0xC2..=0xDF => (2, u32::from(byte & 0x1F)), + 0xE0..=0xEF => (3, u32::from(byte & 0x0F)), + 0xF0..=0xF4 => (4, u32::from(byte & 0x07)), + _ => return None, + }; + Some(Self { + length, + count: 1, + value, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn read(text: &str, script: WritingScript) -> Option { + ScriptLock::new(script).partial_after(PartialCharacter::NOTHING, text.as_bytes()) + } + + #[test] + fn english_writes_latin_letters_digits_and_punctuation() { + let whole = Some(PartialCharacter::NOTHING); + assert_eq!( + read( + "Send the café’s report — 10% off, “naïve” résumé.\n- Item 2", + WritingScript::Latin + ), + whole + ); + assert_eq!(read("Việt Nam, Łódź, ʻŌlelo", WritingScript::Latin), whole); + assert_eq!(read("ක", WritingScript::Latin), None); + assert_eq!(read("我们", WritingScript::Latin), None); + assert_eq!(read("。", WritingScript::Latin), None, "CJK punctuation is CJK's"); + assert_eq!(read("Привет", WritingScript::Latin), None); + } + + /// The Sinhala fine-tune writes English words in English letters, and every lock but Latin's + /// allows them. + #[test] + fn other_languages_write_their_letters_and_english_words() { + let whole = Some(PartialCharacter::NOTHING); + assert_eq!(read("මම Zoom meeting එකට යනවා.", WritingScript::Sinhala), whole); + assert_eq!( + read("ප්‍රශ්නය", WritingScript::Sinhala), + whole, + "with its conjuncts' joiner" + ); + assert_eq!(read("我们", WritingScript::Sinhala), None); + assert_eq!(read("我们开会。Hello", WritingScript::Han), whole); + assert_eq!(read("ひらがなとカタカナと漢字", WritingScript::Japanese), whole); + assert_eq!(read("한국어", WritingScript::Korean), whole); + assert_eq!(read("한국어", WritingScript::Japanese), None); + } + + /// "ක" is E0 B6 9A, and a token can end after its first bytes. + #[test] + fn a_character_split_across_tokens_is_read_across_them() { + let sinhala = ScriptLock::new(WritingScript::Sinhala); + let english = ScriptLock::new(WritingScript::Latin); + let started = sinhala.partial_after(PartialCharacter::NOTHING, &[0xE0, 0xB6]).unwrap(); + assert!(!started.is_whole()); + assert_eq!(sinhala.partial_after(started, &[0x9A]), Some(PartialCharacter::NOTHING)); + assert_eq!( + sinhala.partial_after(started, &[0x61]), + None, + "it must be finished first" + ); + // Told English, the first byte already can't begin anything it allows. + assert_eq!(english.partial_after(PartialCharacter::NOTHING, &[0xE0]), None); + // "—" (U+2014, E2 80 94) is allowed, so its first byte is. + assert!(english.partial_after(PartialCharacter::NOTHING, &[0xE2]).is_some()); + } + + #[test] + fn bytes_that_arent_utf8_are_refused() { + let english = ScriptLock::new(WritingScript::Latin); + for bytes in [&[0x80][..], &[0xC0, 0x80], &[0xFF]] { + assert_eq!( + english.partial_after(PartialCharacter::NOTHING, bytes), + None, + "{bytes:x?}" + ); + } + } + + #[test] + fn scripts_are_read_by_their_iso_15924_codes() { + assert_eq!( + serde_json::from_str::(r#""Sinh""#).unwrap(), + WritingScript::Sinhala + ); + assert!(serde_json::from_str::(r#""Sinhala""#).is_err()); + } +} diff --git a/linux-windows/crates/transcription/src/sherpa.rs b/linux-windows/crates/transcription/src/sherpa.rs index 226af53..7d05a55 100644 --- a/linux-windows/crates/transcription/src/sherpa.rs +++ b/linux-windows/crates/transcription/src/sherpa.rs @@ -42,12 +42,18 @@ impl Family { } } - /// Whether its models are told which language to write: Cohere Transcribe can't find it, and - /// writes nothing at all without one. NeMo transducers find it themselves. + /// Whether its models can be told which language to write: Cohere Transcribe must be. NeMo + /// transducers can't be, and find it themselves. pub fn takes_language(self) -> bool { matches!(self, Self::CohereTranscribe) } + /// Whether its models find the language themselves: Cohere Transcribe can't, and writes + /// nothing at all without one. + pub fn finds_language(self) -> bool { + !matches!(self, Self::CohereTranscribe) + } + /// Points `config` at the files, whose paths `path` gives by role. `language` is the one a /// family that takes a language writes unless a clip is told another. fn configure(self, config: &mut OfflineModelConfig, path: impl Fn(Role) -> String, language: Option<&str>) { @@ -146,20 +152,23 @@ pub struct SherpaTranscriber { recognizer: OfflineRecognizer, family: Family, threads: usize, - /// The languages the model can be told, its default first; empty for a model that finds it. + /// The languages the model can be told; empty for a model that can't be told one. languages: Vec, + /// The code of the language Automatic tells it, for a model that can't find the language. + automatic: Option, } impl SherpaTranscriber { /// Opens the model of `family` whose `files` are in `folder`, to run on `threads` of the CPU's /// threads. Each file must have been checked first (see the module's documentation). A family - /// that takes a language is told one of `languages` for each clip, the first unless the clip - /// asks for another; the catalog gives it at least one. + /// that takes a language is told one of `languages` for each clip: the one the clip asks for, + /// or Automatic's, `automatic`, which the catalog gives a model that can't find it. pub(crate) fn open( folder: &Path, family: Family, files: &[ModelFile], languages: &[LanguageChoice], + automatic: Option<&str>, threads: usize, ) -> Result { let mut paths = Vec::with_capacity(family.roles().len()); @@ -181,13 +190,14 @@ impl SherpaTranscriber { }; // Only a family that takes a language keeps them. - let languages = if family.takes_language() { - languages.to_vec() + let (languages, automatic) = if family.takes_language() { + (languages.to_vec(), automatic.map(str::to_owned)) } else { - Vec::new() + (Vec::new(), None) }; let mut config = OfflineRecognizerConfig::default(); - let default_language = languages.first().map(|choice| choice.code.as_str()); + let default_language = + chosen_language(&languages, automatic.as_deref(), None).map(|choice| choice.code.as_str()); family.configure(&mut config.model_config, path, default_language); let threads = threads.max(1); config.model_config.num_threads = i32::try_from(threads).unwrap_or(i32::MAX); @@ -207,6 +217,7 @@ impl SherpaTranscriber { family, threads, languages, + automatic, }) } @@ -220,9 +231,9 @@ impl SherpaTranscriber { self.threads } - /// The transcript of `samples`, 16 kHz mono. A model told which language to write writes - /// `language` (the Language setting, a code) if it's one of its languages, and otherwise its - /// first ([`chosen_language`]). + /// The transcript of `samples`, 16 kHz mono. A model that can be told which language to write + /// writes `language` (the Language setting, a code) if it's one of its languages, and otherwise + /// Automatic's ([`chosen_language`]). pub fn transcribe(&self, samples: &[f32], language: Option<&str>) -> Result { if samples.len() < MINIMUM_SAMPLES { return Ok(SherpaTranscript { @@ -238,7 +249,7 @@ impl SherpaTranscriber { let stream = self.recognizer.create_stream(); // Read for each clip, over the one the model was opened with: a change of language // doesn't load the model again. - let told = chosen_language(&self.languages, language); + let told = chosen_language(&self.languages, self.automatic.as_deref(), language); if let Some(choice) = told { stream.set_option("language", &choice.code); } diff --git a/linux-windows/crates/transcription/src/speech_to_text.rs b/linux-windows/crates/transcription/src/speech_to_text.rs index 67fdde7..04162a5 100644 --- a/linux-windows/crates/transcription/src/speech_to_text.rs +++ b/linux-windows/crates/transcription/src/speech_to_text.rs @@ -4,7 +4,7 @@ use std::fmt; use std::path::Path; -use crate::catalog::{Engine, VerifiedModel}; +use crate::catalog::{Engine, LanguageChoice, VerifiedModel, chosen_language}; use crate::qwen3_asr::{self, DeviceChoice, OpenVinoModel, Transcriber, open_transcriber}; use crate::sherpa::{self, SherpaTranscriber}; @@ -20,7 +20,11 @@ pub struct Transcript { /// A loaded model. pub enum SpeechToText { - Qwen3Asr(Box>), + Qwen3Asr { + transcriber: Box>, + /// The languages it can be told, from the catalog; none for a model outside it. + languages: Vec, + }, SherpaOnnx(SherpaTranscriber), } @@ -65,12 +69,15 @@ impl SpeechToText { /// keeps compiled models) are for the OpenVINO models; sherpa-onnx runs on the CPU. pub fn open(model: &VerifiedModel<'_>, device: &DeviceChoice, cache: Option<&Path>) -> Result { match &model.model().engine { - Engine::OpenVino { .. } => Self::open_converted(model.folder(), device, cache), + Engine::OpenVino { .. } => { + Self::open_qwen3_asr(model.folder(), device, cache, &model.model().language_choices) + } Engine::SherpaOnnx { family, .. } => SherpaTranscriber::open( model.folder(), *family, &model.model().files, &model.model().language_choices, + model.model().automatic_language.as_deref(), sherpa::default_threads(), ) .map(Self::SherpaOnnx) @@ -78,26 +85,49 @@ impl SpeechToText { } } - /// Opens the Qwen3-ASR model the setup kit converted into `folder`, a catalog one or another. + /// Opens a Qwen3-ASR model the setup kit converted into `folder`, outside the catalog. It + /// isn't told a language. pub fn open_converted(folder: &Path, device: &DeviceChoice, cache: Option<&Path>) -> Result { + Self::open_qwen3_asr(folder, device, cache, &[]) + } + + fn open_qwen3_asr( + folder: &Path, + device: &DeviceChoice, + cache: Option<&Path>, + languages: &[LanguageChoice], + ) -> Result { open_transcriber(folder, device, cache) - .map(|transcriber| Self::Qwen3Asr(Box::new(transcriber))) + .map(|transcriber| Self::Qwen3Asr { + transcriber: Box::new(transcriber), + languages: languages.to_vec(), + }) .map_err(OpenError::Qwen3Asr) } /// The transcript of `samples`, 16 kHz mono. `language` is the Language setting - /// (`sttLanguage`, a code): a model told which language to write, Cohere Transcribe, writes - /// it if it's one of its languages, and otherwise its first; the others find the language. + /// (`sttLanguage`, a code, `None` for Automatic): a model that can be told which language to + /// write writes it if it's one of its languages, and otherwise Automatic's: Qwen3-ASR finds + /// the language, and Cohere Transcribe writes English. Parakeet always finds it. Qwen3-ASR + /// takes the language as a hint, so it's also kept to the language's letters + /// ([`crate::script_lock`]). pub fn transcribe(&mut self, samples: &[f32], language: Option<&str>) -> Result { match self { - Self::Qwen3Asr(transcriber) => transcriber - .transcribe(samples) - .map(|transcription| Transcript { - text: transcription.text, - language: transcription.language, - tokens: transcription.tokens, - }) - .map_err(TranscribeError::Qwen3Asr), + Self::Qwen3Asr { transcriber, languages } => { + let choice = chosen_language(languages, None, language); + transcriber + .transcribe( + samples, + choice.map(|choice| choice.name.as_str()), + choice.and_then(|choice| choice.script), + ) + .map(|transcription| Transcript { + text: transcription.text, + language: transcription.language, + tokens: transcription.tokens, + }) + .map_err(TranscribeError::Qwen3Asr) + } Self::SherpaOnnx(transcriber) => transcriber .transcribe(samples, language) .map(|transcript| Transcript { @@ -112,7 +142,7 @@ impl SpeechToText { /// Where the model's passes run, as the terminal and Settings say it. pub fn placement(&self) -> String { match self { - Self::Qwen3Asr(transcriber) => transcriber.model().placement(), + Self::Qwen3Asr { transcriber, .. } => transcriber.model().placement(), Self::SherpaOnnx(transcriber) => format!("CPU, {} threads (sherpa-onnx)", transcriber.threads()), } }