From c0162495bc8c46a64e154580889a4931245a87d2 Mon Sep 17 00:00:00 2001 From: michael Date: Tue, 14 Jul 2026 12:54:41 +0200 Subject: [PATCH 01/21] Keep ai.onnxruntime classes from R8 stripping in release builds R8 was obfuscating/removing ONNX Runtime's Java classes, which its native JNI layer (libonnxruntime4j_jni.so) looks up by name via reflection. This caused a JNI abort ("java_class == null" in GetMethodID) inside OrtSession.run, crashing the NLLB offline translation path in release builds. --- app/proguard-rules.pro | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/app/proguard-rules.pro b/app/proguard-rules.pro index 9abcb3a..440a43b 100644 --- a/app/proguard-rules.pro +++ b/app/proguard-rules.pro @@ -1,2 +1,8 @@ # Add project specific ProGuard rules here. -keep class com.k2fsa.sherpa.onnx.** { *; } + +# ONNX Runtime's native JNI code (libonnxruntime4j_jni.so) looks up Java classes/methods +# by name via reflection (GetMethodID). R8 stripping/renaming them causes a JNI abort +# (java_class == null) inside OrtSession.run, crashing the NLLB offline translation path. +-keep class ai.onnxruntime.** { *; } +-dontwarn ai.onnxruntime.** From 4e4ad0225b62bf86de3bddf4f863486d71f4b26a Mon Sep 17 00:00:00 2001 From: michael Date: Tue, 14 Jul 2026 12:54:45 +0200 Subject: [PATCH 02/21] Declare TTS_SERVICE queries element for API 30+ package visibility Without this, TextToSpeech cannot see any installed TTS engine on Android 11+ and initialization silently fails with status ERROR. --- app/src/main/AndroidManifest.xml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 340a48a..1153241 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -19,6 +19,14 @@ + + + + + + + Date: Tue, 14 Jul 2026 12:54:52 +0200 Subject: [PATCH 03/21] Fix TTS speaking wrong language via Voice API selection TtsEngine.initialize() had two divergent code paths: the cold-init path checked setLanguage()'s result and fell back to English on missing data, but the warm re-init path (hit every time the target language changes after the first call) discarded the result entirely with no fallback or logging. Some TTS engines also only fully honor the modern Voice API, so setLanguage() alone can report success without actually switching the active synthesis voice. Both paths now go through a shared applyLocale() that checks the setLanguage() result, explicitly selects a matching offline Voice, logs the outcome, and surfaces a warning via TranslationUiState when the requested language falls back to English. --- .../service/TranslationService.kt | 10 ++-- .../instantvoicetranslate/tts/TtsEngine.kt | 56 ++++++++++++++++--- 2 files changed, 53 insertions(+), 13 deletions(-) diff --git a/app/src/main/java/com/example/instantvoicetranslate/service/TranslationService.kt b/app/src/main/java/com/example/instantvoicetranslate/service/TranslationService.kt index 95cba0e..f6eb079 100644 --- a/app/src/main/java/com/example/instantvoicetranslate/service/TranslationService.kt +++ b/app/src/main/java/com/example/instantvoicetranslate/service/TranslationService.kt @@ -220,7 +220,12 @@ class TranslationService : Service() { translator } - // Initialize TTS + uiState.setRunning(true) + uiState.setError(null) + + // Initialize TTS (after the error reset above, so a locale- + // fallback warning set inside applyLocale() isn't immediately + // cleared again). val ttsLocale = Locale.forLanguageTag(settings.targetLanguage) ttsEngine.initialize(ttsLocale) ttsEngine.setSpeechRate(settings.ttsSpeed) @@ -228,9 +233,6 @@ class TranslationService : Service() { ttsEngine.setVolume(settings.ttsVolume) ttsEngine.setDuckingEnabled(settings.duckAudio) - uiState.setRunning(true) - uiState.setError(null) - // Start audio capture using the source passed via intent val rawAudioFlow = audioCaptureManager.startCapture(currentAudioSource) diff --git a/app/src/main/java/com/example/instantvoicetranslate/tts/TtsEngine.kt b/app/src/main/java/com/example/instantvoicetranslate/tts/TtsEngine.kt index 961e5ee..a89e23b 100644 --- a/app/src/main/java/com/example/instantvoicetranslate/tts/TtsEngine.kt +++ b/app/src/main/java/com/example/instantvoicetranslate/tts/TtsEngine.kt @@ -8,6 +8,7 @@ import android.os.Bundle import android.speech.tts.TextToSpeech import android.speech.tts.UtteranceProgressListener import android.util.Log +import com.example.instantvoicetranslate.data.TranslationUiState import dagger.hilt.android.qualifiers.ApplicationContext import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.CoroutineScope @@ -39,7 +40,8 @@ import javax.inject.Singleton */ @Singleton class TtsEngine @Inject constructor( - @param:ApplicationContext private val context: Context + @param:ApplicationContext private val context: Context, + private val uiState: TranslationUiState ) { companion object { private const val TAG = "TtsEngine" @@ -79,20 +81,14 @@ class TtsEngine @Inject constructor( fun initialize(locale: Locale = Locale.forLanguageTag("ru"), onReady: (() -> Unit)? = null) { // If already initialized with a working engine, just update locale if (_isInitialized.value && tts != null) { - tts?.language = locale + applyLocale(locale) onReady?.invoke() return } tts = TextToSpeech(context) { status -> if (status == TextToSpeech.SUCCESS) { - val result = tts?.setLanguage(locale) - if (result == TextToSpeech.LANG_MISSING_DATA || - result == TextToSpeech.LANG_NOT_SUPPORTED - ) { - Log.w(TAG, "Language $locale not supported, trying English") - tts?.language = Locale.US - } + applyLocale(locale) // Use USAGE_ASSISTANT so that TTS audio is NOT captured by // MediaProjection (AudioPlaybackCapture). Android automatically @@ -142,6 +138,48 @@ class TtsEngine @Inject constructor( } } + /** + * Applies [locale] to the active TTS engine and returns whether it was + * actually applied (false if the engine fell back to English). + * + * Sets both the legacy [TextToSpeech.setLanguage] locale AND an explicit + * [android.speech.tts.Voice] match. Some TTS engines (e.g. custom + * offline engines) only fully honor the modern Voice API — their + * setLanguage() compatibility shim can report success without actually + * switching the active synthesis voice, which is what caused this app to + * speak translated text in the wrong language even though setLanguage() + * looked like it succeeded. + */ + private fun applyLocale(locale: Locale): Boolean { + val engine = tts ?: return false + + val result = engine.setLanguage(locale) + val requestedLocaleSupported = result != TextToSpeech.LANG_MISSING_DATA && + result != TextToSpeech.LANG_NOT_SUPPORTED + var effectiveLocale = locale + + if (!requestedLocaleSupported) { + Log.w(TAG, "Language $locale not supported (result=$result), falling back to English") + engine.language = Locale.US + effectiveLocale = Locale.US + val languageName = locale.getDisplayLanguage(Locale.ENGLISH) + uiState.setError("TTS voice for $languageName not available, using English instead") + } + + val matchedVoice = runCatching { engine.voices }.getOrNull() + ?.filter { !it.isNetworkConnectionRequired && it.locale.language == effectiveLocale.language } + ?.let { candidates -> + candidates.firstOrNull { it.locale.country == effectiveLocale.country } + ?: candidates.firstOrNull() + } + if (matchedVoice != null) { + engine.voice = matchedVoice + } + + Log.i(TAG, "TTS active voice: ${engine.voice?.name} (${engine.voice?.locale}), requested=$locale") + return requestedLocaleSupported + } + /** * Splits text into sentences and enqueues each one separately. * This allows Android TTS to begin speaking the first sentence From c5e497f8cf7d118082d8e7031f973d79a524af87 Mon Sep 17 00:00:00 2001 From: michael Date: Tue, 14 Jul 2026 12:55:00 +0200 Subject: [PATCH 04/21] Prepare packaging for bundled offline model assets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gitignore app/src/main/assets/models/ (fetched locally, not committed — multi-GB ASR/NLLB model binaries) and configure the release build to store them uncompressed (required for AssetManager.openFd()-based extraction progress sizing) with a larger Gradle heap to package them. No model files or extraction logic yet — this only prepares the build config ahead of that. --- .gitignore | 3 +++ app/build.gradle.kts | 10 ++++++++++ gradle.properties | 2 +- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index a95ad84..0c05442 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,9 @@ keystore.properties *.apk *.aab +# Bundled model assets (fetched locally, not committed — multi-GB binaries) +app/src/main/assets/models/ + # Android Studio captures/ .externalNativeBuild/ diff --git a/app/build.gradle.kts b/app/build.gradle.kts index de9005b..aca3487 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -82,6 +82,16 @@ android { } } + androidResources { + // Bundled ASR/NLLB model files (app/src/main/assets/models/) are large, + // already-dense binaries (quantized ONNX weights) or small text/config + // files read via AssetManager.openFd() for first-launch-extraction + // progress sizing (openFd requires STORED, i.e. uncompressed, zip + // entries). No other .onnx/.txt/.model/.vocab/.json assets exist in + // this project. + noCompress += listOf("onnx", "txt", "model", "vocab", "json") + } + lint { // onnxruntime-android:1.17.1 native lib is not 16KB-aligned, but the APK // uses libonnxruntime.so from sherpa-onnx.aar (via pickFirsts), not Maven. diff --git a/gradle.properties b/gradle.properties index d09a8d5..7db2c47 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -org.gradle.jvmargs=-Xmx2048m -Dfile.encoding=UTF-8 +org.gradle.jvmargs=-Xmx4096m -XX:MaxMetaspaceSize=1g -Dfile.encoding=UTF-8 android.useAndroidX=true kotlin.code.style=official android.nonTransitiveRClass=true From 234e6a8e4b2d86cb230a908c8175329cb96bac8c Mon Sep 17 00:00:00 2001 From: michael Date: Tue, 14 Jul 2026 13:03:11 +0200 Subject: [PATCH 05/21] Provision ASR/NLLB models from bundled APK assets when present MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds AssetModelProvisioner, which extracts model files from app/src/main/assets/models/ (if bundled) into internal storage — the native ASR/ONNX code needs real filesystem paths, not zipped asset streams. ModelDownloader and NllbModelManager both try this fast local copy first, falling back to the existing HTTP download unchanged when a language's assets aren't bundled (e.g. a language added later without a matching assets rebuild). NllbModelManager.ensureModelAvailable() also gains an optional warmup callback so status-transition/error-handling boilerplate around initializing an NllbTranslator is shared rather than duplicated per call site. No model files are committed — app/src/main/assets/models/ is gitignored and fetched independently per build. --- .../data/AssetModelProvisioner.kt | 100 ++++++++++++++++++ .../data/ModelDownloader.kt | 19 +++- .../translation/NllbModelManager.kt | 41 +++++-- 3 files changed, 151 insertions(+), 9 deletions(-) create mode 100644 app/src/main/java/com/example/instantvoicetranslate/data/AssetModelProvisioner.kt diff --git a/app/src/main/java/com/example/instantvoicetranslate/data/AssetModelProvisioner.kt b/app/src/main/java/com/example/instantvoicetranslate/data/AssetModelProvisioner.kt new file mode 100644 index 0000000..5d7d7b8 --- /dev/null +++ b/app/src/main/java/com/example/instantvoicetranslate/data/AssetModelProvisioner.kt @@ -0,0 +1,100 @@ +package com.example.instantvoicetranslate.data + +import android.content.Context +import android.util.Log +import dagger.hilt.android.qualifiers.ApplicationContext +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import java.io.File +import java.io.FileOutputStream +import javax.inject.Inject +import javax.inject.Singleton + +/** + * Copies model files bundled as APK assets into real filesystem storage + * (native ASR/ONNX code requires real file paths, not zipped asset streams). + */ +@Singleton +class AssetModelProvisioner @Inject constructor( + @param:ApplicationContext private val context: Context +) { + companion object { + private const val TAG = "AssetModelProvisioner" + private const val BUFFER_SIZE = 16 * 1024 + } + + /** + * Copies [expectedFiles] from `assets/$assetSubdir/` into [targetDir] on + * the real filesystem. Returns false if the assets subdirectory is absent + * or doesn't contain every expected file — callers should then fall back + * to a network download, keeping the app functional for any language + * added later without a matching bundled-assets rebuild. + */ + suspend fun installFromAssets( + assetSubdir: String, + targetDir: File, + expectedFiles: List, + onProgress: (ModelStatus.Downloading) -> Unit = {}, + ): Boolean = withContext(Dispatchers.IO) { + val available = runCatching { context.assets.list(assetSubdir)?.toSet() }.getOrNull() + if (available.isNullOrEmpty() || !expectedFiles.all { it in available }) { + Log.i(TAG, "Assets not bundled for '$assetSubdir' (or incomplete); will use network fallback") + return@withContext false + } + + targetDir.mkdirs() + + // Best-effort byte sizes via openFd() (only works for uncompressed/ + // noCompress entries). Falls back to equal-weight-per-file progress + // if unavailable — extraction itself works either way since + // assets.open() transparently inflates compressed entries too. + val sizes = expectedFiles.associateWith { name -> + runCatching { context.assets.openFd("$assetSubdir/$name").use { it.length } }.getOrDefault(-1L) + } + val useByteProgress = sizes.values.all { it > 0 } + val totalUnits = if (useByteProgress) sizes.values.sum() else expectedFiles.size.toLong() + var doneUnits = 0L + + try { + for (name in expectedFiles) { + val target = File(targetDir, name) + if (target.exists() && target.length() > 0) { + doneUnits += if (useByteProgress) sizes.getValue(name) else 1L + continue + } + + onProgress(ModelStatus.Downloading((doneUnits.toFloat() / totalUnits).coerceIn(0f, 1f), name)) + + val tmp = File(targetDir, "$name.tmp") + context.assets.open("$assetSubdir/$name").use { input -> + FileOutputStream(tmp).use { output -> + val buffer = ByteArray(BUFFER_SIZE) + var read: Int + var fileDone = 0L + while (input.read(buffer).also { read = it } != -1) { + output.write(buffer, 0, read) + fileDone += read + if (useByteProgress) { + val units = (doneUnits + fileDone).toFloat() / totalUnits + onProgress(ModelStatus.Downloading(units.coerceIn(0f, 1f), name)) + } + } + output.fd.sync() + } + } + if (!tmp.renameTo(target)) { + Log.e(TAG, "Rename failed for $name") + tmp.delete() + return@withContext false + } + doneUnits += if (useByteProgress) sizes.getValue(name) else 1L + } + val complete = expectedFiles.all { File(targetDir, it).let { f -> f.exists() && f.length() > 0 } } + Log.i(TAG, "Asset extraction for '$assetSubdir' complete=$complete") + complete + } catch (e: Exception) { + Log.e(TAG, "Asset extraction failed for '$assetSubdir'", e) + false + } + } +} diff --git a/app/src/main/java/com/example/instantvoicetranslate/data/ModelDownloader.kt b/app/src/main/java/com/example/instantvoicetranslate/data/ModelDownloader.kt index a84e564..605a974 100644 --- a/app/src/main/java/com/example/instantvoicetranslate/data/ModelDownloader.kt +++ b/app/src/main/java/com/example/instantvoicetranslate/data/ModelDownloader.kt @@ -39,7 +39,8 @@ data class LanguageModelConfig( @Singleton class ModelDownloader @Inject constructor( - @param:ApplicationContext private val context: Context + @param:ApplicationContext private val context: Context, + private val assetModelProvisioner: AssetModelProvisioner ) { companion object { private const val TAG = "ModelDownloader" @@ -193,6 +194,22 @@ class ModelDownloader @Inject constructor( _status.value = ModelStatus.Ready return } + + val config = LANGUAGE_MODELS[language] + if (config != null) { + val installed = assetModelProvisioner.installFromAssets( + assetSubdir = "models/asr/${config.dirName}", + targetDir = File(context.filesDir, config.dirName), + expectedFiles = config.files.map { it.name }, + onProgress = { _status.value = it }, + ) + if (installed) { + _status.value = ModelStatus.Ready + Log.i(TAG, "ASR model for '$language' provisioned from bundled assets") + return + } + } + downloadModel(language) } diff --git a/app/src/main/java/com/example/instantvoicetranslate/translation/NllbModelManager.kt b/app/src/main/java/com/example/instantvoicetranslate/translation/NllbModelManager.kt index 54697e1..21c00fc 100644 --- a/app/src/main/java/com/example/instantvoicetranslate/translation/NllbModelManager.kt +++ b/app/src/main/java/com/example/instantvoicetranslate/translation/NllbModelManager.kt @@ -2,6 +2,7 @@ package com.example.instantvoicetranslate.translation import android.content.Context import android.util.Log +import com.example.instantvoicetranslate.data.AssetModelProvisioner import com.example.instantvoicetranslate.data.ModelStatus import com.example.instantvoicetranslate.data.downloadToFile import dagger.hilt.android.qualifiers.ApplicationContext @@ -22,7 +23,8 @@ import javax.inject.Singleton */ @Singleton class NllbModelManager @Inject constructor( - @param:ApplicationContext private val context: Context + @param:ApplicationContext private val context: Context, + private val assetModelProvisioner: AssetModelProvisioner ) { companion object { private const val TAG = "NllbModelManager" @@ -79,15 +81,38 @@ class NllbModelManager @Inject constructor( } /** - * Downloads the NLLB model if not already present. - * Updates [status] with download progress. + * Provisions the NLLB model if not already present — tries bundled APK + * assets first (fast local copy), falling back to a network download. + * Updates [status] with progress. If [warmup] is given, it runs after + * the model is ready with status transitions around it (e.g. to + * initialize-then-release an [NllbTranslator] instance). */ - suspend fun ensureModelAvailable() { - if (isModelReady()) { - _status.value = ModelStatus.Ready - return + suspend fun ensureModelAvailable(warmup: (suspend () -> Unit)? = null) { + if (!isModelReady()) { + val installed = assetModelProvisioner.installFromAssets( + assetSubdir = "models/nllb", + targetDir = getModelDir(), + expectedFiles = ALL_FILES.map { it.name }, + onProgress = { _status.value = it }, + ) + if (!installed) { + downloadModel() + } + } + + if (!isModelReady()) return + _status.value = ModelStatus.Ready + + if (warmup != null) { + _status.value = ModelStatus.Initializing("Warming up offline translation...") + try { + warmup() + _status.value = ModelStatus.Ready + } catch (e: Throwable) { + Log.e(TAG, "NLLB warmup failed", e) + _status.value = ModelStatus.Error("Warmup failed: ${e.message}") + } } - downloadModel() } /** From 37ac67a65bd0b92cabeb4cec4ae46634518c75c3 Mon Sep 17 00:00:00 2001 From: michael Date: Tue, 14 Jul 2026 13:03:16 +0200 Subject: [PATCH 06/21] Auto-provision NLLB at launch instead of requiring a manual download Previously, offline-mode pre-loading only touched the NLLB model if it was already downloaded, silently skipping provisioning otherwise and requiring a manual trip to Settings. With bundled assets making provisioning a fast local copy rather than a multi-hundred-MB network download, it's now safe to call ensureModelAvailable() unconditionally whenever offline mode is enabled. SettingsViewModel's manual "download" button now goes through the same shared warmup-callback path added to NllbModelManager. --- .../ui/viewmodel/MainViewModel.kt | 16 ++++++---- .../ui/viewmodel/SettingsViewModel.kt | 30 ++++++------------- 2 files changed, 20 insertions(+), 26 deletions(-) diff --git a/app/src/main/java/com/example/instantvoicetranslate/ui/viewmodel/MainViewModel.kt b/app/src/main/java/com/example/instantvoicetranslate/ui/viewmodel/MainViewModel.kt index 35d4e6f..a540505 100644 --- a/app/src/main/java/com/example/instantvoicetranslate/ui/viewmodel/MainViewModel.kt +++ b/app/src/main/java/com/example/instantvoicetranslate/ui/viewmodel/MainViewModel.kt @@ -152,13 +152,19 @@ class MainViewModel @Inject constructor( ttsEngine.initialize(ttsLocale) } - // 5. If offline mode is enabled and NLLB model is downloaded, pre-load it - if (currentSettings.offlineMode && nllbModelManager.isModelReady()) { + // 5. If offline mode is enabled, provision (bundled assets or + // download) and pre-load the NLLB model. No isModelReady() gate + // here: with bundled assets this is a fast local copy, not a + // multi-hundred-MB network download, so it's safe to call + // unconditionally whenever offline mode is on. + if (currentSettings.offlineMode) { modelDownloader.updateStatus(ModelStatus.Initializing("Loading offline translation model...")) try { - if (!nllbTranslator.isInitialized) { - nllbTranslator.initialize() - } + nllbModelManager.ensureModelAvailable(warmup = { + if (!nllbTranslator.isInitialized) { + nllbTranslator.initialize() + } + }) Log.i(TAG, "NLLB translator pre-loaded for offline mode") } catch (e: Throwable) { // Catch Throwable (not just Exception) because UnsatisfiedLinkError diff --git a/app/src/main/java/com/example/instantvoicetranslate/ui/viewmodel/SettingsViewModel.kt b/app/src/main/java/com/example/instantvoicetranslate/ui/viewmodel/SettingsViewModel.kt index 6dfa566..7738e47 100644 --- a/app/src/main/java/com/example/instantvoicetranslate/ui/viewmodel/SettingsViewModel.kt +++ b/app/src/main/java/com/example/instantvoicetranslate/ui/viewmodel/SettingsViewModel.kt @@ -42,27 +42,15 @@ class SettingsViewModel @Inject constructor( fun downloadNllbModel() { viewModelScope.launch { - nllbModelManager.ensureModelAvailable() - - // Warmup: initialize NLLB translator to force DJL native lib - // extraction/caching while we still have internet (if needed). - // This ensures offline mode works later without any network calls. - if (nllbModelManager.isModelReady()) { - nllbModelManager.updateStatus( - ModelStatus.Initializing("Warming up offline translation...") - ) - try { - nllbTranslator.initialize() - Log.i(TAG, "NLLB warmup completed, releasing to free RAM") - nllbTranslator.release() - nllbModelManager.updateStatus(ModelStatus.Ready) - } catch (e: Throwable) { - Log.e(TAG, "NLLB warmup failed", e) - nllbModelManager.updateStatus( - ModelStatus.Error("Warmup failed: ${e.message}") - ) - } - } + // Warmup: initialize NLLB translator to force native lib + // extraction/caching while we still have internet (if needed), + // then release immediately to free RAM since this is just a + // manual "predownload" button, not an active translation session. + nllbModelManager.ensureModelAvailable(warmup = { + nllbTranslator.initialize() + Log.i(TAG, "NLLB warmup completed, releasing to free RAM") + nllbTranslator.release() + }) } } From c9f47cf244df324e22b383e195aab5fdf0c99d8b Mon Sep 17 00:00:00 2001 From: michael Date: Tue, 14 Jul 2026 13:03:22 +0200 Subject: [PATCH 07/21] Default offlineMode to true Only affects fresh installs / cleared app data. Paired with bundled model assets, this makes the app fully offline-capable out of the box with no user setup required. --- .../example/instantvoicetranslate/data/SettingsRepository.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/src/main/java/com/example/instantvoicetranslate/data/SettingsRepository.kt b/app/src/main/java/com/example/instantvoicetranslate/data/SettingsRepository.kt index b8f23bc..24f3696 100644 --- a/app/src/main/java/com/example/instantvoicetranslate/data/SettingsRepository.kt +++ b/app/src/main/java/com/example/instantvoicetranslate/data/SettingsRepository.kt @@ -37,7 +37,7 @@ data class AppSettings( val audioDiagnostics: Boolean = false, /** Custom output directory for diagnostic WAV files (empty = default). */ val diagOutputDir: String = "", - val offlineMode: Boolean = false, + val offlineMode: Boolean = true, ) @Singleton @@ -84,7 +84,7 @@ class SettingsRepository @Inject constructor( duckAudio = prefs[KEY_DUCK_AUDIO] ?: true, audioDiagnostics = prefs[KEY_AUDIO_DIAGNOSTICS] ?: false, diagOutputDir = prefs[KEY_DIAG_OUTPUT_DIR] ?: "", - offlineMode = prefs[KEY_OFFLINE_MODE] ?: false, + offlineMode = prefs[KEY_OFFLINE_MODE] ?: true, ) } From f05f765c7d2612272f380b3c72785321f6a1315f Mon Sep 17 00:00:00 2001 From: michael Date: Tue, 14 Jul 2026 13:10:24 +0200 Subject: [PATCH 08/21] Default mic muting during TTS on, with an acoustic-tail cooldown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously muteMicDuringTts defaulted to false, so on the microphone source the mic kept picking up the phone's own TTS playback acoustically, which got transcribed as gibberish and re-translated — a voice feedback loop. isSpeaking now also stays true for a short cooldown after each utterance ends, covering the speaker/room echo tail that would otherwise still leak into the next recognition window. --- .../data/SettingsRepository.kt | 4 +-- .../instantvoicetranslate/tts/TtsEngine.kt | 34 +++++++++++++++++-- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/app/src/main/java/com/example/instantvoicetranslate/data/SettingsRepository.kt b/app/src/main/java/com/example/instantvoicetranslate/data/SettingsRepository.kt index 24f3696..e299705 100644 --- a/app/src/main/java/com/example/instantvoicetranslate/data/SettingsRepository.kt +++ b/app/src/main/java/com/example/instantvoicetranslate/data/SettingsRepository.kt @@ -30,7 +30,7 @@ data class AppSettings( val showOriginalText: Boolean = true, val showPartialText: Boolean = true, val autoSpeak: Boolean = true, - val muteMicDuringTts: Boolean = false, + val muteMicDuringTts: Boolean = true, /** Request audio focus with ducking so other apps lower their volume during TTS. */ val duckAudio: Boolean = false, /** Record raw audio to WAV files for debugging (off by default). */ @@ -80,7 +80,7 @@ class SettingsRepository @Inject constructor( showOriginalText = prefs[KEY_SHOW_ORIGINAL] ?: true, showPartialText = prefs[KEY_SHOW_PARTIAL] ?: true, autoSpeak = prefs[KEY_AUTO_SPEAK] ?: true, - muteMicDuringTts = prefs[KEY_MUTE_MIC_DURING_TTS] ?: false, + muteMicDuringTts = prefs[KEY_MUTE_MIC_DURING_TTS] ?: true, duckAudio = prefs[KEY_DUCK_AUDIO] ?: true, audioDiagnostics = prefs[KEY_AUDIO_DIAGNOSTICS] ?: false, diagOutputDir = prefs[KEY_DIAG_OUTPUT_DIR] ?: "", diff --git a/app/src/main/java/com/example/instantvoicetranslate/tts/TtsEngine.kt b/app/src/main/java/com/example/instantvoicetranslate/tts/TtsEngine.kt index a89e23b..4496173 100644 --- a/app/src/main/java/com/example/instantvoicetranslate/tts/TtsEngine.kt +++ b/app/src/main/java/com/example/instantvoicetranslate/tts/TtsEngine.kt @@ -13,9 +13,11 @@ import dagger.hilt.android.qualifiers.ApplicationContext import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.cancel import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.delay import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow @@ -47,6 +49,15 @@ class TtsEngine @Inject constructor( private const val TAG = "TtsEngine" private const val UTTERANCE_TIMEOUT_MS = 30_000L + /** + * How long to keep [isSpeaking] true after an utterance finishes. + * Covers the acoustic tail (speaker decay/room echo) that would + * otherwise still be picked up by the microphone right after + * playback ends, causing the ASR to transcribe TTS's own trailing + * audio and feed it back into translation (a feedback loop). + */ + private const val SPEECH_TAIL_COOLDOWN_MS = 400L + /** * Matches sentence-ending punctuation (.!?) followed by whitespace. * Uses a fixed-length lookbehind (single char) which is safe for Java regex. @@ -64,6 +75,7 @@ class TtsEngine @Inject constructor( private var tts: TextToSpeech? = null private val utteranceId = AtomicInteger(0) private var currentDeferred: CompletableDeferred? = null + private var speakingCooldownJob: Job? = null // --- Volume & audio ducking --- private val audioManager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager @@ -103,28 +115,29 @@ class TtsEngine @Inject constructor( tts?.setOnUtteranceProgressListener(object : UtteranceProgressListener() { override fun onStart(utteranceId: String?) { + speakingCooldownJob?.cancel() _isSpeaking.value = true requestAudioFocus() } override fun onDone(utteranceId: String?) { - _isSpeaking.value = false releaseAudioFocus() currentDeferred?.complete(Unit) + scheduleSpeakingCooldown() } @Deprecated("Deprecated in Java") override fun onError(utteranceId: String?) { - _isSpeaking.value = false releaseAudioFocus() currentDeferred?.complete(Unit) + scheduleSpeakingCooldown() } override fun onError(utteranceId: String?, errorCode: Int) { Log.e(TAG, "TTS error: $errorCode") - _isSpeaking.value = false releaseAudioFocus() currentDeferred?.complete(Unit) + scheduleSpeakingCooldown() } }) @@ -195,6 +208,7 @@ class TtsEngine @Inject constructor( fun stop() { tts?.stop() + speakingCooldownJob?.cancel() _isSpeaking.value = false releaseAudioFocus() currentDeferred?.complete(Unit) @@ -249,6 +263,20 @@ class TtsEngine @Inject constructor( } } + /** + * Keeps [isSpeaking] true for [SPEECH_TAIL_COOLDOWN_MS] after an utterance + * ends, so callers muting the mic during TTS also cover the acoustic tail. + * Cancelled by [onStart] if a new utterance begins before it fires. + */ + private fun scheduleSpeakingCooldown() { + speakingCooldownJob?.cancel() + val scope = queueScope ?: return + speakingCooldownJob = scope.launch { + delay(SPEECH_TAIL_COOLDOWN_MS) + _isSpeaking.value = false + } + } + private fun startQueueProcessor() { queueScope?.cancel() val scope = CoroutineScope(Dispatchers.Default + SupervisorJob()) From e01f778f6b4e30d99fb9cb480b835b0242eafaf1 Mon Sep 17 00:00:00 2001 From: michael Date: Tue, 14 Jul 2026 13:27:49 +0200 Subject: [PATCH 09/21] Keep both conversation-direction ASR models warm in a small pool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds ConversationRecognizerPool, holding up to 2 SherpaOnnxRecognizer instances keyed by language (LRU-evicted beyond that). Switching the source language previously always released and reinitialized the ASR recognizer (a multi-second reload); now TranslationService and MainViewModel acquire from this pool instead, so once both directions of a conversation have been loaded once, switching between them is a cache hit. SherpaOnnxRecognizer is no longer Hilt-managed (the pool creates plain instances directly, since @Singleton is incompatible with holding more than one loaded language at a time) — the now-unused bindSpeechRecognizer binding is removed from AppModule. MainViewModel.preloadPipelineComponents() also pre-warms the target language's ASR model at launch (when it has one), not just the source, so both directions are ready before a swap is ever requested. --- .../asr/ConversationRecognizerPool.kt | 57 ++++++++++ .../asr/SherpaOnnxRecognizer.kt | 11 +- .../instantvoicetranslate/di/AppModule.kt | 6 - .../service/TranslationService.kt | 40 +++---- .../ui/viewmodel/MainViewModel.kt | 106 ++++++++++++++---- 5 files changed, 168 insertions(+), 52 deletions(-) create mode 100644 app/src/main/java/com/example/instantvoicetranslate/asr/ConversationRecognizerPool.kt diff --git a/app/src/main/java/com/example/instantvoicetranslate/asr/ConversationRecognizerPool.kt b/app/src/main/java/com/example/instantvoicetranslate/asr/ConversationRecognizerPool.kt new file mode 100644 index 0000000..41fcdc5 --- /dev/null +++ b/app/src/main/java/com/example/instantvoicetranslate/asr/ConversationRecognizerPool.kt @@ -0,0 +1,57 @@ +package com.example.instantvoicetranslate.asr + +import android.util.Log +import javax.inject.Singleton + +/** + * Keeps up to [MAX_WARM] ASR recognizers loaded simultaneously, keyed by + * language, so swapping the conversation direction between two languages + * doesn't pay the multi-second model-load cost every time. + * + * [SherpaOnnxRecognizer] has no shared/static native state — each instance + * owns its own native handles — so holding several concurrently is safe; + * this pool exists purely to bound memory by evicting the least-recently + * acquired language once the cap is reached. + */ +@Singleton +class ConversationRecognizerPool @javax.inject.Inject constructor() { + + companion object { + private const val TAG = "ConversationRecognizerPool" + private const val MAX_WARM = 2 + } + + // Insertion order == recency: re-inserting a key on cache hit moves it to + // the end, so the front entry is always the least-recently-used one. + private val pool = LinkedHashMap() + + suspend fun acquire(language: String, modelDir: String): SpeechRecognizer { + pool[language]?.let { existing -> + if (existing.isReady.value) { + // Move to most-recently-used position. + pool.remove(language) + pool[language] = existing + return existing + } + pool.remove(language) + } + + if (pool.size >= MAX_WARM) { + val lruLanguage = pool.keys.first() + pool.remove(lruLanguage)?.release() + Log.i(TAG, "Evicted warm recognizer for '$lruLanguage' to make room for '$language'") + } + + val recognizer = SherpaOnnxRecognizer() + recognizer.initialize(modelDir, language) + pool[language] = recognizer + return recognizer + } + + fun isWarm(language: String): Boolean = pool[language]?.isReady?.value == true + + fun releaseAll() { + pool.values.forEach { it.release() } + pool.clear() + } +} diff --git a/app/src/main/java/com/example/instantvoicetranslate/asr/SherpaOnnxRecognizer.kt b/app/src/main/java/com/example/instantvoicetranslate/asr/SherpaOnnxRecognizer.kt index f7f1a26..aae91a8 100644 --- a/app/src/main/java/com/example/instantvoicetranslate/asr/SherpaOnnxRecognizer.kt +++ b/app/src/main/java/com/example/instantvoicetranslate/asr/SherpaOnnxRecognizer.kt @@ -26,11 +26,14 @@ import kotlinx.coroutines.flow.asSharedFlow import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.launch import kotlinx.coroutines.withContext -import javax.inject.Inject -import javax.inject.Singleton -@Singleton -class SherpaOnnxRecognizer @Inject constructor() : SpeechRecognizer { +/** + * Not Hilt-managed: instances are created directly (plain constructor) by + * [ConversationRecognizerPool], which can hold more than one warm instance + * at a time (one per conversation direction) — incompatible with a + * `@Singleton` binding. + */ +class SherpaOnnxRecognizer : SpeechRecognizer { companion object { private const val TAG = "SherpaOnnxRecognizer" diff --git a/app/src/main/java/com/example/instantvoicetranslate/di/AppModule.kt b/app/src/main/java/com/example/instantvoicetranslate/di/AppModule.kt index 5393eec..870ccc9 100644 --- a/app/src/main/java/com/example/instantvoicetranslate/di/AppModule.kt +++ b/app/src/main/java/com/example/instantvoicetranslate/di/AppModule.kt @@ -1,7 +1,5 @@ package com.example.instantvoicetranslate.di -import com.example.instantvoicetranslate.asr.SherpaOnnxRecognizer -import com.example.instantvoicetranslate.asr.SpeechRecognizer import com.example.instantvoicetranslate.translation.TextTranslator import com.example.instantvoicetranslate.translation.FreeTranslator import dagger.Binds @@ -14,10 +12,6 @@ import javax.inject.Singleton @InstallIn(SingletonComponent::class) abstract class AppModule { - @Binds - @Singleton - abstract fun bindSpeechRecognizer(impl: SherpaOnnxRecognizer): SpeechRecognizer - @Binds @Singleton abstract fun bindTextTranslator(impl: FreeTranslator): TextTranslator diff --git a/app/src/main/java/com/example/instantvoicetranslate/service/TranslationService.kt b/app/src/main/java/com/example/instantvoicetranslate/service/TranslationService.kt index f6eb079..7edae85 100644 --- a/app/src/main/java/com/example/instantvoicetranslate/service/TranslationService.kt +++ b/app/src/main/java/com/example/instantvoicetranslate/service/TranslationService.kt @@ -13,6 +13,7 @@ import androidx.core.app.NotificationCompat import com.example.instantvoicetranslate.MainActivity import com.example.instantvoicetranslate.R import com.example.instantvoicetranslate.InstantVoiceTranslateApp +import com.example.instantvoicetranslate.asr.ConversationRecognizerPool import com.example.instantvoicetranslate.asr.SpeechRecognizer import com.example.instantvoicetranslate.audio.AudioCaptureManager import com.example.instantvoicetranslate.audio.AudioDiagnostics @@ -55,7 +56,7 @@ class TranslationService : Service() { } @Inject lateinit var audioCaptureManager: AudioCaptureManager - @Inject lateinit var speechRecognizer: SpeechRecognizer + @Inject lateinit var recognizerPool: ConversationRecognizerPool @Inject lateinit var translator: TextTranslator @Inject lateinit var nllbTranslator: NllbTranslator @Inject lateinit var ttsEngine: TtsEngine @@ -65,6 +66,7 @@ class TranslationService : Service() { private val serviceScope = CoroutineScope(Dispatchers.Default + SupervisorJob()) private var pipelineJob: Job? = null + private var activeRecognizer: SpeechRecognizer? = null private var isPaused = false private var currentAudioSource = AudioCaptureManager.Source.MICROPHONE @@ -180,23 +182,17 @@ class TranslationService : Service() { } } - // Initialize recognizer if needed, or reinitialize if language changed - val needsInit = !speechRecognizer.isReady.value || - speechRecognizer.currentLanguage.value != srcLang - if (needsInit) { - if (speechRecognizer.isReady.value) { - speechRecognizer.release() - } - speechRecognizer.initialize( - modelDownloader.getModelDir(srcLang).absolutePath, - srcLang - ) - // Initialize punctuation model if available - if (srcLang == "en" && modelDownloader.isPunctModelReady()) { - speechRecognizer.initializePunctuation( - modelDownloader.getPunctModelDir().absolutePath - ) - } + // Acquire a warm recognizer for the source language from the + // pool — instant if already warm (pre-warmed at launch or + // kept warm from a prior conversation-direction swap), + // avoiding a multi-second release+reinit on every switch. + val recognizer = recognizerPool.acquire( + srcLang, + modelDownloader.getModelDir(srcLang).absolutePath + ) + activeRecognizer = recognizer + if (srcLang == "en" && modelDownloader.isPunctModelReady()) { + recognizer.initializePunctuation(modelDownloader.getPunctModelDir().absolutePath) } // Select translator based on offline mode @@ -252,11 +248,11 @@ class TranslationService : Service() { } // Start recognition (energy-based silence detection inside recognizer) - speechRecognizer.startRecognition(audioFlow) + recognizer.startRecognition(audioFlow) // Collect partial results -> UI launch { - speechRecognizer.partialText.collect { text -> + recognizer.partialText.collect { text -> uiState.updatePartial(text) } } @@ -268,7 +264,7 @@ class TranslationService : Service() { // Limit concurrent HTTP translation requests val translationSemaphore = Semaphore(MAX_CONCURRENT_TRANSLATIONS) - speechRecognizer.recognizedSegments.collect { segment -> + recognizer.recognizedSegments.collect { segment -> if (isPaused) return@collect // Assign a monotonic sequence number to preserve ordering @@ -339,7 +335,7 @@ class TranslationService : Service() { private fun stopPipeline() { pipelineJob?.cancel() pipelineJob = null - speechRecognizer.stopRecognition() + activeRecognizer?.stopRecognition() audioCaptureManager.stopCapture() audioCaptureManager.releaseMediaProjection() audioCaptureManager.diagnostics = null diff --git a/app/src/main/java/com/example/instantvoicetranslate/ui/viewmodel/MainViewModel.kt b/app/src/main/java/com/example/instantvoicetranslate/ui/viewmodel/MainViewModel.kt index a540505..33d60db 100644 --- a/app/src/main/java/com/example/instantvoicetranslate/ui/viewmodel/MainViewModel.kt +++ b/app/src/main/java/com/example/instantvoicetranslate/ui/viewmodel/MainViewModel.kt @@ -5,7 +5,7 @@ import android.content.Intent import android.util.Log import androidx.lifecycle.AndroidViewModel import androidx.lifecycle.viewModelScope -import com.example.instantvoicetranslate.asr.SpeechRecognizer +import com.example.instantvoicetranslate.asr.ConversationRecognizerPool import com.example.instantvoicetranslate.audio.AudioCaptureManager import com.example.instantvoicetranslate.data.AppSettings import com.example.instantvoicetranslate.data.ModelDownloader @@ -16,6 +16,7 @@ import com.example.instantvoicetranslate.service.TranslationService import com.example.instantvoicetranslate.translation.NllbModelManager import com.example.instantvoicetranslate.translation.NllbTranslator import com.example.instantvoicetranslate.tts.TtsEngine +import com.example.instantvoicetranslate.ui.utils.LanguageUtils import dagger.hilt.android.lifecycle.HiltViewModel import kotlinx.coroutines.async import kotlinx.coroutines.delay @@ -24,6 +25,7 @@ import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.first import kotlinx.coroutines.flow.stateIn import kotlinx.coroutines.launch +import kotlinx.coroutines.withTimeoutOrNull import java.util.Locale import javax.inject.Inject @@ -38,7 +40,7 @@ class MainViewModel @Inject constructor( private val translationUiState: TranslationUiState, private val settingsRepository: SettingsRepository, private val modelDownloader: ModelDownloader, - private val speechRecognizer: SpeechRecognizer, + private val recognizerPool: ConversationRecognizerPool, private val ttsEngine: TtsEngine, private val nllbModelManager: NllbModelManager, private val nllbTranslator: NllbTranslator, @@ -109,23 +111,16 @@ class MainViewModel @Inject constructor( // 2. Pre-initialize ASR recognizer (loads ONNX model into memory) modelDownloader.updateStatus(ModelStatus.Initializing("Loading speech model...")) - if (!speechRecognizer.isReady.value || speechRecognizer.currentLanguage.value != srcLang) { - try { - if (speechRecognizer.isReady.value) { - speechRecognizer.release() - } - Log.i(TAG, "Pre-loading ASR model for '$srcLang'...") - speechRecognizer.initialize( - modelDownloader.getModelDir(srcLang).absolutePath, - srcLang - ) - Log.i(TAG, "ASR model pre-loaded successfully") - } catch (e: Exception) { - Log.e(TAG, "Failed to pre-load ASR model", e) - modelDownloader.updateStatus(ModelStatus.Error("ASR init failed: ${e.message}")) - punctJob?.cancel() - return@launch - } + val recognizer = try { + Log.i(TAG, "Pre-loading ASR model for '$srcLang'...") + val r = recognizerPool.acquire(srcLang, modelDownloader.getModelDir(srcLang).absolutePath) + Log.i(TAG, "ASR model pre-loaded successfully") + r + } catch (e: Exception) { + Log.e(TAG, "Failed to pre-load ASR model", e) + modelDownloader.updateStatus(ModelStatus.Error("ASR init failed: ${e.message}")) + punctJob?.cancel() + return@launch } // 3. Wait for punctuation download (started in parallel) and initialize @@ -134,7 +129,7 @@ class MainViewModel @Inject constructor( try { punctJob?.await() if (modelDownloader.isPunctModelReady()) { - speechRecognizer.initializePunctuation( + recognizer.initializePunctuation( modelDownloader.getPunctModelDir().absolutePath ) Log.i(TAG, "Punctuation model pre-loaded") @@ -144,6 +139,22 @@ class MainViewModel @Inject constructor( } } + // 3b. Also pre-warm the target language's ASR model (if it's one of + // the ASR-capable languages) so a conversation-direction swap has + // both sides already loaded and doesn't pay a reload delay. + val tgtLang = currentSettings.targetLanguage + if (tgtLang != srcLang && LanguageUtils.sourceLanguages.any { it.first == tgtLang }) { + try { + modelDownloader.ensureModelAvailable(tgtLang) + if (modelDownloader.isModelReady(tgtLang)) { + recognizerPool.acquire(tgtLang, modelDownloader.getModelDir(tgtLang).absolutePath) + Log.i(TAG, "ASR model for target language '$tgtLang' pre-warmed for fast swap") + } + } catch (e: Exception) { + Log.w(TAG, "Failed to pre-warm target language ASR model '$tgtLang'", e) + } + } + // 4. Pre-initialize TTS engine modelDownloader.updateStatus(ModelStatus.Initializing("Starting TTS engine...")) val ttsLocale = Locale.forLanguageTag(currentSettings.targetLanguage) @@ -187,6 +198,61 @@ class MainViewModel @Inject constructor( } } + /** + * Flips source <-> target language for a conversation-direction swap. + * Only enabled when the current target is itself ASR-capable (otherwise + * there'd be no recognizer model to hear the reply) -- callers should + * gate the UI control on that same condition. + * + * If a translation session is running on the microphone source, restarts + * it so the new pairing takes effect immediately; the restart is fast + * because the target language's ASR model was already pre-warmed by + * [preloadPipelineComponents]. System-audio sessions are left running + * with the old pairing (restarting would require re-requesting + * MediaProjection consent via an Activity), taking effect on the next + * manual start. + */ + fun swapLanguages() { + val current = settings.value + if (current.targetLanguage !in LanguageUtils.sourceLanguages.map { it.first }) { + Log.w(TAG, "Swap ignored: target '${current.targetLanguage}' has no ASR model") + return + } + + val wasRunning = isRunning.value + val canAutoRestart = wasRunning && current.audioSource == AudioCaptureManager.Source.MICROPHONE + if (wasRunning && !canAutoRestart) { + Log.i(TAG, "Swap applied to settings; system-audio session keeps running with the old pairing") + } + + viewModelScope.launch { + if (canAutoRestart) { + stopTranslation() + // stopTranslation()/startTranslation() just fire Android + // Intents at the service (fire-and-forget) -- wait for the + // stop to actually land so the START intent below can't race + // a still-in-flight stopSelf() from the previous session. + withTimeoutOrNull(2_000) { isRunning.first { !it } } + } + + settingsRepository.swapLanguages() + + // Pre-warm the new source (almost always already warm as the + // previous target) so a subsequent start is instant either way. + val newSrcLang = current.targetLanguage + try { + modelDownloader.ensureModelAvailable(newSrcLang) + if (modelDownloader.isModelReady(newSrcLang)) { + recognizerPool.acquire(newSrcLang, modelDownloader.getModelDir(newSrcLang).absolutePath) + } + } catch (e: Exception) { + Log.w(TAG, "Failed to pre-warm swapped source language '$newSrcLang'", e) + } + + if (canAutoRestart) startTranslation() + } + } + fun startTranslation() { translationUiState.setStarting(true) launchStartingTimeout() From fb74ad6e83fd774c9f1985d19aae19bac80674ed Mon Sep 17 00:00:00 2001 From: michael Date: Tue, 14 Jul 2026 13:27:56 +0200 Subject: [PATCH 10/21] Add SettingsRepository.swapLanguages() Atomically flips source/target language in one DataStore edit, instead of two separate update calls, to avoid a torn intermediate state. --- .../instantvoicetranslate/data/SettingsRepository.kt | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/app/src/main/java/com/example/instantvoicetranslate/data/SettingsRepository.kt b/app/src/main/java/com/example/instantvoicetranslate/data/SettingsRepository.kt index e299705..f5c40f0 100644 --- a/app/src/main/java/com/example/instantvoicetranslate/data/SettingsRepository.kt +++ b/app/src/main/java/com/example/instantvoicetranslate/data/SettingsRepository.kt @@ -100,6 +100,16 @@ class SettingsRepository @Inject constructor( context.dataStore.edit { it[KEY_TARGET_LANG] = lang } } + /** Atomically swaps source and target language (conversation-direction flip). */ + suspend fun swapLanguages() { + context.dataStore.edit { prefs -> + val src = prefs[KEY_SOURCE_LANG] ?: "en" + val tgt = prefs[KEY_TARGET_LANG] ?: "ru" + prefs[KEY_SOURCE_LANG] = tgt + prefs[KEY_TARGET_LANG] = src + } + } + suspend fun updateTtsSpeed(speed: Float) { context.dataStore.edit { it[KEY_TTS_SPEED] = speed } } From 60ff1d677ffe990fca2dc319159bcf6295e971ac Mon Sep 17 00:00:00 2001 From: michael Date: Tue, 14 Jul 2026 13:36:12 +0200 Subject: [PATCH 11/21] Fix OOM kill from dual-warming both ASR models on every launch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Eagerly pre-warming the target language's ASR model at every app launch (on top of the already-loaded source model and, in offline mode, NLLB's ~1GB) pushed a loaded device over its memory watermark and got the whole process killed by the Android low-memory killer ("min watermark is breached and swap is low") — reproduced on a Pixel 7a mid-conversation. The warm pool now only builds up lazily: the source language loads as before, and the target only gets pre-warmed the first time swapLanguages() is actually used, not on every launch regardless of whether the user ever swaps. As a second line of defense, ConversationRecognizerPool.trimToMostRecentlyUsed() releases the least-recently-used warm recognizer in response to Application.onTrimMemory(), so once both directions do get warmed during a long conversation, the pool shrinks itself back to one under memory pressure instead of the system killing the whole process. --- .../InstantVoiceTranslateApp.kt | 19 +++++++++++++++ .../asr/ConversationRecognizerPool.kt | 17 +++++++++++++ .../ui/viewmodel/MainViewModel.kt | 24 +++++++------------ 3 files changed, 45 insertions(+), 15 deletions(-) diff --git a/app/src/main/java/com/example/instantvoicetranslate/InstantVoiceTranslateApp.kt b/app/src/main/java/com/example/instantvoicetranslate/InstantVoiceTranslateApp.kt index ffc4708..7ef6e7e 100644 --- a/app/src/main/java/com/example/instantvoicetranslate/InstantVoiceTranslateApp.kt +++ b/app/src/main/java/com/example/instantvoicetranslate/InstantVoiceTranslateApp.kt @@ -3,21 +3,40 @@ package com.example.instantvoicetranslate import android.app.Application import android.app.NotificationChannel import android.app.NotificationManager +import android.util.Log +import com.example.instantvoicetranslate.asr.ConversationRecognizerPool import dagger.hilt.android.HiltAndroidApp +import javax.inject.Inject @HiltAndroidApp class InstantVoiceTranslateApp : Application() { companion object { const val CHANNEL_ID = "translation_service" + private const val TAG = "InstantVoiceTranslateApp" } + @Inject lateinit var recognizerPool: ConversationRecognizerPool + override fun onCreate() { super.onCreate() createNotificationChannel() } + /** + * Under memory pressure, shrink the ASR warm-pool back to one language + * before the system decides to kill this whole process instead — losing + * one pre-warmed recognizer is far cheaper than a full restart. + */ + override fun onTrimMemory(level: Int) { + super.onTrimMemory(level) + if (level >= TRIM_MEMORY_RUNNING_LOW) { + Log.w(TAG, "onTrimMemory(level=$level): trimming ASR recognizer pool") + recognizerPool.trimToMostRecentlyUsed() + } + } + private fun createNotificationChannel() { val channel = NotificationChannel( CHANNEL_ID, diff --git a/app/src/main/java/com/example/instantvoicetranslate/asr/ConversationRecognizerPool.kt b/app/src/main/java/com/example/instantvoicetranslate/asr/ConversationRecognizerPool.kt index 41fcdc5..991bd70 100644 --- a/app/src/main/java/com/example/instantvoicetranslate/asr/ConversationRecognizerPool.kt +++ b/app/src/main/java/com/example/instantvoicetranslate/asr/ConversationRecognizerPool.kt @@ -50,6 +50,23 @@ class ConversationRecognizerPool @javax.inject.Inject constructor() { fun isWarm(language: String): Boolean = pool[language]?.isReady?.value == true + /** + * Releases every warm recognizer except the most-recently-used one, in + * response to a system memory-pressure signal (see + * [InstantVoiceTranslateApp.onTrimMemory]). Keeping two ~30-190MB ASR + * models plus NLLB (~1GB) resident can otherwise push a loaded device + * over its memory watermark and get the whole process killed by the + * Android low-memory killer instead of just this pool shrinking back to + * one warm language. + */ + fun trimToMostRecentlyUsed() { + while (pool.size > 1) { + val lruLanguage = pool.keys.first() + pool.remove(lruLanguage)?.release() + Log.i(TAG, "Trimmed warm recognizer for '$lruLanguage' due to memory pressure") + } + } + fun releaseAll() { pool.values.forEach { it.release() } pool.clear() diff --git a/app/src/main/java/com/example/instantvoicetranslate/ui/viewmodel/MainViewModel.kt b/app/src/main/java/com/example/instantvoicetranslate/ui/viewmodel/MainViewModel.kt index 33d60db..5cbc691 100644 --- a/app/src/main/java/com/example/instantvoicetranslate/ui/viewmodel/MainViewModel.kt +++ b/app/src/main/java/com/example/instantvoicetranslate/ui/viewmodel/MainViewModel.kt @@ -139,21 +139,15 @@ class MainViewModel @Inject constructor( } } - // 3b. Also pre-warm the target language's ASR model (if it's one of - // the ASR-capable languages) so a conversation-direction swap has - // both sides already loaded and doesn't pay a reload delay. - val tgtLang = currentSettings.targetLanguage - if (tgtLang != srcLang && LanguageUtils.sourceLanguages.any { it.first == tgtLang }) { - try { - modelDownloader.ensureModelAvailable(tgtLang) - if (modelDownloader.isModelReady(tgtLang)) { - recognizerPool.acquire(tgtLang, modelDownloader.getModelDir(tgtLang).absolutePath) - Log.i(TAG, "ASR model for target language '$tgtLang' pre-warmed for fast swap") - } - } catch (e: Exception) { - Log.w(TAG, "Failed to pre-warm target language ASR model '$tgtLang'", e) - } - } + // Note: the target language's ASR model is intentionally NOT + // pre-warmed here. Keeping two ASR models resident for the + // entire app lifetime (on top of NLLB's ~1GB in offline mode) + // pushed a loaded device over its memory watermark and got the + // whole process killed by the Android low-memory killer. Instead + // the pool builds up lazily: the first swapLanguages() call pays + // a normal cold-load for the new direction, and only + // conversations that actually swap back and forth keep both + // directions warm (see ConversationRecognizerPool). // 4. Pre-initialize TTS engine modelDownloader.updateStatus(ModelStatus.Initializing("Starting TTS engine...")) From 3f1f744da21f4ac2b04283acad950e86903786be Mon Sep 17 00:00:00 2001 From: michael Date: Tue, 14 Jul 2026 13:36:22 +0200 Subject: [PATCH 12/21] Add language-swap button to MainScreen Icon button between the source/target language labels, flipping the conversation direction via MainViewModel.swapLanguages(). Disabled when the current target language has no ASR model, since there'd be nothing to recognize a reply in that language. --- .../ui/screens/MainScreen.kt | 15 ++++++++++----- app/src/main/res/values/strings.xml | 1 + 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/app/src/main/java/com/example/instantvoicetranslate/ui/screens/MainScreen.kt b/app/src/main/java/com/example/instantvoicetranslate/ui/screens/MainScreen.kt index a373253..3d95acc 100644 --- a/app/src/main/java/com/example/instantvoicetranslate/ui/screens/MainScreen.kt +++ b/app/src/main/java/com/example/instantvoicetranslate/ui/screens/MainScreen.kt @@ -25,6 +25,7 @@ import androidx.compose.material.icons.Icons import androidx.compose.material.icons.filled.Mic import androidx.compose.material.icons.filled.Settings import androidx.compose.material.icons.filled.Stop +import androidx.compose.material.icons.filled.SwapHoriz import androidx.compose.material.icons.automirrored.filled.VolumeUp import androidx.compose.material3.Card import androidx.compose.material3.CardDefaults @@ -109,11 +110,15 @@ fun MainScreen( text = LanguageUtils.displayName(settings.sourceLanguage), style = MaterialTheme.typography.titleMedium ) - Text( - text = " \u2192 ", - style = MaterialTheme.typography.titleMedium, - color = MaterialTheme.colorScheme.primary - ) + IconButton( + onClick = { viewModel.swapLanguages() }, + enabled = settings.targetLanguage in LanguageUtils.sourceLanguages.map { it.first }, + ) { + Icon( + Icons.Default.SwapHoriz, + contentDescription = stringResource(R.string.action_swap_languages), + ) + } Text( text = LanguageUtils.displayName(settings.targetLanguage), style = MaterialTheme.typography.titleMedium diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 5437cd9..8fea61a 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -4,6 +4,7 @@ Settings + Swap languages Stop Start Translation From ea1e3f0432a88f5ee99bcea0c39c3ed741bfb9c3 Mon Sep 17 00:00:00 2001 From: michael Date: Tue, 14 Jul 2026 13:51:32 +0200 Subject: [PATCH 13/21] Reduce ASR pool thrashing and add a locale retry for TTS engine restarts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two distinct issues surfaced testing conversation swaps on a memory-constrained device: 1. onTrimMemory reacted at TRIM_MEMORY_RUNNING_LOW, which fires very often under system-wide (not just this app's own) memory pressure — discarding the other direction's warm ASR recognizer right after it loaded and forcing a reload on essentially every swap. Now only reacts at TRIM_MEMORY_RUNNING_CRITICAL. 2. The external TTS engine app itself can be killed by the system under memory pressure (observed directly: "Kill 'org.woheller69. ttsengine' ... min watermark is breached"). After Android auto-restarts and rebinds it, there's a window where it hasn't finished reloading its voice data yet, so setLanguage() reports the requested language as missing even though it's actually installed — TtsEngine now retries once after a short delay before permanently falling back to English. --- .../InstantVoiceTranslateApp.kt | 15 +++++-- .../instantvoicetranslate/tts/TtsEngine.kt | 41 ++++++++++++++++++- 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/app/src/main/java/com/example/instantvoicetranslate/InstantVoiceTranslateApp.kt b/app/src/main/java/com/example/instantvoicetranslate/InstantVoiceTranslateApp.kt index 7ef6e7e..faf7315 100644 --- a/app/src/main/java/com/example/instantvoicetranslate/InstantVoiceTranslateApp.kt +++ b/app/src/main/java/com/example/instantvoicetranslate/InstantVoiceTranslateApp.kt @@ -25,13 +25,20 @@ class InstantVoiceTranslateApp : Application() { } /** - * Under memory pressure, shrink the ASR warm-pool back to one language - * before the system decides to kill this whole process instead — losing - * one pre-warmed recognizer is far cheaper than a full restart. + * Under severe memory pressure, shrink the ASR warm-pool back to one + * language before the system decides to kill this whole process instead. + * + * Reacts only to RUNNING_CRITICAL, not the much more frequent + * RUNNING_LOW/MODERATE levels: on a device that's chronically close to + * its memory watermark (many background apps, a memory-hungry external + * TTS engine also competing for RAM), those fire on essentially every + * turn of a conversation, which was thrashing the pool -- discarding the + * other direction's warm recognizer right after it was loaded and + * forcing a reload on the very next swap. */ override fun onTrimMemory(level: Int) { super.onTrimMemory(level) - if (level >= TRIM_MEMORY_RUNNING_LOW) { + if (level >= TRIM_MEMORY_RUNNING_CRITICAL) { Log.w(TAG, "onTrimMemory(level=$level): trimming ASR recognizer pool") recognizerPool.trimToMostRecentlyUsed() } diff --git a/app/src/main/java/com/example/instantvoicetranslate/tts/TtsEngine.kt b/app/src/main/java/com/example/instantvoicetranslate/tts/TtsEngine.kt index 4496173..6e47344 100644 --- a/app/src/main/java/com/example/instantvoicetranslate/tts/TtsEngine.kt +++ b/app/src/main/java/com/example/instantvoicetranslate/tts/TtsEngine.kt @@ -58,6 +58,16 @@ class TtsEngine @Inject constructor( */ private const val SPEECH_TAIL_COOLDOWN_MS = 400L + /** + * Delay before retrying a locale that initially came back unsupported. + * Covers external TTS engine apps (e.g. a separate neural-voice app) + * that can themselves be killed by the system under memory pressure + * and take a moment to reload their voice data after an automatic + * restart -- during that window setLanguage() reports the voice + * missing even though it's actually installed. + */ + private const val LOCALE_RETRY_DELAY_MS = 1_500L + /** * Matches sentence-ending punctuation (.!?) followed by whitespace. * Uses a fixed-length lookbehind (single char) which is safe for Java regex. @@ -77,6 +87,11 @@ class TtsEngine @Inject constructor( private var currentDeferred: CompletableDeferred? = null private var speakingCooldownJob: Job? = null + // Lives for the singleton's lifetime, independent of [queueScope] (which + // is cancelled/replaced on every initialize() call) so a scheduled + // locale retry can't be cancelled out from under it by a later re-init. + private val engineScope = CoroutineScope(Dispatchers.Default + SupervisorJob()) + // --- Volume & audio ducking --- private val audioManager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager private var volume = 1.0f @@ -163,7 +178,7 @@ class TtsEngine @Inject constructor( * speak translated text in the wrong language even though setLanguage() * looked like it succeeded. */ - private fun applyLocale(locale: Locale): Boolean { + private fun applyLocale(locale: Locale, allowRetry: Boolean = true): Boolean { val engine = tts ?: return false val result = engine.setLanguage(locale) @@ -172,7 +187,12 @@ class TtsEngine @Inject constructor( var effectiveLocale = locale if (!requestedLocaleSupported) { - Log.w(TAG, "Language $locale not supported (result=$result), falling back to English") + if (allowRetry) { + Log.w(TAG, "Language $locale not supported (result=$result) -- retrying once shortly") + scheduleLocaleRetry(locale) + } else { + Log.w(TAG, "Language $locale still not supported after retry (result=$result), falling back to English") + } engine.language = Locale.US effectiveLocale = Locale.US val languageName = locale.getDisplayLanguage(Locale.ENGLISH) @@ -193,6 +213,23 @@ class TtsEngine @Inject constructor( return requestedLocaleSupported } + /** + * Retries [locale] once after a short delay -- covers an external TTS + * engine app that was itself killed by the system under memory pressure + * and is still reloading its voice data right after an automatic + * restart. Uses [engineScope] (not [queueScope], which gets cancelled + * and replaced on every initialize() call) so this survives a + * conversation-direction swap that happens to land in the same window. + */ + private fun scheduleLocaleRetry(locale: Locale) { + engineScope.launch { + delay(LOCALE_RETRY_DELAY_MS) + if (applyLocale(locale, allowRetry = false)) { + Log.i(TAG, "Language $locale became available on retry") + } + } + } + /** * Splits text into sentences and enqueues each one separately. * This allows Android TTS to begin speaking the first sentence From 5dec1261243380e6f874a42653a397045fd4f43e Mon Sep 17 00:00:00 2001 From: michael Date: Tue, 14 Jul 2026 14:12:00 +0200 Subject: [PATCH 14/21] Document RAM requirements for offline mode Offline mode (NLLB) plus the ASR conversation-swap cache and a memory-hungry third-party neural TTS engine were observed getting killed by the Android low-memory killer on an 8GB Pixel 7a under normal background app load. Document a 6-8GB RAM recommendation for offline mode so users aren't surprised by this on constrained devices. --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index 02bf7e4..badb6fd 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,20 @@ or **offline** (NLLB-200-distilled-600M via ONNX Runtime, no internet needed aft - **Android SDK 36** (compileSdk) - **Device or emulator** with API 29+ (Android 10 required for AudioPlaybackCapture) +### Device RAM + +- **Online mode** (Yandex API translation): a single ~26--190 MB ASR model resident -- fine on any modern device. +- **Offline mode** (NLLB-200 translation): NLLB itself uses **~900 MB--1 GB RAM** once loaded, on top of the active + ASR model(s). **At least 6--8 GB of total device RAM is recommended** for offline mode to run reliably alongside + normal background app usage. +- Switching translation direction keeps up to two ASR models warm at once (source + target language) for fast + swaps, adding up to ~350 MB more in the worst case (e.g. English + Spanish). +- On memory-constrained devices, or devices with many background apps, the Android low-memory killer can terminate + the app (offline mode + a memory-hungry third-party neural TTS engine were observed being killed under pressure + on an 8 GB Pixel 7a with many other apps installed). The app defends against this by shrinking its own ASR cache + under memory pressure (`ComponentCallbacks2.onTrimMemory`), but a chronically RAM-starved device may still see + occasional TTS voice hiccups or ASR reloads. + ## Build & Run ```bash From 69f68db46efbe80325b8b034a28e400755c6fa04 Mon Sep 17 00:00:00 2001 From: michael Date: Wed, 15 Jul 2026 16:03:19 +0200 Subject: [PATCH 15/21] Make ConversationRecognizerPool concurrency-safe and evict stale languages acquire() had no synchronization: a language swap's pre-warm call and the subsequent pipeline restart's own acquire() could race, each seeing the pool empty and building a duplicate SherpaOnnxRecognizer for the same language, with the second write silently orphaning the first. Observed crashing the pipeline (JobCancellationException followed by a colliding AudioRecord IllegalStateException). Added a Mutex around acquire/reconcile. Also add reconcile(desiredLanguages), releasing any warm recognizer left over from a since-changed source/target setting -- previously a stale entry from before a settings change stayed resident indefinitely alongside the new one. --- .../asr/ConversationRecognizerPool.kt | 39 +++++++++++++++++-- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/app/src/main/java/com/example/instantvoicetranslate/asr/ConversationRecognizerPool.kt b/app/src/main/java/com/example/instantvoicetranslate/asr/ConversationRecognizerPool.kt index 991bd70..f1c29ca 100644 --- a/app/src/main/java/com/example/instantvoicetranslate/asr/ConversationRecognizerPool.kt +++ b/app/src/main/java/com/example/instantvoicetranslate/asr/ConversationRecognizerPool.kt @@ -1,6 +1,8 @@ package com.example.instantvoicetranslate.asr import android.util.Log +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock import javax.inject.Singleton /** @@ -25,13 +27,23 @@ class ConversationRecognizerPool @javax.inject.Inject constructor() { // the end, so the front entry is always the least-recently-used one. private val pool = LinkedHashMap() - suspend fun acquire(language: String, modelDir: String): SpeechRecognizer { + // Guards `pool` against concurrent acquire()/reconcile() calls. Without + // this, a swap (which pre-warms the new source directly from + // MainViewModel while the pipeline restart's own acquire() call races it + // from TranslationService) could both see the language missing and each + // build+initialize their own SherpaOnnxRecognizer, with the second + // write silently orphaning the first -- observed as a crashed pipeline + // (JobCancellationException + a subsequent AudioRecord IllegalStateException + // from the colliding native lifecycles). + private val mutex = Mutex() + + suspend fun acquire(language: String, modelDir: String): SpeechRecognizer = mutex.withLock { pool[language]?.let { existing -> if (existing.isReady.value) { // Move to most-recently-used position. pool.remove(language) pool[language] = existing - return existing + return@withLock existing } pool.remove(language) } @@ -45,11 +57,32 @@ class ConversationRecognizerPool @javax.inject.Inject constructor() { val recognizer = SherpaOnnxRecognizer() recognizer.initialize(modelDir, language) pool[language] = recognizer - return recognizer + recognizer } fun isWarm(language: String): Boolean = pool[language]?.isReady?.value == true + /** + * Releases any warm recognizer whose language is not in [desiredLanguages]. + * + * Settings can change the active source/target language (via the + * Settings screen or the swap button) without ever releasing whatever + * was warmed under the old settings, since neither of those call sites + * knows what's still relevant. Left unchecked, a stale recognizer from + * before the change stays resident indefinitely alongside the new one -- + * on top of NLLB (~1GB in offline mode) this was observed pushing a + * loaded device over its memory watermark and getting the whole process + * killed. Call this right before a translation session actually starts, + * when the true set of languages worth keeping warm is known. + */ + suspend fun reconcile(desiredLanguages: Set) = mutex.withLock { + val stale = pool.keys.filterNot { it in desiredLanguages } + stale.forEach { language -> + pool.remove(language)?.release() + Log.i(TAG, "Released stale warm recognizer for '$language', no longer relevant to current settings") + } + } + /** * Releases every warm recognizer except the most-recently-used one, in * response to a system memory-pressure signal (see From a5ef1c9cf2415bb5aa69c3e75a5fbfc2e5bca711 Mon Sep 17 00:00:00 2001 From: michael Date: Wed, 15 Jul 2026 16:03:30 +0200 Subject: [PATCH 16/21] Flush in-flight ASR text on stop and pause instead of dropping it Stop (and now pause) tore down the recognizer's collection loop via plain cancellation, discarding whatever was still being transcribed if the user pressed Stop/Pause before sustained silence or Sherpa's own endpoint detection had a chance to finalize a segment -- the most common case, since users typically stop right after finishing a sentence rather than waiting through an extra pause first. The recognition coroutine's finally block now recomputes and flushes whatever's buffered, wrapped in withContext(NonCancellable) so the (already-cancelling) coroutine can still emit it. Also add pauseAndFlush(), returning the flushed text directly rather than via recognizedSegments -- that flow's collector is gated on "not paused", and a normally-emitted segment would arrive there right as isPaused turns true, silently dropped by the same guard it needs to get past. --- .../asr/SherpaOnnxRecognizer.kt | 52 +++++++++++++++++++ .../asr/SpeechRecognizer.kt | 9 ++++ 2 files changed, 61 insertions(+) diff --git a/app/src/main/java/com/example/instantvoicetranslate/asr/SherpaOnnxRecognizer.kt b/app/src/main/java/com/example/instantvoicetranslate/asr/SherpaOnnxRecognizer.kt index aae91a8..0df058d 100644 --- a/app/src/main/java/com/example/instantvoicetranslate/asr/SherpaOnnxRecognizer.kt +++ b/app/src/main/java/com/example/instantvoicetranslate/asr/SherpaOnnxRecognizer.kt @@ -12,9 +12,11 @@ import com.k2fsa.sherpa.onnx.OnlinePunctuation import com.k2fsa.sherpa.onnx.OnlinePunctuationConfig import com.k2fsa.sherpa.onnx.OnlinePunctuationModelConfig import com.k2fsa.sherpa.onnx.OnlineTransducerModelConfig +import kotlinx.coroutines.CompletableDeferred import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job +import kotlinx.coroutines.NonCancellable import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.cancel import kotlinx.coroutines.flow.Flow @@ -96,6 +98,16 @@ class SherpaOnnxRecognizer : SpeechRecognizer { private var recognitionScope: CoroutineScope? = null private var recognitionJob: Job? = null + /** + * Set by [pauseAndFlush] just before cancelling, so the recognition + * coroutine's finally block delivers the flushed text here instead of + * emitting it to [recognizedSegments]. That flow is gated on "not + * paused" by its collector -- by the time a normally-emitted segment + * got there, isPaused would already be true, silently dropping it. + */ + @Volatile + private var pendingFlushDeferred: CompletableDeferred? = null + override suspend fun initialize(modelDir: String, language: String): Unit = withContext(Dispatchers.IO) { try { val langConfig = ModelDownloader.getLanguageConfig(language) @@ -293,6 +305,34 @@ class SherpaOnnxRecognizer : SpeechRecognizer { } catch (e: Exception) { Log.e(TAG, "Recognition error", e) } finally { + // Flush whatever's still buffered so far. Stop is commonly + // pressed right after the user finishes speaking, before + // sustained silence/endpoint detection above ever gets a + // chance to fire -- without this, that last utterance is + // simply discarded instead of being translated. NonCancellable + // because this coroutine's job is normally already cancelling + // by the time we get here (see stopRecognition()), and a plain + // suspend call would otherwise throw immediately. + withContext(NonCancellable) { + var flushed: String? = null + try { + val currentText = rec.getResult(stream).text.trim() + val fullText = combineText(emittedText, currentText) + if (fullText.isNotBlank()) { + flushed = applyPunctuation(fullText) + } + } catch (e: Exception) { + Log.w(TAG, "Failed to flush final segment", e) + } + val flushDeferred = pendingFlushDeferred + if (flushDeferred != null) { + pendingFlushDeferred = null + flushDeferred.complete(flushed) + } else if (flushed != null) { + Log.i(TAG, "Stop: flushing final segment: $flushed") + _recognizedSegments.emit(flushed) + } + } stream.release() } } @@ -357,6 +397,18 @@ class SherpaOnnxRecognizer : SpeechRecognizer { _partialText.value = "" } + override suspend fun pauseAndFlush(): String? { + val job = recognitionJob ?: return null + val deferred = CompletableDeferred() + pendingFlushDeferred = deferred + recognitionJob = null + job.cancel() + recognitionScope?.cancel() + recognitionScope = null + _partialText.value = "" + return deferred.await() + } + override fun release() { stopRecognition() recognizer?.release() diff --git a/app/src/main/java/com/example/instantvoicetranslate/asr/SpeechRecognizer.kt b/app/src/main/java/com/example/instantvoicetranslate/asr/SpeechRecognizer.kt index 5cf8b80..cba389d 100644 --- a/app/src/main/java/com/example/instantvoicetranslate/asr/SpeechRecognizer.kt +++ b/app/src/main/java/com/example/instantvoicetranslate/asr/SpeechRecognizer.kt @@ -26,6 +26,15 @@ interface SpeechRecognizer { /** Stop ongoing recognition. */ fun stopRecognition() + /** + * Stops recognition and returns whatever was still being transcribed as + * a final segment (or null if nothing was pending), WITHOUT emitting it + * to [recognizedSegments]. For use by pause: that flow's collector + * drops anything recognized while paused, which would otherwise + * silently discard this in-flight utterance too. + */ + suspend fun pauseAndFlush(): String? = null + /** Initialize punctuation restoration model (optional, no-op by default). */ fun initializePunctuation(modelDir: String) {} From 636d27fa499d78f2b69ce8726645ac3cd260d53d Mon Sep 17 00:00:00 2001 From: michael Date: Wed, 15 Jul 2026 16:03:37 +0200 Subject: [PATCH 17/21] Add isPaused state for the pause/resume UI Plumbing only -- TranslationService will set this once pause becomes a distinct action from stop. --- .../instantvoicetranslate/data/TranslationUiState.kt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/src/main/java/com/example/instantvoicetranslate/data/TranslationUiState.kt b/app/src/main/java/com/example/instantvoicetranslate/data/TranslationUiState.kt index ebe07ce..d9aca4e 100644 --- a/app/src/main/java/com/example/instantvoicetranslate/data/TranslationUiState.kt +++ b/app/src/main/java/com/example/instantvoicetranslate/data/TranslationUiState.kt @@ -26,6 +26,9 @@ class TranslationUiState @Inject constructor() { private val _isRunning = MutableStateFlow(false) val isRunning: StateFlow = _isRunning.asStateFlow() + private val _isPaused = MutableStateFlow(false) + val isPaused: StateFlow = _isPaused.asStateFlow() + private val _partialText = MutableStateFlow("") val partialText: StateFlow = _partialText.asStateFlow() @@ -85,9 +88,14 @@ class TranslationUiState @Inject constructor() { } if (!running) { _partialText.value = "" + _isPaused.value = false } } + fun setPaused(paused: Boolean) { + _isPaused.value = paused + } + fun clearTexts() { _partialText.value = "" _originalText.value = "" From a192cf46d72466daa47774e8bf3e9aed5e7e067d Mon Sep 17 00:00:00 2001 From: michael Date: Wed, 15 Jul 2026 16:03:48 +0200 Subject: [PATCH 18/21] Stop pipeline lifecycle from cancelling work it should let finish Several related fixes to how starting/stopping/pausing tear down (or don't) in-flight work: - Reconcile the ASR pool to the current source/target before acquiring a recognizer, so a language changed via Settings or a swap doesn't leave its old recognizer orphaned but resident (see ConversationRecognizerPool commit) -- this is what was causing an OOM kill shortly after launch. - Translation coroutines now run on serviceScope directly instead of as children of the per-session pipelineJob, tracked in pendingTranslationJobs. stopPipeline() used to cancel pipelineJob immediately, tearing down a translation already in flight for the segment recognized right before Stop was pressed -- it now gives those jobs a bounded grace period to reach TTS before finalizing the stop. - pausePipeline/resumePipeline rewritten: pause now flushes and translates whatever the user was mid-sentence saying (via SpeechRecognizer.pauseAndFlush(), see previous commit) instead of silently dropping it, and resume re-opens audio capture from scratch (extracted into beginListening()) since pausing tears down the underlying AudioRecord too. --- .../service/TranslationService.kt | 162 ++++++++++++++---- 1 file changed, 133 insertions(+), 29 deletions(-) diff --git a/app/src/main/java/com/example/instantvoicetranslate/service/TranslationService.kt b/app/src/main/java/com/example/instantvoicetranslate/service/TranslationService.kt index 7edae85..5bae85d 100644 --- a/app/src/main/java/com/example/instantvoicetranslate/service/TranslationService.kt +++ b/app/src/main/java/com/example/instantvoicetranslate/service/TranslationService.kt @@ -17,12 +17,14 @@ import com.example.instantvoicetranslate.asr.ConversationRecognizerPool import com.example.instantvoicetranslate.asr.SpeechRecognizer import com.example.instantvoicetranslate.audio.AudioCaptureManager import com.example.instantvoicetranslate.audio.AudioDiagnostics +import com.example.instantvoicetranslate.data.AppSettings import com.example.instantvoicetranslate.data.ModelDownloader import com.example.instantvoicetranslate.data.SettingsRepository import com.example.instantvoicetranslate.data.TranslationUiState import com.example.instantvoicetranslate.translation.NllbTranslator import com.example.instantvoicetranslate.translation.TextTranslator import com.example.instantvoicetranslate.tts.TtsEngine +import com.example.instantvoicetranslate.ui.utils.LanguageUtils import dagger.hilt.android.AndroidEntryPoint import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers @@ -31,10 +33,13 @@ import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.cancel import kotlinx.coroutines.flow.filter import kotlinx.coroutines.flow.first +import kotlinx.coroutines.joinAll import kotlinx.coroutines.launch import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.withTimeoutOrNull import java.util.Locale import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.ConcurrentLinkedQueue import java.util.concurrent.atomic.AtomicLong import javax.inject.Inject @@ -53,6 +58,12 @@ class TranslationService : Service() { private const val NOTIFICATION_ID = 1 /** Maximum number of concurrent translation requests. */ private const val MAX_CONCURRENT_TRANSLATIONS = 3 + /** + * How long stopPipeline() waits for a translation already in flight + * (for the segment recognized right before Stop was pressed) before + * giving up and finalizing the stop anyway. + */ + private const val FINAL_SEGMENT_GRACE_MS = 15_000L } @Inject lateinit var audioCaptureManager: AudioCaptureManager @@ -89,6 +100,13 @@ class TranslationService : Service() { private val nextTtsSeqNum = AtomicLong(0) private val ttsFlushLock = Any() + /** + * Translation coroutines launched directly on [serviceScope] (not as + * children of [pipelineJob]) so that stopping the pipeline can't cancel + * one mid-flight -- see [stopPipeline]. + */ + private val pendingTranslationJobs = ConcurrentLinkedQueue() + /** Last translated segment, used as context for the next translation. */ @Volatile private var lastTranslatedSegment: String? = null @@ -107,8 +125,8 @@ class TranslationService : Service() { startPipeline(intent) } ACTION_STOP -> stopPipeline() - ACTION_PAUSE -> pausePipeline() - ACTION_RESUME -> resumePipeline() + ACTION_PAUSE -> serviceScope.launch { pausePipeline() } + ACTION_RESUME -> serviceScope.launch { resumePipeline() } } return START_STICKY } @@ -182,6 +200,16 @@ class TranslationService : Service() { } } + // Drop any warm recognizer left over from a since-changed + // source/target setting before acquiring -- otherwise a + // stale language stays resident indefinitely alongside the + // new one (see ConversationRecognizerPool.reconcile). + val desiredWarm = mutableSetOf(srcLang) + if (settings.targetLanguage in LanguageUtils.sourceLanguages.map { it.first }) { + desiredWarm += settings.targetLanguage + } + recognizerPool.reconcile(desiredWarm) + // Acquire a warm recognizer for the source language from the // pool — instant if already warm (pre-warmed at launch or // kept warm from a prior conversation-direction swap), @@ -229,26 +257,7 @@ class TranslationService : Service() { ttsEngine.setVolume(settings.ttsVolume) ttsEngine.setDuckingEnabled(settings.duckAudio) - // Start audio capture using the source passed via intent - val rawAudioFlow = audioCaptureManager.startCapture(currentAudioSource) - - // Filter out audio while TTS is speaking to prevent feedback loop - val audioFlow = if (settings.muteMicDuringTts) { - rawAudioFlow.filter { !ttsEngine.isSpeaking.value } - } else { - rawAudioFlow - } - - // Attach audio diagnostics if enabled in settings. - // Recordings saved to: /sdcard/Android/data//files/audio_diag/ - audioCaptureManager.diagnostics = if (settings.audioDiagnostics) { - AudioDiagnostics(this@TranslationService, settings.diagOutputDir) - } else { - null - } - - // Start recognition (energy-based silence detection inside recognizer) - recognizer.startRecognition(audioFlow) + beginListening(recognizer, settings) // Collect partial results -> UI launch { @@ -279,8 +288,13 @@ class TranslationService : Service() { // Capture context snapshot before launching parallel coroutine val contextSegment = lastTranslatedSegment - // Launch translation in a separate coroutine (parallel) - launch { + // Launch translation in a separate coroutine (parallel), + // directly on serviceScope rather than as a child of + // this collector -- stopPipeline() cancels pipelineJob + // immediately on Stop, and a child coroutine would be + // torn down mid-translation, silently dropping the + // last segment the user spoke before Stop. + val translationJob = serviceScope.launch { translationSemaphore.acquire() try { val result = activeTranslator.translate( @@ -304,6 +318,8 @@ class TranslationService : Service() { translationSemaphore.release() } } + pendingTranslationJobs += translationJob + translationJob.invokeOnCompletion { pendingTranslationJobs -= translationJob } } } } catch (e: Exception) { @@ -314,6 +330,36 @@ class TranslationService : Service() { } } + /** + * Starts audio capture and hands it to the recognizer. Used both by the + * initial [startPipeline] and by [resumePipeline] -- cancelling the + * recognizer's collection (as pause does, via + * [SpeechRecognizer.pauseAndFlush]) tears down the underlying + * AudioRecord too (it's released in the capture flow's `awaitClose`), + * so resuming needs a fresh capture, not just a fresh recognition loop. + */ + private suspend fun beginListening(recognizer: SpeechRecognizer, settings: AppSettings) { + val rawAudioFlow = audioCaptureManager.startCapture(currentAudioSource) + + // Filter out audio while TTS is speaking to prevent feedback loop + val audioFlow = if (settings.muteMicDuringTts) { + rawAudioFlow.filter { !ttsEngine.isSpeaking.value } + } else { + rawAudioFlow + } + + // Attach audio diagnostics if enabled in settings. + // Recordings saved to: /sdcard/Android/data//files/audio_diag/ + audioCaptureManager.diagnostics = if (settings.audioDiagnostics) { + AudioDiagnostics(this@TranslationService, settings.diagOutputDir) + } else { + null + } + + // Start recognition (energy-based silence detection inside recognizer) + recognizer.startRecognition(audioFlow) + } + /** * Send all consecutive ready translations to TTS in order. * @@ -339,21 +385,79 @@ class TranslationService : Service() { audioCaptureManager.stopCapture() audioCaptureManager.releaseMediaProjection() audioCaptureManager.diagnostics = null - ttsEngine.stop() - translationBuffer.clear() uiState.setRunning(false) stopForeground(STOP_FOREGROUND_REMOVE) - stopSelf() + + // Give a translation already in flight for the segment recognized + // right before Stop was pressed a bounded grace period to finish and + // reach TTS (its coroutine lives on serviceScope, untouched by the + // pipelineJob.cancel() above) before actually tearing everything + // down. Without this, the last thing the user said got translated + // but never spoken. + serviceScope.launch { + withTimeoutOrNull(FINAL_SEGMENT_GRACE_MS) { + pendingTranslationJobs.toList().joinAll() + } + ttsEngine.stop() + translationBuffer.clear() + stopSelf() + } } - private fun pausePipeline() { + private suspend fun pausePipeline() { + // Flush whatever the user was mid-sentence saying when they hit + // pause, and translate it -- same reasoning as the Stop-button fix: + // without this, the last thing spoken before pausing was silently + // dropped (recognized right as/after isPaused became true, and the + // segment collector below discards anything recognized while paused). + val flushed = activeRecognizer?.pauseAndFlush() + if (!flushed.isNullOrBlank()) { + val settings = settingsRepository.settings.first() + val seqNum = segmentCounter.getAndIncrement() + uiState.updateOriginal(flushed) + val contextSegment = lastTranslatedSegment + val activeTranslator: TextTranslator = if (settings.offlineMode) nllbTranslator else translator + val translationJob = serviceScope.launch { + try { + val result = activeTranslator.translate( + flushed, settings.sourceLanguage, settings.targetLanguage, contextSegment + ) + result.onSuccess { translated -> + lastTranslatedSegment = translated + uiState.updateTranslated(translated) + if (settings.autoSpeak) { + translationBuffer[seqNum] = translated + flushTtsBuffer() + } + }.onFailure { error -> + Log.e(TAG, "Translation failed for pause-flushed segment", error) + uiState.setError("Translation failed: ${error.message}") + } + } catch (e: Exception) { + Log.e(TAG, "Translation failed for pause-flushed segment", e) + } + } + pendingTranslationJobs += translationJob + translationJob.invokeOnCompletion { pendingTranslationJobs -= translationJob } + } + isPaused = true ttsEngine.stop() + uiState.setPaused(true) updateNotification(paused = true) } - private fun resumePipeline() { + private suspend fun resumePipeline() { isPaused = false + // pauseAndFlush() cancelled the recognizer's collection, which tears + // down the underlying AudioRecord too (released in the capture + // flow's awaitClose) -- resume needs a fresh capture, not just a + // fresh recognition loop. + activeRecognizer?.let { recognizer -> + val settings = settingsRepository.settings.first() + beginListening(recognizer, settings) + } + uiState.setPaused(false) updateNotification(paused = false) } From 130198c70c7d86a2a3e1b2710b79635595b06351 Mon Sep 17 00:00:00 2001 From: michael Date: Wed, 15 Jul 2026 16:03:58 +0200 Subject: [PATCH 19/21] Fix onTrimMemory over-reacting to ordinary backgrounding Checked level >= TRIM_MEMORY_RUNNING_CRITICAL, but the docstring's stated intent was "only RUNNING_CRITICAL" -- >= also matched TRIM_MEMORY_UI_HIDDEN (20) and everything above it, which fire on any simple app switch or screen lock with no memory pressure implied, unlike RUNNING_CRITICAL (15) specifically. The pool was being trimmed and reloaded on ordinary backgrounding instead of only under real pressure. --- .../instantvoicetranslate/InstantVoiceTranslateApp.kt | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/app/src/main/java/com/example/instantvoicetranslate/InstantVoiceTranslateApp.kt b/app/src/main/java/com/example/instantvoicetranslate/InstantVoiceTranslateApp.kt index faf7315..6880504 100644 --- a/app/src/main/java/com/example/instantvoicetranslate/InstantVoiceTranslateApp.kt +++ b/app/src/main/java/com/example/instantvoicetranslate/InstantVoiceTranslateApp.kt @@ -38,7 +38,13 @@ class InstantVoiceTranslateApp : Application() { */ override fun onTrimMemory(level: Int) { super.onTrimMemory(level) - if (level >= TRIM_MEMORY_RUNNING_CRITICAL) { + // Must be == not >=: TRIM_MEMORY_UI_HIDDEN (20) and everything above + // it fire on ordinary backgrounding (task switch, screen lock) with + // no memory pressure implied, unlike RUNNING_CRITICAL (15) which is + // specifically the foreground-process pressure signal. `>=` matched + // UI_HIDDEN too, so the pool was being trimmed and reloaded on every + // simple app switch instead of only under real pressure. + if (level == TRIM_MEMORY_RUNNING_CRITICAL) { Log.w(TAG, "onTrimMemory(level=$level): trimming ASR recognizer pool") recognizerPool.trimToMostRecentlyUsed() } From aa4f6152be2eff885ff9b0aa424180186a0dbee6 Mon Sep 17 00:00:00 2001 From: michael Date: Wed, 15 Jul 2026 16:04:08 +0200 Subject: [PATCH 20/21] Add pause/resume, unblock Ready from punctuation, release models before reload - pauseTranslation()/resumeTranslation() send the existing ACTION_PAUSE/ ACTION_RESUME intents, exposing isPaused as a StateFlow for the UI. - Punctuation model download+extraction is no longer awaited during preload: it's optional (English sentence-casing only), but the pipeline wouldn't reach ModelStatus.Ready until it finished, and its download has been observed stalling for minutes under memory/CPU pressure from the ASR+NLLB loads happening alongside it, making the app appear hung. Now fire-and-forget: it wires into the recognizer whenever/if it lands. - releaseWarmRecognizersThen() releases every warm ASR recognizer before running a start or swap, called unconditionally rather than only under low memory: loading a new ASR model on top of an already-warm one was observed reliably tipping memory-constrained devices into a full system-wide low-memory-killer cascade. swapLanguages() calls this at the point it's safe to (after stopping and waiting out any running session), trading the pool's instant swap-back for a full reload every time (a few seconds) -- confirmed an acceptable default on a memory-constrained device. --- .../ui/viewmodel/MainViewModel.kt | 92 +++++++++++++++---- 1 file changed, 73 insertions(+), 19 deletions(-) diff --git a/app/src/main/java/com/example/instantvoicetranslate/ui/viewmodel/MainViewModel.kt b/app/src/main/java/com/example/instantvoicetranslate/ui/viewmodel/MainViewModel.kt index 5cbc691..aad271a 100644 --- a/app/src/main/java/com/example/instantvoicetranslate/ui/viewmodel/MainViewModel.kt +++ b/app/src/main/java/com/example/instantvoicetranslate/ui/viewmodel/MainViewModel.kt @@ -54,6 +54,7 @@ class MainViewModel @Inject constructor( val isStarting: StateFlow = translationUiState.isStarting val isRunning: StateFlow = translationUiState.isRunning + val isPaused: StateFlow = translationUiState.isPaused val partialText: StateFlow = translationUiState.partialText val originalText: StateFlow = translationUiState.originalText val translatedText: StateFlow = translationUiState.translatedText @@ -123,19 +124,28 @@ class MainViewModel @Inject constructor( return@launch } - // 3. Wait for punctuation download (started in parallel) and initialize + // 3. Wire punctuation into the now-ready recognizer once its + // download (started in parallel above) finishes -- fire-and- + // forget, NOT awaited here. Punctuation is optional (English + // sentence-casing only); its download/extraction has been + // observed stalling for minutes under memory/CPU pressure from + // the ASR+NLLB loads happening alongside it, and blocking Ready + // on it meant the whole app appeared hung. Users can start + // translating immediately now; punctuation kicks in whenever it + // lands, if it lands. if (loadPunct) { - modelDownloader.updateStatus(ModelStatus.Initializing("Loading punctuation model...")) - try { - punctJob?.await() - if (modelDownloader.isPunctModelReady()) { - recognizer.initializePunctuation( - modelDownloader.getPunctModelDir().absolutePath - ) - Log.i(TAG, "Punctuation model pre-loaded") + viewModelScope.launch { + try { + punctJob?.await() + if (modelDownloader.isPunctModelReady()) { + recognizer.initializePunctuation( + modelDownloader.getPunctModelDir().absolutePath + ) + Log.i(TAG, "Punctuation model pre-loaded") + } + } catch (e: Exception) { + Log.w(TAG, "Punctuation model not available, continuing without it", e) } - } catch (e: Exception) { - Log.w(TAG, "Punctuation model not available, continuing without it", e) } } @@ -192,6 +202,23 @@ class MainViewModel @Inject constructor( } } + /** + * Releases every warm ASR recognizer before running [action] -- trades + * away the pool's "keep both swap directions warm" convenience (instant + * swap-back) for a lower peak memory footprint, so the upcoming model + * load never has to coexist with whatever's already resident. Called + * unconditionally before every start/swap (not just when memory happens + * to be low): on memory-constrained devices, loading a new ASR model on + * top of an already-warm one was observed reliably tipping the whole + * system into a low-memory-killer thrashing cascade, and the full-reload + * cost this trades in for (a few seconds) was confirmed an acceptable + * default rather than something to ask about every time. + */ + fun releaseWarmRecognizersThen(action: () -> Unit) { + recognizerPool.releaseAll() + action() + } + /** * Flips source <-> target language for a conversation-direction swap. * Only enabled when the current target is itself ASR-capable (otherwise @@ -199,12 +226,13 @@ class MainViewModel @Inject constructor( * gate the UI control on that same condition. * * If a translation session is running on the microphone source, restarts - * it so the new pairing takes effect immediately; the restart is fast - * because the target language's ASR model was already pre-warmed by - * [preloadPipelineComponents]. System-audio sessions are left running - * with the old pairing (restarting would require re-requesting - * MediaProjection consent via an Activity), taking effect on the next - * manual start. + * it so the new pairing takes effect immediately -- the restart pays a + * full model reload (a few seconds), since [releaseWarmRecognizersThen]'s + * reasoning applies here too: the previous direction's model is released + * before the new one loads, rather than kept warm alongside it. System- + * audio sessions are left running with the old pairing (restarting would + * require re-requesting MediaProjection consent via an Activity), taking + * effect on the next manual start. */ fun swapLanguages() { val current = settings.value @@ -226,13 +254,19 @@ class MainViewModel @Inject constructor( // Intents at the service (fire-and-forget) -- wait for the // stop to actually land so the START intent below can't race // a still-in-flight stopSelf() from the previous session. + // This also guarantees the recognizer released just below + // isn't still actively processing audio when released. withTimeoutOrNull(2_000) { isRunning.first { !it } } } settingsRepository.swapLanguages() - // Pre-warm the new source (almost always already warm as the - // previous target) so a subsequent start is instant either way. + // Release the previous direction's warm recognizer before + // loading the new one instead of keeping both resident -- see + // releaseWarmRecognizersThen's doc for why. Safe here: any + // running session was already stopped and waited on above. + recognizerPool.releaseAll() + val newSrcLang = current.targetLanguage try { modelDownloader.ensureModelAvailable(newSrcLang) @@ -299,6 +333,26 @@ class MainViewModel @Inject constructor( application.startService(intent) } + /** + * Pauses recognition/TTS without tearing down the pipeline (models, + * audio capture, and the foreground session all stay alive) so a quick + * tap can resume exactly where it left off -- unlike [stopTranslation], + * which ends the session. + */ + fun pauseTranslation() { + val intent = Intent(application, TranslationService::class.java).apply { + action = TranslationService.ACTION_PAUSE + } + application.startService(intent) + } + + fun resumeTranslation() { + val intent = Intent(application, TranslationService::class.java).apply { + action = TranslationService.ACTION_RESUME + } + application.startService(intent) + } + fun updateAudioSource(source: AudioCaptureManager.Source) { viewModelScope.launch { settingsRepository.updateAudioSource(source) From 610fa588f77588874d8050500df983895f9e79bf Mon Sep 17 00:00:00 2001 From: michael Date: Wed, 15 Jul 2026 16:04:16 +0200 Subject: [PATCH 21/21] Rework the record button: tap pauses/resumes, long-press stops The button previously toggled start/stop on every tap. It's now a Surface with combinedClickable (FloatingActionButton's own onClick fought with an overlaid combinedClickable, silently swallowing every tap when tried first): short tap starts, or pauses/resumes while running; long-press fully stops. Ending the session is otherwise only triggered by a language swap, which needs a real restart anyway. Icon reflects current state, not the next action: Mic while actively recording, Pause once actually paused (this was backwards in an earlier iteration -- tapping to start showed the Pause icon immediately). The "Listening..." status line likewise switches to a static "Paused" label and icon instead of a misleading spinner. --- .../ui/screens/MainScreen.kt | 164 +++++++++++++----- app/src/main/res/values-ru/strings.xml | 4 +- app/src/main/res/values/strings.xml | 4 +- 3 files changed, 122 insertions(+), 50 deletions(-) diff --git a/app/src/main/java/com/example/instantvoicetranslate/ui/screens/MainScreen.kt b/app/src/main/java/com/example/instantvoicetranslate/ui/screens/MainScreen.kt index 3d95acc..5ec0d19 100644 --- a/app/src/main/java/com/example/instantvoicetranslate/ui/screens/MainScreen.kt +++ b/app/src/main/java/com/example/instantvoicetranslate/ui/screens/MainScreen.kt @@ -8,6 +8,7 @@ import androidx.activity.result.contract.ActivityResultContracts import androidx.compose.animation.AnimatedVisibility import androidx.compose.animation.fadeIn import androidx.compose.animation.fadeOut +import androidx.compose.foundation.combinedClickable import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.Column @@ -23,16 +24,17 @@ import androidx.compose.foundation.rememberScrollState import androidx.compose.foundation.verticalScroll import androidx.compose.material.icons.Icons import androidx.compose.material.icons.filled.Mic +import androidx.compose.material.icons.filled.Pause import androidx.compose.material.icons.filled.Settings -import androidx.compose.material.icons.filled.Stop import androidx.compose.material.icons.filled.SwapHoriz import androidx.compose.material.icons.automirrored.filled.VolumeUp import androidx.compose.material3.Card import androidx.compose.material3.CardDefaults import androidx.compose.material3.CircularProgressIndicator +import androidx.compose.material3.contentColorFor import androidx.compose.material3.ExperimentalMaterial3Api import androidx.compose.material3.FilledTonalButton -import androidx.compose.material3.FloatingActionButton +import androidx.compose.material3.FloatingActionButtonDefaults import androidx.compose.material3.HorizontalDivider import androidx.compose.material3.Icon import androidx.compose.material3.IconButton @@ -44,6 +46,7 @@ import androidx.compose.material3.SegmentedButtonDefaults import androidx.compose.material3.SingleChoiceSegmentedButtonRow import androidx.compose.material3.SnackbarHost import androidx.compose.material3.SnackbarHostState +import androidx.compose.material3.Surface import androidx.compose.material3.Text import androidx.compose.material3.TopAppBar import androidx.compose.runtime.Composable @@ -73,6 +76,7 @@ fun MainScreen( ) { val isStarting by viewModel.isStarting.collectAsStateWithLifecycle() val isRunning by viewModel.isRunning.collectAsStateWithLifecycle() + val isPaused by viewModel.isPaused.collectAsStateWithLifecycle() val partialText by viewModel.partialText.collectAsStateWithLifecycle() val originalText by viewModel.originalText.collectAsStateWithLifecycle() val translatedText by viewModel.translatedText.collectAsStateWithLifecycle() @@ -94,6 +98,31 @@ fun MainScreen( } } + fun startTranslationFlow() { + if (settings.audioSource == AudioCaptureManager.Source.SYSTEM_AUDIO) { + val projectionManager = context.getSystemService( + Context.MEDIA_PROJECTION_SERVICE + ) as MediaProjectionManager + mediaProjectionLauncher.launch(projectionManager.createScreenCaptureIntent()) + } else { + viewModel.startTranslation() + } + } + + // Always release whatever's currently warm before loading the next + // language, rather than keeping the previous direction resident too (the + // pool's normal "keep both swap directions warm" behavior). Loading a + // new ASR model on top of an already-warm one was observed reliably + // tipping memory-constrained devices into a full system-wide OOM + // cascade; paying a full reload every swap (a few seconds) instead of an + // instant one is a trade Michael confirmed is worth making by default. + // (swapLanguages() does its own release internally, at the point where + // it's guaranteed safe -- after stopping any running session -- rather + // than upfront here.) + fun requestStartTranslation() { + viewModel.releaseWarmRecognizersThen(::startTranslationFlow) + } + LaunchedEffect(error) { error?.let { snackbarHostState.showSnackbar(it) @@ -135,43 +164,67 @@ fun MainScreen( snackbarHost = { SnackbarHost(snackbarHostState) }, floatingActionButton = { if (modelStatus is ModelStatus.Ready) { - FloatingActionButton( - onClick = { - if (isRunning) { - viewModel.stopTranslation() - } else if (!isStarting) { - if (settings.audioSource == AudioCaptureManager.Source.SYSTEM_AUDIO) { - val projectionManager = context.getSystemService( - Context.MEDIA_PROJECTION_SERVICE - ) as MediaProjectionManager - mediaProjectionLauncher.launch( - projectionManager.createScreenCaptureIntent() - ) - } else { - viewModel.startTranslation() - } - } - }, - containerColor = when { - isRunning -> MaterialTheme.colorScheme.error - isStarting -> MaterialTheme.colorScheme.surfaceVariant - else -> MaterialTheme.colorScheme.primary - } + // Tap: pause/resume without ending the session (or start, when + // idle). Long-press: fully stop -- ends the foreground session, + // releases audio capture. Ending the session is otherwise only + // triggered by a language swap, which needs a real restart. + // + // Built from Surface rather than FloatingActionButton: layering + // Modifier.combinedClickable on top of a FloatingActionButton's + // own (no-op) onClick created two competing gesture detectors -- + // the FAB's internal one silently won, so taps never reached our + // handler at all. A single combinedClickable on a plain Surface + // has only one gesture detector, so this is unambiguous. + val fabContainerColor = when { + isRunning && isPaused -> MaterialTheme.colorScheme.tertiary + isRunning -> MaterialTheme.colorScheme.error + isStarting -> MaterialTheme.colorScheme.surfaceVariant + else -> MaterialTheme.colorScheme.primary + } + Surface( + modifier = Modifier + .size(56.dp) + .combinedClickable( + onClick = { + when { + isRunning && isPaused -> viewModel.resumeTranslation() + isRunning -> viewModel.pauseTranslation() + !isStarting -> requestStartTranslation() + } + }, + onLongClick = { + if (isRunning) viewModel.stopTranslation() + }, + ), + shape = FloatingActionButtonDefaults.shape, + color = fabContainerColor, + contentColor = MaterialTheme.colorScheme.contentColorFor(fabContainerColor), + tonalElevation = 6.dp, + shadowElevation = 6.dp, ) { - when { - isStarting -> CircularProgressIndicator( - modifier = Modifier.size(24.dp), - strokeWidth = 2.5.dp, - color = MaterialTheme.colorScheme.onSurfaceVariant, - ) - isRunning -> Icon( - imageVector = Icons.Default.Stop, - contentDescription = stringResource(R.string.action_stop), - ) - else -> Icon( - imageVector = Icons.Default.Mic, - contentDescription = stringResource(R.string.action_start), - ) + Box(contentAlignment = Alignment.Center, modifier = Modifier.fillMaxSize()) { + when { + isStarting -> CircularProgressIndicator( + modifier = Modifier.size(24.dp), + strokeWidth = 2.5.dp, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + // Icon reflects the CURRENT state, not the next + // action: Mic while actively recording, Pause + // once actually paused. + isRunning && isPaused -> Icon( + imageVector = Icons.Default.Pause, + contentDescription = stringResource(R.string.action_resume), + ) + isRunning -> Icon( + imageVector = Icons.Default.Mic, + contentDescription = stringResource(R.string.action_pause), + ) + else -> Icon( + imageVector = Icons.Default.Mic, + contentDescription = stringResource(R.string.action_start), + ) + } } } } @@ -352,16 +405,31 @@ fun MainScreen( verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.Center, ) { - CircularProgressIndicator( - modifier = Modifier.size(16.dp), - strokeWidth = 2.dp - ) - Spacer(modifier = Modifier.width(8.dp)) - Text( - stringResource(R.string.status_listening), - style = MaterialTheme.typography.bodySmall, - color = MaterialTheme.colorScheme.primary, - ) + if (isPaused) { + Icon( + imageVector = Icons.Default.Pause, + contentDescription = null, + modifier = Modifier.size(16.dp), + tint = MaterialTheme.colorScheme.tertiary, + ) + Spacer(modifier = Modifier.width(8.dp)) + Text( + stringResource(R.string.status_paused), + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.tertiary, + ) + } else { + CircularProgressIndicator( + modifier = Modifier.size(16.dp), + strokeWidth = 2.dp + ) + Spacer(modifier = Modifier.width(8.dp)) + Text( + stringResource(R.string.status_listening), + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.primary, + ) + } } } diff --git a/app/src/main/res/values-ru/strings.xml b/app/src/main/res/values-ru/strings.xml index 0003af4..38c4ec5 100644 --- a/app/src/main/res/values-ru/strings.xml +++ b/app/src/main/res/values-ru/strings.xml @@ -4,14 +4,16 @@ Настройки - Стоп Старт + Пауза + Продолжить Перевод Нажмите кнопку микрофона для начала перевода Оригинал Частичное распознавание … Слушаю… + Пауза Микрофон Системный звук diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 8fea61a..f5966bd 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -5,14 +5,16 @@ Settings Swap languages - Stop Start + Pause + Resume Translation Press the mic button to start translating Original Partial ASR … Listening… + Paused Microphone System Audio