From a74fc9b1a94c901d551c18bb6313915db3609eeb Mon Sep 17 00:00:00 2001 From: michael Date: Tue, 14 Jul 2026 15:44:22 +0200 Subject: [PATCH] Fix audio crackle at the start of every synthesized utterance TtsService called callback.start() before any audio had been generated, so the system AudioTrack began draining an empty buffer while the model computed its first inference chunk (slower than steady-state, since onnxruntime warms up its execution graph on first use). The race caused an audible crackle/underrun at the start of every utterance played through the system TTS API (confirmed via AudioTrack "disabled due to previous underrun, restarting" in logcat). Likely the same root cause as #90, which reports it as a clipped first word instead of a crackle. Buffer ~300ms of audio before calling callback.start(), giving the AudioTrack a pre-roll cushion so inference no longer races playback. Short utterances that finish generating before the threshold is reached flush whatever was buffered once generation completes, so added latency stays proportional to utterance length rather than a flat per-utterance cost. Tested on a Pixel 7a: before the fix, every utterance played through the system TTS API produced an audible crackle and an AudioTrack underrun warning in logcat; after the fix neither reproduces across repeated test utterances in different languages. Co-Authored-By: Claude Sonnet 5 --- .../sherpa/onnx/tts/engine/TtsService.kt | 51 ++++++++++++++++--- 1 file changed, 45 insertions(+), 6 deletions(-) diff --git a/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/TtsService.kt b/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/TtsService.kt index 68c3c23..05e9ab6 100644 --- a/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/TtsService.kt +++ b/app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/TtsService.kt @@ -7,13 +7,25 @@ import android.speech.tts.SynthesisRequest import android.speech.tts.TextToSpeech import android.speech.tts.TextToSpeechService import android.util.Log +import java.io.ByteArrayOutputStream class TtsService : TextToSpeechService() { + // How much audio to accumulate before calling callback.start(), so the + // system AudioTrack isn't left draining an empty buffer while the model + // computes the first chunk. Without this, the first inference call + // (slower than steady-state, since onnxruntime warms up its execution + // graph on first use) races the AudioTrack and causes an audible + // crackle/underrun at the start of every utterance. See issue #90 for + // the same root cause manifesting as a clipped first word instead. + private val PRE_ROLL_MS = 300 + // Member variables to hold state for the callback private var currentPitch = 100f private var currentSynthesisCallback: SynthesisCallback? = null + private var preRollBuffer: ByteArrayOutputStream? = null + private var preRollThresholdBytes = 0 override fun onCreate() { Log.i(TAG, "onCreate tts service") @@ -97,10 +109,10 @@ class TtsService : TextToSpeechService() { val tts = TtsEngine.tts!! - - callback.start(tts.sampleRate(), AudioFormat.ENCODING_PCM_16BIT, 1) + val sampleRate = tts.sampleRate() if (text.isBlank() || text.isEmpty()) { + callback.start(sampleRate, AudioFormat.ENCODING_PCM_16BIT, 1) callback.done() return } @@ -108,6 +120,8 @@ class TtsService : TextToSpeechService() { // Store state in member variables so the function reference can access them currentPitch = pitch currentSynthesisCallback = callback + preRollBuffer = ByteArrayOutputStream() + preRollThresholdBytes = sampleRate * 2 * PRE_ROLL_MS / 1000 // 16-bit mono PCM // FIX: Use a function reference (::ttsCallback) instead of an inline lambda. // This forces the Kotlin compiler to generate the correct JNI signature: ([F)Ljava/lang/Integer; @@ -117,6 +131,10 @@ class TtsService : TextToSpeechService() { callback = ::ttsCallback, ) + // Short utterances may finish generating before the pre-roll threshold is + // ever reached; flush whatever was buffered so playback still starts. + flushPreRoll(callback, sampleRate) + callback.done() // Clear state after synthesis is complete @@ -150,6 +168,31 @@ class TtsService : TextToSpeechService() { samples = floatArrayToByteArray(floatSamples) } + val buffer = preRollBuffer + if (buffer != null) { + buffer.write(samples) + if (buffer.size() >= preRollThresholdBytes) { + flushPreRoll(cb, TtsEngine.tts!!.sampleRate()) + } + } else { + writeAudio(cb, samples) + } + + // 1 means to continue + // 0 means to stop + return 1 + } + + // Calls callback.start() (if not already called for this utterance) and flushes + // whatever has been buffered so far. After this, ttsCallback streams directly. + private fun flushPreRoll(cb: SynthesisCallback, sampleRate: Int) { + val buffer = preRollBuffer ?: return + preRollBuffer = null + cb.start(sampleRate, AudioFormat.ENCODING_PCM_16BIT, 1) + writeAudio(cb, buffer.toByteArray()) + } + + private fun writeAudio(cb: SynthesisCallback, samples: ByteArray) { val maxBufferSize: Int = cb.maxBufferSize var offset = 0 while (offset < samples.size) { @@ -157,10 +200,6 @@ class TtsService : TextToSpeechService() { cb.audioAvailable(samples, offset, bytesToWrite) offset += bytesToWrite } - - // 1 means to continue - // 0 means to stop - return 1 } private fun floatArrayToByteArray(audio: FloatArray): ByteArray {