Skip to content
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 45 additions & 6 deletions app/src/main/java/com/k2fsa/sherpa/onnx/tts/engine/TtsService.kt
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,25 @@ import android.speech.tts.SynthesisRequest
import android.speech.tts.TextToSpeech
import android.speech.tts.TextToSpeechService
import android.util.Log
import java.io.ByteArrayOutputStream


class TtsService : TextToSpeechService() {

// How much audio to accumulate before calling callback.start(), so the
// system AudioTrack isn't left draining an empty buffer while the model
// computes the first chunk. Without this, the first inference call
// (slower than steady-state, since onnxruntime warms up its execution
// graph on first use) races the AudioTrack and causes an audible
// crackle/underrun at the start of every utterance. See issue #90 for
// the same root cause manifesting as a clipped first word instead.
private val PRE_ROLL_MS = 300

// Member variables to hold state for the callback
private var currentPitch = 100f
private var currentSynthesisCallback: SynthesisCallback? = null
private var preRollBuffer: ByteArrayOutputStream? = null
private var preRollThresholdBytes = 0

override fun onCreate() {
Log.i(TAG, "onCreate tts service")
Expand Down Expand Up @@ -97,17 +109,19 @@ class TtsService : TextToSpeechService() {


val tts = TtsEngine.tts!!

callback.start(tts.sampleRate(), AudioFormat.ENCODING_PCM_16BIT, 1)
val sampleRate = tts.sampleRate()

if (text.isBlank() || text.isEmpty()) {
callback.start(sampleRate, AudioFormat.ENCODING_PCM_16BIT, 1)
callback.done()
return
}

// Store state in member variables so the function reference can access them
currentPitch = pitch
currentSynthesisCallback = callback
preRollBuffer = ByteArrayOutputStream()
preRollThresholdBytes = sampleRate * 2 * PRE_ROLL_MS / 1000 // 16-bit mono PCM

// FIX: Use a function reference (::ttsCallback) instead of an inline lambda.
// This forces the Kotlin compiler to generate the correct JNI signature: ([F)Ljava/lang/Integer;
Expand All @@ -117,6 +131,10 @@ class TtsService : TextToSpeechService() {
callback = ::ttsCallback,
)

// Short utterances may finish generating before the pre-roll threshold is
// ever reached; flush whatever was buffered so playback still starts.
flushPreRoll(callback, sampleRate)

callback.done()

// Clear state after synthesis is complete
Expand Down Expand Up @@ -150,17 +168,38 @@ class TtsService : TextToSpeechService() {
samples = floatArrayToByteArray(floatSamples)
}

val buffer = preRollBuffer
if (buffer != null) {
buffer.write(samples)
if (buffer.size() >= preRollThresholdBytes) {
flushPreRoll(cb, TtsEngine.tts!!.sampleRate())
}
} else {
writeAudio(cb, samples)
}

// 1 means to continue
// 0 means to stop
return 1
}

// Calls callback.start() (if not already called for this utterance) and flushes
// whatever has been buffered so far. After this, ttsCallback streams directly.
private fun flushPreRoll(cb: SynthesisCallback, sampleRate: Int) {
val buffer = preRollBuffer ?: return
preRollBuffer = null
cb.start(sampleRate, AudioFormat.ENCODING_PCM_16BIT, 1)
writeAudio(cb, buffer.toByteArray())
}

private fun writeAudio(cb: SynthesisCallback, samples: ByteArray) {
val maxBufferSize: Int = cb.maxBufferSize
var offset = 0
while (offset < samples.size) {
val bytesToWrite = Math.min(maxBufferSize, samples.size - offset)
cb.audioAvailable(samples, offset, bytesToWrite)
offset += bytesToWrite
}

// 1 means to continue
// 0 means to stop
return 1
}

private fun floatArrayToByteArray(audio: FloatArray): ByteArray {
Expand Down