The output of the model is unstable, and the timbre will change.

[pengpengtao]

The generated text will always have the sound of enenene humming, just like it feels like it has collapsed.

[zhenye234]

hi, can you show me your code

[pengpengtao]

您好，您能给我看一下您的代码吗

from transformers import AutoTokenizer, AutoModelForCausalLM
from faster_whisper import WhisperModel
import torch
from xcodec2.modeling_xcodec2 import XCodec2Model
import torchaudio
import numpy as np
import random

初始化随机种子

def set_seed(seed=49):
np.random.seed(seed)
random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed(seed)
torch.cuda.manual_seed_all(seed)
torch.backends.cudnn.benchmark = False
torch.backends.cudnn.deterministic = True

初始化ASR模型

def init_asr_model():
return WhisperModel("faster-whisper-large-v3", device="cpu")

语音转录功能

def transcribe_audio(audio_path):
model = init_asr_model()
try:
segments, _ = model.transcribe(
audio=audio_path,
beam_size=5,
vad_filter=True,
vad_parameters=dict(min_silence_duration_ms=700)
)
return "\n".join([segment.text for segment in segments])
except Exception as e:
print(f"ASR Error: {str(e)}")
return ""

初始化语音生成模型

def init_tts_models():
# 初始化Codec模型
codec_model = XCodec2Model.from_pretrained(
"/data/user/gk/audio/Llasa/xcodec2",
device_map='cuda'
)
codec_model.eval()

# 初始化LLM模型
tokenizer = AutoTokenizer.from_pretrained("/data/user/gk/audio/Llasa/Llasa-1B")
llm_model = AutoModelForCausalLM.from_pretrained(
    "/data/user/gk/audio/Llasa/Llasa-1B",
    torch_dtype=torch.float16,
    device_map='cuda'
)
llm_model.eval()

return codec_model, tokenizer, llm_model

核心生成函数

def generate_speech(
reference_audio_path: str,
reference_text: str,
target_text: str,
output_path: str = "output.wav",
sample_rate: int = 24000,
temperature: float = 1.0,
top_k: int = 50,
top_p: float = 0.9,
repetition_penalty: float = 1.2
):
set_seed()

# 加载模型
codec_model, tokenizer, llm_model = init_tts_models()

# 处理输入音频
try:
    waveform, orig_sr = torchaudio.load(reference_audio_path)
    if waveform.size(0) > 1:
        waveform = torch.mean(waveform, dim=0, keepdim=True)
    resampled_wav = torchaudio.transforms.Resample(orig_sr, sample_rate)(waveform)
except Exception as e:
    print(f"Audio processing error: {str(e)}")
    return

# 生成语音编码
with torch.no_grad():
    vq_code = codec_model.encode_code(resampled_wav.cuda())
    speech_ids = [f"<|s_{int(i)}|>" for i in vq_code[0,0,:].cpu().numpy()]

# 构建生成提示
prompt = [
    {"role": "user", "content": f"Convert the text to speech:<|TEXT_UNDERSTANDING_START|>{reference_text} {target_text}<|TEXT_UNDERSTANDING_END|>"},
    {"role": "assistant", "content": f"<|SPEECH_GENERATION_START|>{''.join(speech_ids)}"}
]

# 生成语音标记
input_ids = tokenizer.apply_chat_template(
    prompt, 
    tokenize=True, 
    return_tensors='pt', 
    continue_final_message=True
).cuda()

generated_ids = llm_model.generate(
    input_ids,
    max_length=2048,
    eos_token_id=tokenizer.convert_tokens_to_ids('<|SPEECH_GENERATION_END|>'),
    do_sample=True,
    top_p=top_p,
    top_k=top_k,
    temperature=temperature,
    repetition_penalty=repetition_penalty
)

# 解码生成结果
speech_tokens = tokenizer.batch_decode(generated_ids[:, input_ids.shape[1]:-1], skip_special_tokens=True)
speech_ids = [int(t[4:-2]) for t in speech_tokens if t.startswith('<|s_')]

# 生成音频波形
try:
    speech_tensor = torch.tensor(speech_ids, device='cuda').unsqueeze(0).unsqueeze(0)
    gen_wav = codec_model.decode_code(speech_tensor)
    torchaudio.save(output_path, gen_wav[0,0].cpu(), sample_rate)
    print(f"Successfully generated: {output_path}")
except Exception as e:
    print(f"Audio generation failed: {str(e)}")

使用示例

if name == "main":
#tts_engine = TTSGenerator()

# 生成语音
generate_speech(
    sample_audio_path="1.wav",
    sample_text=",",
    target_text="",
    output_path="generated_audio.wav",
    temperature=0.7,
    top_k=30
)

#print(f"Generated audio saved to: {result_path}")

[Petriorlz]

we also face the same question, use the offical code to prompt zero-shot tts task, the model often output silence or some unstable voice