[runtime] rename vocab to g2p prosody vocab (#239)

Author: robin1001

runtime/core/bin/tts_main.cc

@@ -33,9 +33,6 @@ DEFINE_string(vits_flags, "", "vits flags file");
 DEFINE_string(tagger, "", "tagger fst file");
 DEFINE_string(verbalizer, "", "verbalizer fst file");
 
-// Tokenizer
-DEFINE_string(vocab, "", "tokenizer vocab file");
-
 // G2P for English
 DEFINE_string(cmudict, "", "cmudict for english words");
 DEFINE_string(g2p_en_model, "", "english g2p fst model for oov");
@@ -46,6 +43,7 @@ DEFINE_string(char2pinyin, "", "chinese character to pinyin");
 DEFINE_string(pinyin2id, "", "pinyin to id");
 DEFINE_string(pinyin2phones, "", "pinyin to phones");
 DEFINE_string(g2p_prosody_model, "", "g2p prosody model file");
+DEFINE_string(g2p_prosody_vocab, "", "g2p prosody vocab file");
 
 // VITS
 DEFINE_string(speaker2id, "", "speaker to id");
@@ -77,8 +75,8 @@ int main(int argc, char* argv[]) {
              : nullptr;
 
   auto g2p_prosody = std::make_shared<wetts::G2pProsody>(
-      FLAGS_g2p_prosody_model, FLAGS_vocab, FLAGS_char2pinyin, FLAGS_pinyin2id,
-      FLAGS_pinyin2phones, g2p_en);
+      FLAGS_g2p_prosody_model, FLAGS_g2p_prosody_vocab, FLAGS_char2pinyin,
+      FLAGS_pinyin2id, FLAGS_pinyin2phones, g2p_en);
   auto model = std::make_shared<wetts::TtsModel>(
       FLAGS_vits_encoder_model, FLAGS_vits_decoder_model, FLAGS_speaker2id,
       FLAGS_phone2id, FLAGS_sampling_rate, tn, g2p_prosody, FLAGS_chunk_size,

runtime/core/frontend/g2p_prosody.cc

@@ -31,17 +31,17 @@
 namespace wetts {
 
 G2pProsody::G2pProsody(const std::string& g2p_prosody_model,
-                       const std::string& vocab,
+                       const std::string& g2p_prosody_vocab,
                        const std::string& lexicon_file,
                        const std::string& pinyin2id,
                        const std::string& pinyin2phones,
                        std::shared_ptr<G2pEn> g2p_en)
     : g2p_en_(std::move(g2p_en)), model_(g2p_prosody_model) {
-  std::ifstream in(vocab);
+  std::ifstream in(g2p_prosody_vocab);
   std::string line;
   int id = 0;
   while (getline(in, line)) {
-    vocab_[line] = id;
+    g2p_vocab_[line] = id;
     id++;
   }
   lexicon_ = std::make_shared<Lexicon>(lexicon_file);
@@ -61,7 +61,7 @@ void G2pProsody::Tokenize(const std::vector<std::string>& words,
                           std::vector<int64_t>* token_ids,
                           std::vector<int>* token_offsets) {
   token_ids->clear();
-  token_ids->emplace_back(vocab_.at(CLS_));
+  token_ids->emplace_back(g2p_vocab_.at(CLS_));
   token_offsets->clear();
   int offset = 1;  // 0 is taken by CLS_
   for (const std::string& word : words) {
@@ -71,20 +71,20 @@ void G2pProsody::Tokenize(const std::vector<std::string>& words,
       std::vector<std::string> chars;
       SplitUTF8StringToChars(word, &chars);
       for (const std::string& ch : chars) {
-        token_ids->emplace_back(vocab_.at(ch));
+        token_ids->emplace_back(g2p_vocab_.at(ch));
         offset++;
       }
     } else if (word[0] < 128 && std::isalnum(word[0])) {
       // English or digit word, Convert english word to UNK
-      token_ids->emplace_back(vocab_.at(UNK_));
+      token_ids->emplace_back(g2p_vocab_.at(UNK_));
       offset++;
     } else {
-      std::string v = vocab_.find(word) != vocab_.end() ? word : UNK_;
-      token_ids->emplace_back(vocab_.at(v));
+      std::string v = g2p_vocab_.find(word) != g2p_vocab_.end() ? word : UNK_;
+      token_ids->emplace_back(g2p_vocab_.at(v));
       offset++;
     }
   }
-  token_ids->emplace_back(vocab_.at(SEP_));
+  token_ids->emplace_back(g2p_vocab_.at(SEP_));
 }
 
 void G2pProsody::Forward(const std::vector<std::string>& words,

runtime/core/frontend/g2p_prosody.h

@@ -33,7 +33,8 @@ namespace wetts {
 class G2pProsody {
  public:
   explicit G2pProsody(const std::string& g2p_prosody_model,
-                      const std::string& vocab, const std::string& lexicon_file,
+                      const std::string& g2p_prosody_vocab,
+                      const std::string& lexicon_file,
                       const std::string& pinyin2id,
                       const std::string& pinyin2phones,
                       std::shared_ptr<G2pEn> g2p_en = nullptr);
@@ -52,7 +53,7 @@ class G2pProsody {
   const std::string SEP_ = "[SEP]";
   const std::string UNK_ = "[UNK]";
   OnnxModel model_;
-  std::unordered_map<std::string, int> vocab_;
+  std::unordered_map<std::string, int> g2p_vocab_;
   std::unordered_map<std::string, int> phones_;
   std::shared_ptr<G2pEn> g2p_en_;
   std::shared_ptr<Lexicon> lexicon_;