From 4284d88db8dee737b72c3de2642df93561e3a7b9 Mon Sep 17 00:00:00 2001 From: Tanuj Jain Date: Tue, 31 Aug 2021 14:31:55 +0200 Subject: [PATCH 01/10] Update gitignore. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 6ad59d2..0e06443 100644 --- a/.gitignore +++ b/.gitignore @@ -137,3 +137,4 @@ dmypy.json # Pyre type checker .pyre/ +.ljspeech \ No newline at end of file From 8068ef0b1538011f6667919590a80e68a9a6fe6c Mon Sep 17 00:00:00 2001 From: Tanuj Jain Date: Tue, 31 Aug 2021 14:33:15 +0200 Subject: [PATCH 02/10] Add symbols from updated phonemizer (deep phonemizer) --- data/text/symbols.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/data/text/symbols.py b/data/text/symbols.py index 809e4ec..ef6402c 100644 --- a/data/text/symbols.py +++ b/data/text/symbols.py @@ -4,8 +4,9 @@ _suprasegmentals = 'ˈˌːˑ' _other_symbols = 'ʍwɥʜʢʡɕʑɺɧ' _diacrilics = 'ɚ˞ɫ' +_extra_phons = ['g', 'ɝ', '̃', '̍', '̥', '̩', '̯', '͡'] # some extra symbols that I found in from wiktionary ipa annotations _phonemes = sorted(list( - _vowels + _non_pulmonic_consonants + _pulmonic_consonants + _suprasegmentals + _other_symbols + _diacrilics)) + _vowels + _non_pulmonic_consonants + _pulmonic_consonants + _suprasegmentals + _other_symbols + _diacrilics + _extra_phons)) _punctuations = '!,-.:;? \'()' _alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzäüößÄÖÜ' From 4e6241c5510dabc3516c81477e10474f57af6a21 Mon Sep 17 00:00:00 2001 From: Tanuj Jain Date: Thu, 16 Sep 2021 18:14:52 +0200 Subject: [PATCH 03/10] Most changes should work here. --- .gitignore | 5 ++- config/training_config.yaml | 20 ++++++------ create_training_data.py | 62 +++++++++++++++++++++++++++++++++++-- data/text/symbols.py | 2 +- data/text/tokenizer.py | 25 ++++++++------- predict_tts.py | 3 +- 6 files changed, 90 insertions(+), 27 deletions(-) diff --git a/.gitignore b/.gitignore index 0e06443..68a7689 100644 --- a/.gitignore +++ b/.gitignore @@ -137,4 +137,7 @@ dmypy.json # Pyre type checker .pyre/ -.ljspeech \ No newline at end of file +.ljspeech + +# custom +outputs \ No newline at end of file diff --git a/config/training_config.yaml b/config/training_config.yaml index 2a12af0..16c4b89 100644 --- a/config/training_config.yaml +++ b/config/training_config.yaml @@ -1,9 +1,9 @@ paths: # PATHS: change accordingly - wav_directory: '/path/to/wav_directory' # path to directory cointaining the wavs - metadata_path: '/path/to/metadata.csv' # name of metadata file under wav_directory - log_directory: '/path/to/logs_directory' # weights and logs are stored here - train_data_directory: 'transformer_tts_data' # training data is stored here + wav_directory: '/Users/tjain1/Desktop/Projects/tts/data/data_small' #'/Users/tjain1/Desktop/Projects/tts/data/ASVoice4_incl_english' # path to directory cointaining the wavs + metadata_path: '/Users/tjain1/Desktop/Projects/tts/data/ph_metadata_small.csv' #'/Users/tjain1/Desktop/Projects/tts/data/metadata_flagged_phonemized_bind_nostress.csv' # name of metadata file under wav_directory + log_directory: '/Users/tjain1/Desktop/Projects/tts/results/logs_txtts_ph_dev_small' # weights and logs are stored here + train_data_directory: '/Users/tjain1/Desktop/Projects/tts/results/logs_txtts_ph_dev_small/transformer_tts_data' # training data is stored here naming: data_name: ljspeech # raw data naming for default data reader (select function from data/metadata_readers.py) @@ -51,7 +51,7 @@ audio_settings: text_settings: # TOKENIZER - phoneme_language: 'en-us' + phoneme_language: 'en-us' # only relevant if non-phonemized data is provided as input, else, a placeholder value here will suffice with_stress: True # use stress symbols in phonemization model_breathing: false # add a token for the initial breathing @@ -78,10 +78,8 @@ aligner_settings: - [0, 1.0e-4] reduction_factor_schedule: - [0, 10] - - [80_000, 5] - - [100_000, 2] - - [130_000, 1] - max_steps: 260_000 + - [5_000, 1] + max_steps: 8_000 force_encoder_diagonal_steps: 500 force_decoder_diagonal_steps: 7_000 extract_attention_weighted: False # weighted average between last layer decoder attention heads when extracting durations @@ -128,7 +126,7 @@ tts_settings: dropout_rate: 0.1 learning_rate_schedule: - [0, 1.0e-4] - max_steps: 100_000 + max_steps: 10_000 debug: False # LOGGING @@ -138,7 +136,7 @@ tts_settings: weights_save_starting_step: 5_000 train_images_plotting_frequency: 1_000 keep_n_weights: 5 - keep_checkpoint_every_n_hours: 12 + keep_checkpoint_every_n_hours: 1 n_steps_avg_losses: [100, 500, 1_000, 5_000] # command line display of average loss values for the last n steps prediction_start_step: 4_000 text_prediction: diff --git a/create_training_data.py b/create_training_data.py index bacd22e..77458cc 100644 --- a/create_training_data.py +++ b/create_training_data.py @@ -16,7 +16,7 @@ parser = argparse.ArgumentParser() parser.add_argument('--config', type=str, required=True) -parser.add_argument('--skip_phonemes', action='store_true') +parser.add_argument('--skip_phonemization', action='store_true') parser.add_argument('--skip_mels', action='store_true') args = parser.parse_args() @@ -98,7 +98,7 @@ def process_pitches(item: tuple): summary_manager.display_scalar('Total duration (hours)', scalar_value=total_wav_len / audio.config['sampling_rate'] / 60. ** 2) -if not args.skip_phonemes: +if not args.skip_phonemization: remove_files = pickle.load(open(cm.data_dir / 'under-over_sized_mels.pkl', 'rb')) phonemized_metadata_path = cm.phonemized_metadata_path train_metadata_path = cm.train_metadata_path @@ -176,5 +176,63 @@ def process_phonemes(file_id): f'Length of metadata ({metadata_len}) does not match the length of the phoneme array ({len(set(list(phonemized_data.keys())))}). Check for empty text lines in metadata.' assert len(train_metadata) + len(test_metadata) == metadata_len, \ f'Train and/or validation lengths incorrect. ({len(train_metadata)} + {len(test_metadata)} != {metadata_len})' +else: + print('Skipped phonemization, assuming data is already phonemized ..') + remove_files = pickle.load(open(cm.data_dir / 'under-over_sized_mels.pkl', 'rb')) + phonemized_metadata_path = cm.phonemized_metadata_path # Stores data in a particular format + train_metadata_path = cm.train_metadata_path + test_metadata_path = cm.valid_metadata_path + print(f'\nReading metadata from {metadatareader.metadata_path}') + print(f'\nFound {len(metadatareader.filenames)} lines.') + print(f'\nRemoving {len(remove_files)} line(s) due to mel filtering.') + metadata_file_ids = [fname for fname in cross_file_ids if fname not in remove_files] + metadata_len = len(metadata_file_ids) + sample_items = np.random.choice(metadata_file_ids, 5) + test_len = cm.config['n_test'] + train_len = metadata_len - test_len + print(f'\nMetadata contains {metadata_len} lines.') + print(f'\nFiles will be stored under {cm.data_dir}') + print(f' - {train_len} training lines: {train_metadata_path}') + print(f' - {test_len} validation lines: {test_metadata_path}') + + # run cleaner on raw text + text_proc = TextToTokens.default(cm.config['phoneme_language'], add_start_end=False, + with_stress=cm.config['with_stress'], model_breathing=cm.config['model_breathing'], + njobs=1) + + + def process_phonemes(file_id): + text = metadatareader.text_dict[file_id] + try: + phon = text_proc.phonemizer(text, only_preprocess=True) + except Exception as e: + print(f'{e}\nFile id {file_id}') + raise BrokenPipeError + return (file_id, phon) + + + print('\ Processing Phonemes .. ') + phonemized_data = {} + phon_iter = p_uimap(process_phonemes, metadata_file_ids) + for (file_id, phonemes) in phon_iter: + phonemized_data.update({file_id: phonemes}) + + + print('\nMetadata samples:') + for i in sample_items: + print(f'{i}:{metadatareader.text_dict[i]}') + summary_manager.add_text(f'{i}/phonemes', text=metadatareader.text_dict[i]) + + new_metadata = [f'{k}|{v}\n' for k, v in phonemized_data.items()] #replaced phonemized_data + shuffled_metadata = np.random.permutation(new_metadata) + train_metadata = shuffled_metadata[0:train_len] + test_metadata = shuffled_metadata[-test_len:] + + with open(phonemized_metadata_path, 'w+', encoding='utf-8') as file: + file.writelines(new_metadata) + with open(train_metadata_path, 'w+', encoding='utf-8') as file: + file.writelines(train_metadata) + with open(test_metadata_path, 'w+', encoding='utf-8') as file: + file.writelines(test_metadata) print('\nDone') diff --git a/data/text/symbols.py b/data/text/symbols.py index ef6402c..4786ab6 100644 --- a/data/text/symbols.py +++ b/data/text/symbols.py @@ -6,7 +6,7 @@ _diacrilics = 'ɚ˞ɫ' _extra_phons = ['g', 'ɝ', '̃', '̍', '̥', '̩', '̯', '͡'] # some extra symbols that I found in from wiktionary ipa annotations _phonemes = sorted(list( - _vowels + _non_pulmonic_consonants + _pulmonic_consonants + _suprasegmentals + _other_symbols + _diacrilics + _extra_phons)) + _vowels + _non_pulmonic_consonants + _pulmonic_consonants + _suprasegmentals + _other_symbols + _diacrilics)) + _extra_phons _punctuations = '!,-.:;? \'()' _alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzäüößÄÖÜ' diff --git a/data/text/tokenizer.py b/data/text/tokenizer.py index 98a176c..b1366bc 100644 --- a/data/text/tokenizer.py +++ b/data/text/tokenizer.py @@ -57,21 +57,24 @@ def __init__(self, language: str, with_stress: bool, njobs=4): self._whitespace_re = re.compile(r'\s+') self._whitespace_punctuation_re = re.compile(f'\s*([{_punctuations}])\s*') - def __call__(self, text: Union[str, list], with_stress=None, njobs=None, language=None) -> Union[str, list]: + def __call__(self, text: Union[str, list], with_stress=None, njobs=None, language=None, only_preprocess=False) -> Union[str, list]: language = language or self.language njobs = njobs or self.njobs with_stress = with_stress or self.with_stress # phonemizer does not like hyphens. - text = self._preprocess(text) - phonemes = phonemize(text, - language=language, - backend='espeak', - strip=True, - preserve_punctuation=True, - with_stress=with_stress, - punctuation_marks=self.punctuation, - njobs=njobs, - language_switch='remove-flags') + if not only_preprocess: + text = self._preprocess(text) + phonemes = phonemize(text, + language=language, + backend='espeak', + strip=True, + preserve_punctuation=True, + with_stress=with_stress, + punctuation_marks=self.punctuation, + njobs=njobs, + language_switch='remove-flags') + else: + phonemes = text return self._postprocess(phonemes) def _preprocess_string(self, text: str): diff --git a/predict_tts.py b/predict_tts.py index 5063e7a..d1428d2 100644 --- a/predict_tts.py +++ b/predict_tts.py @@ -17,6 +17,7 @@ parser.add_argument('--store_mel', '-m', dest='store_mel', action='store_true') parser.add_argument('--verbose', '-v', dest='verbose', action='store_true') parser.add_argument('--single', '-s', dest='single', action='store_true') + parser.add_argument('--phonemized', dest='phonemized', action='store_false') args = parser.parse_args() if args.file is not None: @@ -46,7 +47,7 @@ print(f'Output wav under {output_path.parent}') wavs = [] for i, text_line in enumerate(text): - phons = model.text_pipeline.phonemizer(text_line) + phons = model.text_pipeline.phonemizer(text_line, only_preprocess=args.phonemized) tokens = model.text_pipeline.tokenizer(phons) if args.verbose: print(f'Predicting {text_line}') From 6caa43934134333b3e46c8b6bf2c33965b350329 Mon Sep 17 00:00:00 2001 From: Tanuj Jain Date: Thu, 16 Sep 2021 19:23:34 +0200 Subject: [PATCH 04/10] Clean create_training script. --- config/training_config.yaml | 6 +- create_training_data.py | 211 ++++++++++++++---------------------- predict_tts.py | 2 +- 3 files changed, 84 insertions(+), 135 deletions(-) diff --git a/config/training_config.yaml b/config/training_config.yaml index 16c4b89..bb18a9f 100644 --- a/config/training_config.yaml +++ b/config/training_config.yaml @@ -1,7 +1,7 @@ paths: # PATHS: change accordingly - wav_directory: '/Users/tjain1/Desktop/Projects/tts/data/data_small' #'/Users/tjain1/Desktop/Projects/tts/data/ASVoice4_incl_english' # path to directory cointaining the wavs - metadata_path: '/Users/tjain1/Desktop/Projects/tts/data/ph_metadata_small.csv' #'/Users/tjain1/Desktop/Projects/tts/data/metadata_flagged_phonemized_bind_nostress.csv' # name of metadata file under wav_directory + wav_directory: '/Users/tjain1/Desktop/Projects/tts/data/ASVoice4_incl_english' #'/Users/tjain1/Desktop/Projects/tts/data/data_small' # path to directory cointaining the wavs + metadata_path: '/Users/tjain1/Desktop/Projects/tts/data/metadata_flagged_phonemized_bind_nostress.csv' # '/Users/tjain1/Desktop/Projects/tts/data/ph_metadata_small.csv' # name of metadata file under wav_directory log_directory: '/Users/tjain1/Desktop/Projects/tts/results/logs_txtts_ph_dev_small' # weights and logs are stored here train_data_directory: '/Users/tjain1/Desktop/Projects/tts/results/logs_txtts_ph_dev_small/transformer_tts_data' # training data is stored here @@ -14,7 +14,7 @@ naming: # TRAINING DATA SETTINGS training_data_settings: - n_test: 100 + n_test: 2 mel_start_value: .5 mel_end_value: -.5 max_mel_len: 1_200 diff --git a/create_training_data.py b/create_training_data.py index 77458cc..e1dcf09 100644 --- a/create_training_data.py +++ b/create_training_data.py @@ -10,7 +10,7 @@ from data.datasets import DataReader from utils.training_config_manager import TrainingConfigManager from data.audio import Audio -from data.text.symbols import _alphabet +from data.text.symbols import _alphabet, all_phonemes np.random.seed(42) @@ -23,6 +23,8 @@ for arg in vars(args): print('{}: {}'.format(arg, getattr(args, arg))) +phonemized_flag = args.skip_phonemization # If the input is already in the form of phonemes, this flag will be set (User wanting to skip phonemization indicates that the input is already phonemized) + cm = TrainingConfigManager(args.config, aligner=True) cm.create_remove_dirs() metadatareader = DataReader.from_config(cm, kind='original', scan_wavs=True) @@ -97,142 +99,89 @@ def process_pitches(item: tuple): total_wav_len = total_mel_len * audio.config['hop_length'] summary_manager.display_scalar('Total duration (hours)', scalar_value=total_wav_len / audio.config['sampling_rate'] / 60. ** 2) - -if not args.skip_phonemization: - remove_files = pickle.load(open(cm.data_dir / 'under-over_sized_mels.pkl', 'rb')) - phonemized_metadata_path = cm.phonemized_metadata_path - train_metadata_path = cm.train_metadata_path - test_metadata_path = cm.valid_metadata_path - print(f'\nReading metadata from {metadatareader.metadata_path}') - print(f'\nFound {len(metadatareader.filenames)} lines.') + +def get_short_files(phonemized=False): + if not phonemized: + symbol_list = _alphabet + else: + symbol_list = all_phonemes + filter_metadata = [] for fname in cross_file_ids: item = metadatareader.text_dict[fname] - non_p = [c for c in item if c in _alphabet] + non_p = [c for c in item if c in symbol_list] if len(non_p) < 1: filter_metadata.append(fname) if len(filter_metadata) > 0: print(f'Removing {len(filter_metadata)} suspiciously short line(s):') for fname in filter_metadata: print(f'{fname}: {metadatareader.text_dict[fname]}') - print(f'\nRemoving {len(remove_files)} line(s) due to mel filtering.') - remove_files += filter_metadata - metadata_file_ids = [fname for fname in cross_file_ids if fname not in remove_files] - metadata_len = len(metadata_file_ids) - sample_items = np.random.choice(metadata_file_ids, 5) - test_len = cm.config['n_test'] - train_len = metadata_len - test_len - print(f'\nMetadata contains {metadata_len} lines.') - print(f'\nFiles will be stored under {cm.data_dir}') - print(f' - all: {phonemized_metadata_path}') - print(f' - {train_len} training lines: {train_metadata_path}') - print(f' - {test_len} validation lines: {test_metadata_path}') - - print('\nMetadata samples:') - for i in sample_items: - print(f'{i}:{metadatareader.text_dict[i]}') - summary_manager.add_text(f'{i}/text', text=metadatareader.text_dict[i]) - - # run cleaner on raw text - text_proc = TextToTokens.default(cm.config['phoneme_language'], add_start_end=False, - with_stress=cm.config['with_stress'], model_breathing=cm.config['model_breathing'], - njobs=1) - - - def process_phonemes(file_id): - text = metadatareader.text_dict[file_id] - try: - phon = text_proc.phonemizer(text) - except Exception as e: - print(f'{e}\nFile id {file_id}') - raise BrokenPipeError - return (file_id, phon) - - - print('\nPHONEMIZING') - phonemized_data = {} - phon_iter = p_uimap(process_phonemes, metadata_file_ids) - for (file_id, phonemes) in phon_iter: - phonemized_data.update({file_id: phonemes}) - - print('\nPhonemized metadata samples:') - for i in sample_items: - print(f'{i}:{phonemized_data[i]}') - summary_manager.add_text(f'{i}/phonemes', text=phonemized_data[i]) - - new_metadata = [f'{k}|{v}\n' for k, v in phonemized_data.items()] - shuffled_metadata = np.random.permutation(new_metadata) - train_metadata = shuffled_metadata[0:train_len] - test_metadata = shuffled_metadata[-test_len:] - - with open(phonemized_metadata_path, 'w+', encoding='utf-8') as file: - file.writelines(new_metadata) - with open(train_metadata_path, 'w+', encoding='utf-8') as file: - file.writelines(train_metadata) - with open(test_metadata_path, 'w+', encoding='utf-8') as file: - file.writelines(test_metadata) - # some checks - assert metadata_len == len(set(list(phonemized_data.keys()))), \ - f'Length of metadata ({metadata_len}) does not match the length of the phoneme array ({len(set(list(phonemized_data.keys())))}). Check for empty text lines in metadata.' - assert len(train_metadata) + len(test_metadata) == metadata_len, \ - f'Train and/or validation lengths incorrect. ({len(train_metadata)} + {len(test_metadata)} != {metadata_len})' -else: - print('Skipped phonemization, assuming data is already phonemized ..') - remove_files = pickle.load(open(cm.data_dir / 'under-over_sized_mels.pkl', 'rb')) - phonemized_metadata_path = cm.phonemized_metadata_path # Stores data in a particular format - train_metadata_path = cm.train_metadata_path - test_metadata_path = cm.valid_metadata_path - print(f'\nReading metadata from {metadatareader.metadata_path}') - print(f'\nFound {len(metadatareader.filenames)} lines.') - print(f'\nRemoving {len(remove_files)} line(s) due to mel filtering.') - metadata_file_ids = [fname for fname in cross_file_ids if fname not in remove_files] - metadata_len = len(metadata_file_ids) - sample_items = np.random.choice(metadata_file_ids, 5) - test_len = cm.config['n_test'] - train_len = metadata_len - test_len - print(f'\nMetadata contains {metadata_len} lines.') - print(f'\nFiles will be stored under {cm.data_dir}') - print(f' - {train_len} training lines: {train_metadata_path}') - print(f' - {test_len} validation lines: {test_metadata_path}') - - # run cleaner on raw text - text_proc = TextToTokens.default(cm.config['phoneme_language'], add_start_end=False, - with_stress=cm.config['with_stress'], model_breathing=cm.config['model_breathing'], - njobs=1) - - - def process_phonemes(file_id): - text = metadatareader.text_dict[file_id] - try: - phon = text_proc.phonemizer(text, only_preprocess=True) - except Exception as e: - print(f'{e}\nFile id {file_id}') - raise BrokenPipeError - return (file_id, phon) - - - print('\ Processing Phonemes .. ') - phonemized_data = {} - phon_iter = p_uimap(process_phonemes, metadata_file_ids) - for (file_id, phonemes) in phon_iter: - phonemized_data.update({file_id: phonemes}) - - - print('\nMetadata samples:') - for i in sample_items: - print(f'{i}:{metadatareader.text_dict[i]}') - summary_manager.add_text(f'{i}/phonemes', text=metadatareader.text_dict[i]) - - new_metadata = [f'{k}|{v}\n' for k, v in phonemized_data.items()] #replaced phonemized_data - shuffled_metadata = np.random.permutation(new_metadata) - train_metadata = shuffled_metadata[0:train_len] - test_metadata = shuffled_metadata[-test_len:] - - with open(phonemized_metadata_path, 'w+', encoding='utf-8') as file: - file.writelines(new_metadata) - with open(train_metadata_path, 'w+', encoding='utf-8') as file: - file.writelines(train_metadata) - with open(test_metadata_path, 'w+', encoding='utf-8') as file: - file.writelines(test_metadata) - -print('\nDone') + return filter_metadata + +remove_files = pickle.load(open(cm.data_dir / 'under-over_sized_mels.pkl', 'rb')) +phonemized_metadata_path = cm.phonemized_metadata_path +train_metadata_path = cm.train_metadata_path +test_metadata_path = cm.valid_metadata_path +print(f'\nReading metadata from {metadatareader.metadata_path}') +print(f'\nFound {len(metadatareader.filenames)} lines.') + +filter_metadata = get_short_files(phonemized=phonemized_flag) +remove_files += filter_metadata +print(f'\nRemoving {len(remove_files)} line(s) due to mel filtering.') +metadata_file_ids = [fname for fname in cross_file_ids if fname not in remove_files] +metadata_len = len(metadata_file_ids) +sample_items = np.random.choice(metadata_file_ids, 5) +test_len = cm.config['n_test'] +train_len = metadata_len - test_len +print(f'\nMetadata contains {metadata_len} lines.') +print(f'\nFiles will be stored under {cm.data_dir}') +print(f' - all: {phonemized_metadata_path}') +print(f' - {train_len} training lines: {train_metadata_path}') +print(f' - {test_len} validation lines: {test_metadata_path}') + +# run cleaner on raw text +text_proc = TextToTokens.default(cm.config['phoneme_language'], add_start_end=False, + with_stress=cm.config['with_stress'], model_breathing=cm.config['model_breathing'], + njobs=1) + + +def process_phonemes(file_id): + text = metadatareader.text_dict[file_id] + try: + phon = text_proc.phonemizer(text, only_preprocess=phonemized_flag) + except Exception as e: + print(f'{e}\nFile id {file_id}') + raise BrokenPipeError + return (file_id, phon) + + +print('\nPHONEMIZING') +phonemized_data = {} +phon_iter = p_uimap(process_phonemes, metadata_file_ids) +for (file_id, phonemes) in phon_iter: + phonemized_data.update({file_id: phonemes}) + +print('\nPhonemized metadata samples:') +for i in sample_items: + print(f'{i}:{phonemized_data[i]}') + summary_manager.add_text(f'{i}/phonemes', text=phonemized_data[i]) + +new_metadata = [f'{k}|{v}\n' for k, v in phonemized_data.items()] +shuffled_metadata = np.random.permutation(new_metadata) +train_metadata = shuffled_metadata[0:train_len] +test_metadata = shuffled_metadata[-test_len:] + +with open(phonemized_metadata_path, 'w+', encoding='utf-8') as file: + file.writelines(new_metadata) +with open(train_metadata_path, 'w+', encoding='utf-8') as file: + file.writelines(train_metadata) +with open(test_metadata_path, 'w+', encoding='utf-8') as file: + file.writelines(test_metadata) + +# some checks +assert metadata_len == len(set(list(phonemized_data.keys()))), \ + f'Length of metadata ({metadata_len}) does not match the length of the phoneme array ({len(set(list(phonemized_data.keys())))}). Check for empty text lines in metadata.' +assert len(train_metadata) + len(test_metadata) == metadata_len, \ + f'Train and/or validation lengths incorrect. ({len(train_metadata)} + {len(test_metadata)} != {metadata_len})' + +print('\n Done') diff --git a/predict_tts.py b/predict_tts.py index d1428d2..ba1cd76 100644 --- a/predict_tts.py +++ b/predict_tts.py @@ -43,7 +43,7 @@ outdir = outdir / 'outputs' / f'{fname}' outdir.mkdir(exist_ok=True, parents=True) output_path = (outdir / file_name).with_suffix('.wav') - audio = Audio.from_config(model.config) + audio = Audio.from_config(model.config) print(f'Output wav under {output_path.parent}') wavs = [] for i, text_line in enumerate(text): From d708547be06f9d6ccc4ac9d0140d27b12fe1500d Mon Sep 17 00:00:00 2001 From: Tanuj Jain Date: Fri, 17 Sep 2021 09:55:30 +0200 Subject: [PATCH 05/10] Works with external flags, next commits will try to use config file. Rollback to this commit if the config file inputs dont work. --- data/text/__init__.py | 4 ++-- model/models.py | 17 +++++++++-------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/data/text/__init__.py b/data/text/__init__.py index bc9dbf7..3ec3482 100644 --- a/data/text/__init__.py +++ b/data/text/__init__.py @@ -9,8 +9,8 @@ def __init__(self, phonemizer: Phonemizer, tokenizer: Tokenizer): self.phonemizer = phonemizer self.tokenizer = tokenizer - def __call__(self, input_text: Union[str, list]) -> list: - phons = self.phonemizer(input_text) + def __call__(self, input_text: Union[str, list], only_preprocess=False) -> list: + phons = self.phonemizer(input_text, only_preprocess=only_preprocess) tokens = self.tokenizer(phons) return tokens diff --git a/model/models.py b/model/models.py index df47ec6..c36fcc7 100644 --- a/model/models.py +++ b/model/models.py @@ -268,9 +268,9 @@ def align(self, text, mel, mels_have_start_end_vectors=False, phonemize=False, e attn_weights = model_out['decoder_attention']['Decoder_LastBlock_CrossAttention'] return attn_weights, model_out - def predict(self, inp, max_length=1000, encode=True, verbose=True): + def predict(self, inp, max_length=1000, encode=True, verbose=True, phonemized=True): if encode: - inp = self.encode_text(inp) + inp = self.encode_text(inp, phonemized=phonemized) inp = tf.cast(tf.expand_dims(inp, 0), tf.int32) output = tf.cast(tf.expand_dims(self.start_vec, 0), tf.float32) output_concat = tf.cast(tf.expand_dims(self.start_vec, 0), tf.float32) @@ -311,8 +311,8 @@ def set_constants(self, if force_decoder_diagonal is not None: self._set_force_decoder_diagonal(force_decoder_diagonal) - def encode_text(self, text): - return self.text_pipeline(text) + def encode_text(self, text, phonemized=True): + return self.text_pipeline(text, only_preprocess=phonemized) def build_model_weights(self) -> None: _ = self(tf.zeros((1, 1)), tf.zeros((1, 1, self.mel_channels)), training=False) @@ -553,13 +553,14 @@ def set_constants(self, learning_rate: float = None, **kwargs): if learning_rate is not None: self.optimizer.lr.assign(learning_rate) - def encode_text(self, text): - return self.text_pipeline(text) + def encode_text(self, text, phonemized=False): + return self.text_pipeline(text, only_preprocess=phonemized) def predict(self, inp, encode=True, speed_regulator=1., phoneme_max_duration=None, phoneme_min_duration=None, - max_durations_mask=None, min_durations_mask=None, phoneme_durations=None, phoneme_pitch=None): + max_durations_mask=None, min_durations_mask=None, phoneme_durations=None, phoneme_pitch=None, + phonemized=False): if encode: - inp = self.encode_text(inp) + inp = self.encode_text(inp, phonemized=phonemized) if len(tf.shape(inp)) < 2: inp = tf.expand_dims(inp, 0) inp = tf.cast(inp, tf.int32) From a90d5566f0d249b8163c02e56f2c81010f159a90 Mon Sep 17 00:00:00 2001 From: Tanuj Jain Date: Wed, 13 Oct 2021 15:00:06 +0200 Subject: [PATCH 06/10] Allow phonemized data to be passed directly to the training logic. --- config/training_config.yaml | 6 +++--- create_training_data.py | 9 +++++---- data/text/__init__.py | 4 ++-- data/text/tokenizer.py | 4 ++-- model/models.py | 17 ++++++++--------- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/config/training_config.yaml b/config/training_config.yaml index bb18a9f..edae754 100644 --- a/config/training_config.yaml +++ b/config/training_config.yaml @@ -1,7 +1,7 @@ paths: # PATHS: change accordingly - wav_directory: '/Users/tjain1/Desktop/Projects/tts/data/ASVoice4_incl_english' #'/Users/tjain1/Desktop/Projects/tts/data/data_small' # path to directory cointaining the wavs - metadata_path: '/Users/tjain1/Desktop/Projects/tts/data/metadata_flagged_phonemized_bind_nostress.csv' # '/Users/tjain1/Desktop/Projects/tts/data/ph_metadata_small.csv' # name of metadata file under wav_directory + wav_directory: '/Users/tjain1/Desktop/Projects/tts/data/ASVoice4_incl_english' # '/Users/tjain1/Desktop/Projects/tts/data/data_small' # path to directory cointaining the wavs + metadata_path: '/Users/tjain1/Desktop/Projects/tts/data/metadata_flagged_phonemized_bind_nostress.csv' # '/Users/tjain1/Desktop/Projects/tts/data/ph_metadata_small.csv' # # name of metadata file under wav_directory log_directory: '/Users/tjain1/Desktop/Projects/tts/results/logs_txtts_ph_dev_small' # weights and logs are stored here train_data_directory: '/Users/tjain1/Desktop/Projects/tts/results/logs_txtts_ph_dev_small/transformer_tts_data' # training data is stored here @@ -51,7 +51,7 @@ audio_settings: text_settings: # TOKENIZER - phoneme_language: 'en-us' # only relevant if non-phonemized data is provided as input, else, a placeholder value here will suffice + phoneme_language: 'de' #false with_stress: True # use stress symbols in phonemization model_breathing: false # add a token for the initial breathing diff --git a/create_training_data.py b/create_training_data.py index e1dcf09..876700d 100644 --- a/create_training_data.py +++ b/create_training_data.py @@ -16,17 +16,18 @@ parser = argparse.ArgumentParser() parser.add_argument('--config', type=str, required=True) -parser.add_argument('--skip_phonemization', action='store_true') +# parser.add_argument('--skip_phonemization', action='store_true') parser.add_argument('--skip_mels', action='store_true') args = parser.parse_args() for arg in vars(args): print('{}: {}'.format(arg, getattr(args, arg))) -phonemized_flag = args.skip_phonemization # If the input is already in the form of phonemes, this flag will be set (User wanting to skip phonemization indicates that the input is already phonemized) +# phonemized_flag = args.skip_phonemization # If the input is already in the form of phonemes, this flag will be set (User wanting to skip phonemization indicates that the input is already phonemized) cm = TrainingConfigManager(args.config, aligner=True) cm.create_remove_dirs() +#phonemized_flag = not cm.config['phoneme_language'] # a value of None means the data is pre-phonemized metadatareader = DataReader.from_config(cm, kind='original', scan_wavs=True) summary_manager = SummaryManager(model=None, log_dir=cm.log_dir / 'data_preprocessing', config=cm.config, default_writer='data_preprocessing') @@ -125,7 +126,7 @@ def get_short_files(phonemized=False): print(f'\nReading metadata from {metadatareader.metadata_path}') print(f'\nFound {len(metadatareader.filenames)} lines.') -filter_metadata = get_short_files(phonemized=phonemized_flag) +filter_metadata = get_short_files(phonemized=not cm.config['phoneme_language']) remove_files += filter_metadata print(f'\nRemoving {len(remove_files)} line(s) due to mel filtering.') metadata_file_ids = [fname for fname in cross_file_ids if fname not in remove_files] @@ -148,7 +149,7 @@ def get_short_files(phonemized=False): def process_phonemes(file_id): text = metadatareader.text_dict[file_id] try: - phon = text_proc.phonemizer(text, only_preprocess=phonemized_flag) + phon = text_proc.phonemizer(text) # , only_preprocess=phonemized_flag except Exception as e: print(f'{e}\nFile id {file_id}') raise BrokenPipeError diff --git a/data/text/__init__.py b/data/text/__init__.py index 3ec3482..c531b59 100644 --- a/data/text/__init__.py +++ b/data/text/__init__.py @@ -9,8 +9,8 @@ def __init__(self, phonemizer: Phonemizer, tokenizer: Tokenizer): self.phonemizer = phonemizer self.tokenizer = tokenizer - def __call__(self, input_text: Union[str, list], only_preprocess=False) -> list: - phons = self.phonemizer(input_text, only_preprocess=only_preprocess) + def __call__(self, input_text: Union[str, list]) -> list: # , only_preprocess=False + phons = self.phonemizer(input_text) # , only_preprocess=only_preprocess tokens = self.tokenizer(phons) return tokens diff --git a/data/text/tokenizer.py b/data/text/tokenizer.py index b1366bc..4f2bb98 100644 --- a/data/text/tokenizer.py +++ b/data/text/tokenizer.py @@ -57,12 +57,12 @@ def __init__(self, language: str, with_stress: bool, njobs=4): self._whitespace_re = re.compile(r'\s+') self._whitespace_punctuation_re = re.compile(f'\s*([{_punctuations}])\s*') - def __call__(self, text: Union[str, list], with_stress=None, njobs=None, language=None, only_preprocess=False) -> Union[str, list]: + def __call__(self, text: Union[str, list], with_stress=None, njobs=None, language=None) -> Union[str, list]: # , only_preprocess=False language = language or self.language njobs = njobs or self.njobs with_stress = with_stress or self.with_stress # phonemizer does not like hyphens. - if not only_preprocess: + if language: text = self._preprocess(text) phonemes = phonemize(text, language=language, diff --git a/model/models.py b/model/models.py index c36fcc7..a3ce6d4 100644 --- a/model/models.py +++ b/model/models.py @@ -268,9 +268,9 @@ def align(self, text, mel, mels_have_start_end_vectors=False, phonemize=False, e attn_weights = model_out['decoder_attention']['Decoder_LastBlock_CrossAttention'] return attn_weights, model_out - def predict(self, inp, max_length=1000, encode=True, verbose=True, phonemized=True): + def predict(self, inp, max_length=1000, encode=True, verbose=True):#, phonemized=True if encode: - inp = self.encode_text(inp, phonemized=phonemized) + inp = self.encode_text(inp) inp = tf.cast(tf.expand_dims(inp, 0), tf.int32) output = tf.cast(tf.expand_dims(self.start_vec, 0), tf.float32) output_concat = tf.cast(tf.expand_dims(self.start_vec, 0), tf.float32) @@ -311,8 +311,8 @@ def set_constants(self, if force_decoder_diagonal is not None: self._set_force_decoder_diagonal(force_decoder_diagonal) - def encode_text(self, text, phonemized=True): - return self.text_pipeline(text, only_preprocess=phonemized) + def encode_text(self, text):#, phonemized=True + return self.text_pipeline(text) # , only_preprocess=phonemized def build_model_weights(self) -> None: _ = self(tf.zeros((1, 1)), tf.zeros((1, 1, self.mel_channels)), training=False) @@ -553,14 +553,13 @@ def set_constants(self, learning_rate: float = None, **kwargs): if learning_rate is not None: self.optimizer.lr.assign(learning_rate) - def encode_text(self, text, phonemized=False): - return self.text_pipeline(text, only_preprocess=phonemized) + def encode_text(self, text):#, phonemized=False + return self.text_pipeline(text) #, only_preprocess=phonemized def predict(self, inp, encode=True, speed_regulator=1., phoneme_max_duration=None, phoneme_min_duration=None, - max_durations_mask=None, min_durations_mask=None, phoneme_durations=None, phoneme_pitch=None, - phonemized=False): + max_durations_mask=None, min_durations_mask=None, phoneme_durations=None, phoneme_pitch=None): if encode: - inp = self.encode_text(inp, phonemized=phonemized) + inp = self.encode_text(inp) #, phonemized=phonemized if len(tf.shape(inp)) < 2: inp = tf.expand_dims(inp, 0) inp = tf.cast(inp, tf.int32) From 079742f63628e7dd16e8f83c2a4731b0e5e5d210 Mon Sep 17 00:00:00 2001 From: Tanuj Jain Date: Wed, 19 Jan 2022 18:39:02 +0100 Subject: [PATCH 07/10] Update tokenizer test to pass with the latest tokenizer. --- tests/test_char_tokenizer.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_char_tokenizer.py b/tests/test_char_tokenizer.py index a9ab2fa..e52f06d 100644 --- a/tests/test_char_tokenizer.py +++ b/tests/test_char_tokenizer.py @@ -12,12 +12,12 @@ def test_tokenizer(self): tokenizer = Tokenizer(alphabet=list('ab c')) self.assertEqual(5, tokenizer.start_token_index) self.assertEqual(6, tokenizer.end_token_index) - self.assertEqual(7, tokenizer.vocab_size) - - seq = tokenizer('a b d') - self.assertEqual([5, 1, 3, 2, 3, 6], seq) - - seq = np.array([5, 1, 3, 2, 8, 6]) + self.assertEqual(8, tokenizer.vocab_size) + + seq = tokenizer('a b c') + self.assertEqual([5, 7, 2, 1, 7, 3, 1, 7, 4, 6], seq) + + seq = np.array([5, 2, 1, 3, 6]) seq = tf.convert_to_tensor(seq) text = tokenizer.decode(seq) self.assertEqual('>a b<', text) From adbbd49515e300e38f1e133c9d4d086f61b1b806 Mon Sep 17 00:00:00 2001 From: Tanuj Jain Date: Wed, 19 Jan 2022 19:28:24 +0100 Subject: [PATCH 08/10] Add test to verify that false flag bypasses phonemization step. Also do some comment cleanup. --- config/training_config.yaml | 2 +- create_training_data.py | 16 ++++++++-------- data/text/__init__.py | 4 ++-- data/text/symbols.py | 2 +- data/text/tokenizer.py | 4 ++-- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/config/training_config.yaml b/config/training_config.yaml index edae754..ab7896b 100644 --- a/config/training_config.yaml +++ b/config/training_config.yaml @@ -51,7 +51,7 @@ audio_settings: text_settings: # TOKENIZER - phoneme_language: 'de' #false + phoneme_language: 'de' # use false if the input data is already phonemized with_stress: True # use stress symbols in phonemization model_breathing: false # add a token for the initial breathing diff --git a/create_training_data.py b/create_training_data.py index 876700d..00cf1b6 100644 --- a/create_training_data.py +++ b/create_training_data.py @@ -16,18 +16,14 @@ parser = argparse.ArgumentParser() parser.add_argument('--config', type=str, required=True) -# parser.add_argument('--skip_phonemization', action='store_true') parser.add_argument('--skip_mels', action='store_true') args = parser.parse_args() for arg in vars(args): print('{}: {}'.format(arg, getattr(args, arg))) -# phonemized_flag = args.skip_phonemization # If the input is already in the form of phonemes, this flag will be set (User wanting to skip phonemization indicates that the input is already phonemized) - cm = TrainingConfigManager(args.config, aligner=True) cm.create_remove_dirs() -#phonemized_flag = not cm.config['phoneme_language'] # a value of None means the data is pre-phonemized metadatareader = DataReader.from_config(cm, kind='original', scan_wavs=True) summary_manager = SummaryManager(model=None, log_dir=cm.log_dir / 'data_preprocessing', config=cm.config, default_writer='data_preprocessing') @@ -101,6 +97,7 @@ def process_pitches(item: tuple): summary_manager.display_scalar('Total duration (hours)', scalar_value=total_wav_len / audio.config['sampling_rate'] / 60. ** 2) + def get_short_files(phonemized=False): if not phonemized: symbol_list = _alphabet @@ -119,6 +116,7 @@ def get_short_files(phonemized=False): print(f'{fname}: {metadatareader.text_dict[fname]}') return filter_metadata + remove_files = pickle.load(open(cm.data_dir / 'under-over_sized_mels.pkl', 'rb')) phonemized_metadata_path = cm.phonemized_metadata_path train_metadata_path = cm.train_metadata_path @@ -141,15 +139,17 @@ def get_short_files(phonemized=False): print(f' - {test_len} validation lines: {test_metadata_path}') # run cleaner on raw text -text_proc = TextToTokens.default(cm.config['phoneme_language'], add_start_end=False, - with_stress=cm.config['with_stress'], model_breathing=cm.config['model_breathing'], - njobs=1) +text_proc = TextToTokens.default(cm.config['phoneme_language'], + add_start_end=False, + with_stress=cm.config['with_stress'], + model_breathing=cm.config['model_breathing'], + njobs=1) def process_phonemes(file_id): text = metadatareader.text_dict[file_id] try: - phon = text_proc.phonemizer(text) # , only_preprocess=phonemized_flag + phon = text_proc.phonemizer(text) except Exception as e: print(f'{e}\nFile id {file_id}') raise BrokenPipeError diff --git a/data/text/__init__.py b/data/text/__init__.py index c531b59..bc9dbf7 100644 --- a/data/text/__init__.py +++ b/data/text/__init__.py @@ -9,8 +9,8 @@ def __init__(self, phonemizer: Phonemizer, tokenizer: Tokenizer): self.phonemizer = phonemizer self.tokenizer = tokenizer - def __call__(self, input_text: Union[str, list]) -> list: # , only_preprocess=False - phons = self.phonemizer(input_text) # , only_preprocess=only_preprocess + def __call__(self, input_text: Union[str, list]) -> list: + phons = self.phonemizer(input_text) tokens = self.tokenizer(phons) return tokens diff --git a/data/text/symbols.py b/data/text/symbols.py index 4786ab6..9eb233d 100644 --- a/data/text/symbols.py +++ b/data/text/symbols.py @@ -4,7 +4,7 @@ _suprasegmentals = 'ˈˌːˑ' _other_symbols = 'ʍwɥʜʢʡɕʑɺɧ' _diacrilics = 'ɚ˞ɫ' -_extra_phons = ['g', 'ɝ', '̃', '̍', '̥', '̩', '̯', '͡'] # some extra symbols that I found in from wiktionary ipa annotations +_extra_phons = ['g', 'ɝ', '̃', '̍', '̥', '̩', '̯', '͡'] # some extra symbols from wiktionary ipa annotations _phonemes = sorted(list( _vowels + _non_pulmonic_consonants + _pulmonic_consonants + _suprasegmentals + _other_symbols + _diacrilics)) + _extra_phons _punctuations = '!,-.:;? \'()' diff --git a/data/text/tokenizer.py b/data/text/tokenizer.py index 4f2bb98..5b98892 100644 --- a/data/text/tokenizer.py +++ b/data/text/tokenizer.py @@ -33,7 +33,7 @@ def __init__(self, start_token='>', end_token='<', pad_token='/', add_start_end= self.breathing_token = '@' self.idx_to_token[self.breathing_token_index] = self.breathing_token self.token_to_idx[self.breathing_token] = [self.breathing_token_index] - + def __call__(self, sentence: str) -> list: sequence = [self.token_to_idx[c] for c in sentence] # No filtering: text should only contain known chars. sequence = [item for items in sequence for item in items] @@ -57,7 +57,7 @@ def __init__(self, language: str, with_stress: bool, njobs=4): self._whitespace_re = re.compile(r'\s+') self._whitespace_punctuation_re = re.compile(f'\s*([{_punctuations}])\s*') - def __call__(self, text: Union[str, list], with_stress=None, njobs=None, language=None) -> Union[str, list]: # , only_preprocess=False + def __call__(self, text: Union[str, list], with_stress=None, njobs=None, language=None) -> Union[str, list]: language = language or self.language njobs = njobs or self.njobs with_stress = with_stress or self.with_stress From 2d562e0acfe4ef790f9f2bd2ad8c22ffe3bd38f2 Mon Sep 17 00:00:00 2001 From: Tanuj Jain Date: Wed, 19 Jan 2022 19:44:50 +0100 Subject: [PATCH 09/10] Revert to generic paths. --- config/training_config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/config/training_config.yaml b/config/training_config.yaml index ab7896b..4dd63a8 100644 --- a/config/training_config.yaml +++ b/config/training_config.yaml @@ -1,9 +1,9 @@ paths: # PATHS: change accordingly - wav_directory: '/Users/tjain1/Desktop/Projects/tts/data/ASVoice4_incl_english' # '/Users/tjain1/Desktop/Projects/tts/data/data_small' # path to directory cointaining the wavs - metadata_path: '/Users/tjain1/Desktop/Projects/tts/data/metadata_flagged_phonemized_bind_nostress.csv' # '/Users/tjain1/Desktop/Projects/tts/data/ph_metadata_small.csv' # # name of metadata file under wav_directory - log_directory: '/Users/tjain1/Desktop/Projects/tts/results/logs_txtts_ph_dev_small' # weights and logs are stored here - train_data_directory: '/Users/tjain1/Desktop/Projects/tts/results/logs_txtts_ph_dev_small/transformer_tts_data' # training data is stored here + wav_directory: '/path/to/wav_directory' # path to directory cointaining the wavs + metadata_path: '/path/to/metadata.csv' # name of metadata file under wav_directory + log_directory: '/path/to/logs_directory' # weights and logs are stored here + train_data_directory: 'transformer_tts_data' # training data is stored here naming: data_name: ljspeech # raw data naming for default data reader (select function from data/metadata_readers.py) From 984c1af99f37c3999ad0e7e942e7b2e78516b160 Mon Sep 17 00:00:00 2001 From: Tanuj Jain Date: Wed, 19 Jan 2022 20:02:55 +0100 Subject: [PATCH 10/10] Restore defaults for config file and additional cleanup. --- config/training_config.yaml | 14 ++++++++------ model/models.py | 12 ++++++------ predict_tts.py | 3 +-- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/config/training_config.yaml b/config/training_config.yaml index 4dd63a8..0368c38 100644 --- a/config/training_config.yaml +++ b/config/training_config.yaml @@ -14,7 +14,7 @@ naming: # TRAINING DATA SETTINGS training_data_settings: - n_test: 2 + n_test: 100 mel_start_value: .5 mel_end_value: -.5 max_mel_len: 1_200 @@ -51,7 +51,7 @@ audio_settings: text_settings: # TOKENIZER - phoneme_language: 'de' # use false if the input data is already phonemized + phoneme_language: 'en-us' # use 'de' for german, use false if the input data is already phonemized with_stress: True # use stress symbols in phonemization model_breathing: false # add a token for the initial breathing @@ -78,8 +78,10 @@ aligner_settings: - [0, 1.0e-4] reduction_factor_schedule: - [0, 10] - - [5_000, 1] - max_steps: 8_000 + - [80_000, 5] + - [100_000, 2] + - [130_000, 1] + max_steps: 260_000 force_encoder_diagonal_steps: 500 force_decoder_diagonal_steps: 7_000 extract_attention_weighted: False # weighted average between last layer decoder attention heads when extracting durations @@ -126,7 +128,7 @@ tts_settings: dropout_rate: 0.1 learning_rate_schedule: - [0, 1.0e-4] - max_steps: 10_000 + max_steps: 100_000 debug: False # LOGGING @@ -136,7 +138,7 @@ tts_settings: weights_save_starting_step: 5_000 train_images_plotting_frequency: 1_000 keep_n_weights: 5 - keep_checkpoint_every_n_hours: 1 + keep_checkpoint_every_n_hours: 12 n_steps_avg_losses: [100, 500, 1_000, 5_000] # command line display of average loss values for the last n steps prediction_start_step: 4_000 text_prediction: diff --git a/model/models.py b/model/models.py index a3ce6d4..df47ec6 100644 --- a/model/models.py +++ b/model/models.py @@ -268,7 +268,7 @@ def align(self, text, mel, mels_have_start_end_vectors=False, phonemize=False, e attn_weights = model_out['decoder_attention']['Decoder_LastBlock_CrossAttention'] return attn_weights, model_out - def predict(self, inp, max_length=1000, encode=True, verbose=True):#, phonemized=True + def predict(self, inp, max_length=1000, encode=True, verbose=True): if encode: inp = self.encode_text(inp) inp = tf.cast(tf.expand_dims(inp, 0), tf.int32) @@ -311,8 +311,8 @@ def set_constants(self, if force_decoder_diagonal is not None: self._set_force_decoder_diagonal(force_decoder_diagonal) - def encode_text(self, text):#, phonemized=True - return self.text_pipeline(text) # , only_preprocess=phonemized + def encode_text(self, text): + return self.text_pipeline(text) def build_model_weights(self) -> None: _ = self(tf.zeros((1, 1)), tf.zeros((1, 1, self.mel_channels)), training=False) @@ -553,13 +553,13 @@ def set_constants(self, learning_rate: float = None, **kwargs): if learning_rate is not None: self.optimizer.lr.assign(learning_rate) - def encode_text(self, text):#, phonemized=False - return self.text_pipeline(text) #, only_preprocess=phonemized + def encode_text(self, text): + return self.text_pipeline(text) def predict(self, inp, encode=True, speed_regulator=1., phoneme_max_duration=None, phoneme_min_duration=None, max_durations_mask=None, min_durations_mask=None, phoneme_durations=None, phoneme_pitch=None): if encode: - inp = self.encode_text(inp) #, phonemized=phonemized + inp = self.encode_text(inp) if len(tf.shape(inp)) < 2: inp = tf.expand_dims(inp, 0) inp = tf.cast(inp, tf.int32) diff --git a/predict_tts.py b/predict_tts.py index ba1cd76..feb4ebd 100644 --- a/predict_tts.py +++ b/predict_tts.py @@ -17,7 +17,6 @@ parser.add_argument('--store_mel', '-m', dest='store_mel', action='store_true') parser.add_argument('--verbose', '-v', dest='verbose', action='store_true') parser.add_argument('--single', '-s', dest='single', action='store_true') - parser.add_argument('--phonemized', dest='phonemized', action='store_false') args = parser.parse_args() if args.file is not None: @@ -47,7 +46,7 @@ print(f'Output wav under {output_path.parent}') wavs = [] for i, text_line in enumerate(text): - phons = model.text_pipeline.phonemizer(text_line, only_preprocess=args.phonemized) + phons = model.text_pipeline.phonemizer(text_line) tokens = model.text_pipeline.tokenizer(phons) if args.verbose: print(f'Predicting {text_line}')