{ "cells": [ { "cell_type": "code", "execution_count": 4, "id": "80ca0f5c", "metadata": {}, "outputs": [], "source": [ "import gradio as gr\n", "import numpy as np\n", "import torch\n", "import torch.nn.functional as F\n", "from pathlib import Path\n", "import tempfile\n", "\n", "from TTS.api import TTS\n", "from TTS.utils.manage import ModelManager" ] }, { "cell_type": "code", "execution_count": 6, "id": "4ad622cb", "metadata": { "scrolled": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " > voice_conversion_models/multilingual/vctk/freevc24 is already downloaded.\n", " > Model's license - MIT\n", " > Check https://choosealicense.com/licenses/mit/ for more info.\n", " > Using model: freevc\n", " > Loading pretrained speaker encoder model ...\n", "Loaded the voice encoder model on cpu in 0.01 seconds.\n", "Running on local URL: http://127.0.0.1:7863\n", "\n", "To create a public link, set `share=True` in `launch()`.\n" ] }, { "data": { "text/html": [ "
" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/plain": [] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" }, { "name": "stdout", "output_type": "stream", "text": [ " > tts_models/en/ljspeech/tacotron2-DDC_ph is already downloaded.\n", " > Model's license - apache 2.0\n", " > Check https://choosealicense.com/licenses/apache-2.0/ for more info.\n", " > vocoder_models/en/ljspeech/univnet is already downloaded.\n", " > Model's license - apache 2.0\n", " > Check https://choosealicense.com/licenses/apache-2.0/ for more info.\n", " > Using model: Tacotron2\n", " > Setting up Audio Processor...\n", " | > sample_rate:22050\n", " | > resample:False\n", " | > num_mels:80\n", " | > log_func:np.log10\n", " | > min_level_db:-100\n", " | > frame_shift_ms:None\n", " | > frame_length_ms:None\n", " | > ref_level_db:20\n", " | > fft_size:1024\n", " | > power:1.5\n", " | > preemphasis:0.0\n", " | > griffin_lim_iters:60\n", " | > signal_norm:True\n", " | > symmetric_norm:True\n", " | > mel_fmin:50.0\n", " | > mel_fmax:7600.0\n", " | > pitch_fmin:0.0\n", " | > pitch_fmax:640.0\n", " | > spec_gain:1.0\n", " | > stft_pad_mode:reflect\n", " | > max_norm:4.0\n", " | > clip_norm:True\n", " | > do_trim_silence:True\n", " | > trim_db:60\n", " | > do_sound_norm:False\n", " | > do_amp_to_db_linear:True\n", " | > do_amp_to_db_mel:True\n", " | > do_rms_norm:False\n", " | > db_level:None\n", " | > stats_path:C:\\Users\\Torch\\AppData\\Local\\tts\\tts_models--en--ljspeech--tacotron2-DDC_ph\\scale_stats.npy\n", " | > base:10\n", " | > hop_length:256\n", " | > win_length:1024\n", " > Model's reduction rate `r` is set to: 2\n", " > Vocoder Model: univnet\n", " > Setting up Audio Processor...\n", " | > sample_rate:22050\n", " | > resample:False\n", " | > num_mels:80\n", " | > log_func:np.log10\n", " | > min_level_db:-100\n", " | > frame_shift_ms:None\n", " | > frame_length_ms:None\n", " | > ref_level_db:20\n", " | > fft_size:1024\n", " | > power:1.5\n", " | > preemphasis:0.0\n", " | > griffin_lim_iters:60\n", " | > signal_norm:True\n", " | > symmetric_norm:True\n", " | > mel_fmin:50.0\n", " | > mel_fmax:7600.0\n", " | > pitch_fmin:1.0\n", " | > pitch_fmax:640.0\n", " | > spec_gain:1.0\n", " | > stft_pad_mode:reflect\n", " | > max_norm:4.0\n", " | > clip_norm:True\n", " | > do_trim_silence:True\n", " | > trim_db:60\n", " | > do_sound_norm:False\n", " | > do_amp_to_db_linear:True\n", " | > do_amp_to_db_mel:True\n", " | > do_rms_norm:False\n", " | > db_level:None\n", " | > stats_path:C:\\Users\\Torch\\AppData\\Local\\tts\\vocoder_models--en--ljspeech--univnet\\scale_stats.npy\n", " | > base:10\n", " | > hop_length:256\n", " | > win_length:1024\n", " > Generator Model: univnet_generator\n", " > Discriminator Model: univnet_discriminator\n", "model: tts_models/en/ljspeech/tacotron2-DDC_ph\n", "language: \n", "speaker: \n", "voice cloning with the voice conversion model\n", " > Text splitted to sentences.\n", "['Mary had a little lamb,', 'Its fleece was white as snow.', 'Everywhere the child went,', 'The little lamb was sure to go.']\n", "ɛvɹiwɛɹ ðə t͡ʃaɪld wɛnt,\n", " [!] Character '͡' not found in the vocabulary. Discarding it.\n", " > Processing time: 3.3410003185272217\n", " > Real-time factor: 0.38459038289093944\n", "model: tts_models/en/ljspeech/tacotron2-DDC_ph\n", "language: \n", "speaker: \n", "voice cloning with the voice conversion model\n", " > Text splitted to sentences.\n", "['Mary had a little lamb,', 'Its fleece was white as snow.', 'Everywhere the child went,', 'The little lamb was sure to go.']\n", " > Processing time: 2.9179999828338623\n", " > Real-time factor: 0.3358978221135079\n", " > tts_models/en/ljspeech/tacotron2-DDC_ph is already downloaded.\n", " > Model's license - apache 2.0\n", " > Check https://choosealicense.com/licenses/apache-2.0/ for more info.\n", " > vocoder_models/en/ljspeech/univnet is already downloaded.\n", " > Model's license - apache 2.0\n", " > Check https://choosealicense.com/licenses/apache-2.0/ for more info.\n", " > Using model: Tacotron2\n", " > Setting up Audio Processor...\n", " | > sample_rate:22050\n", " | > resample:False\n", " | > num_mels:80\n", " | > log_func:np.log10\n", " | > min_level_db:-100\n", " | > frame_shift_ms:None\n", " | > frame_length_ms:None\n", " | > ref_level_db:20\n", " | > fft_size:1024\n", " | > power:1.5\n", " | > preemphasis:0.0\n", " | > griffin_lim_iters:60\n", " | > signal_norm:True\n", " | > symmetric_norm:True\n", " | > mel_fmin:50.0\n", " | > mel_fmax:7600.0\n", " | > pitch_fmin:0.0\n", " | > pitch_fmax:640.0\n", " | > spec_gain:1.0\n", " | > stft_pad_mode:reflect\n", " | > max_norm:4.0\n", " | > clip_norm:True\n", " | > do_trim_silence:True\n", " | > trim_db:60\n", " | > do_sound_norm:False\n", " | > do_amp_to_db_linear:True\n", " | > do_amp_to_db_mel:True\n", " | > do_rms_norm:False\n", " | > db_level:None\n", " | > stats_path:C:\\Users\\Torch\\AppData\\Local\\tts\\tts_models--en--ljspeech--tacotron2-DDC_ph\\scale_stats.npy\n", " | > base:10\n", " | > hop_length:256\n", " | > win_length:1024\n", " > Model's reduction rate `r` is set to: 2\n", " > Vocoder Model: univnet\n", " > Setting up Audio Processor...\n", " | > sample_rate:22050\n", " | > resample:False\n", " | > num_mels:80\n", " | > log_func:np.log10\n", " | > min_level_db:-100\n", " | > frame_shift_ms:None\n", " | > frame_length_ms:None\n", " | > ref_level_db:20\n", " | > fft_size:1024\n", " | > power:1.5\n", " | > preemphasis:0.0\n", " | > griffin_lim_iters:60\n", " | > signal_norm:True\n", " | > symmetric_norm:True\n", " | > mel_fmin:50.0\n", " | > mel_fmax:7600.0\n", " | > pitch_fmin:1.0\n", " | > pitch_fmax:640.0\n", " | > spec_gain:1.0\n", " | > stft_pad_mode:reflect\n", " | > max_norm:4.0\n", " | > clip_norm:True\n", " | > do_trim_silence:True\n", " | > trim_db:60\n", " | > do_sound_norm:False\n", " | > do_amp_to_db_linear:True\n", " | > do_amp_to_db_mel:True\n", " | > do_rms_norm:False\n", " | > db_level:None\n", " | > stats_path:C:\\Users\\Torch\\AppData\\Local\\tts\\vocoder_models--en--ljspeech--univnet\\scale_stats.npy\n", " | > base:10\n", " | > hop_length:256\n", " | > win_length:1024\n", " > Generator Model: univnet_generator\n", " > Discriminator Model: univnet_discriminator\n", "model: tts_models/en/ljspeech/tacotron2-DDC_ph\n", "language: \n", "speaker: \n", "voice cloning with the voice conversion model\n", " > Text splitted to sentences.\n", "['Mary had a little lamb,', 'Its fleece was white as snow.', 'Everywhere the child went,', 'The little lamb was sure to go.']\n", "ɛvɹiwɛɹ ðə t͡ʃaɪld wɛnt,\n", " [!] Character '͡' not found in the vocabulary. Discarding it.\n", " > Processing time: 3.021000385284424\n", " > Real-time factor: 0.3477544400242312\n", "model: tts_models/en/ljspeech/tacotron2-DDC_ph\n", "language: \n", "speaker: \n", "voice cloning with the voice conversion model\n", " > Text splitted to sentences.\n", "['Mary had a little lamb,', 'Its fleece was white as snow.', 'Everywhere the child went,', 'The little lamb was sure to go.']\n", " > Processing time: 2.9099998474121094\n", " > Real-time factor: 0.33497690776101013\n", "model: tts_models/en/ljspeech/tacotron2-DDC_ph\n", "language: \n", "speaker: \n", "voice cloning with the voice conversion model\n", " > Text splitted to sentences.\n", "['Mary had a little lamb,', 'Its fleece was white as snow.', 'Everywhere the child went,', 'The little lamb was sure to go.']\n", " > Processing time: 2.933000087738037\n", " > Real-time factor: 0.33762451937136506\n", " > tts_models/en/ljspeech/tacotron2-DDC is already downloaded.\n", " > Model's license - apache 2.0\n", " > Check https://choosealicense.com/licenses/apache-2.0/ for more info.\n", " > vocoder_models/en/ljspeech/hifigan_v2 is already downloaded.\n", " > Model's license - apache 2.0\n", " > Check https://choosealicense.com/licenses/apache-2.0/ for more info.\n", " > Using model: Tacotron2\n", " > Setting up Audio Processor...\n", " | > sample_rate:22050\n", " | > resample:False\n", " | > num_mels:80\n", " | > log_func:np.log\n", " | > min_level_db:-100\n", " | > frame_shift_ms:None\n", " | > frame_length_ms:None\n", " | > ref_level_db:20\n", " | > fft_size:1024\n", " | > power:1.5\n", " | > preemphasis:0.0\n", " | > griffin_lim_iters:60\n", " | > signal_norm:False\n", " | > symmetric_norm:True\n", " | > mel_fmin:0\n", " | > mel_fmax:8000.0\n", " | > pitch_fmin:1.0\n", " | > pitch_fmax:640.0\n", " | > spec_gain:1.0\n", " | > stft_pad_mode:reflect\n", " | > max_norm:4.0\n", " | > clip_norm:True\n", " | > do_trim_silence:True\n", " | > trim_db:60\n", " | > do_sound_norm:False\n", " | > do_amp_to_db_linear:True\n", " | > do_amp_to_db_mel:True\n", " | > do_rms_norm:False\n", " | > db_level:None\n", " | > stats_path:None\n", " | > base:2.718281828459045\n", " | > hop_length:256\n", " | > win_length:1024\n", " > Model's reduction rate `r` is set to: 1\n", " > Vocoder Model: hifigan\n", " > Setting up Audio Processor...\n", " | > sample_rate:22050\n", " | > resample:False\n", " | > num_mels:80\n", " | > log_func:np.log\n", " | > min_level_db:-100\n", " | > frame_shift_ms:None\n", " | > frame_length_ms:None\n", " | > ref_level_db:20\n", " | > fft_size:1024\n", " | > power:1.5\n", " | > preemphasis:0.0\n", " | > griffin_lim_iters:60\n", " | > signal_norm:False\n", " | > symmetric_norm:True\n", " | > mel_fmin:0\n", " | > mel_fmax:8000.0\n", " | > pitch_fmin:1.0\n", " | > pitch_fmax:640.0\n", " | > spec_gain:1.0\n", " | > stft_pad_mode:reflect\n", " | > max_norm:4.0\n", " | > clip_norm:True\n", " | > do_trim_silence:False\n", " | > trim_db:60\n", " | > do_sound_norm:False\n", " | > do_amp_to_db_linear:True\n", " | > do_amp_to_db_mel:True\n", " | > do_rms_norm:False\n", " | > db_level:None\n", " | > stats_path:None\n", " | > base:2.718281828459045\n", " | > hop_length:256\n", " | > win_length:1024\n", " > Generator Model: hifigan_generator\n", " > Discriminator Model: hifigan_discriminator\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Removing weight norm...\n", "model: tts_models/en/ljspeech/tacotron2-DDC\n", "language: \n", "speaker: \n", "voice cloning with the voice conversion model\n", " > Text splitted to sentences.\n", "['Mary had a little lamb,', 'Its fleece was white as snow.', 'Everywhere the child went,', 'The little lamb was sure to go.']\n", " > Processing time: 4.28600001335144\n", " > Real-time factor: 0.42371906516498953\n", " > tts_models/en/ek1/tacotron2 is already downloaded.\n", " > Model's license - apache 2.0\n", " > Check https://choosealicense.com/licenses/apache-2.0/ for more info.\n", " > vocoder_models/en/ek1/wavegrad is already downloaded.\n", " > Model's license - apache 2.0\n", " > Check https://choosealicense.com/licenses/apache-2.0/ for more info.\n", " > Using model: Tacotron2\n", " > Setting up Audio Processor...\n", " | > sample_rate:22050\n", " | > resample:False\n", " | > num_mels:80\n", " | > log_func:np.log10\n", " | > min_level_db:-10\n", " | > frame_shift_ms:None\n", " | > frame_length_ms:None\n", " | > ref_level_db:0\n", " | > fft_size:1024\n", " | > power:1.8\n", " | > preemphasis:0.99\n", " | > griffin_lim_iters:60\n", " | > signal_norm:True\n", " | > symmetric_norm:True\n", " | > mel_fmin:0\n", " | > mel_fmax:8000.0\n", " | > pitch_fmin:1.0\n", " | > pitch_fmax:640.0\n", " | > spec_gain:1.0\n", " | > stft_pad_mode:reflect\n", " | > max_norm:4.0\n", " | > clip_norm:True\n", " | > do_trim_silence:True\n", " | > trim_db:60\n", " | > do_sound_norm:False\n", " | > do_amp_to_db_linear:True\n", " | > do_amp_to_db_mel:True\n", " | > do_rms_norm:False\n", " | > db_level:None\n", " | > stats_path:None\n", " | > base:10\n", " | > hop_length:256\n", " | > win_length:1024\n", " > Model's reduction rate `r` is set to: 2\n", " > Vocoder Model: wavegrad\n", "model: tts_models/en/ek1/tacotron2\n", "language: \n", "speaker: \n", "voice cloning with the voice conversion model\n", " > Text splitted to sentences.\n", "['Mary had a little lamb,', 'Its fleece was white as snow.', 'Everywhere the child went,', 'The little lamb was sure to go.']\n", "ɛvɹiwɛɹ ðə t͡ʃaɪld wɛnt,\n", " [!] Character '͡' not found in the vocabulary. Discarding it.\n", " > Processing time: 224.84099984169006\n", " > Real-time factor: 29.51038122922182\n", " > tts_models/en/ek1/tacotron2 is already downloaded.\n", " > Model's license - apache 2.0\n", " > Check https://choosealicense.com/licenses/apache-2.0/ for more info.\n", " > vocoder_models/en/ek1/wavegrad is already downloaded.\n", " > Model's license - apache 2.0\n", " > Check https://choosealicense.com/licenses/apache-2.0/ for more info.\n", " > Using model: Tacotron2\n", " > Setting up Audio Processor...\n", " | > sample_rate:22050\n", " | > resample:False\n", " | > num_mels:80\n", " | > log_func:np.log10\n", " | > min_level_db:-10\n", " | > frame_shift_ms:None\n", " | > frame_length_ms:None\n", " | > ref_level_db:0\n", " | > fft_size:1024\n", " | > power:1.8\n", " | > preemphasis:0.99\n", " | > griffin_lim_iters:60\n", " | > signal_norm:True\n", " | > symmetric_norm:True\n", " | > mel_fmin:0\n", " | > mel_fmax:8000.0\n", " | > pitch_fmin:1.0\n", " | > pitch_fmax:640.0\n", " | > spec_gain:1.0\n", " | > stft_pad_mode:reflect\n", " | > max_norm:4.0\n", " | > clip_norm:True\n", " | > do_trim_silence:True\n", " | > trim_db:60\n", " | > do_sound_norm:False\n", " | > do_amp_to_db_linear:True\n", " | > do_amp_to_db_mel:True\n", " | > do_rms_norm:False\n", " | > db_level:None\n", " | > stats_path:None\n", " | > base:10\n", " | > hop_length:256\n", " | > win_length:1024\n", " > Model's reduction rate `r` is set to: 2\n", " > Vocoder Model: wavegrad\n", "model: tts_models/en/ek1/tacotron2\n", "language: \n", "speaker: \n", "voice cloning with the voice conversion model\n", " > Text splitted to sentences.\n", "['Mary had a little lamb,', 'Its fleece was white as snow.', 'Everywhere the child went,', 'The little lamb was sure to go.']\n", "ɛvɹiwɛɹ ðə t͡ʃaɪld wɛnt,\n", " [!] Character '͡' not found in the vocabulary. Discarding it.\n", " > Processing time: 266.6489999294281\n", " > Real-time factor: 34.99768124073744\n", " > tts_models/en/ljspeech/tacotron2-DDC_ph is already downloaded.\n", " > Model's license - apache 2.0\n", " > Check https://choosealicense.com/licenses/apache-2.0/ for more info.\n", " > vocoder_models/en/ljspeech/univnet is already downloaded.\n", " > Model's license - apache 2.0\n", " > Check https://choosealicense.com/licenses/apache-2.0/ for more info.\n", " > Using model: Tacotron2\n", " > Setting up Audio Processor...\n", " | > sample_rate:22050\n", " | > resample:False\n", " | > num_mels:80\n", " | > log_func:np.log10\n", " | > min_level_db:-100\n", " | > frame_shift_ms:None\n", " | > frame_length_ms:None\n", " | > ref_level_db:20\n", " | > fft_size:1024\n", " | > power:1.5\n", " | > preemphasis:0.0\n", " | > griffin_lim_iters:60\n", " | > signal_norm:True\n", " | > symmetric_norm:True\n", " | > mel_fmin:50.0\n", " | > mel_fmax:7600.0\n", " | > pitch_fmin:0.0\n", " | > pitch_fmax:640.0\n", " | > spec_gain:1.0\n", " | > stft_pad_mode:reflect\n", " | > max_norm:4.0\n", " | > clip_norm:True\n", " | > do_trim_silence:True\n", " | > trim_db:60\n", " | > do_sound_norm:False\n", " | > do_amp_to_db_linear:True\n", " | > do_amp_to_db_mel:True\n", " | > do_rms_norm:False\n", " | > db_level:None\n", " | > stats_path:C:\\Users\\Torch\\AppData\\Local\\tts\\tts_models--en--ljspeech--tacotron2-DDC_ph\\scale_stats.npy\n", " | > base:10\n", " | > hop_length:256\n", " | > win_length:1024\n", " > Model's reduction rate `r` is set to: 2\n", " > Vocoder Model: univnet\n", " > Setting up Audio Processor...\n", " | > sample_rate:22050\n", " | > resample:False\n", " | > num_mels:80\n", " | > log_func:np.log10\n", " | > min_level_db:-100\n", " | > frame_shift_ms:None\n", " | > frame_length_ms:None\n", " | > ref_level_db:20\n", " | > fft_size:1024\n", " | > power:1.5\n", " | > preemphasis:0.0\n", " | > griffin_lim_iters:60\n", " | > signal_norm:True\n", " | > symmetric_norm:True\n", " | > mel_fmin:50.0\n", " | > mel_fmax:7600.0\n", " | > pitch_fmin:1.0\n", " | > pitch_fmax:640.0\n", " | > spec_gain:1.0\n", " | > stft_pad_mode:reflect\n", " | > max_norm:4.0\n", " | > clip_norm:True\n", " | > do_trim_silence:True\n", " | > trim_db:60\n", " | > do_sound_norm:False\n", " | > do_amp_to_db_linear:True\n", " | > do_amp_to_db_mel:True\n", " | > do_rms_norm:False\n", " | > db_level:None\n", " | > stats_path:C:\\Users\\Torch\\AppData\\Local\\tts\\vocoder_models--en--ljspeech--univnet\\scale_stats.npy\n", " | > base:10\n", " | > hop_length:256\n", " | > win_length:1024\n", " > Generator Model: univnet_generator\n", " > Discriminator Model: univnet_discriminator\n", "model: tts_models/en/ljspeech/tacotron2-DDC_ph\n", "language: \n", "speaker: \n", "voice cloning with the voice conversion model\n", " > Text splitted to sentences.\n", "['Mary had a little lamb,', 'Its fleece was white as snow.', 'Everywhere the child went,', 'The little lamb was sure to go.']\n", "ɛvɹiwɛɹ ðə t͡ʃaɪld wɛnt,\n", " [!] Character '͡' not found in the vocabulary. Discarding it.\n", " > Processing time: 2.885999917984009\n", " > Real-time factor: 0.3322142195933605\n", " > Downloading model to C:\\Users\\Torch\\AppData\\Local\\tts\\tts_models--en--ljspeech--speedy-speech\n", " > Model's license - apache 2.0\n", " > Check https://choosealicense.com/licenses/apache-2.0/ for more info.\n", " > vocoder_models/en/ljspeech/hifigan_v2 is already downloaded.\n", " > Model's license - apache 2.0\n", " > Check https://choosealicense.com/licenses/apache-2.0/ for more info.\n", " > Using model: speedy_speech\n", " > Setting up Audio Processor...\n", " | > sample_rate:22050\n", " | > resample:False\n", " | > num_mels:80\n", " | > log_func:np.log\n", " | > min_level_db:-100\n", " | > frame_shift_ms:None\n", " | > frame_length_ms:None\n", " | > ref_level_db:20\n", " | > fft_size:1024\n", " | > power:1.5\n", " | > preemphasis:0.0\n", " | > griffin_lim_iters:60\n", " | > signal_norm:False\n", " | > symmetric_norm:True\n", " | > mel_fmin:0\n", " | > mel_fmax:8000.0\n", " | > pitch_fmin:1.0\n", " | > pitch_fmax:640.0\n", " | > spec_gain:1.0\n", " | > stft_pad_mode:reflect\n", " | > max_norm:4.0\n", " | > clip_norm:True\n", " | > do_trim_silence:True\n", " | > trim_db:60\n", " | > do_sound_norm:False\n", " | > do_amp_to_db_linear:True\n", " | > do_amp_to_db_mel:True\n", " | > do_rms_norm:False\n", " | > db_level:None\n", " | > stats_path:None\n", " | > base:2.718281828459045\n", " | > hop_length:256\n", " | > win_length:1024\n", " > Vocoder Model: hifigan\n", " > Setting up Audio Processor...\n", " | > sample_rate:22050\n", " | > resample:False\n", " | > num_mels:80\n", " | > log_func:np.log\n", " | > min_level_db:-100\n", " | > frame_shift_ms:None\n", " | > frame_length_ms:None\n", " | > ref_level_db:20\n", " | > fft_size:1024\n", " | > power:1.5\n", " | > preemphasis:0.0\n", " | > griffin_lim_iters:60\n", " | > signal_norm:False\n", " | > symmetric_norm:True\n", " | > mel_fmin:0\n", " | > mel_fmax:8000.0\n", " | > pitch_fmin:1.0\n", " | > pitch_fmax:640.0\n", " | > spec_gain:1.0\n", " | > stft_pad_mode:reflect\n", " | > max_norm:4.0\n", " | > clip_norm:True\n", " | > do_trim_silence:False\n", " | > trim_db:60\n", " | > do_sound_norm:False\n", " | > do_amp_to_db_linear:True\n", " | > do_amp_to_db_mel:True\n", " | > do_rms_norm:False\n", " | > db_level:None\n", " | > stats_path:None\n", " | > base:2.718281828459045\n", " | > hop_length:256\n", " | > win_length:1024\n", " > Generator Model: hifigan_generator\n", " > Discriminator Model: hifigan_discriminator\n", "Removing weight norm...\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "model: tts_models/en/ljspeech/speedy-speech\n", "language: \n", "speaker: \n", "Using original voice\n", " > Text splitted to sentences.\n", "['Mary had a little lamb,', 'Its fleece was white as snow.', 'Everywhere the child went,', 'The little lamb was sure to go.']\n", "ɛvɹiwɛɹ ðə t͡ʃaɪld wɛnt,\n", " [!] Character '͡' not found in the vocabulary. Discarding it.\n", " > Processing time: 0.9679999351501465\n", " > Real-time factor: 0.11673301633083617\n", "model: tts_models/en/ljspeech/speedy-speech\n", "language: \n", "speaker: \n", "voice cloning with the voice conversion model\n", " > Text splitted to sentences.\n", "['Mary had a little lamb,', 'Its fleece was white as snow.', 'Everywhere the child went,', 'The little lamb was sure to go.']\n", " > Processing time: 0.9630000591278076\n", " > Real-time factor: 0.11613007144605443\n", " > tts_models/en/ljspeech/tacotron2-DCA is already downloaded.\n", " > Model's license - MPL\n", " > Check https://www.mozilla.org/en-US/MPL/2.0/ for more info.\n", " > vocoder_models/en/ljspeech/multiband-melgan is already downloaded.\n", " > Model's license - MPL\n", " > Check https://www.mozilla.org/en-US/MPL/2.0/ for more info.\n", " > Using model: Tacotron2\n", " > Setting up Audio Processor...\n", " | > sample_rate:22050\n", " | > resample:False\n", " | > num_mels:80\n", " | > log_func:np.log10\n", " | > min_level_db:-100\n", " | > frame_shift_ms:None\n", " | > frame_length_ms:None\n", " | > ref_level_db:20\n", " | > fft_size:1024\n", " | > power:1.5\n", " | > preemphasis:0.0\n", " | > griffin_lim_iters:60\n", " | > signal_norm:True\n", " | > symmetric_norm:True\n", " | > mel_fmin:50.0\n", " | > mel_fmax:7600.0\n", " | > pitch_fmin:0.0\n", " | > pitch_fmax:640.0\n", " | > spec_gain:1.0\n", " | > stft_pad_mode:reflect\n", " | > max_norm:4.0\n", " | > clip_norm:True\n", " | > do_trim_silence:True\n", " | > trim_db:60\n", " | > do_sound_norm:False\n", " | > do_amp_to_db_linear:True\n", " | > do_amp_to_db_mel:True\n", " | > do_rms_norm:False\n", " | > db_level:None\n", " | > stats_path:C:\\Users\\Torch\\AppData\\Local\\tts\\tts_models--en--ljspeech--tacotron2-DCA\\scale_stats.npy\n", " | > base:10\n", " | > hop_length:256\n", " | > win_length:1024\n", " > Model's reduction rate `r` is set to: 2\n", " > Vocoder Model: multiband_melgan\n", " > Setting up Audio Processor...\n", " | > sample_rate:22050\n", " | > resample:False\n", " | > num_mels:80\n", " | > log_func:np.log10\n", " | > min_level_db:-100\n", " | > frame_shift_ms:None\n", " | > frame_length_ms:None\n", " | > ref_level_db:0\n", " | > fft_size:1024\n", " | > power:1.5\n", " | > preemphasis:0.0\n", " | > griffin_lim_iters:60\n", " | > signal_norm:True\n", " | > symmetric_norm:True\n", " | > mel_fmin:50.0\n", " | > mel_fmax:7600.0\n", " | > pitch_fmin:0.0\n", " | > pitch_fmax:640.0\n", " | > spec_gain:1.0\n", " | > stft_pad_mode:reflect\n", " | > max_norm:4.0\n", " | > clip_norm:True\n", " | > do_trim_silence:True\n", " | > trim_db:60\n", " | > do_sound_norm:False\n", " | > do_amp_to_db_linear:True\n", " | > do_amp_to_db_mel:True\n", " | > do_rms_norm:False\n", " | > db_level:None\n", " | > stats_path:C:\\Users\\Torch\\AppData\\Local\\tts\\vocoder_models--en--ljspeech--multiband-melgan\\scale_stats.npy\n", " | > base:10\n", " | > hop_length:256\n", " | > win_length:1024\n", " > Generator Model: multiband_melgan_generator\n", " > Discriminator Model: melgan_multiscale_discriminator\n", "model: tts_models/en/ljspeech/tacotron2-DCA\n", "language: \n", "speaker: \n", "Using original voice\n", " > Text splitted to sentences.\n", "['Mary had a little lamb,', 'Its fleece was white as snow.', 'Everywhere the child went,', 'The little lamb was sure to go.']\n", "ɛvɹiwɛɹ ðə t͡ʃaɪld wɛnt,\n", " [!] Character '͡' not found in the vocabulary. Discarding it.\n", " > Processing time: 2.067000150680542\n", " > Real-time factor: 0.23295588670728015\n", "model: tts_models/en/ljspeech/tacotron2-DCA\n", "language: \n", "speaker: \n", "voice cloning with the voice conversion model\n", " > Text splitted to sentences.\n", "['Mary had a little lamb,', 'Its fleece was white as snow.', 'Everywhere the child went,', 'The little lamb was sure to go.']\n", " > Processing time: 2.1570000648498535\n", " > Real-time factor: 0.2430990934225715\n", "model: tts_models/en/ljspeech/tacotron2-DCA\n", "language: \n", "speaker: \n", "voice cloning with the voice conversion model\n", " > Text splitted to sentences.\n", "['Mary had a little lamb,', 'Its fleece was white as snow.', 'Everywhere the child went,', 'The little lamb was sure to go.']\n", " > Processing time: 2.0920000076293945\n", " > Real-time factor: 0.23577343069302087\n", " > Downloading model to C:\\Users\\Torch\\AppData\\Local\\tts\\tts_models--en--ljspeech--fast_pitch\n", " > Model's license - apache 2.0\n", " > Check https://choosealicense.com/licenses/apache-2.0/ for more info.\n", " > vocoder_models/en/ljspeech/hifigan_v2 is already downloaded.\n", " > Model's license - apache 2.0\n", " > Check https://choosealicense.com/licenses/apache-2.0/ for more info.\n", " > Using model: fast_pitch\n", " > Setting up Audio Processor...\n", " | > sample_rate:22050\n", " | > resample:False\n", " | > num_mels:80\n", " | > log_func:np.log\n", " | > min_level_db:-100\n", " | > frame_shift_ms:None\n", " | > frame_length_ms:None\n", " | > ref_level_db:20\n", " | > fft_size:1024\n", " | > power:1.5\n", " | > preemphasis:0.0\n", " | > griffin_lim_iters:60\n", " | > signal_norm:False\n", " | > symmetric_norm:True\n", " | > mel_fmin:0\n", " | > mel_fmax:8000.0\n", " | > pitch_fmin:1.0\n", " | > pitch_fmax:640.0\n", " | > spec_gain:1.0\n", " | > stft_pad_mode:reflect\n", " | > max_norm:4.0\n", " | > clip_norm:True\n", " | > do_trim_silence:True\n", " | > trim_db:60\n", " | > do_sound_norm:False\n", " | > do_amp_to_db_linear:True\n", " | > do_amp_to_db_mel:True\n", " | > do_rms_norm:False\n", " | > db_level:None\n", " | > stats_path:None\n", " | > base:2.718281828459045\n", " | > hop_length:256\n", " | > win_length:1024\n", " > Vocoder Model: hifigan\n", " > Setting up Audio Processor...\n", " | > sample_rate:22050\n", " | > resample:False\n", " | > num_mels:80\n", " | > log_func:np.log\n", " | > min_level_db:-100\n", " | > frame_shift_ms:None\n", " | > frame_length_ms:None\n", " | > ref_level_db:20\n", " | > fft_size:1024\n", " | > power:1.5\n", " | > preemphasis:0.0\n", " | > griffin_lim_iters:60\n", " | > signal_norm:False\n", " | > symmetric_norm:True\n", " | > mel_fmin:0\n", " | > mel_fmax:8000.0\n", " | > pitch_fmin:1.0\n", " | > pitch_fmax:640.0\n", " | > spec_gain:1.0\n", " | > stft_pad_mode:reflect\n", " | > max_norm:4.0\n", " | > clip_norm:True\n", " | > do_trim_silence:False\n", " | > trim_db:60\n", " | > do_sound_norm:False\n", " | > do_amp_to_db_linear:True\n", " | > do_amp_to_db_mel:True\n", " | > do_rms_norm:False\n", " | > db_level:None\n", " | > stats_path:None\n", " | > base:2.718281828459045\n", " | > hop_length:256\n", " | > win_length:1024\n", " > Generator Model: hifigan_generator\n", " > Discriminator Model: hifigan_discriminator\n", "Removing weight norm...\n", "model: tts_models/en/ljspeech/fast_pitch\n", "language: \n", "speaker: \n", "Using original voice\n", " > Text splitted to sentences.\n", "['Mary had a little lamb,', 'Its fleece was white as snow.', 'Everywhere the child went,', 'The little lamb was sure to go.']\n", "ɛvɹiwɛɹ ðə t͡ʃaɪld wɛnt,\n", " [!] Character '͡' not found in the vocabulary. Discarding it.\n", " > Processing time: 1.8829996585845947\n", " > Real-time factor: 0.19894272496832988\n", "model: tts_models/en/ljspeech/fast_pitch\n", "language: \n", "speaker: \n", "voice cloning with the voice conversion model\n", " > Text splitted to sentences.\n", "['Mary had a little lamb,', 'Its fleece was white as snow.', 'Everywhere the child went,', 'The little lamb was sure to go.']\n", " > Processing time: 1.8359999656677246\n", " > Real-time factor: 0.19397711228808903\n", "model: tts_models/en/ljspeech/fast_pitch\n", "language: \n", "speaker: \n", "voice cloning with the voice conversion model\n", " > Text splitted to sentences.\n", "['Mary had a little lamb,', 'Its fleece was white as snow.', 'Everywhere the child went,', 'The little lamb was sure to go.']\n", " > Processing time: 1.8659999370574951\n", " > Real-time factor: 0.19714666998293168\n", "model: voice_conversion_models/multilingual/vctk/freevc24\n", "source_wav: C:\\Users\\Torch\\AppData\\Local\\Temp\\gradio\\b6e9c24083a878478ebbecd7bc42e1f631c05df6\\henry5-0-100.wav\n", "target_wav: C:\\Users\\Torch\\AppData\\Local\\Temp\\gradio\\11c82c70d145ea630f81dfa541de52bf615719ae\\yearn_for_time-0-100.wav\n", " > Downloading model to C:\\Users\\Torch\\AppData\\Local\\tts\\tts_models--en--ljspeech--overflow\n", " > Model's license - apache 2.0\n", " > Check https://choosealicense.com/licenses/apache-2.0/ for more info.\n", " > vocoder_models/en/ljspeech/hifigan_v2 is already downloaded.\n", " > Model's license - apache 2.0\n", " > Check https://choosealicense.com/licenses/apache-2.0/ for more info.\n", " > Using model: OverFlow\n", " > Setting up Audio Processor...\n", " | > sample_rate:22050\n", " | > resample:False\n", " | > num_mels:80\n", " | > log_func:np.log\n", " | > min_level_db:-100\n", " | > frame_shift_ms:None\n", " | > frame_length_ms:None\n", " | > ref_level_db:20\n", " | > fft_size:1024\n", " | > power:1.5\n", " | > preemphasis:0.0\n", " | > griffin_lim_iters:60\n", " | > signal_norm:False\n", " | > symmetric_norm:True\n", " | > mel_fmin:0\n", " | > mel_fmax:8000.0\n", " | > pitch_fmin:1.0\n", " | > pitch_fmax:640.0\n", " | > spec_gain:1.0\n", " | > stft_pad_mode:reflect\n", " | > max_norm:4.0\n", " | > clip_norm:True\n", " | > do_trim_silence:True\n", " | > trim_db:60\n", " | > do_sound_norm:False\n", " | > do_amp_to_db_linear:True\n", " | > do_amp_to_db_mel:True\n", " | > do_rms_norm:False\n", " | > db_level:None\n", " | > stats_path:None\n", " | > base:2.718281828459045\n", " | > hop_length:256\n", " | > win_length:1024\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " > Vocoder Model: hifigan\n", " > Setting up Audio Processor...\n", " | > sample_rate:22050\n", " | > resample:False\n", " | > num_mels:80\n", " | > log_func:np.log\n", " | > min_level_db:-100\n", " | > frame_shift_ms:None\n", " | > frame_length_ms:None\n", " | > ref_level_db:20\n", " | > fft_size:1024\n", " | > power:1.5\n", " | > preemphasis:0.0\n", " | > griffin_lim_iters:60\n", " | > signal_norm:False\n", " | > symmetric_norm:True\n", " | > mel_fmin:0\n", " | > mel_fmax:8000.0\n", " | > pitch_fmin:1.0\n", " | > pitch_fmax:640.0\n", " | > spec_gain:1.0\n", " | > stft_pad_mode:reflect\n", " | > max_norm:4.0\n", " | > clip_norm:True\n", " | > do_trim_silence:False\n", " | > trim_db:60\n", " | > do_sound_norm:False\n", " | > do_amp_to_db_linear:True\n", " | > do_amp_to_db_mel:True\n", " | > do_rms_norm:False\n", " | > db_level:None\n", " | > stats_path:None\n", " | > base:2.718281828459045\n", " | > hop_length:256\n", " | > win_length:1024\n", " > Generator Model: hifigan_generator\n", " > Discriminator Model: hifigan_discriminator\n", "Removing weight norm...\n", "model: tts_models/en/ljspeech/overflow\n", "language: \n", "speaker: \n", "Using original voice\n", " > Text splitted to sentences.\n", "['Mary had a little lamb,', 'Its fleece was white as snow.', 'Everywhere the child went,', 'The little lamb was sure to go.']\n", " > Processing time: 2.4030001163482666\n", " > Real-time factor: 0.26459208495864933\n", "model: tts_models/en/ljspeech/overflow\n", "language: \n", "speaker: \n", "voice cloning with the voice conversion model\n", " > Text splitted to sentences.\n", "['Mary had a little lamb,', 'Its fleece was white as snow.', 'Everywhere the child went,', 'The little lamb was sure to go.']\n", " > Processing time: 2.4769999980926514\n", " > Real-time factor: 0.27343925203231617\n" ] } ], "source": [ "title = \"\"\n", "description = \"\"\"\"\"\"\n", "article = \"\"\"\"\"\"\n", "\n", "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", "GPU = device == \"cuda\"\n", "INT16MAX = np.iinfo(np.int16).max\n", "VC_MODEL = TTS(model_name='voice_conversion_models/multilingual/vctk/freevc24', progress_bar=False, gpu=GPU)\n", "\n", "\n", "model_ids = ModelManager(verbose=False).list_models()\n", "model_tts_ids = [model for model in model_ids if 'tts_models' in model and ('/multilingual/' in model or '/en/' in model)]\n", "model_voc_ids = [model for model in model_ids if 'vocoder_models' in model and ('/universal/' in model or '/en/' in model)]\n", "model_vc_ids = [model for model in model_ids if 'voice_conversion_models' in model and ('/multilingual/' in model or '/en/' in model)]\n", "examples_pt = 'examples'\n", "allowed_extentions = ['.mp3', '.wav']\n", "examples = {f.name: f for f in Path(examples_pt).glob('*') if f.suffix in allowed_extentions}\n", "verse = \"\"\"Mary had a little lamb,\n", "Its fleece was white as snow.\n", "Everywhere the child went,\n", "The little lamb was sure to go.\"\"\"\n", "\n", "\n", "def on_model_tts_select(model_name):\n", " tts_var = TTS(model_name=model_name, progress_bar=False, gpu=GPU)\n", " languages = tts_var.languages if tts_var.is_multi_lingual else ['']\n", " speakers = [s.replace('\\n', '-n') for s in tts_var.speakers] if tts_var.is_multi_speaker else [''] # there's weird speaker formatting\n", " language = languages[0]\n", " speaker = speakers[0]\n", " return tts_var, gr.update(choices=languages, value=language, interactive=tts_var.is_multi_lingual),\\\n", " gr.update(choices=speakers, value=speaker, interactive=tts_var.is_multi_speaker)\n", "\n", "\n", "def on_voicedropdown(x):\n", " return examples[x]\n", "\n", "\n", "def voice_clone(source_wav, target_wav):\n", " print(f'model: {VC_MODEL.model_name}\\nsource_wav: {source_wav}\\ntarget_wav: {target_wav}')\n", " sample_rate = VC_MODEL.voice_converter.output_sample_rate\n", " if source_wav is None or target_wav is None:\n", " return (sample_rate, np.zeros(0).astype(np.int16))\n", "\n", " speech = VC_MODEL.voice_conversion(source_wav=source_wav, target_wav=target_wav)\n", " speech = (np.array(speech) * INT16MAX).astype(np.int16)\n", " return (sample_rate, speech)\n", "\n", "\n", "def text_to_speech(text, tts_model, language, speaker, target_wav, use_original_voice):\n", " if len(text.strip()) == 0 or tts_model is None or (target_wav is None and not use_original_voice):\n", " return (16000, np.zeros(0).astype(np.int16))\n", "\n", " sample_rate = tts_model.synthesizer.output_sample_rate\n", " if tts_model.is_multi_speaker:\n", " speaker = {s.replace('\\n', '-n'): s for s in tts_model.speakers}[speaker] # there's weird speaker formatting\n", " print(f'model: {tts_model.model_name}\\nlanguage: {language}\\nspeaker: {speaker}')\n", "\n", " language = None if language == '' else language\n", " speaker = None if speaker == '' else speaker\n", " if use_original_voice:\n", " print('Using original voice')\n", " speech = tts_model.tts(text, language=language, speaker=speaker) \n", " elif tts_model.synthesizer.tts_model.speaker_manager:\n", " print('voice cloning with the tts')\n", " speech = tts_model.tts(text, language=language, speaker_wav=target_wav)\n", " else:\n", " print('voice cloning with the voice conversion model')\n", "# speech = tts_model.tts_with_vc(text, language=language, speaker_wav=target_wav)\n", " with tempfile.NamedTemporaryFile(suffix=\".wav\", delete=False) as fp:\n", " # Lazy code... save it to a temp file to resample it while reading it for VC\n", " tts_model.tts_to_file(text, language=language, speaker=speaker, file_path=fp.name)\n", " speech = VC_MODEL.voice_conversion(source_wav=fp.name, target_wav=target_wav)\n", " \n", "\n", " speech = (np.array(speech) * INT16MAX).astype(np.int16)\n", " return (sample_rate, speech)\n", "\n", "\n", "with gr.Blocks() as demo:\n", " tts_model = gr.State(None)\n", "# vc_model = gr.State(None)\n", " def activate(*args):\n", " return gr.update(interactive=True) if len(args) == 1 else [gr.update(interactive=True)] * len(args)\n", " def deactivate(*args):\n", " return gr.update(interactive=False) if len(args) == 1 else [gr.update(interactive=False)] * len(args)\n", "\n", " \n", " gr.Markdown(description)\n", "\n", " with gr.Row(equal_height=True):\n", " with gr.Column(scale=5, min_width=50):\n", " model_tts_dropdown = gr.Dropdown(model_tts_ids, value=None, label='Text-to-speech model', interactive=True)\n", " with gr.Column(scale=1, min_width=10):\n", " language_dropdown = gr.Dropdown(None, value=None, label='Language', interactive=False, visible=True)\n", " with gr.Column(scale=1, min_width=10):\n", " speaker_dropdown = gr.Dropdown(None, value=None, label='Speaker', interactive=False, visible=True)\n", " \n", " with gr.Accordion(\"Target voice\", open=False) as accordion:\n", " gr.Markdown(\"Upload target voice...\")\n", " with gr.Row(equal_height=True):\n", " voice_upload = gr.Audio(label='Upload target voice', source='upload', type='filepath')\n", " voice_dropdown = gr.Dropdown(examples, label='Examples', interactive=True)\n", "\n", " with gr.Row(equal_height=True):\n", " with gr.Column(scale=2):\n", " with gr.Row(equal_height=True):\n", " with gr.Column():\n", " text_to_convert = gr.Textbox(verse)\n", " orig_voice = gr.Checkbox(label='Use original voice')\n", " voice_to_convert = gr.Audio(label=\"Upload voice to convert\", source='upload', type='filepath')\n", " with gr.Row(equal_height=True):\n", " button_text = gr.Button('Text to speech', interactive=True)\n", " button_audio = gr.Button('Convert audio', interactive=True)\n", " with gr.Row(equal_height=True):\n", " speech = gr.Audio(label='Converted Speech', type='numpy', visible=True, interactive=False) \n", " \n", " # actions\n", " model_tts_dropdown.change(deactivate, [button_text, button_audio], [button_text, button_audio]).\\\n", " then(fn=on_model_tts_select, inputs=[model_tts_dropdown], outputs=[tts_model, language_dropdown, speaker_dropdown]).\\\n", " then(activate, [button_text, button_audio], [button_text, button_audio])\n", " voice_dropdown.change(deactivate, [button_text, button_audio], [button_text, button_audio]).\\\n", " then(fn=on_voicedropdown, inputs=voice_dropdown, outputs=voice_upload).\\\n", " then(activate, [button_text, button_audio], [button_text, button_audio])\n", "\n", " button_text.click(deactivate, [button_text, button_audio], [button_text, button_audio]).\\\n", " then(fn=text_to_speech, inputs=[text_to_convert, tts_model, language_dropdown, speaker_dropdown, voice_upload, orig_voice], \n", " outputs=speech).\\\n", " then(activate, [button_text, button_audio], [button_text, button_audio])\n", "\n", " button_audio.click(deactivate, [button_text, button_audio], [button_text, button_audio]).\\\n", " then(fn=voice_clone, inputs=[voice_to_convert, voice_upload], outputs=speech).\\\n", " then(activate, [button_text, button_audio], [button_text, button_audio])\n", " \n", " gr.HTML(article)\n", "demo.launch(share=False)" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.9" } }, "nbformat": 4, "nbformat_minor": 5 }