Implement changes to speak accent agnostic and in many languages

InferenceInterfaces/ControllableInterface.py

@@ -1,5 +1,4 @@
 import os
-
 import torch
 from huggingface_hub import hf_hub_download
 
@@ -30,88 +29,62 @@ class ControllableInterface:
     def read(self,
              prompt,
              reference_audio,
-             language,
-             accent,
              voice_seed,
              prosody_creativity,
-             duration_scaling_factor,
-             pause_duration_scaling_factor,
-             pitch_variance_scale,
-             energy_variance_scale,
-             emb_slider_1,
-             emb_slider_2,
-             emb_slider_3,
-             emb_slider_4,
-             emb_slider_5,
-             emb_slider_6,
              loudness_in_db
              ):
-        if self.current_language != language:
-            self.model.set_phonemizer_language(language)
-            print(f"switched phonemizer language to {language}")
-            self.current_language = language
-        if self.current_accent != accent:
-            self.model.set_accent_language(accent)
-            print(f"switched accent language to {accent}")
-            self.current_accent = accent
+        print(prompt + "\n\n")
+        self.model.set_accent_language(accent)
+
         if reference_audio is None:
-            self.wgan.set_latent(voice_seed)
-            controllability_vector = torch.tensor([emb_slider_1,
-                                                   emb_slider_2,
-                                                   emb_slider_3,
-                                                   emb_slider_4,
-                                                   emb_slider_5,
-                                                   emb_slider_6], dtype=torch.float32)
-            embedding = self.wgan.modify_embed(controllability_vector)
-            self.model.set_utterance_embedding(embedding=embedding)
+            if not voice_seed:
+                self.wgan.set_latent(7)
+                controllability_vector = torch.tensor([0.0,
+                                                       0.0,
+                                                       0.0,
+                                                       0.0,
+                                                       0.0,
+                                                       0.0], dtype=torch.float32)
+                embedding = self.wgan.modify_embed(controllability_vector)
+                self.model.set_utterance_embedding(embedding=embedding)
+            else:
+                wavs = list()
+                pitch, energy, durations = None, None, None
+                for i in range(10):
+                    self.wgan.set_latent(i)
+                    controllability_vector = torch.tensor([0.0,
+                                                           0.0,
+                                                           0.0,
+                                                           0.0,
+                                                           0.0,
+                                                           0.0], dtype=torch.float32)
+                    embedding = self.wgan.modify_embed(controllability_vector)
+                    self.model.set_utterance_embedding(embedding=embedding)
+                    wav, sr, fig, pitch, energy, durations = self.model(prompt,
+                                                                        input_is_phones=True,
+                                                                        duration_scaling_factor=1.0,
+                                                                        pitch_variance_scale=1.0,
+                                                                        energy_variance_scale=1.0,
+                                                                        pause_duration_scaling_factor=1.0,
+                                                                        return_plot_as_filepath=True,
+                                                                        prosody_creativity=prosody_creativity,
+                                                                        loudness_in_db=loudness_in_db,
+                                                                        pitch=pitch,
+                                                                        energy=energy,
+                                                                        durations=durations)
+                    wavs.append(wav)
+                wav = wavs.mean()
         else:
             self.model.set_utterance_embedding(reference_audio)
 
-        phones = self.model.text2phone.get_phone_string(prompt)
-        if len(phones) > 1800:
-            if language == "deu":
-                prompt = "Deine Eingabe war zu lang. Bitte versuche es entweder mit einem kürzeren Text oder teile ihn in mehrere Teile auf."
-            elif language == "ell":
-                prompt = "Η εισήγησή σας ήταν πολύ μεγάλη. Παρακαλώ δοκιμάστε είτε ένα μικρότερο κείμενο είτε χωρίστε το σε διάφορα μέρη."
-            elif language == "spa":
-                prompt = "Su entrada es demasiado larga. Por favor, intente un texto más corto o divídalo en varias partes."
-            elif language == "fin":
-                prompt = "Vastauksesi oli liian pitkä. Kokeile joko lyhyempää tekstiä tai jaa se useampaan osaan."
-            elif language == "rus":
-                prompt = "Ваш текст слишком длинный. Пожалуйста, попробуйте либо сократить текст, либо разделить его на несколько частей."
-            elif language == "hun":
-                prompt = "Túl hosszú volt a bevitele. Kérjük, próbáljon meg rövidebb szöveget írni, vagy ossza több részre."
-            elif language == "nld":
-                prompt = "Uw input was te lang. Probeer een kortere tekst of splits het in verschillende delen."
-            elif language == "fra":
-                prompt = "Votre saisie était trop longue. Veuillez essayer un texte plus court ou le diviser en plusieurs parties."
-            elif language == 'pol':
-                prompt = "Twój wpis był zbyt długi. Spróbuj skrócić tekst lub podzielić go na kilka części."
-            elif language == 'por':
-                prompt = "O seu contributo foi demasiado longo. Por favor, tente um texto mais curto ou divida-o em várias partes."
-            elif language == 'ita':
-                prompt = "Il tuo input era troppo lungo. Per favore, prova un testo più corto o dividilo in più parti."
-            elif language == 'cmn':
-                prompt = "你的输入太长了。请尝试使用较短的文本或将其拆分为多个部分。"
-            elif language == 'vie':
-                prompt = "Đầu vào của bạn quá dài. Vui lòng thử một văn bản ngắn hơn hoặc chia nó thành nhiều phần."
-            else:
-                prompt = "Your input was too long. Please try either a shorter text or split it into several parts."
-                if self.current_language != "eng":
-                    self.model.set_phonemizer_language("eng")
-                    self.current_language = "eng"
-                if self.current_accent != "eng":
-                    self.model.set_accent_language("eng")
-                    self.current_accent = "eng"
-
-        print(prompt + "\n\n")
-        wav, sr, fig = self.model(prompt,
-                                  input_is_phones=False,
-                                  duration_scaling_factor=duration_scaling_factor,
-                                  pitch_variance_scale=pitch_variance_scale,
-                                  energy_variance_scale=energy_variance_scale,
-                                  pause_duration_scaling_factor=pause_duration_scaling_factor,
-                                  return_plot_as_filepath=True,
-                                  prosody_creativity=prosody_creativity,
-                                  loudness_in_db=loudness_in_db)
+        if not voice_seed and reference_audio is not None:
+            wav, sr, fig, pitch, energy, durations = self.model(prompt,
+                                                                input_is_phones=True,
+                                                                duration_scaling_factor=1.0,
+                                                                pitch_variance_scale=1.0,
+                                                                energy_variance_scale=1.0,
+                                                                pause_duration_scaling_factor=1.0,
+                                                                return_plot_as_filepath=True,
+                                                                prosody_creativity=prosody_creativity,
+                                                                loudness_in_db=loudness_in_db)
         return sr, wav, fig

InferenceInterfaces/ToucanTTSInterface.py

@@ -231,7 +231,7 @@ class ToucanTTSInterface(torch.nn.Module):
             if return_plot_as_filepath:
                 plt.savefig("tmp.png")
                 plt.close()
-                return wave, sr, "tmp.png"
+                return wave, sr, "tmp.png", pitch, energy, durations
         return wave, sr
 
     def read_to_file(self,

Modules/GeneralLayers/Conformer.py

@@ -130,10 +130,13 @@ class Conformer(torch.nn.Module):
             xs = self.art_embed_norm(xs)
 
         if lang_ids is not None:
-            lang_embs = self.language_embedding(lang_ids)
-            projected_lang_embs = self.language_embedding_projection(lang_embs).unsqueeze(-1).transpose(1, 2)
-            projected_lang_embs = self.language_emb_norm(projected_lang_embs)
-            xs = xs + projected_lang_embs  # offset phoneme representation by language specific offset
+            proj_lang_embs_s = list()
+            for lang_id in [1448, 1709, 1250, 6356, 1809, 2450, 5540, 4447, 334, 1685, 2211, 2574, 5222]:
+                lang_embs = self.language_embedding(torch.LongTensor([lang_id]))
+                projected_lang_embs = self.language_embedding_projection(lang_embs).unsqueeze(-1).transpose(1, 2)
+                projected_lang_embs = self.language_emb_norm(projected_lang_embs)
+                proj_lang_embs_s.append(projected_lang_embs)
+            xs = xs + proj_lang_embs_s.mean()  # offset phoneme representation by language specific offset
 
         xs = self.pos_enc(xs)
 

app.py

@@ -33,17 +33,11 @@ class TTSWebUI:
                                                      embedding_gan_path=embedding_gan_path)
         self.iface = gr.Interface(fn=self.read,
                                   inputs=[gr.Textbox(lines=2,
-                                                     placeholder="write what you want the synthesis to read here...",
-                                                     value="What I cannot create, I do not understand.",
-                                                     label="Text input"),
-                                          gr.Dropdown(text_selection,
-                                                      type="value",
-                                                      value='English (eng)',
-                                                      label="Select the Language of the Text (type on your keyboard to find it quickly)"),
+                                                     placeholder="put in IPA symbols here...",
+                                                     value="~tə ɡɛt ɐ pˈɔːz~ plˈeɪs ɐ tˈɪldə sˈɪmbəl. jˈuːs pʌŋktʃuːˈeɪʃən~ æz ɪf ðɪs wʌz tˈɛkst.~#",
+                                                     label="IPA Input"),
                                           gr.Slider(minimum=0.0, maximum=0.8, step=0.1, value=0.5, label="Prosody Creativity"),
-                                          gr.Slider(minimum=0.7, maximum=1.3, step=0.1, value=1.0, label="Faster - Slower"),
-                                          gr.Slider(minimum=0, maximum=available_artificial_voices, step=1, value=5, label="Random Seed for the artificial Voice"),
-                                          gr.Slider(minimum=-10.0, maximum=10.0, step=0.1, value=0.0, label="Gender of artificial Voice"),
+                                          gr.Checkbox(value=True, label="Speak in many Voices"),
                                           gr.Audio(type="filepath", show_label=True, container=True, label="[OPTIONAL] Voice to Clone (if left empty, will use an artificial voice instead)"),
                                           # gr.Slider(minimum=0.5, maximum=1.5, step=0.1, value=1.0, label="Pitch Variance Scale"),
                                           # gr.Slider(minimum=0.5, maximum=1.5, step=0.1, value=1.0, label="Energy Variance Scale"),
@@ -59,32 +53,14 @@ class TTSWebUI:
 
     def read(self,
              prompt,
-             language,
              prosody_creativity,
-             duration_scaling_factor,
              voice_seed,
-             emb1,
              reference_audio,
-             # pitch_variance_scale,
-             # energy_variance_scale,
-             # emb2
              ):
         sr, wav, fig = self.controllable_ui.read(prompt,
                                                  reference_audio,
-                                                 language.split(" ")[-1].split("(")[1].split(")")[0],
-                                                 language.split(" ")[-1].split("(")[1].split(")")[0],
                                                  voice_seed,
                                                  prosody_creativity,
-                                                 duration_scaling_factor,
-                                                 1.,
-                                                 1.0,
-                                                 1.0,
-                                                 emb1,
-                                                 0.,
-                                                 0.,
-                                                 0.,
-                                                 0.,
-                                                 0.,
                                                  -24.)
         return (sr, float2pcm(wav)), fig
 

requirements.txt

@@ -1,6 +1,3 @@
-torch==2.2.0
-torchaudio==2.2.0
-torchvision==0.17.0
 torch_complex~=0.4.3
 epitran==1.24
 tqdm~=4.64.1