"""
Gradio UI for Text-to-Speech using HiggsAudioServeEngine
"""
import argparse
import base64
import os
import uuid
import json
from typing import Optional
import gradio as gr
from loguru import logger
import numpy as np
import time
from functools import lru_cache
import re
import spaces
import torch
# Import HiggsAudio components
from higgs_audio.serve.serve_engine import HiggsAudioServeEngine
from higgs_audio.data_types import ChatMLSample, AudioContent, Message
# Global engine instance
engine = None
# Default model configuration
DEFAULT_MODEL_PATH = "bosonai/higgs-audio-v2-generation-3B-base"
DEFAULT_AUDIO_TOKENIZER_PATH = "bosonai/higgs-audio-v2-tokenizer"
SAMPLE_RATE = 24000
DEFAULT_SYSTEM_PROMPT = (
"Generate audio following instruction.\n\n"
"<|scene_desc_start|>\n"
"Audio is recorded from a quiet room.\n"
"<|scene_desc_end|>"
)
DEFAULT_STOP_STRINGS = ["<|end_of_text|>", "<|eot_id|>"]
# Predefined examples for system and input messages
PREDEFINED_EXAMPLES = {
"voice-clone": {
"system_prompt": "",
"input_text": "Hey there! I'm your friendly voice twin in the making. Pick a voice preset below or upload your own audio - let's clone some vocals and bring your voice to life! ",
"description": "Voice clone to clone the reference audio. Leave the system prompt empty.",
},
"smart-voice": {
"system_prompt": DEFAULT_SYSTEM_PROMPT,
"input_text": "The sun rises in the east and sets in the west. This simple fact has been observed by humans for thousands of years.",
"description": "Smart voice to generate speech based on the context",
},
"multispeaker-voice-description": {
"system_prompt": "You are an AI assistant designed to convert text into speech.\n"
"If the user's message includes a [SPEAKER*] tag, do not read out the tag and generate speech for the following text, using the specified voice.\n"
"If no speaker tag is present, select a suitable voice on your own.\n\n"
"<|scene_desc_start|>\n"
"SPEAKER0: feminine\n"
"SPEAKER1: masculine\n"
"<|scene_desc_end|>",
"input_text": "[SPEAKER0] I can't believe you did that without even asking me first!\n"
"[SPEAKER1] Oh, come on! It wasn't a big deal, and I knew you would overreact like this.\n"
"[SPEAKER0] Overreact? You made a decision that affects both of us without even considering my opinion!\n"
"[SPEAKER1] Because I didn't have time to sit around waiting for you to make up your mind! Someone had to act.",
"description": "Multispeaker with different voice descriptions in the system prompt",
},
"single-speaker-voice-description": {
"system_prompt": "Generate audio following instruction.\n\n"
"<|scene_desc_start|>\n"
"SPEAKER0: He speaks with a clear British accent and a conversational, inquisitive tone. His delivery is articulate and at a moderate pace, and very clear audio.\n"
"<|scene_desc_end|>",
"input_text": "Hey, everyone! Welcome back to Tech Talk Tuesdays.\n"
"It's your host, Alex, and today, we're diving into a topic that's become absolutely crucial in the tech world — deep learning.\n"
"And let's be honest, if you've been even remotely connected to tech, AI, or machine learning lately, you know that deep learning is everywhere.\n"
"\n"
"So here's the big question: Do you want to understand how deep learning works?\n",
"description": "Single speaker with voice description in the system prompt",
},
"single-speaker-zh": {
"system_prompt": "Generate audio following instruction.\n\n"
"<|scene_desc_start|>\n"
"Audio is recorded from a quiet room.\n"
"<|scene_desc_end|>",
"input_text": "大家好, 欢迎收听本期的跟李沐学AI. 今天沐哥在忙着洗数据, 所以由我, 希格斯主播代替他讲这期视频.\n"
"今天我们要聊的是一个你绝对不能忽视的话题: 多模态学习.\n"
"那么, 问题来了, 你真的了解多模态吗? 你知道如何自己动手构建多模态大模型吗.\n"
"或者说, 你能察觉到我其实是个机器人吗?",
"description": "Single speaker speaking Chinese",
},
"single-speaker-bgm": {
"system_prompt": DEFAULT_SYSTEM_PROMPT,
"input_text": "
{PREDEFINED_EXAMPLES[default_template]["description"]}
', visible=True, ) system_prompt = gr.TextArea( label="System Prompt", placeholder="Enter system prompt to guide the model...", value=PREDEFINED_EXAMPLES[default_template]["system_prompt"], lines=2, ) input_text = gr.TextArea( label="Input Text", placeholder="Type the text you want to convert to speech...", value=PREDEFINED_EXAMPLES[default_template]["input_text"], lines=5, ) voice_preset = gr.Dropdown( label="Voice Preset", choices=list(VOICE_PRESETS.keys()), value="EMPTY", interactive=False, # Disabled by default since default template is not voice-clone visible=False, ) with gr.Accordion( "Custom Reference (Optional)", open=False, visible=False ) as custom_reference_accordion: reference_audio = gr.Audio(label="Reference Audio", type="filepath") reference_text = gr.TextArea( label="Reference Text (transcript of the reference audio)", placeholder="Enter the transcript of your reference audio...", lines=3, ) with gr.Accordion("Advanced Parameters", open=False): max_completion_tokens = gr.Slider( minimum=128, maximum=4096, value=1024, step=10, label="Max Completion Tokens", ) temperature = gr.Slider( minimum=0.0, maximum=1.5, value=1.0, step=0.1, label="Temperature", ) top_p = gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top P") top_k = gr.Slider(minimum=-1, maximum=100, value=50, step=1, label="Top K") ras_win_len = gr.Slider( minimum=0, maximum=10, value=0, step=1, label="RAS Window Length", info="Window length for repetition avoidance sampling", ) ras_win_max_num_repeat = gr.Slider( minimum=1, maximum=10, value=2, step=1, label="RAS Max Num Repeat", info="Maximum number of repetitions allowed in the window", ) # Add stop strings component stop_strings = gr.Dataframe( label="Stop Strings", headers=["stops"], datatype=["str"], value=[[s] for s in DEFAULT_STOP_STRINGS], interactive=True, col_count=(1, "fixed"), ) submit_btn = gr.Button("Generate Speech", variant="primary", scale=1) with gr.Column(scale=2): output_text = gr.TextArea(label="Model Response", lines=2) # Audio output output_audio = gr.Audio(label="Generated Audio", interactive=False, autoplay=True) stop_btn = gr.Button("Stop Playback", variant="primary") # Example voice with gr.Row(visible=False) as voice_samples_section: voice_samples_table = gr.Dataframe( headers=["Voice Preset", "Sample Text"], datatype=["str", "str"], value=[[preset, text] for preset, text in VOICE_PRESETS.items() if preset != "EMPTY"], interactive=False, ) sample_audio = gr.Audio(label="Voice Sample") # Function to play voice sample when clicking on a row def play_voice_sample(evt: gr.SelectData): try: # Get the preset name from the clicked row preset_names = [preset for preset in VOICE_PRESETS.keys() if preset != "EMPTY"] if evt.index[0] < len(preset_names): preset = preset_names[evt.index[0]] voice_path, _ = get_voice_present(preset) if voice_path and os.path.exists(voice_path): return voice_path else: gr.Warning(f"Voice sample file not found for preset: {preset}") return None else: gr.Warning("Invalid voice preset selection") return None except Exception as e: logger.error(f"Error playing voice sample: {e}") gr.Error(f"Error playing voice sample: {e}") return None voice_samples_table.select(fn=play_voice_sample, outputs=[sample_audio]) # Function to handle template selection def apply_template(template_name): if template_name in PREDEFINED_EXAMPLES: template = PREDEFINED_EXAMPLES[template_name] # Enable voice preset and custom reference only for voice-clone template is_voice_clone = template_name == "voice-clone" voice_preset_value = "belinda" if is_voice_clone else "EMPTY" description_text = f'{template["description"]}
' return ( template["system_prompt"], # system_prompt template["input_text"], # input_text description_text, # template_description gr.update( value=voice_preset_value, interactive=is_voice_clone, visible=is_voice_clone ), # voice_preset (value and interactivity) gr.update(visible=is_voice_clone), # custom reference accordion visibility gr.update(visible=is_voice_clone), # voice samples section visibility ) else: return ( gr.update(), gr.update(), gr.update(), gr.update(), gr.update(), gr.update(), ) # No change if template not found # Set up event handlers # Connect template dropdown to handler template_dropdown.change( fn=apply_template, inputs=[template_dropdown], outputs=[ system_prompt, input_text, template_description, voice_preset, custom_reference_accordion, voice_samples_section, ], ) # Connect submit button to the TTS function submit_btn.click( fn=text_to_speech, inputs=[ input_text, voice_preset, reference_audio, reference_text, max_completion_tokens, temperature, top_p, top_k, system_prompt, stop_strings, ras_win_len, ras_win_max_num_repeat, ], outputs=[output_text, output_audio], api_name="generate_speech", ) # Stop button functionality stop_btn.click( fn=lambda: None, inputs=[], outputs=[output_audio], js="() => {const audio = document.querySelector('audio'); if(audio) audio.pause(); return null;}", ) return demo def main(): """Main function to parse arguments and launch the UI.""" global DEFAULT_MODEL_PATH, DEFAULT_AUDIO_TOKENIZER_PATH, VOICE_PRESETS parser = argparse.ArgumentParser(description="Gradio UI for Text-to-Speech using HiggsAudioServeEngine") parser.add_argument( "--device", type=str, default="cuda", choices=["cuda", "cpu"], help="Device to run the model on.", ) parser.add_argument("--host", type=str, default="0.0.0.0", help="Host for the Gradio interface.") parser.add_argument("--port", type=int, default=7860, help="Port for the Gradio interface.") args = parser.parse_args() # Update default values if provided via command line VOICE_PRESETS = load_voice_presets() # Create and launch the UI demo = create_ui() demo.launch(server_name=args.host, server_port=args.port) if __name__ == "__main__": main()