Intial Config

Dockerfile

@@ -0,0 +1,16 @@
+# Read the doc: https://huggingface.co/docs/hub/spaces-sdks-docker
+# you will also find guides on how best to write your Dockerfile
+
+FROM python:3.10.0
+
+RUN useradd -m -u 1000 user
+USER user
+ENV PATH="/home/user/.local/bin:$PATH"
+
+WORKDIR /app
+
+COPY --chown=user ./requirements.txt requirements.txt
+RUN pip install --no-cache-dir --upgrade -r requirements.txt
+
+COPY --chown=user . /app
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

app.py

@@ -0,0 +1,975 @@
+import gradio as gr
+import numpy as np
+import queue
+import torch
+import time
+import threading
+import os
+import urllib.request
+import torchaudio
+from scipy.spatial.distance import cosine
+from scipy.signal import resample
+from RealtimeSTT import AudioToTextRecorder
+from fastapi import FastAPI, APIRouter
+from fastrtc import Stream, AsyncStreamHandler
+import json
+import asyncio
+import uvicorn
+from queue import Queue
+import logging
+
+# Set up logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Simplified configuration parameters
+SILENCE_THRESHS = [0, 0.4]
+FINAL_TRANSCRIPTION_MODEL = "distil-large-v3"
+FINAL_BEAM_SIZE = 5
+REALTIME_TRANSCRIPTION_MODEL = "distil-small.en"
+REALTIME_BEAM_SIZE = 5
+TRANSCRIPTION_LANGUAGE = "en"
+SILERO_SENSITIVITY = 0.4
+WEBRTC_SENSITIVITY = 3
+MIN_LENGTH_OF_RECORDING = 0.7
+PRE_RECORDING_BUFFER_DURATION = 0.35
+
+# Speaker change detection parameters
+DEFAULT_CHANGE_THRESHOLD = 0.65
+EMBEDDING_HISTORY_SIZE = 5
+MIN_SEGMENT_DURATION = 1.5
+DEFAULT_MAX_SPEAKERS = 4
+ABSOLUTE_MAX_SPEAKERS = 8
+
+# Global variables
+SAMPLE_RATE = 16000
+BUFFER_SIZE = 1024
+CHANNELS = 1
+
+# Speaker colors - more distinguishable colors
+SPEAKER_COLORS = [
+    "#FF6B6B",  # Red
+    "#4ECDC4",  # Teal
+    "#45B7D1",  # Blue
+    "#96CEB4",  # Green
+    "#FFEAA7",  # Yellow
+    "#DDA0DD",  # Plum
+    "#98D8C8",  # Mint
+    "#F7DC6F",  # Gold
+]
+
+SPEAKER_COLOR_NAMES = [
+    "Red", "Teal", "Blue", "Green", "Yellow", "Plum", "Mint", "Gold"
+]
+
+
+class SpeechBrainEncoder:
+    """ECAPA-TDNN encoder from SpeechBrain for speaker embeddings"""
+    def __init__(self, device="cpu"):
+        self.device = device
+        self.model = None
+        self.embedding_dim = 192
+        self.model_loaded = False
+        self.cache_dir = os.path.join(os.path.expanduser("~"), ".cache", "speechbrain")
+        os.makedirs(self.cache_dir, exist_ok=True)
+    
+    def load_model(self):
+        """Load the ECAPA-TDNN model"""
+        try:
+            # Import SpeechBrain
+            from speechbrain.pretrained import EncoderClassifier
+            
+            # Get model path
+            model_path = self._download_model()
+            
+            # Load the pre-trained model
+            self.model = EncoderClassifier.from_hparams(
+                source="speechbrain/spkrec-ecapa-voxceleb",
+                savedir=self.cache_dir,
+                run_opts={"device": self.device}
+            )
+            
+            self.model_loaded = True
+            return True
+        except Exception as e:
+            print(f"Error loading ECAPA-TDNN model: {e}")
+            return False
+    
+    def embed_utterance(self, audio, sr=16000):
+        """Extract speaker embedding from audio"""
+        if not self.model_loaded:
+            raise ValueError("Model not loaded. Call load_model() first.")
+        
+        try:
+            if isinstance(audio, np.ndarray):
+                # Ensure audio is float32 and properly normalized
+                audio = audio.astype(np.float32)
+                if np.max(np.abs(audio)) > 1.0:
+                    audio = audio / np.max(np.abs(audio))
+                waveform = torch.tensor(audio).unsqueeze(0)
+            else:
+                waveform = audio.unsqueeze(0)
+            
+            # Resample if necessary
+            if sr != 16000:
+                waveform = torchaudio.functional.resample(waveform, orig_freq=sr, new_freq=16000)
+            
+            with torch.no_grad():
+                embedding = self.model.encode_batch(waveform)
+                
+            return embedding.squeeze().cpu().numpy()
+        except Exception as e:
+            logger.error(f"Error extracting embedding: {e}")
+            return np.zeros(self.embedding_dim)
+
+
+class AudioProcessor:
+    """Processes audio data to extract speaker embeddings"""
+    def __init__(self, encoder):
+        self.encoder = encoder
+        self.audio_buffer = []
+        self.min_audio_length = int(SAMPLE_RATE * 1.0)  # Minimum 1 second of audio
+    
+    def add_audio_chunk(self, audio_chunk):
+        """Add audio chunk to buffer"""
+        self.audio_buffer.extend(audio_chunk)
+        
+        # Keep buffer from getting too large
+        max_buffer_size = int(SAMPLE_RATE * 10)  # 10 seconds max
+        if len(self.audio_buffer) > max_buffer_size:
+            self.audio_buffer = self.audio_buffer[-max_buffer_size:]
+    
+    def extract_embedding_from_buffer(self):
+        """Extract embedding from current audio buffer"""
+        if len(self.audio_buffer) < self.min_audio_length:
+            return None
+            
+        try:
+            # Use the last portion of the buffer for embedding
+            audio_segment = np.array(self.audio_buffer[-self.min_audio_length:], dtype=np.float32)
+            
+            # Normalize audio
+            if np.max(np.abs(audio_segment)) > 0:
+                audio_segment = audio_segment / np.max(np.abs(audio_segment))
+            else:
+                return None
+            
+            embedding = self.encoder.embed_utterance(audio_segment)
+            return embedding
+        except Exception as e:
+            logger.error(f"Embedding extraction error: {e}")
+            return None
+
+
+class SpeakerChangeDetector:
+    """Improved speaker change detector"""
+    def __init__(self, embedding_dim=192, change_threshold=DEFAULT_CHANGE_THRESHOLD, max_speakers=DEFAULT_MAX_SPEAKERS):
+        self.embedding_dim = embedding_dim
+        self.change_threshold = change_threshold
+        self.max_speakers = min(max_speakers, ABSOLUTE_MAX_SPEAKERS)
+        self.current_speaker = 0
+        self.speaker_embeddings = [[] for _ in range(self.max_speakers)]
+        self.speaker_centroids = [None] * self.max_speakers
+        self.last_change_time = time.time()
+        self.last_similarity = 1.0
+        self.active_speakers = set([0])
+        self.segment_counter = 0
+        
+    def set_max_speakers(self, max_speakers):
+        """Update the maximum number of speakers"""
+        new_max = min(max_speakers, ABSOLUTE_MAX_SPEAKERS)
+        
+        if new_max < self.max_speakers:
+            # Remove speakers beyond the new limit
+            for speaker_id in list(self.active_speakers):
+                if speaker_id >= new_max:
+                    self.active_speakers.discard(speaker_id)
+            
+            if self.current_speaker >= new_max:
+                self.current_speaker = 0
+        
+        # Resize arrays
+        if new_max > self.max_speakers:
+            self.speaker_embeddings.extend([[] for _ in range(new_max - self.max_speakers)])
+            self.speaker_centroids.extend([None] * (new_max - self.max_speakers))
+        else:
+            self.speaker_embeddings = self.speaker_embeddings[:new_max]
+            self.speaker_centroids = self.speaker_centroids[:new_max]
+        
+        self.max_speakers = new_max
+        
+    def set_change_threshold(self, threshold):
+        """Update the threshold for detecting speaker changes"""
+        self.change_threshold = max(0.1, min(threshold, 0.95))
+        
+    def add_embedding(self, embedding, timestamp=None):
+        """Add a new embedding and detect speaker changes"""
+        current_time = timestamp or time.time()
+        self.segment_counter += 1
+        
+        # Initialize first speaker
+        if not self.speaker_embeddings[0]:
+            self.speaker_embeddings[0].append(embedding)
+            self.speaker_centroids[0] = embedding.copy()
+            self.active_speakers.add(0)
+            return 0, 1.0
+        
+        # Calculate similarity with current speaker
+        current_centroid = self.speaker_centroids[self.current_speaker]
+        if current_centroid is not None:
+            similarity = 1.0 - cosine(embedding, current_centroid)
+        else:
+            similarity = 0.5
+        
+        self.last_similarity = similarity
+        
+        # Check for speaker change
+        time_since_last_change = current_time - self.last_change_time
+        speaker_changed = False
+        
+        if time_since_last_change >= MIN_SEGMENT_DURATION and similarity < self.change_threshold:
+            # Find best matching speaker
+            best_speaker = self.current_speaker
+            best_similarity = similarity
+            
+            for speaker_id in self.active_speakers:
+                if speaker_id == self.current_speaker:
+                    continue
+                    
+                centroid = self.speaker_centroids[speaker_id]
+                if centroid is not None:
+                    speaker_similarity = 1.0 - cosine(embedding, centroid)
+                    if speaker_similarity > best_similarity and speaker_similarity > self.change_threshold:
+                        best_similarity = speaker_similarity
+                        best_speaker = speaker_id
+            
+            # If no good match found and we can add a new speaker
+            if best_speaker == self.current_speaker and len(self.active_speakers) < self.max_speakers:
+                for new_id in range(self.max_speakers):
+                    if new_id not in self.active_speakers:
+                        best_speaker = new_id
+                        self.active_speakers.add(new_id)
+                        break
+            
+            if best_speaker != self.current_speaker:
+                self.current_speaker = best_speaker
+                self.last_change_time = current_time
+                speaker_changed = True
+        
+        # Update speaker embeddings and centroids
+        self.speaker_embeddings[self.current_speaker].append(embedding)
+        
+        # Keep only recent embeddings (sliding window)
+        max_embeddings = 20
+        if len(self.speaker_embeddings[self.current_speaker]) > max_embeddings:
+            self.speaker_embeddings[self.current_speaker] = self.speaker_embeddings[self.current_speaker][-max_embeddings:]
+        
+        # Update centroid
+        if self.speaker_embeddings[self.current_speaker]:
+            self.speaker_centroids[self.current_speaker] = np.mean(
+                self.speaker_embeddings[self.current_speaker], axis=0
+            )
+        
+        return self.current_speaker, similarity
+    
+    def get_color_for_speaker(self, speaker_id):
+        """Return color for speaker ID"""
+        if 0 <= speaker_id < len(SPEAKER_COLORS):
+            return SPEAKER_COLORS[speaker_id]
+        return "#FFFFFF"
+    
+    def get_status_info(self):
+        """Return status information"""
+        speaker_counts = [len(self.speaker_embeddings[i]) for i in range(self.max_speakers)]
+        
+        return {
+            "current_speaker": self.current_speaker,
+            "speaker_counts": speaker_counts,
+            "active_speakers": len(self.active_speakers),
+            "max_speakers": self.max_speakers,
+            "last_similarity": self.last_similarity,
+            "threshold": self.change_threshold,
+            "segment_counter": self.segment_counter
+        }
+
+
+class RealtimeSpeakerDiarization:
+    def __init__(self):
+        self.encoder = None
+        self.audio_processor = None
+        self.speaker_detector = None
+        self.recorder = None
+        self.sentence_queue = queue.Queue()
+        self.full_sentences = []
+        self.sentence_speakers = []
+        self.pending_sentences = []
+        self.current_conversation = ""
+        self.is_running = False
+        self.change_threshold = DEFAULT_CHANGE_THRESHOLD
+        self.max_speakers = DEFAULT_MAX_SPEAKERS
+        self.last_transcription = ""
+        self.transcription_lock = threading.Lock()
+        
+    def initialize_models(self):
+        """Initialize the speaker encoder model"""
+        try:
+            device_str = "cuda" if torch.cuda.is_available() else "cpu"
+            logger.info(f"Using device: {device_str}")
+            
+            self.encoder = SpeechBrainEncoder(device=device_str)
+            success = self.encoder.load_model()
+            
+            if success:
+                self.audio_processor = AudioProcessor(self.encoder)
+                self.speaker_detector = SpeakerChangeDetector(
+                    embedding_dim=self.encoder.embedding_dim,
+                    change_threshold=self.change_threshold,
+                    max_speakers=self.max_speakers
+                )
+                logger.info("Models initialized successfully!")
+                return True
+            else:
+                logger.error("Failed to load models")
+                return False
+        except Exception as e:
+            logger.error(f"Model initialization error: {e}")
+            return False
+    
+    def live_text_detected(self, text):
+        """Callback for real-time transcription updates"""
+        with self.transcription_lock:
+            self.last_transcription = text.strip()
+    
+    def process_final_text(self, text):
+        """Process final transcribed text with speaker embedding"""
+        text = text.strip()
+        if text:
+            try:
+                # Get audio data for this transcription
+                audio_bytes = getattr(self.recorder, 'last_transcription_bytes', None)
+                if audio_bytes:
+                    self.sentence_queue.put((text, audio_bytes))
+                else:
+                    # If no audio bytes, use current speaker
+                    self.sentence_queue.put((text, None))
+                    
+            except Exception as e:
+                logger.error(f"Error processing final text: {e}")
+    
+    def process_sentence_queue(self):
+        """Process sentences in the queue for speaker detection"""
+        while self.is_running:
+            try:
+                text, audio_bytes = self.sentence_queue.get(timeout=1)
+                
+                current_speaker = self.speaker_detector.current_speaker
+                
+                if audio_bytes:
+                    # Convert audio data and extract embedding
+                    audio_int16 = np.frombuffer(audio_bytes, dtype=np.int16)
+                    audio_float = audio_int16.astype(np.float32) / 32768.0
+                    
+                    # Extract embedding
+                    embedding = self.audio_processor.encoder.embed_utterance(audio_float)
+                    if embedding is not None:
+                        current_speaker, similarity = self.speaker_detector.add_embedding(embedding)
+                
+                # Store sentence with speaker
+                with self.transcription_lock:
+                    self.full_sentences.append((text, current_speaker))
+                    self.update_conversation_display()
+                    
+            except queue.Empty:
+                continue
+            except Exception as e:
+                logger.error(f"Error processing sentence: {e}")
+    
+    def update_conversation_display(self):
+        """Update the conversation display"""
+        try:
+            sentences_with_style = []
+            
+            for sentence_text, speaker_id in self.full_sentences:
+                color = self.speaker_detector.get_color_for_speaker(speaker_id)
+                speaker_name = f"Speaker {speaker_id + 1}"
+                sentences_with_style.append(
+                    f'<span style="color:{color}; font-weight: bold;">{speaker_name}:</span> '
+                    f'<span style="color:#333333;">{sentence_text}</span>'
+                )
+            
+            # Add current transcription if available
+            if self.last_transcription:
+                current_color = self.speaker_detector.get_color_for_speaker(self.speaker_detector.current_speaker)
+                current_speaker = f"Speaker {self.speaker_detector.current_speaker + 1}"
+                sentences_with_style.append(
+                    f'<span style="color:{current_color}; font-weight: bold; opacity: 0.7;">{current_speaker}:</span> '
+                    f'<span style="color:#666666; font-style: italic;">{self.last_transcription}...</span>'
+                )
+            
+            if sentences_with_style:
+                self.current_conversation = "<br><br>".join(sentences_with_style)
+            else:
+                self.current_conversation = "<i>Waiting for speech input...</i>"
+                
+        except Exception as e:
+            logger.error(f"Error updating conversation display: {e}")
+            self.current_conversation = f"<i>Error: {str(e)}</i>"
+    
+    def start_recording(self):
+        """Start the recording and transcription process"""
+        if self.encoder is None:
+            return "Please initialize models first!"
+        
+        try:
+            # Setup recorder configuration
+            recorder_config = {
+                'spinner': False,
+                'use_microphone': False,  # Using FastRTC for audio input
+                'model': FINAL_TRANSCRIPTION_MODEL,
+                'language': TRANSCRIPTION_LANGUAGE,
+                'silero_sensitivity': SILERO_SENSITIVITY,
+                'webrtc_sensitivity': WEBRTC_SENSITIVITY,
+                'post_speech_silence_duration': SILENCE_THRESHS[1],
+                'min_length_of_recording': MIN_LENGTH_OF_RECORDING,
+                'pre_recording_buffer_duration': PRE_RECORDING_BUFFER_DURATION,
+                'min_gap_between_recordings': 0,
+                'enable_realtime_transcription': True,
+                'realtime_processing_pause': 0.1,
+                'realtime_model_type': REALTIME_TRANSCRIPTION_MODEL,
+                'on_realtime_transcription_update': self.live_text_detected,
+                'beam_size': FINAL_BEAM_SIZE,
+                'beam_size_realtime': REALTIME_BEAM_SIZE,
+                'sample_rate': SAMPLE_RATE,
+            }
+
+            self.recorder = AudioToTextRecorder(**recorder_config)
+            
+            # Start processing threads
+            self.is_running = True
+            self.sentence_thread = threading.Thread(target=self.process_sentence_queue, daemon=True)
+            self.sentence_thread.start()
+            
+            self.transcription_thread = threading.Thread(target=self.run_transcription, daemon=True)
+            self.transcription_thread.start()
+            
+            return "Recording started successfully!"
+            
+        except Exception as e:
+            logger.error(f"Error starting recording: {e}")
+            return f"Error starting recording: {e}"
+    
+    def run_transcription(self):
+        """Run the transcription loop"""
+        try:
+            logger.info("Starting transcription thread")
+            while self.is_running:
+                # Just check for final text from recorder, audio is fed externally via FastRTC
+                text = self.recorder.text(self.process_final_text)
+                time.sleep(0.01)  # Small sleep to prevent CPU hogging
+        except Exception as e:
+            logger.error(f"Transcription error: {e}")
+    
+    def stop_recording(self):
+        """Stop the recording process"""
+        self.is_running = False
+        if self.recorder:
+            self.recorder.stop()
+        return "Recording stopped!"
+    
+    def clear_conversation(self):
+        """Clear all conversation data"""
+        with self.transcription_lock:
+            self.full_sentences = []
+            self.last_transcription = ""
+            self.current_conversation = "Conversation cleared!"
+        
+        if self.speaker_detector:
+            self.speaker_detector = SpeakerChangeDetector(
+                embedding_dim=self.encoder.embedding_dim,
+                change_threshold=self.change_threshold,
+                max_speakers=self.max_speakers
+            )
+        
+        return "Conversation cleared!"
+    
+    def update_settings(self, threshold, max_speakers):
+        """Update speaker detection settings"""
+        self.change_threshold = threshold
+        self.max_speakers = max_speakers
+        
+        if self.speaker_detector:
+            self.speaker_detector.set_change_threshold(threshold)
+            self.speaker_detector.set_max_speakers(max_speakers)
+        
+        return f"Settings updated: Threshold={threshold:.2f}, Max Speakers={max_speakers}"
+    
+    def get_formatted_conversation(self):
+        """Get the formatted conversation"""
+        return self.current_conversation
+    
+    def get_status_info(self):
+        """Get current status information"""
+        if not self.speaker_detector:
+            return "Speaker detector not initialized"
+        
+        try:
+            status = self.speaker_detector.get_status_info()
+            
+            status_lines = [
+                f"**Current Speaker:** {status['current_speaker'] + 1}",
+                f"**Active Speakers:** {status['active_speakers']} of {status['max_speakers']}",
+                f"**Last Similarity:** {status['last_similarity']:.3f}",
+                f"**Change Threshold:** {status['threshold']:.2f}",
+                f"**Total Sentences:** {len(self.full_sentences)}",
+                f"**Segments Processed:** {status['segment_counter']}",
+                "",
+                "**Speaker Activity:**"
+            ]
+            
+            for i in range(status['max_speakers']):
+                color_name = SPEAKER_COLOR_NAMES[i] if i < len(SPEAKER_COLOR_NAMES) else f"Speaker {i+1}"
+                count = status['speaker_counts'][i]
+                active = "🟢" if count > 0 else "⚫"
+                status_lines.append(f"{active} Speaker {i+1} ({color_name}): {count} segments")
+            
+            return "\n".join(status_lines)
+            
+        except Exception as e:
+            return f"Error getting status: {e}"
+
+    def process_audio_chunk(self, audio_data, sample_rate=16000):
+        """Process audio chunk from FastRTC input"""
+        if not self.is_running or self.audio_processor is None:
+            return
+            
+        try:
+            # Ensure audio is float32
+            if isinstance(audio_data, np.ndarray):
+                if audio_data.dtype != np.float32:
+                    audio_data = audio_data.astype(np.float32)
+            else:
+                audio_data = np.array(audio_data, dtype=np.float32)
+            
+            # Ensure mono
+            if len(audio_data.shape) > 1:
+                audio_data = np.mean(audio_data, axis=1) if audio_data.shape[1] > 1 else audio_data.flatten()
+            
+            # Normalize if needed
+            if np.max(np.abs(audio_data)) > 1.0:
+                audio_data = audio_data / np.max(np.abs(audio_data))
+            
+            # Add to audio processor buffer for speaker detection
+            self.audio_processor.add_audio_chunk(audio_data)
+            
+            # Periodically extract embeddings for speaker detection
+            if len(self.audio_processor.audio_buffer) % (SAMPLE_RATE // 2) == 0:  # Every 0.5 seconds
+                embedding = self.audio_processor.extract_embedding_from_buffer()
+                if embedding is not None:
+                    self.speaker_detector.add_embedding(embedding)
+            
+            # Feed audio to RealtimeSTT recorder
+            if self.recorder and self.is_running:
+                # Convert float32 [-1.0, 1.0] to int16 for RealtimeSTT
+                int16_data = (audio_data * 32768.0).astype(np.int16).tobytes()
+                if sample_rate != 16000:
+                    int16_data = self.resample_audio(int16_data, sample_rate, 16000)
+                self.recorder.feed_audio(int16_data)
+                    
+        except Exception as e:
+            logger.error(f"Error processing audio chunk: {e}")
+    
+    def resample_audio(self, audio_bytes, from_rate, to_rate):
+        """Resample audio to target sample rate"""
+        try:
+            audio_np = np.frombuffer(audio_bytes, dtype=np.int16)
+            num_samples = len(audio_np)
+            num_target_samples = int(num_samples * to_rate / from_rate)
+            
+            resampled = resample(audio_np, num_target_samples)
+            
+            return resampled.astype(np.int16).tobytes()
+        except Exception as e:
+            logger.error(f"Error resampling audio: {e}")
+            return audio_bytes
+
+
+# FastRTC Audio Handler
+class DiarizationHandler(AsyncStreamHandler):
+    def __init__(self, diarization_system):
+        super().__init__()
+        self.diarization_system = diarization_system
+        self.audio_buffer = []
+        self.buffer_size = BUFFER_SIZE
+        
+    def copy(self):
+        """Return a fresh handler for each new stream connection"""
+        return DiarizationHandler(self.diarization_system)
+    
+    async def emit(self):
+        """Not used - we only receive audio"""
+        return None
+    
+    async def receive(self, frame):
+        """Receive audio data from FastRTC"""
+        try:
+            if not self.diarization_system.is_running:
+                return
+                
+            # Extract audio data
+            audio_data = getattr(frame, 'data', frame)
+            
+            # Convert to numpy array
+            if isinstance(audio_data, bytes):
+                audio_array = np.frombuffer(audio_data, dtype=np.int16).astype(np.float32) / 32768.0
+            elif isinstance(audio_data, (list, tuple)):
+                sample_rate, audio_array = audio_data
+                if isinstance(audio_array, (list, tuple)):
+                    audio_array = np.array(audio_array, dtype=np.float32)
+            else:
+                audio_array = np.array(audio_data, dtype=np.float32)
+            
+            # Ensure 1D
+            if len(audio_array.shape) > 1:
+                audio_array = audio_array.flatten()
+            
+            # Buffer audio chunks
+            self.audio_buffer.extend(audio_array)
+            
+            # Process in chunks
+            while len(self.audio_buffer) >= self.buffer_size:
+                chunk = np.array(self.audio_buffer[:self.buffer_size])
+                self.audio_buffer = self.audio_buffer[self.buffer_size:]
+                
+                # Process asynchronously
+                await self.process_audio_async(chunk)
+                
+        except Exception as e:
+            logger.error(f"Error in FastRTC receive: {e}")
+    
+    async def process_audio_async(self, audio_data):
+        """Process audio data asynchronously"""
+        try:
+            loop = asyncio.get_event_loop()
+            await loop.run_in_executor(
+                None, 
+                self.diarization_system.process_audio_chunk, 
+                audio_data, 
+                SAMPLE_RATE
+            )
+        except Exception as e:
+            logger.error(f"Error in async audio processing: {e}")
+
+
+# Global instances
+diarization_system = RealtimeSpeakerDiarization()
+audio_handler = None
+
+def initialize_system():
+    """Initialize the diarization system"""
+    global audio_handler
+    try:
+        success = diarization_system.initialize_models()
+        if success:
+            audio_handler = DiarizationHandler(diarization_system)
+            return "✅ System initialized successfully!"
+        else:
+            return "❌ Failed to initialize system. Check logs for details."
+    except Exception as e:
+        logger.error(f"Initialization error: {e}")
+        return f"❌ Initialization error: {str(e)}"
+
+def start_recording():
+    """Start recording and transcription"""
+    try:
+        result = diarization_system.start_recording()
+        return f"🎙️ {result}"
+    except Exception as e:
+        return f"❌ Failed to start recording: {str(e)}"
+
+def stop_recording():
+    """Stop recording and transcription"""
+    try:
+        result = diarization_system.stop_recording()
+        return f"⏹️ {result}"
+    except Exception as e:
+        return f"❌ Failed to stop recording: {str(e)}"
+
+def clear_conversation():
+    """Clear the conversation"""
+    try:
+        result = diarization_system.clear_conversation()
+        return f"🗑️ {result}"
+    except Exception as e:
+        return f"❌ Failed to clear conversation: {str(e)}"
+
+def update_settings(threshold, max_speakers):
+    """Update system settings"""
+    try:
+        result = diarization_system.update_settings(threshold, max_speakers)
+        return f"⚙️ {result}"
+    except Exception as e:
+        return f"❌ Failed to update settings: {str(e)}"
+
+def get_conversation():
+    """Get the current conversation"""
+    try:
+        return diarization_system.get_formatted_conversation()
+    except Exception as e:
+        return f"<i>Error getting conversation: {str(e)}</i>"
+
+def get_status():
+    """Get system status"""
+    try:
+        return diarization_system.get_status_info()
+    except Exception as e:
+        return f"Error getting status: {str(e)}"
+
+# Create Gradio interface
+def create_interface():
+    with gr.Blocks(title="Real-time Speaker Diarization", theme=gr.themes.Soft()) as interface:
+        gr.Markdown("# 🎤 Real-time Speech Recognition with Speaker Diarization")
+        gr.Markdown("Live transcription with automatic speaker identification using FastRTC audio streaming.")
+        
+        with gr.Row():
+            with gr.Column(scale=2):
+                # Conversation display
+                conversation_output = gr.HTML(
+                    value="<div style='padding: 20px; background: #f8f9fa; border-radius: 10px; min-height: 300px;'><i>Click 'Initialize System' to start...</i></div>",
+                    label="Live Conversation"
+                )
+                
+                # Control buttons
+                with gr.Row():
+                    init_btn = gr.Button("🔧 Initialize System", variant="secondary", size="lg")
+                    start_btn = gr.Button("🎙️ Start", variant="primary", size="lg", interactive=False)
+                    stop_btn = gr.Button("⏹️ Stop", variant="stop", size="lg", interactive=False)
+                    clear_btn = gr.Button("🗑️ Clear", variant="secondary", size="lg", interactive=False)
+                
+                # Status display
+                status_output = gr.Textbox(
+                    label="System Status",
+                    value="Ready to initialize...",
+                    lines=8,
+                    interactive=False
+                )
+            
+            with gr.Column(scale=1):
+                # Settings
+                gr.Markdown("## ⚙️ Settings")
+                
+                threshold_slider = gr.Slider(
+                    minimum=0.3,
+                    maximum=0.9,
+                    step=0.05,
+                    value=DEFAULT_CHANGE_THRESHOLD,
+                    label="Speaker Change Sensitivity",
+                    info="Lower = more sensitive"
+                )
+                
+                max_speakers_slider = gr.Slider(
+                    minimum=2,
+                    maximum=ABSOLUTE_MAX_SPEAKERS,
+                    step=1,
+                    value=DEFAULT_MAX_SPEAKERS,
+                    label="Maximum Speakers"
+                )
+                
+                update_btn = gr.Button("Update Settings", variant="secondary")
+                
+                # Instructions
+                gr.Markdown("""
+                ## 📋 Instructions
+                1. **Initialize** the system (loads AI models)
+                2. **Start** recording 
+                3. **Speak** - system will transcribe and identify speakers
+                4. **Monitor** real-time results below
+                
+                ## 🎨 Speaker Colors
+                - 🔴 Speaker 1 (Red)
+                - 🟢 Speaker 2 (Teal) 
+                - 🔵 Speaker 3 (Blue)
+                - 🟡 Speaker 4 (Green)
+                - 🟣 Speaker 5 (Yellow)
+                - 🟤 Speaker 6 (Plum)
+                - 🟫 Speaker 7 (Mint)
+                - 🟨 Speaker 8 (Gold)
+                """)
+        
+        # Event handlers
+        def on_initialize():
+            result = initialize_system()
+            if "✅" in result:
+                return result, gr.update(interactive=True), gr.update(interactive=True), gr.update(interactive=True)
+            else:
+                return result, gr.update(interactive=False), gr.update(interactive=False), gr.update(interactive=False)
+        
+        def on_start():
+            result = start_recording()
+            return result, gr.update(interactive=False), gr.update(interactive=True)
+        
+        def on_stop():
+            result = stop_recording()
+            return result, gr.update(interactive=True), gr.update(interactive=False)
+        
+        def on_clear():
+            result = clear_conversation()
+            return result
+        
+        def on_update_settings(threshold, max_speakers):
+            result = update_settings(threshold, int(max_speakers))
+            return result
+        
+        def refresh_conversation():
+            return get_conversation()
+        
+        def refresh_status():
+            return get_status()
+        
+        # Button click handlers
+        init_btn.click(
+            fn=on_initialize,
+            outputs=[status_output, start_btn, stop_btn, clear_btn]
+        )
+        
+        start_btn.click(
+            fn=on_start,
+            outputs=[status_output, start_btn, stop_btn]
+        )
+        
+        stop_btn.click(
+            fn=on_stop,
+            outputs=[status_output, start_btn, stop_btn]
+        )
+        
+        clear_btn.click(
+            fn=on_clear,
+            outputs=[status_output]
+        )
+        
+        update_btn.click(
+            fn=on_update_settings,
+            inputs=[threshold_slider, max_speakers_slider],
+            outputs=[status_output]
+        )
+        
+        # Auto-refresh conversation display every 1 second
+        conversation_timer = gr.Timer(1)
+        conversation_timer.tick(refresh_conversation, outputs=[conversation_output])
+        
+        # Auto-refresh status every 2 seconds  
+        status_timer = gr.Timer(2)
+        status_timer.tick(refresh_status, outputs=[status_output])
+    
+    return interface
+
+
+# FastAPI setup for FastRTC integration
+app = FastAPI()
+
+@app.get("/")
+async def root():
+    return {"message": "Real-time Speaker Diarization API"}
+
+@app.get("/health")
+async def health_check():
+    return {"status": "healthy", "system_running": diarization_system.is_running}
+
+@app.post("/initialize")
+async def api_initialize():
+    result = initialize_system()
+    return {"result": result, "success": "✅" in result}
+
+@app.post("/start")
+async def api_start():
+    result = start_recording()
+    return {"result": result, "success": "🎙️" in result}
+
+@app.post("/stop")
+async def api_stop():
+    result = stop_recording()
+    return {"result": result, "success": "⏹️" in result}
+
+@app.post("/clear")
+async def api_clear():
+    result = clear_conversation()
+    return {"result": result}
+
+@app.get("/conversation")
+async def api_get_conversation():
+    return {"conversation": get_conversation()}
+
+@app.get("/status")
+async def api_get_status():
+    return {"status": get_status()}
+
+@app.post("/settings")
+async def api_update_settings(threshold: float, max_speakers: int):
+    result = update_settings(threshold, max_speakers)
+    return {"result": result}
+
+# FastRTC Stream setup
+if audio_handler:
+    stream = Stream(handler=audio_handler)
+    app.include_router(stream.router, prefix="/stream")
+
+
+# Main execution
+if __name__ == "__main__":
+    import argparse
+    
+    parser = argparse.ArgumentParser(description="Real-time Speaker Diarization System")
+    parser.add_argument("--mode", choices=["gradio", "api", "both"], default="gradio", 
+                       help="Run mode: gradio interface, API only, or both")
+    parser.add_argument("--host", default="0.0.0.0", help="Host to bind to")
+    parser.add_argument("--port", type=int, default=7860, help="Port to bind to")
+    parser.add_argument("--api-port", type=int, default=8000, help="API port (when running both)")
+    
+    args = parser.parse_args()
+    
+    if args.mode == "gradio":
+        # Run Gradio interface only
+        interface = create_interface()
+        interface.launch(
+            server_name=args.host,
+            server_port=args.port,
+            share=True,
+            show_error=True
+        )
+    
+    elif args.mode == "api":
+        # Run FastAPI only
+        uvicorn.run(
+            app, 
+            host=args.host, 
+            port=args.port,
+            log_level="info"
+        )
+    
+    elif args.mode == "both":
+        # Run both Gradio and FastAPI
+        import multiprocessing
+        import threading
+        
+        def run_gradio():
+            interface = create_interface()
+            interface.launch(
+                server_name=args.host,
+                server_port=args.port,
+                share=True,
+                show_error=True
+            )
+        
+        def run_fastapi():
+            uvicorn.run(
+                app,
+                host=args.host,
+                port=args.api_port,
+                log_level="info"
+            )
+        
+        # Start FastAPI in a separate thread
+        api_thread = threading.Thread(target=run_fastapi, daemon=True)
+        api_thread.start()
+        
+        # Start Gradio in main thread
+        run_gradio()

packages.txt

@@ -0,0 +1,5 @@
+portaudio19-dev
+libasound2-dev
+libjack-jackd2-dev
+pulseaudio
+pulseaudio-utils

realtime_diarize.py

@@ -0,0 +1,970 @@
+from PyQt6.QtWidgets import (QApplication, QTextEdit, QMainWindow, QLabel, QVBoxLayout, QWidget, 
+                            QHBoxLayout, QPushButton, QSizePolicy, QGroupBox, QSlider, QSpinBox)
+from PyQt6.QtCore import Qt, pyqtSignal, QThread, QEvent, QTimer
+from scipy.spatial.distance import cosine
+from RealtimeSTT import AudioToTextRecorder
+import numpy as np
+import soundcard as sc
+import queue
+import torch
+import time
+import sys
+import os
+import urllib.request
+import torchaudio
+
+# Simplified configuration parameters
+SILENCE_THRESHS = [0, 0.4]
+FINAL_TRANSCRIPTION_MODEL = "distil-large-v3"
+FINAL_BEAM_SIZE = 5
+REALTIME_TRANSCRIPTION_MODEL = "distil-small.en"
+REALTIME_BEAM_SIZE = 5
+TRANSCRIPTION_LANGUAGE = "en" # Accuracy in languages ​​other than English is very low.
+SILERO_SENSITIVITY = 0.4
+WEBRTC_SENSITIVITY = 3
+MIN_LENGTH_OF_RECORDING = 0.7
+PRE_RECORDING_BUFFER_DURATION = 0.35
+
+# Speaker change detection parameters
+DEFAULT_CHANGE_THRESHOLD = 0.7  # Threshold for detecting speaker change
+EMBEDDING_HISTORY_SIZE = 5  # Number of embeddings to keep for comparison
+MIN_SEGMENT_DURATION = 1.0  # Minimum duration before considering a speaker change
+DEFAULT_MAX_SPEAKERS = 4  # Default maximum number of speakers
+ABSOLUTE_MAX_SPEAKERS = 10  # Absolute maximum number of speakers allowed
+
+# Global variables
+FAST_SENTENCE_END = True
+USE_MICROPHONE = False
+SAMPLE_RATE = 16000
+BUFFER_SIZE = 512
+CHANNELS = 1
+
+# Speaker colors - now we have colors for up to 10 speakers
+SPEAKER_COLORS = [
+    "#FFFF00",  # Yellow
+    "#FF0000",  # Red
+    "#00FF00",  # Green
+    "#00FFFF",  # Cyan
+    "#FF00FF",  # Magenta
+    "#0000FF",  # Blue
+    "#FF8000",  # Orange
+    "#00FF80",  # Spring Green
+    "#8000FF",  # Purple
+    "#FFFFFF",  # White
+]
+
+# Color names for display
+SPEAKER_COLOR_NAMES = [
+    "Yellow",
+    "Red",
+    "Green",
+    "Cyan",
+    "Magenta",
+    "Blue",
+    "Orange",
+    "Spring Green",
+    "Purple",
+    "White"
+]
+
+
+class SpeechBrainEncoder:
+    """ECAPA-TDNN encoder from SpeechBrain for speaker embeddings"""
+    def __init__(self, device="cpu"):
+        self.device = device
+        self.model = None
+        self.embedding_dim = 192  # ECAPA-TDNN default dimension
+        self.model_loaded = False
+        self.cache_dir = os.path.join(os.path.expanduser("~"), ".cache", "speechbrain")
+        os.makedirs(self.cache_dir, exist_ok=True)
+    
+    def _download_model(self):
+        """Download pre-trained SpeechBrain ECAPA-TDNN model if not present"""
+        model_url = "https://huggingface.co/speechbrain/spkrec-ecapa-voxceleb/resolve/main/embedding_model.ckpt"
+        model_path = os.path.join(self.cache_dir, "embedding_model.ckpt")
+        
+        if not os.path.exists(model_path):
+            print(f"Downloading ECAPA-TDNN model to {model_path}...")
+            urllib.request.urlretrieve(model_url, model_path)
+        
+        return model_path
+    
+    def load_model(self):
+        """Load the ECAPA-TDNN model"""
+        try:
+            # Import SpeechBrain
+            from speechbrain.pretrained import EncoderClassifier
+            
+            # Get model path
+            model_path = self._download_model()
+            
+            # Load the pre-trained model
+            self.model = EncoderClassifier.from_hparams(
+                source="speechbrain/spkrec-ecapa-voxceleb",
+                savedir=self.cache_dir,
+                run_opts={"device": self.device}
+            )
+            
+            self.model_loaded = True
+            return True
+        except Exception as e:
+            print(f"Error loading ECAPA-TDNN model: {e}")
+            return False
+    
+    def embed_utterance(self, audio, sr=16000):
+        """Extract speaker embedding from audio"""
+        if not self.model_loaded:
+            raise ValueError("Model not loaded. Call load_model() first.")
+        
+        try:
+            # Convert numpy array to torch tensor
+            if isinstance(audio, np.ndarray):
+                waveform = torch.tensor(audio, dtype=torch.float32).unsqueeze(0)
+            else:
+                waveform = audio.unsqueeze(0)
+            
+            # Ensure sample rate matches model expected rate
+            if sr != 16000:
+                waveform = torchaudio.functional.resample(waveform, orig_freq=sr, new_freq=16000)
+            
+            # Get embedding
+            with torch.no_grad():
+                embedding = self.model.encode_batch(waveform)
+                
+            return embedding.squeeze().cpu().numpy()
+        except Exception as e:
+            print(f"Error extracting embedding: {e}")
+            return np.zeros(self.embedding_dim)
+
+
+class AudioProcessor:
+    """Processes audio data to extract speaker embeddings"""
+    def __init__(self, encoder):
+        self.encoder = encoder
+    
+    def extract_embedding(self, audio_int16):
+        try:
+            # Convert int16 audio data to float32
+            float_audio = audio_int16.astype(np.float32) / 32768.0
+            
+            # Normalize if needed
+            if np.abs(float_audio).max() > 1.0:
+                float_audio = float_audio / np.abs(float_audio).max()
+            
+            # Extract embedding using the loaded encoder
+            embedding = self.encoder.embed_utterance(float_audio)
+            
+            return embedding
+        except Exception as e:
+            print(f"Embedding extraction error: {e}")
+            return np.zeros(self.encoder.embedding_dim)
+
+
+class EncoderLoaderThread(QThread):
+    """Thread for loading the speaker encoder model"""
+    model_loaded = pyqtSignal(object)
+    progress_update = pyqtSignal(str)
+    
+    def run(self):
+        try:
+            self.progress_update.emit("Initializing speaker encoder model...")
+            
+            # Check device
+            device_str = "cuda" if torch.cuda.is_available() else "cpu"
+            self.progress_update.emit(f"Using device: {device_str}")
+            
+            # Create SpeechBrain encoder
+            self.progress_update.emit("Loading ECAPA-TDNN model...")
+            encoder = SpeechBrainEncoder(device=device_str)
+            
+            # Load the model
+            success = encoder.load_model()
+            
+            if success:
+                self.progress_update.emit("ECAPA-TDNN model loading complete!")
+                self.model_loaded.emit(encoder)
+            else:
+                self.progress_update.emit("Failed to load ECAPA-TDNN model. Using fallback...")
+                self.model_loaded.emit(None)
+        except Exception as e:
+            self.progress_update.emit(f"Model loading error: {e}")
+            self.model_loaded.emit(None)
+
+
+class SpeakerChangeDetector:
+    """Modified speaker change detector that supports a configurable number of speakers"""
+    def __init__(self, embedding_dim=192, change_threshold=DEFAULT_CHANGE_THRESHOLD, max_speakers=DEFAULT_MAX_SPEAKERS):
+        self.embedding_dim = embedding_dim
+        self.change_threshold = change_threshold
+        self.max_speakers = min(max_speakers, ABSOLUTE_MAX_SPEAKERS)  # Ensure we don't exceed absolute max
+        self.current_speaker = 0  # Initial speaker (0 to max_speakers-1)
+        self.previous_embeddings = []
+        self.last_change_time = time.time()
+        self.mean_embeddings = [None] * self.max_speakers  # Mean embeddings for each speaker
+        self.speaker_embeddings = [[] for _ in range(self.max_speakers)]  # All embeddings for each speaker
+        self.last_similarity = 0.0
+        self.active_speakers = set([0])  # Track which speakers have been detected
+        
+    def set_max_speakers(self, max_speakers):
+        """Update the maximum number of speakers"""
+        new_max = min(max_speakers, ABSOLUTE_MAX_SPEAKERS)
+        
+        # If reducing the number of speakers
+        if new_max < self.max_speakers:
+            # Remove any speakers beyond the new max
+            for speaker_id in list(self.active_speakers):
+                if speaker_id >= new_max:
+                    self.active_speakers.discard(speaker_id)
+            
+            # Ensure current speaker is valid
+            if self.current_speaker >= new_max:
+                self.current_speaker = 0
+        
+        # Expand arrays if increasing max speakers
+        if new_max > self.max_speakers:
+            # Extend mean_embeddings array
+            self.mean_embeddings.extend([None] * (new_max - self.max_speakers))
+            
+            # Extend speaker_embeddings array
+            self.speaker_embeddings.extend([[] for _ in range(new_max - self.max_speakers)])
+        
+        # Truncate arrays if decreasing max speakers
+        else:
+            self.mean_embeddings = self.mean_embeddings[:new_max]
+            self.speaker_embeddings = self.speaker_embeddings[:new_max]
+        
+        self.max_speakers = new_max
+        
+    def set_change_threshold(self, threshold):
+        """Update the threshold for detecting speaker changes"""
+        self.change_threshold = max(0.1, min(threshold, 0.99))
+        
+    def add_embedding(self, embedding, timestamp=None):
+        """Add a new embedding and check if there's a speaker change"""
+        current_time = timestamp or time.time()
+        
+        # Initialize first speaker if no embeddings yet
+        if not self.previous_embeddings:
+            self.previous_embeddings.append(embedding)
+            self.speaker_embeddings[self.current_speaker].append(embedding)
+            if self.mean_embeddings[self.current_speaker] is None:
+                self.mean_embeddings[self.current_speaker] = embedding.copy()
+            return self.current_speaker, 1.0
+        
+        # Calculate similarity with current speaker's mean embedding
+        current_mean = self.mean_embeddings[self.current_speaker]
+        if current_mean is not None:
+            similarity = 1.0 - cosine(embedding, current_mean)
+        else:
+            # If no mean yet, compare with most recent embedding
+            similarity = 1.0 - cosine(embedding, self.previous_embeddings[-1])
+        
+        self.last_similarity = similarity
+        
+        # Decide if this is a speaker change
+        time_since_last_change = current_time - self.last_change_time
+        is_speaker_change = False
+        
+        # Only consider change if minimum time has passed since last change
+        if time_since_last_change >= MIN_SEGMENT_DURATION:
+            # Check similarity against threshold
+            if similarity < self.change_threshold:
+                # Compare with all other speakers' means if available
+                best_speaker = self.current_speaker
+                best_similarity = similarity
+                
+                # Check each active speaker
+                for speaker_id in range(self.max_speakers):
+                    if speaker_id == self.current_speaker:
+                        continue
+                        
+                    speaker_mean = self.mean_embeddings[speaker_id]
+                    
+                    if speaker_mean is not None:
+                        # Calculate similarity with this speaker
+                        speaker_similarity = 1.0 - cosine(embedding, speaker_mean)
+                        
+                        # If more similar to this speaker, update best match
+                        if speaker_similarity > best_similarity:
+                            best_similarity = speaker_similarity
+                            best_speaker = speaker_id
+                
+                # If best match is different from current speaker, change speaker
+                if best_speaker != self.current_speaker:
+                    is_speaker_change = True
+                    self.current_speaker = best_speaker
+                # If no good match with existing speakers and we haven't used all speakers yet
+                elif len(self.active_speakers) < self.max_speakers:
+                    # Find the next unused speaker ID
+                    for new_id in range(self.max_speakers):
+                        if new_id not in self.active_speakers:
+                            is_speaker_change = True
+                            self.current_speaker = new_id
+                            self.active_speakers.add(new_id)
+                            break
+        
+        # Handle speaker change
+        if is_speaker_change:
+            self.last_change_time = current_time
+        
+        # Update embeddings
+        self.previous_embeddings.append(embedding)
+        if len(self.previous_embeddings) > EMBEDDING_HISTORY_SIZE:
+            self.previous_embeddings.pop(0)
+        
+        # Update current speaker's embeddings and mean
+        self.speaker_embeddings[self.current_speaker].append(embedding)
+        self.active_speakers.add(self.current_speaker)
+        
+        if len(self.speaker_embeddings[self.current_speaker]) > 30:  # Limit history size
+            self.speaker_embeddings[self.current_speaker] = self.speaker_embeddings[self.current_speaker][-30:]
+            
+        # Update mean embedding for current speaker
+        if self.speaker_embeddings[self.current_speaker]:
+            self.mean_embeddings[self.current_speaker] = np.mean(
+                self.speaker_embeddings[self.current_speaker], axis=0
+            )
+        
+        return self.current_speaker, similarity
+    
+    def get_color_for_speaker(self, speaker_id):
+        """Return color for speaker ID (0 to max_speakers-1)"""
+        if 0 <= speaker_id < len(SPEAKER_COLORS):
+            return SPEAKER_COLORS[speaker_id]
+        return "#FFFFFF"  # Default to white if out of range
+    
+    def get_status_info(self):
+        """Return status information about the speaker change detector"""
+        speaker_counts = [len(self.speaker_embeddings[i]) for i in range(self.max_speakers)]
+        
+        return {
+            "current_speaker": self.current_speaker,
+            "speaker_counts": speaker_counts,
+            "active_speakers": len(self.active_speakers),
+            "max_speakers": self.max_speakers,
+            "last_similarity": self.last_similarity,
+            "threshold": self.change_threshold
+        }
+
+
+class TextUpdateThread(QThread):
+    text_update_signal = pyqtSignal(str)
+
+    def __init__(self, text):
+        super().__init__()
+        self.text = text
+
+    def run(self):
+        self.text_update_signal.emit(self.text)
+
+
+class SentenceWorker(QThread):
+    sentence_update_signal = pyqtSignal(list, list)
+    status_signal = pyqtSignal(str)
+
+    def __init__(self, queue, encoder, change_threshold=DEFAULT_CHANGE_THRESHOLD, max_speakers=DEFAULT_MAX_SPEAKERS):
+        super().__init__()
+        self.queue = queue
+        self.encoder = encoder
+        self._is_running = True
+        self.full_sentences = []
+        self.sentence_speakers = []
+        self.change_threshold = change_threshold
+        self.max_speakers = max_speakers
+        
+        # Initialize audio processor for embedding extraction
+        self.audio_processor = AudioProcessor(self.encoder)
+        
+        # Initialize speaker change detector
+        self.speaker_detector = SpeakerChangeDetector(
+            embedding_dim=self.encoder.embedding_dim,
+            change_threshold=self.change_threshold,
+            max_speakers=self.max_speakers
+        )
+        
+        # Setup monitoring timer
+        self.monitoring_timer = QTimer()
+        self.monitoring_timer.timeout.connect(self.report_status)
+        self.monitoring_timer.start(2000)  # Report every 2 seconds
+    
+    def set_change_threshold(self, threshold):
+        """Update change detection threshold"""
+        self.change_threshold = threshold
+        self.speaker_detector.set_change_threshold(threshold)
+        
+    def set_max_speakers(self, max_speakers):
+        """Update maximum number of speakers"""
+        self.max_speakers = max_speakers
+        self.speaker_detector.set_max_speakers(max_speakers)
+    
+    def run(self):
+        """Main worker thread loop"""
+        while self._is_running:
+            try:
+                text, bytes = self.queue.get(timeout=1)
+                self.process_item(text, bytes)
+            except queue.Empty:
+                continue
+    
+    def report_status(self):
+        """Report status information"""
+        # Get status information from speaker detector
+        status = self.speaker_detector.get_status_info()
+        
+        # Prepare status message with information for all speakers
+        status_text = f"Current speaker: {status['current_speaker'] + 1}\n"
+        status_text += f"Active speakers: {status['active_speakers']} of {status['max_speakers']}\n"
+        
+        # Show segment counts for each speaker
+        for i in range(status['max_speakers']):
+            if i < len(SPEAKER_COLOR_NAMES):
+                color_name = SPEAKER_COLOR_NAMES[i]
+            else:
+                color_name = f"Speaker {i+1}"
+            status_text += f"Speaker {i+1} ({color_name}) segments: {status['speaker_counts'][i]}\n"
+        
+        status_text += f"Last similarity score: {status['last_similarity']:.3f}\n"
+        status_text += f"Change threshold: {status['threshold']:.2f}\n"
+        status_text += f"Total sentences: {len(self.full_sentences)}"
+        
+        # Send to UI
+        self.status_signal.emit(status_text)
+    
+    def process_item(self, text, bytes):
+        """Process a new text-audio pair"""
+        # Convert audio data to int16
+        audio_int16 = np.int16(bytes * 32767)
+        
+        # Extract speaker embedding
+        speaker_embedding = self.audio_processor.extract_embedding(audio_int16)
+        
+        # Store sentence and embedding
+        self.full_sentences.append((text, speaker_embedding))
+        
+        # Fill in any missing speaker assignments
+        if len(self.sentence_speakers) < len(self.full_sentences) - 1:
+            while len(self.sentence_speakers) < len(self.full_sentences) - 1:
+                self.sentence_speakers.append(0)  # Default to first speaker
+        
+        # Detect speaker changes
+        speaker_id, similarity = self.speaker_detector.add_embedding(speaker_embedding)
+        self.sentence_speakers.append(speaker_id)
+        
+        # Send updated data to UI
+        self.sentence_update_signal.emit(self.full_sentences, self.sentence_speakers)
+    
+    def stop(self):
+        """Stop the worker thread"""
+        self._is_running = False
+        if self.monitoring_timer.isActive():
+            self.monitoring_timer.stop()
+
+
+class RecordingThread(QThread):
+    def __init__(self, recorder):
+        super().__init__()
+        self.recorder = recorder
+        self._is_running = True
+        
+        # Determine input source
+        if USE_MICROPHONE:
+            self.device_id = str(sc.default_microphone().name)
+            self.include_loopback = False
+        else:
+            self.device_id = str(sc.default_speaker().name)
+            self.include_loopback = True
+
+    def updateDevice(self, device_id, include_loopback):
+        self.device_id = device_id
+        self.include_loopback = include_loopback
+
+    def run(self):
+        while self._is_running:
+            try:
+                with sc.get_microphone(id=self.device_id, include_loopback=self.include_loopback).recorder(
+                    samplerate=SAMPLE_RATE, blocksize=BUFFER_SIZE
+                ) as mic:
+                    # Process audio chunks while device hasn't changed
+                    current_device = self.device_id
+                    current_loopback = self.include_loopback
+                    
+                    while self._is_running and current_device == self.device_id and current_loopback == self.include_loopback:
+                        # Record audio chunk
+                        audio_data = mic.record(numframes=BUFFER_SIZE)
+                        
+                        # Convert stereo to mono if needed
+                        if audio_data.shape[1] > 1 and CHANNELS == 1:
+                            audio_data = audio_data[:, 0]
+                        
+                        # Convert to int16
+                        audio_int16 = (audio_data.flatten() * 32767).astype(np.int16)
+                        
+                        # Feed to recorder
+                        audio_bytes = audio_int16.tobytes()
+                        self.recorder.feed_audio(audio_bytes)
+                    
+            except Exception as e:
+                print(f"Recording error: {e}")
+                # Wait before retry on error
+                time.sleep(1)
+
+    def stop(self):
+        self._is_running = False
+
+
+class TextRetrievalThread(QThread):
+    textRetrievedFinal = pyqtSignal(str, np.ndarray)
+    textRetrievedLive = pyqtSignal(str)
+    recorderStarted = pyqtSignal()
+
+    def __init__(self):
+        super().__init__()
+
+    def live_text_detected(self, text):
+        self.textRetrievedLive.emit(text)
+
+    def run(self):
+        recorder_config = {
+            'spinner': False,
+            'use_microphone': False,
+            'model': FINAL_TRANSCRIPTION_MODEL,
+            'language': TRANSCRIPTION_LANGUAGE,
+            'silero_sensitivity': SILERO_SENSITIVITY,
+            'webrtc_sensitivity': WEBRTC_SENSITIVITY,
+            'post_speech_silence_duration': SILENCE_THRESHS[1],
+            'min_length_of_recording': MIN_LENGTH_OF_RECORDING,
+            'pre_recording_buffer_duration': PRE_RECORDING_BUFFER_DURATION,
+            'min_gap_between_recordings': 0,
+            'enable_realtime_transcription': True,
+            'realtime_processing_pause': 0,
+            'realtime_model_type': REALTIME_TRANSCRIPTION_MODEL,
+            'on_realtime_transcription_update': self.live_text_detected,
+            'beam_size': FINAL_BEAM_SIZE,
+            'beam_size_realtime': REALTIME_BEAM_SIZE,
+            'buffer_size': BUFFER_SIZE,
+            'sample_rate': SAMPLE_RATE,
+        }
+
+        self.recorder = AudioToTextRecorder(**recorder_config)
+        self.recorderStarted.emit()
+
+        def process_text(text):
+            bytes = self.recorder.last_transcription_bytes
+            self.textRetrievedFinal.emit(text, bytes)
+
+        while True:
+            self.recorder.text(process_text)
+
+
+class MainWindow(QMainWindow):
+    def __init__(self):
+        super().__init__()
+
+        self.setWindowTitle("Real-time Speaker Change Detection")
+
+        self.encoder = None
+        self.initialized = False
+        self.displayed_text = ""
+        self.last_realtime_text = ""
+        self.full_sentences = []
+        self.sentence_speakers = []
+        self.pending_sentences = []
+        self.queue = queue.Queue()
+        self.recording_thread = None
+        self.change_threshold = DEFAULT_CHANGE_THRESHOLD
+        self.max_speakers = DEFAULT_MAX_SPEAKERS
+
+        # Create main horizontal layout
+        self.mainLayout = QHBoxLayout()
+
+        # Add text edit area to main layout
+        self.text_edit = QTextEdit(self)
+        self.mainLayout.addWidget(self.text_edit, 1)
+
+        # Create right layout for controls
+        self.rightLayout = QVBoxLayout()
+        self.rightLayout.setAlignment(Qt.AlignmentFlag.AlignTop)
+        
+        # Create all controls
+        self.create_controls()
+
+        # Create container for right layout
+        self.rightContainer = QWidget()
+        self.rightContainer.setLayout(self.rightLayout)
+        self.mainLayout.addWidget(self.rightContainer, 0)
+
+        # Set main layout as central widget
+        self.centralWidget = QWidget()
+        self.centralWidget.setLayout(self.mainLayout)
+        self.setCentralWidget(self.centralWidget)
+
+        self.setStyleSheet("""
+            QGroupBox {
+                border: 1px solid #555;
+                border-radius: 3px;
+                margin-top: 10px;
+                padding-top: 10px;
+                color: #ddd;
+            }
+            QGroupBox::title {
+                subcontrol-origin: margin;
+                subcontrol-position: top center;
+                padding: 0 5px;
+            }
+            QLabel {
+                color: #ddd;
+            }
+            QPushButton {
+                background: #444;
+                color: #ddd;
+                border: 1px solid #555;
+                padding: 5px;
+                margin-bottom: 10px;
+            }
+            QPushButton:hover {
+                background: #555;
+            }
+            QTextEdit {
+                background-color: #1e1e1e;
+                color: #ffffff;
+                font-family: 'Arial';
+                font-size: 16pt;
+            }
+            QSlider {
+                height: 30px;
+            }
+            QSlider::groove:horizontal {
+                height: 8px;
+                background: #333;
+                margin: 2px 0;
+            }
+            QSlider::handle:horizontal {
+                background: #666;
+                border: 1px solid #777;
+                width: 18px;
+                margin: -8px 0;
+                border-radius: 9px;
+            }
+        """)
+
+    def create_controls(self):
+        # Speaker change threshold control
+        self.threshold_group = QGroupBox("Speaker Change Sensitivity")
+        threshold_layout = QVBoxLayout()
+        
+        self.threshold_label = QLabel(f"Change threshold: {self.change_threshold:.2f}")
+        threshold_layout.addWidget(self.threshold_label)
+        
+        self.threshold_slider = QSlider(Qt.Orientation.Horizontal)
+        self.threshold_slider.setMinimum(10)
+        self.threshold_slider.setMaximum(95)
+        self.threshold_slider.setValue(int(self.change_threshold * 100))
+        self.threshold_slider.valueChanged.connect(self.update_threshold)
+        threshold_layout.addWidget(self.threshold_slider)
+        
+        self.threshold_explanation = QLabel(
+            "If the speakers have similar voices, it would be better to set it above 0.5, and if they have different voices, it would be lower."
+        )
+        self.threshold_explanation.setWordWrap(True)
+        threshold_layout.addWidget(self.threshold_explanation)
+        
+        self.threshold_group.setLayout(threshold_layout)
+        self.rightLayout.addWidget(self.threshold_group)
+        
+        # Max speakers control
+        self.max_speakers_group = QGroupBox("Maximum Number of Speakers")
+        max_speakers_layout = QVBoxLayout()
+        
+        self.max_speakers_label = QLabel(f"Max speakers: {self.max_speakers}")
+        max_speakers_layout.addWidget(self.max_speakers_label)
+        
+        self.max_speakers_spinbox = QSpinBox()
+        self.max_speakers_spinbox.setMinimum(2)
+        self.max_speakers_spinbox.setMaximum(ABSOLUTE_MAX_SPEAKERS)
+        self.max_speakers_spinbox.setValue(self.max_speakers)
+        self.max_speakers_spinbox.valueChanged.connect(self.update_max_speakers)
+        max_speakers_layout.addWidget(self.max_speakers_spinbox)
+        
+        self.max_speakers_explanation = QLabel(
+            f"You can set between 2 and {ABSOLUTE_MAX_SPEAKERS} speakers.\n"
+            "Changes will apply immediately."
+        )
+        self.max_speakers_explanation.setWordWrap(True)
+        max_speakers_layout.addWidget(self.max_speakers_explanation)
+        
+        self.max_speakers_group.setLayout(max_speakers_layout)
+        self.rightLayout.addWidget(self.max_speakers_group)
+        
+        # Speaker color legend - dynamic based on max speakers
+        self.legend_group = QGroupBox("Speaker Colors")
+        self.legend_layout = QVBoxLayout()
+        
+        # Create speaker labels dynamically
+        self.speaker_labels = []
+        for i in range(ABSOLUTE_MAX_SPEAKERS):
+            color = SPEAKER_COLORS[i]
+            color_name = SPEAKER_COLOR_NAMES[i]
+            label = QLabel(f"Speaker {i+1} ({color_name}): <span style='color:{color};'>■■■■■</span>")
+            self.speaker_labels.append(label)
+            if i < self.max_speakers:
+                self.legend_layout.addWidget(label)
+        
+        self.legend_group.setLayout(self.legend_layout)
+        self.rightLayout.addWidget(self.legend_group)
+        
+        # Status display area
+        self.status_group = QGroupBox("Status")
+        status_layout = QVBoxLayout()
+        
+        self.status_label = QLabel("Status information will be displayed here.")
+        self.status_label.setWordWrap(True)
+        status_layout.addWidget(self.status_label)
+        
+        self.status_group.setLayout(status_layout)
+        self.rightLayout.addWidget(self.status_group)
+
+        # Clear button
+        self.clear_button = QPushButton("Clear Conversation")
+        self.clear_button.clicked.connect(self.clear_state)
+        self.clear_button.setEnabled(False)
+        self.rightLayout.addWidget(self.clear_button)
+    
+    def update_threshold(self, value):
+        """Update speaker change detection threshold"""
+        threshold = value / 100.0
+        self.change_threshold = threshold
+        self.threshold_label.setText(f"Change threshold: {threshold:.2f}")
+        
+        # Update in worker if it exists
+        if hasattr(self, 'worker_thread'):
+            self.worker_thread.set_change_threshold(threshold)
+            
+    def update_max_speakers(self, value):
+        """Update maximum number of speakers"""
+        self.max_speakers = value
+        self.max_speakers_label.setText(f"Max speakers: {value}")
+        
+        # Update visible speaker labels
+        self.update_speaker_labels()
+        
+        # Update in worker if it exists
+        if hasattr(self, 'worker_thread'):
+            self.worker_thread.set_max_speakers(value)
+    
+    def update_speaker_labels(self):
+        """Update which speaker labels are visible based on max_speakers"""
+        # Clear all labels first
+        for i in range(len(self.speaker_labels)):
+            label = self.speaker_labels[i]
+            if label.parent():
+                self.legend_layout.removeWidget(label)
+                label.setParent(None)
+        
+        # Add only the labels for the current max_speakers
+        for i in range(min(self.max_speakers, len(self.speaker_labels))):
+            self.legend_layout.addWidget(self.speaker_labels[i])
+
+    def clear_state(self):
+        # Clear text edit area
+        self.text_edit.clear()
+
+        # Reset state variables
+        self.displayed_text = ""
+        self.last_realtime_text = ""
+        self.full_sentences = []
+        self.sentence_speakers = []
+        self.pending_sentences = []
+        
+        if hasattr(self, 'worker_thread'):
+            self.worker_thread.full_sentences = []
+            self.worker_thread.sentence_speakers = []
+            # Reset speaker detector with current threshold and max_speakers
+            self.worker_thread.speaker_detector = SpeakerChangeDetector(
+                embedding_dim=self.encoder.embedding_dim,
+                change_threshold=self.change_threshold,
+                max_speakers=self.max_speakers
+            )
+
+        # Display message
+        self.text_edit.setHtml("<i>All content cleared. Waiting for new input...</i>")
+    
+    def update_status(self, status_text):
+        self.status_label.setText(status_text)
+
+    def showEvent(self, event):
+        super().showEvent(event)
+        if event.type() == QEvent.Type.Show:
+            if not self.initialized:
+                self.initialized = True
+                self.resize(1200, 800)
+                self.update_text("<i>Initializing application...</i>")
+
+                QTimer.singleShot(500, self.init)
+
+    def process_live_text(self, text):
+        text = text.strip()
+
+        if text:
+            sentence_delimiters = '.?!。'
+            prob_sentence_end = (
+                len(self.last_realtime_text) > 0
+                and text[-1] in sentence_delimiters
+                and self.last_realtime_text[-1] in sentence_delimiters
+            )
+
+            self.last_realtime_text = text
+
+            if prob_sentence_end:
+                if FAST_SENTENCE_END:
+                    self.text_retrieval_thread.recorder.stop()
+                else:
+                    self.text_retrieval_thread.recorder.post_speech_silence_duration = SILENCE_THRESHS[0]
+            else:
+                self.text_retrieval_thread.recorder.post_speech_silence_duration = SILENCE_THRESHS[1]
+
+        self.text_detected(text)
+
+    def text_detected(self, text):
+        try:
+            sentences_with_style = []
+            for i, sentence in enumerate(self.full_sentences):
+                sentence_text, _ = sentence
+                if i >= len(self.sentence_speakers):
+                    color = "#FFFFFF"  # Default white
+                else:
+                    speaker_id = self.sentence_speakers[i]
+                    color = self.worker_thread.speaker_detector.get_color_for_speaker(speaker_id)
+
+                sentences_with_style.append(
+                    f'<span style="color:{color};">{sentence_text}</span>')
+
+            for pending_sentence in self.pending_sentences:
+                sentences_with_style.append(
+                    f'<span style="color:#60FFFF;">{pending_sentence}</span>')
+
+            new_text = " ".join(sentences_with_style).strip() + " " + text if len(sentences_with_style) > 0 else text
+
+            if new_text != self.displayed_text:
+                self.displayed_text = new_text
+                self.update_text(new_text)
+        except Exception as e:
+            print(f"Error: {e}")
+
+    def process_final(self, text, bytes):
+        text = text.strip()
+        if text:
+            try:
+                self.pending_sentences.append(text)
+                self.queue.put((text, bytes))
+            except Exception as e:
+                print(f"Error: {e}")
+
+    def capture_output_and_feed_to_recorder(self):
+        # Use default device settings
+        device_id = str(sc.default_speaker().name)
+        include_loopback = True
+        
+        self.recording_thread = RecordingThread(self.text_retrieval_thread.recorder)
+        # Update with current device settings
+        self.recording_thread.updateDevice(device_id, include_loopback)
+        self.recording_thread.start()
+
+    def recorder_ready(self):
+        self.update_text("<i>Recording ready</i>")
+        self.capture_output_and_feed_to_recorder()
+
+    def init(self):
+        self.update_text("<i>Loading ECAPA-TDNN model... Please wait.</i>")
+        
+        # Start model loading in background thread
+        self.start_encoder()
+
+    def update_loading_status(self, message):
+        self.update_text(f"<i>{message}</i>")
+
+    def start_encoder(self):
+        # Create and start encoder loader thread
+        self.encoder_loader_thread = EncoderLoaderThread()
+        self.encoder_loader_thread.model_loaded.connect(self.on_model_loaded)
+        self.encoder_loader_thread.progress_update.connect(self.update_loading_status)
+        self.encoder_loader_thread.start()
+
+    def on_model_loaded(self, encoder):
+        # Store loaded encoder model
+        self.encoder = encoder
+        
+        if self.encoder is None:
+            self.update_text("<i>Failed to load ECAPA-TDNN model. Please check your configuration.</i>")
+            return
+        
+        # Enable all controls after model is loaded
+        self.clear_button.setEnabled(True)
+        self.threshold_slider.setEnabled(True)
+        
+        # Continue initialization
+        self.update_text("<i>ECAPA-TDNN model loaded. Starting recorder...</i>")
+        
+        self.text_retrieval_thread = TextRetrievalThread()
+        self.text_retrieval_thread.recorderStarted.connect(
+            self.recorder_ready)
+        self.text_retrieval_thread.textRetrievedLive.connect(
+            self.process_live_text)
+        self.text_retrieval_thread.textRetrievedFinal.connect(
+            self.process_final)
+        self.text_retrieval_thread.start()
+        
+        self.worker_thread = SentenceWorker(
+            self.queue, 
+            self.encoder, 
+            change_threshold=self.change_threshold,
+            max_speakers=self.max_speakers
+        )
+        self.worker_thread.sentence_update_signal.connect(
+            self.sentence_updated)
+        self.worker_thread.status_signal.connect(
+            self.update_status)
+        self.worker_thread.start()
+
+    def sentence_updated(self, full_sentences, sentence_speakers):
+        self.pending_text = ""
+        self.full_sentences = full_sentences
+        self.sentence_speakers = sentence_speakers
+        for sentence in self.full_sentences:
+            sentence_text, _ = sentence
+            if sentence_text in self.pending_sentences:
+                self.pending_sentences.remove(sentence_text)
+        self.text_detected("")
+
+    def set_text(self, text):
+        self.update_thread = TextUpdateThread(text)
+        self.update_thread.text_update_signal.connect(self.update_text)
+        self.update_thread.start()
+
+    def update_text(self, text):
+        self.text_edit.setHtml(text)
+        self.text_edit.verticalScrollBar().setValue(
+            self.text_edit.verticalScrollBar().maximum())
+
+
+def main():
+    app = QApplication(sys.argv)
+
+    dark_stylesheet = """
+    QMainWindow {
+        background-color: #323232;
+    }
+    QTextEdit {
+        background-color: #1e1e1e;
+        color: #ffffff;
+    }
+    """
+    app.setStyleSheet(dark_stylesheet)
+
+    main_window = MainWindow()
+    main_window.show()
+
+    sys.exit(app.exec())
+
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1,188 @@
+fastapi
+uvicorn[standard]
+absl-py
+aiohttp
+aiosignal
+annotated-types
+anyascii
+anyio
+asttokens
+attrs
+audioread
+av
+azure-cognitiveservices-speech
+Babel
+bangla
+blinker
+blis
+bnnumerizer
+bnunicodenormalizer
+catalogue
+certifi
+cffi
+charset-normalizer
+click
+cloudpathlib
+colorama
+coloredlogs
+comtypes
+confection
+contourpy
+coqpit
+ctranslate2
+cycler
+cymem
+Cython
+dateparser
+decorator
+distro
+docopt
+einops
+elevenlabs
+emoji
+encodec
+enum34
+executing
+faster-whisper
+fastrtc==0.0.25
+ffmpeg-python
+filelock
+Flask
+flatbuffers
+fonttools
+frozenlist
+fsspec
+future
+g2pkk
+grpcio
+gruut
+gruut-ipa
+gruut_lang_de
+gruut_lang_en
+gruut_lang_es
+gruut_lang_fr
+h11
+halo
+hangul-romanize
+httpcore
+httpx
+huggingface-hub
+humanfriendly
+idna
+inflect
+ipython
+itsdangerous
+jamo
+jedi
+jieba
+Jinja2
+joblib
+jsonlines
+kiwisolver
+langcodes
+lazy_loader
+librosa
+llvmlite
+log-symbols
+Markdown
+MarkupSafe
+matplotlib
+matplotlib-inline
+more-itertools
+mpmath
+msgpack
+multidict
+murmurhash
+networkx
+nltk
+num2words
+numba
+numpy
+onnxruntime
+openai
+openai-whisper
+packaging
+pandas
+parso
+pillow
+platformdirs
+pooch
+preshed
+prompt-toolkit
+protobuf
+psutil
+pure-eval
+pvporcupine
+pyannote-audio
+pycparser
+pydantic
+pydantic_core
+pydub
+Pygments
+pynndescent
+pyparsing
+pypinyin
+pyreadline3
+pysbd
+python-crfsuite
+python-dateutil
+pyttsx3
+pytz
+PyYAML
+RealTimeSTT
+regex
+requests
+safetensors
+scikit-learn
+scipy
+six
+smart-open
+sniffio
+SoundCard
+soundfile
+soxr
+spacy
+spacy-legacy
+spacy-loggers
+spinners
+srsly
+stable-ts
+stack-data
+stanza
+stream2sentence
+SudachiDict-core
+SudachiPy
+sympy
+tensorboard
+tensorboard-data-server
+termcolor
+thinc
+threadpoolctl
+tiktoken
+tokenizers
+gradio-webrtc==0.0.8
+
+--extra-index-url https://download.pytorch.org/whl/cpu
+gradio
+# … any other non-PyTorch dependencies …
+torch
+torchaudio
+tqdm
+trainer
+traitlets
+transformers
+typer
+typing_extensions
+tzdata
+tzlocal
+umap-learn
+Unidecode
+urllib3
+wasabi
+wcwidth
+weasel
+webrtcvad
+websockets
+Werkzeug
+yarl
+yt-dlp