Spaces:
Sleeping
Sleeping
Commit
·
17cb251
1
Parent(s):
6e5c27e
experiment 2
Browse files
ui.py
CHANGED
@@ -1,31 +1,32 @@
|
|
1 |
import gradio as gr
|
2 |
-
import requests
|
3 |
from fastapi import FastAPI
|
4 |
from shared import DEFAULT_CHANGE_THRESHOLD, DEFAULT_MAX_SPEAKERS, ABSOLUTE_MAX_SPEAKERS, FINAL_TRANSCRIPTION_MODEL, REALTIME_TRANSCRIPTION_MODEL
|
5 |
-
|
|
|
6 |
# Connection configuration (separate signaling server from model server)
|
7 |
-
# These will be replaced
|
8 |
-
RENDER_SIGNALING_URL = "wss://render-signal-audio.onrender.com/stream"
|
9 |
-
HF_SPACE_URL = "https://androidguy-speaker-diarization.hf.space"
|
10 |
|
11 |
def build_ui():
|
12 |
-
"""Build Gradio UI for speaker diarization"""
|
13 |
with gr.Blocks(title="Real-time Speaker Diarization", theme=gr.themes.Soft()) as demo:
|
14 |
# Add configuration variables to page using custom component
|
15 |
gr.HTML(
|
16 |
f"""
|
|
|
17 |
<script>
|
18 |
window.RENDER_SIGNALING_URL = "{RENDER_SIGNALING_URL}";
|
19 |
window.HF_SPACE_URL = "{HF_SPACE_URL}";
|
20 |
-
window.FINAL_TRANSCRIPTION_MODEL = "{FINAL_TRANSCRIPTION_MODEL}";
|
21 |
-
window.REALTIME_TRANSCRIPTION_MODEL = "{REALTIME_TRANSCRIPTION_MODEL}";
|
22 |
</script>
|
23 |
"""
|
24 |
)
|
25 |
|
26 |
# Header and description
|
27 |
gr.Markdown("# 🎤 Live Speaker Diarization")
|
28 |
-
gr.Markdown(
|
29 |
|
30 |
# Add transcription model info
|
31 |
gr.Markdown(f"**Using Models:** Final: {FINAL_TRANSCRIPTION_MODEL}, Realtime: {REALTIME_TRANSCRIPTION_MODEL}")
|
@@ -36,7 +37,8 @@ def build_ui():
|
|
36 |
<span id="status-text" style="color:#888;">Waiting to connect...</span>
|
37 |
<span id="status-icon" style="width:10px; height:10px; display:inline-block;
|
38 |
background-color:#888; border-radius:50%; margin-left:5px;"></span>
|
39 |
-
</div>"""
|
|
|
40 |
)
|
41 |
|
42 |
with gr.Row():
|
@@ -55,11 +57,18 @@ def build_ui():
|
|
55 |
let mediaStream;
|
56 |
let wsConnection;
|
57 |
let statusUpdateInterval;
|
|
|
58 |
|
59 |
-
// Check connection to HF space
|
60 |
async function checkHfConnection() {
|
61 |
try {
|
62 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
63 |
return response.ok;
|
64 |
} catch (err) {
|
65 |
console.warn("HF Space connection failed:", err);
|
@@ -67,43 +76,46 @@ def build_ui():
|
|
67 |
}
|
68 |
}
|
69 |
|
70 |
-
//
|
71 |
async function startStreaming() {
|
72 |
try {
|
73 |
// Update status
|
74 |
updateStatus('connecting');
|
75 |
|
76 |
-
//
|
77 |
-
|
78 |
-
|
79 |
-
noiseSuppression: true,
|
80 |
-
autoGainControl: true
|
81 |
-
}});
|
82 |
-
|
83 |
-
// Check if we can connect to backend
|
84 |
-
let backendAvailable = await checkHfConnection();
|
85 |
-
|
86 |
-
try {
|
87 |
-
// Try WebRTC connection, might fail if backend is down
|
88 |
-
await setupWebRTC();
|
89 |
-
} catch (rtcErr) {
|
90 |
-
console.error("WebRTC setup failed:", rtcErr);
|
91 |
-
// Continue even if WebRTC fails
|
92 |
-
}
|
93 |
|
|
|
94 |
try {
|
95 |
-
|
96 |
-
|
97 |
-
|
98 |
-
|
99 |
-
|
|
|
|
|
|
|
|
|
100 |
}
|
101 |
|
102 |
-
// Start status update interval regardless
|
103 |
-
statusUpdateInterval = setInterval(updateConnectionInfo, 5000);
|
104 |
-
|
105 |
-
// Update status - either connected or offline mode
|
106 |
if (backendAvailable) {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
107 |
updateStatus('connected');
|
108 |
document.getElementById("conversation").innerHTML = "<i>Connected! Start speaking...</i>";
|
109 |
} else {
|
@@ -111,6 +123,10 @@ def build_ui():
|
|
111 |
document.getElementById("conversation").innerHTML =
|
112 |
"<i>Backend connection failed. Microphone active but transcription unavailable.</i>";
|
113 |
}
|
|
|
|
|
|
|
|
|
114 |
} catch (err) {
|
115 |
console.error('Error starting stream:', err);
|
116 |
updateStatus('error', err.message);
|
@@ -126,7 +142,10 @@ def build_ui():
|
|
126 |
|
127 |
// Use FastRTC's connection approach
|
128 |
const pc = new RTCPeerConnection({
|
129 |
-
iceServers: [
|
|
|
|
|
|
|
130 |
});
|
131 |
|
132 |
// Add audio track
|
@@ -134,22 +153,46 @@ def build_ui():
|
|
134 |
pc.addTrack(track, mediaStream);
|
135 |
});
|
136 |
|
137 |
-
// Connect to FastRTC signaling via WebSocket
|
138 |
-
const signalWs = new WebSocket(window.RENDER_SIGNALING_URL
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
139 |
|
140 |
// Handle signaling messages
|
141 |
signalWs.onmessage = async (event) => {
|
142 |
-
|
143 |
-
|
144 |
-
|
145 |
-
|
146 |
-
|
147 |
-
|
148 |
-
|
149 |
-
|
150 |
-
if (message.candidate) {
|
151 |
-
|
|
|
|
|
152 |
}
|
|
|
|
|
153 |
}
|
154 |
};
|
155 |
|
@@ -166,130 +209,175 @@ def build_ui():
|
|
166 |
// Keep connection reference
|
167 |
rtcConnection = pc;
|
168 |
|
169 |
-
// Wait for connection to be established
|
170 |
await new Promise((resolve, reject) => {
|
171 |
-
const timeout = setTimeout(() => reject(new Error("WebRTC connection timeout")),
|
|
|
172 |
pc.onconnectionstatechange = () => {
|
|
|
173 |
if (pc.connectionState === 'connected') {
|
174 |
clearTimeout(timeout);
|
175 |
resolve();
|
176 |
-
} else if (pc.connectionState === 'failed' || pc.connectionState === 'disconnected') {
|
177 |
clearTimeout(timeout);
|
178 |
reject(new Error("WebRTC connection failed"));
|
179 |
}
|
180 |
};
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
181 |
});
|
182 |
|
183 |
updateStatus('connected');
|
|
|
|
|
184 |
} catch (err) {
|
185 |
console.error('WebRTC setup error:', err);
|
186 |
-
updateStatus('
|
|
|
187 |
}
|
188 |
}
|
189 |
|
190 |
// Set up WebSocket connection to HF Space for conversation updates
|
191 |
function setupWebSocket() {
|
192 |
-
|
193 |
-
|
194 |
-
|
195 |
-
|
196 |
-
|
197 |
-
|
198 |
-
|
199 |
-
|
200 |
-
|
201 |
-
|
202 |
-
|
203 |
-
|
204 |
-
|
205 |
-
|
206 |
-
|
207 |
-
|
208 |
-
|
209 |
-
|
210 |
-
|
211 |
-
|
212 |
-
|
213 |
-
|
214 |
-
|
215 |
-
|
216 |
-
|
217 |
-
|
218 |
-
|
219 |
-
|
220 |
-
|
221 |
-
|
222 |
-
|
223 |
-
|
224 |
-
|
225 |
-
|
226 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
227 |
}
|
|
|
|
|
228 |
}
|
229 |
-
|
230 |
-
|
231 |
-
|
232 |
-
|
233 |
-
|
234 |
-
|
235 |
-
|
236 |
-
|
237 |
-
|
238 |
-
|
239 |
-
|
240 |
-
|
241 |
-
|
242 |
-
|
243 |
-
|
244 |
-
|
245 |
-
|
246 |
-
|
247 |
-
|
248 |
-
|
249 |
-
|
250 |
-
|
251 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
252 |
}
|
253 |
-
|
254 |
-
|
255 |
-
case 'conversation_cleared':
|
256 |
-
document.getElementById("conversation").innerHTML =
|
257 |
-
"<i>Conversation cleared. Start speaking again...</i>";
|
258 |
-
break;
|
259 |
-
|
260 |
-
case 'error':
|
261 |
-
console.error('Error message from server:', message.message);
|
262 |
-
updateStatus('warning', message.message);
|
263 |
-
break;
|
264 |
-
|
265 |
-
default:
|
266 |
-
// If it's just HTML content without proper JSON structure (legacy format)
|
267 |
-
document.getElementById("conversation").innerHTML = event.data;
|
268 |
}
|
269 |
-
|
270 |
-
|
271 |
-
|
272 |
-
|
273 |
-
|
274 |
-
// Fallback for non-JSON messages (legacy format)
|
275 |
-
document.getElementById("conversation").innerHTML = event.data;
|
276 |
-
|
277 |
-
// Auto-scroll to bottom
|
278 |
-
const container = document.getElementById("conversation");
|
279 |
-
container.scrollTop = container.scrollHeight;
|
280 |
-
}
|
281 |
-
};
|
282 |
-
|
283 |
-
wsConnection.onerror = (error) => {
|
284 |
-
console.error('WebSocket error:', error);
|
285 |
-
updateStatus('warning', 'WebSocket error');
|
286 |
-
};
|
287 |
-
|
288 |
-
wsConnection.onclose = () => {
|
289 |
-
console.log('WebSocket connection closed');
|
290 |
-
// Try to reconnect after a delay
|
291 |
-
setTimeout(setupWebSocket, 3000);
|
292 |
-
};
|
293 |
}
|
294 |
|
295 |
// Update connection info in the UI with better error handling
|
@@ -298,33 +386,58 @@ def build_ui():
|
|
298 |
const hfConnected = await checkHfConnection();
|
299 |
|
300 |
if (!hfConnected) {
|
301 |
-
// If
|
302 |
-
|
303 |
-
|
304 |
-
|
305 |
-
if (!wsConnection || wsConnection.readyState !== WebSocket.OPEN) {
|
306 |
-
try {
|
307 |
-
setupWebSocket();
|
308 |
-
} catch (e) {
|
309 |
-
console.warn("Failed to reconnect WebSocket:", e);
|
310 |
-
}
|
311 |
}
|
312 |
-
} else if (rtcConnection?.connectionState === 'connected' ||
|
313 |
-
rtcConnection?.iceConnectionState === 'connected') {
|
314 |
-
updateStatus('connected');
|
315 |
} else {
|
316 |
-
|
317 |
-
|
318 |
-
|
319 |
-
|
|
|
320 |
try {
|
321 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
322 |
} catch (e) {
|
323 |
-
console.warn("Failed to reconnect
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
324 |
}
|
325 |
}
|
326 |
}
|
327 |
-
|
328 |
console.error('Error updating connection info:', err);
|
329 |
// Don't update status here to avoid flickering
|
330 |
}
|
@@ -335,6 +448,8 @@ def build_ui():
|
|
335 |
const statusText = document.getElementById('status-text');
|
336 |
const statusIcon = document.getElementById('status-icon');
|
337 |
|
|
|
|
|
338 |
switch(status) {
|
339 |
case 'connected':
|
340 |
statusText.textContent = 'Connected';
|
@@ -399,29 +514,38 @@ def build_ui():
|
|
399 |
"<i>Conversation cleared. Start speaking again...</i>";
|
400 |
|
401 |
// Then try to update on the backend if available
|
402 |
-
|
403 |
-
|
404 |
-
|
405 |
-
|
406 |
-
|
407 |
-
|
408 |
-
|
409 |
-
|
410 |
-
|
411 |
-
|
412 |
-
|
413 |
-
|
414 |
-
|
415 |
-
|
416 |
-
|
417 |
-
|
418 |
-
|
|
|
|
|
419 |
}
|
420 |
|
421 |
// Update settings with better error handling and offline mode support
|
422 |
function updateSettings() {
|
423 |
-
const threshold = document.querySelector('input[data-testid="threshold-slider"]')
|
424 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
425 |
|
426 |
// First update the UI immediately regardless of API success
|
427 |
const statusOutput = document.getElementById('status-output');
|
@@ -441,24 +565,26 @@ def build_ui():
|
|
441 |
`;
|
442 |
}
|
443 |
|
444 |
-
// Then try to update on the backend if available
|
445 |
-
|
446 |
-
|
447 |
-
|
448 |
-
|
449 |
-
|
450 |
-
|
451 |
-
|
452 |
-
|
453 |
-
|
454 |
-
|
455 |
-
|
456 |
-
|
457 |
-
|
458 |
-
|
459 |
-
|
460 |
-
|
461 |
-
|
|
|
|
|
462 |
}
|
463 |
|
464 |
// Set up event listeners when the DOM is loaded
|
@@ -476,9 +602,15 @@ def build_ui():
|
|
476 |
// Fallback to aria-label if IDs aren't found
|
477 |
if (!startBtn) startBtn = document.querySelector('button[aria-label="Start Listening"]');
|
478 |
if (!stopBtn) stopBtn = document.querySelector('button[aria-label="Stop"]');
|
479 |
-
if (!clearBtn)
|
480 |
if (!updateBtn) updateBtn = document.querySelector('button[aria-label="Update Settings"]');
|
481 |
|
|
|
|
|
|
|
|
|
|
|
|
|
482 |
// Check if all buttons are found
|
483 |
const buttonsFound = startBtn && stopBtn && clearBtn && updateBtn;
|
484 |
|
@@ -508,8 +640,8 @@ def build_ui():
|
|
508 |
if (!findAndBindButtons()) {
|
509 |
// If not successful, set up a retry mechanism
|
510 |
let retryCount = 0;
|
511 |
-
const maxRetries =
|
512 |
-
const retryInterval =
|
513 |
|
514 |
const retryBinding = setInterval(() => {
|
515 |
if (findAndBindButtons() || ++retryCount >= maxRetries) {
|
@@ -536,7 +668,7 @@ def build_ui():
|
|
536 |
status_output = gr.Markdown(
|
537 |
"""
|
538 |
## System Status
|
539 |
-
|
540 |
|
541 |
*Click Start Listening to begin*
|
542 |
""",
|
@@ -588,21 +720,22 @@ def build_ui():
|
|
588 |
- 🟠 Speaker 8 (Gold)
|
589 |
""")
|
590 |
|
591 |
-
#
|
592 |
-
# We removed the separate gr.HTML for button bindings since it's now handled with more robust code
|
593 |
-
|
594 |
-
# Set up periodic status updates
|
595 |
def get_status():
|
596 |
"""API call to get system status - called periodically"""
|
597 |
import requests
|
598 |
try:
|
599 |
-
|
|
|
600 |
if resp.status_code == 200:
|
601 |
-
return resp.json().get('
|
602 |
return "Error getting status"
|
603 |
except Exception as e:
|
604 |
-
return f"
|
605 |
-
|
|
|
|
|
|
|
606 |
|
607 |
return demo
|
608 |
|
@@ -615,4 +748,4 @@ def mount_ui(app: FastAPI):
|
|
615 |
|
616 |
# For standalone testing
|
617 |
if __name__ == "__main__":
|
618 |
-
demo.launch()
|
|
|
1 |
import gradio as gr
|
|
|
2 |
from fastapi import FastAPI
|
3 |
from shared import DEFAULT_CHANGE_THRESHOLD, DEFAULT_MAX_SPEAKERS, ABSOLUTE_MAX_SPEAKERS, FINAL_TRANSCRIPTION_MODEL, REALTIME_TRANSCRIPTION_MODEL
|
4 |
+
import os
|
5 |
+
|
6 |
# Connection configuration (separate signaling server from model server)
|
7 |
+
# These will be replaced with environment variables or defaults
|
8 |
+
RENDER_SIGNALING_URL = os.environ.get("RENDER_SIGNALING_URL", "wss://render-signal-audio.onrender.com/stream")
|
9 |
+
HF_SPACE_URL = os.environ.get("HF_SPACE_URL", "https://androidguy-speaker-diarization.hf.space")
|
10 |
|
11 |
def build_ui():
|
12 |
+
"""Build Gradio UI for speaker diarization with improved reliability"""
|
13 |
with gr.Blocks(title="Real-time Speaker Diarization", theme=gr.themes.Soft()) as demo:
|
14 |
# Add configuration variables to page using custom component
|
15 |
gr.HTML(
|
16 |
f"""
|
17 |
+
<!-- Configuration parameters -->
|
18 |
<script>
|
19 |
window.RENDER_SIGNALING_URL = "{RENDER_SIGNALING_URL}";
|
20 |
window.HF_SPACE_URL = "{HF_SPACE_URL}";
|
21 |
+
window.FINAL_TRANSCRIPTION_MODEL = "{FINAL_TRANSCRIPTION_MODEL}";
|
22 |
+
window.REALTIME_TRANSCRIPTION_MODEL = "{REALTIME_TRANSCRIPTION_MODEL}";
|
23 |
</script>
|
24 |
"""
|
25 |
)
|
26 |
|
27 |
# Header and description
|
28 |
gr.Markdown("# 🎤 Live Speaker Diarization")
|
29 |
+
gr.Markdown("Real-time speech recognition with automatic speaker identification")
|
30 |
|
31 |
# Add transcription model info
|
32 |
gr.Markdown(f"**Using Models:** Final: {FINAL_TRANSCRIPTION_MODEL}, Realtime: {REALTIME_TRANSCRIPTION_MODEL}")
|
|
|
37 |
<span id="status-text" style="color:#888;">Waiting to connect...</span>
|
38 |
<span id="status-icon" style="width:10px; height:10px; display:inline-block;
|
39 |
background-color:#888; border-radius:50%; margin-left:5px;"></span>
|
40 |
+
</div>""",
|
41 |
+
elem_id="connection-status"
|
42 |
)
|
43 |
|
44 |
with gr.Row():
|
|
|
57 |
let mediaStream;
|
58 |
let wsConnection;
|
59 |
let statusUpdateInterval;
|
60 |
+
let isOfflineMode = false;
|
61 |
|
62 |
+
// Check connection to HF space with timeout
|
63 |
async function checkHfConnection() {
|
64 |
try {
|
65 |
+
const controller = new AbortController();
|
66 |
+
const timeoutId = setTimeout(() => controller.abort(), 5000);
|
67 |
+
|
68 |
+
const response = await fetch(`${window.HF_SPACE_URL}/health`, {
|
69 |
+
signal: controller.signal
|
70 |
+
});
|
71 |
+
clearTimeout(timeoutId);
|
72 |
return response.ok;
|
73 |
} catch (err) {
|
74 |
console.warn("HF Space connection failed:", err);
|
|
|
76 |
}
|
77 |
}
|
78 |
|
79 |
+
// Start the connection and audio streaming with robust error handling
|
80 |
async function startStreaming() {
|
81 |
try {
|
82 |
// Update status
|
83 |
updateStatus('connecting');
|
84 |
|
85 |
+
// First check backend connectivity
|
86 |
+
const backendAvailable = await checkHfConnection();
|
87 |
+
isOfflineMode = !backendAvailable;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
88 |
|
89 |
+
// Request microphone access - this works even offline
|
90 |
try {
|
91 |
+
mediaStream = await navigator.mediaDevices.getUserMedia({audio: {
|
92 |
+
echoCancellation: true,
|
93 |
+
noiseSuppression: true,
|
94 |
+
autoGainControl: true
|
95 |
+
}});
|
96 |
+
} catch (micErr) {
|
97 |
+
console.error('Microphone access error:', micErr);
|
98 |
+
updateStatus('error', 'Microphone access denied: ' + micErr.message);
|
99 |
+
return;
|
100 |
}
|
101 |
|
|
|
|
|
|
|
|
|
102 |
if (backendAvailable) {
|
103 |
+
// Try WebRTC connection
|
104 |
+
try {
|
105 |
+
await setupWebRTC();
|
106 |
+
} catch (rtcErr) {
|
107 |
+
console.error("WebRTC setup failed:", rtcErr);
|
108 |
+
// Continue even if WebRTC fails
|
109 |
+
}
|
110 |
+
|
111 |
+
// Try WebSocket connection
|
112 |
+
try {
|
113 |
+
setupWebSocket();
|
114 |
+
} catch (wsErr) {
|
115 |
+
console.error("WebSocket setup failed:", wsErr);
|
116 |
+
// Continue even if WebSocket fails
|
117 |
+
}
|
118 |
+
|
119 |
updateStatus('connected');
|
120 |
document.getElementById("conversation").innerHTML = "<i>Connected! Start speaking...</i>";
|
121 |
} else {
|
|
|
123 |
document.getElementById("conversation").innerHTML =
|
124 |
"<i>Backend connection failed. Microphone active but transcription unavailable.</i>";
|
125 |
}
|
126 |
+
|
127 |
+
// Start status update interval regardless
|
128 |
+
statusUpdateInterval = setInterval(updateConnectionInfo, 5000);
|
129 |
+
|
130 |
} catch (err) {
|
131 |
console.error('Error starting stream:', err);
|
132 |
updateStatus('error', err.message);
|
|
|
142 |
|
143 |
// Use FastRTC's connection approach
|
144 |
const pc = new RTCPeerConnection({
|
145 |
+
iceServers: [
|
146 |
+
{ urls: 'stun:stun.l.google.com:19302' },
|
147 |
+
{ urls: 'stun:stun1.l.google.com:19302' }
|
148 |
+
]
|
149 |
});
|
150 |
|
151 |
// Add audio track
|
|
|
153 |
pc.addTrack(track, mediaStream);
|
154 |
});
|
155 |
|
156 |
+
// Connect to FastRTC signaling via WebSocket with timeout
|
157 |
+
const signalWs = new WebSocket(window.RENDER_SIGNALING_URL);
|
158 |
+
|
159 |
+
// Set connection timeout
|
160 |
+
const connectionTimeout = setTimeout(() => {
|
161 |
+
if (signalWs.readyState !== WebSocket.OPEN) {
|
162 |
+
signalWs.close();
|
163 |
+
throw new Error("WebRTC signaling connection timeout");
|
164 |
+
}
|
165 |
+
}, 10000);
|
166 |
+
|
167 |
+
// Wait for connection to open
|
168 |
+
await new Promise((resolve, reject) => {
|
169 |
+
signalWs.onopen = () => {
|
170 |
+
clearTimeout(connectionTimeout);
|
171 |
+
resolve();
|
172 |
+
};
|
173 |
+
signalWs.onerror = (err) => {
|
174 |
+
clearTimeout(connectionTimeout);
|
175 |
+
reject(new Error("WebRTC signaling connection failed"));
|
176 |
+
};
|
177 |
+
});
|
178 |
|
179 |
// Handle signaling messages
|
180 |
signalWs.onmessage = async (event) => {
|
181 |
+
try {
|
182 |
+
const message = JSON.parse(event.data);
|
183 |
+
|
184 |
+
if (message.type === 'offer') {
|
185 |
+
await pc.setRemoteDescription(new RTCSessionDescription(message));
|
186 |
+
const answer = await pc.createAnswer();
|
187 |
+
await pc.setLocalDescription(answer);
|
188 |
+
signalWs.send(JSON.stringify(pc.localDescription));
|
189 |
+
} else if (message.type === 'candidate') {
|
190 |
+
if (message.candidate) {
|
191 |
+
await pc.addIceCandidate(new RTCIceCandidate(message));
|
192 |
+
}
|
193 |
}
|
194 |
+
} catch (err) {
|
195 |
+
console.error("Error processing signaling message:", err);
|
196 |
}
|
197 |
};
|
198 |
|
|
|
209 |
// Keep connection reference
|
210 |
rtcConnection = pc;
|
211 |
|
212 |
+
// Wait for connection to be established with timeout
|
213 |
await new Promise((resolve, reject) => {
|
214 |
+
const timeout = setTimeout(() => reject(new Error("WebRTC connection timeout")), 15000);
|
215 |
+
|
216 |
pc.onconnectionstatechange = () => {
|
217 |
+
console.log("WebRTC connection state:", pc.connectionState);
|
218 |
if (pc.connectionState === 'connected') {
|
219 |
clearTimeout(timeout);
|
220 |
resolve();
|
221 |
+
} else if (pc.connectionState === 'failed' || pc.connectionState === 'disconnected' || pc.connectionState === 'closed') {
|
222 |
clearTimeout(timeout);
|
223 |
reject(new Error("WebRTC connection failed"));
|
224 |
}
|
225 |
};
|
226 |
+
|
227 |
+
// Also check ice connection state as fallback
|
228 |
+
pc.oniceconnectionstatechange = () => {
|
229 |
+
console.log("ICE connection state:", pc.iceConnectionState);
|
230 |
+
if (pc.iceConnectionState === 'connected' || pc.iceConnectionState === 'completed') {
|
231 |
+
clearTimeout(timeout);
|
232 |
+
resolve();
|
233 |
+
} else if (pc.iceConnectionState === 'failed' || pc.iceConnectionState === 'disconnected' || pc.iceConnectionState === 'closed') {
|
234 |
+
clearTimeout(timeout);
|
235 |
+
reject(new Error("ICE connection failed"));
|
236 |
+
}
|
237 |
+
};
|
238 |
});
|
239 |
|
240 |
updateStatus('connected');
|
241 |
+
console.log("WebRTC connection established successfully");
|
242 |
+
|
243 |
} catch (err) {
|
244 |
console.error('WebRTC setup error:', err);
|
245 |
+
updateStatus('warning', 'WebRTC setup issue: ' + err.message);
|
246 |
+
throw err;
|
247 |
}
|
248 |
}
|
249 |
|
250 |
// Set up WebSocket connection to HF Space for conversation updates
|
251 |
function setupWebSocket() {
|
252 |
+
try {
|
253 |
+
// Close existing connection if any
|
254 |
+
if (wsConnection) {
|
255 |
+
wsConnection.close();
|
256 |
+
}
|
257 |
+
|
258 |
+
const wsUrl = window.RENDER_SIGNALING_URL.replace('stream', 'ws_relay');
|
259 |
+
wsConnection = new WebSocket(wsUrl);
|
260 |
+
|
261 |
+
// Set connection timeout
|
262 |
+
const connectionTimeout = setTimeout(() => {
|
263 |
+
if (wsConnection.readyState !== WebSocket.OPEN) {
|
264 |
+
wsConnection.close();
|
265 |
+
throw new Error("WebSocket connection timeout");
|
266 |
+
}
|
267 |
+
}, 10000);
|
268 |
+
|
269 |
+
wsConnection.onopen = () => {
|
270 |
+
clearTimeout(connectionTimeout);
|
271 |
+
console.log('WebSocket connection established');
|
272 |
+
};
|
273 |
+
|
274 |
+
wsConnection.onmessage = (event) => {
|
275 |
+
try {
|
276 |
+
// Parse the JSON message
|
277 |
+
const message = JSON.parse(event.data);
|
278 |
+
|
279 |
+
// Process different message types
|
280 |
+
switch(message.type) {
|
281 |
+
case 'transcription':
|
282 |
+
// Handle transcription data
|
283 |
+
if (message && message.data && typeof message.data === 'object') {
|
284 |
+
document.getElementById("conversation").innerHTML = message.data.conversation_html ||
|
285 |
+
JSON.stringify(message.data);
|
286 |
+
}
|
287 |
+
break;
|
288 |
+
|
289 |
+
case 'processing_result':
|
290 |
+
// Handle individual audio chunk processing result
|
291 |
+
console.log('Processing result:', message.data);
|
292 |
+
|
293 |
+
// Update status info if needed
|
294 |
+
if (message.data && message.data.status === "processed") {
|
295 |
+
const statusElem = document.getElementById('status-text');
|
296 |
+
if (statusElem) {
|
297 |
+
const speakerId = message.data.speaker_id !== undefined ?
|
298 |
+
`Speaker ${message.data.speaker_id + 1}` : '';
|
299 |
+
|
300 |
+
if (speakerId) {
|
301 |
+
statusElem.textContent = `Connected - ${speakerId} active`;
|
302 |
+
}
|
303 |
}
|
304 |
+
} else if (message.data && message.data.status === "error") {
|
305 |
+
updateStatus('error', message.data.message || 'Processing error');
|
306 |
}
|
307 |
+
break;
|
308 |
+
|
309 |
+
case 'connection':
|
310 |
+
console.log('Connection status:', message.status);
|
311 |
+
updateStatus(message.status === 'connected' ? 'connected' : 'warning');
|
312 |
+
break;
|
313 |
+
|
314 |
+
case 'connection_established':
|
315 |
+
console.log('Connection established:', message);
|
316 |
+
updateStatus('connected');
|
317 |
+
|
318 |
+
// If initial conversation is provided, display it
|
319 |
+
if (message.conversation) {
|
320 |
+
document.getElementById("conversation").innerHTML = message.conversation;
|
321 |
+
}
|
322 |
+
break;
|
323 |
+
|
324 |
+
case 'conversation_update':
|
325 |
+
if (message.conversation_html) {
|
326 |
+
document.getElementById("conversation").innerHTML = message.conversation_html;
|
327 |
+
}
|
328 |
+
break;
|
329 |
+
|
330 |
+
case 'conversation_cleared':
|
331 |
+
document.getElementById("conversation").innerHTML =
|
332 |
+
"<i>Conversation cleared. Start speaking again...</i>";
|
333 |
+
break;
|
334 |
+
|
335 |
+
case 'error':
|
336 |
+
console.error('Error message from server:', message.message);
|
337 |
+
updateStatus('warning', message.message);
|
338 |
+
break;
|
339 |
+
|
340 |
+
default:
|
341 |
+
// If it's just HTML content without proper JSON structure (legacy format)
|
342 |
+
document.getElementById("conversation").innerHTML = event.data;
|
343 |
+
}
|
344 |
+
|
345 |
+
// Auto-scroll to bottom
|
346 |
+
const container = document.getElementById("conversation");
|
347 |
+
container.scrollTop = container.scrollHeight;
|
348 |
+
} catch (e) {
|
349 |
+
// Fallback for non-JSON messages (legacy format)
|
350 |
+
document.getElementById("conversation").innerHTML = event.data;
|
351 |
+
|
352 |
+
// Auto-scroll to bottom
|
353 |
+
const container = document.getElementById("conversation");
|
354 |
+
container.scrollTop = container.scrollHeight;
|
355 |
+
}
|
356 |
+
};
|
357 |
+
|
358 |
+
wsConnection.onerror = (error) => {
|
359 |
+
clearTimeout(connectionTimeout);
|
360 |
+
console.error('WebSocket error:', error);
|
361 |
+
updateStatus('warning', 'WebSocket error');
|
362 |
+
};
|
363 |
+
|
364 |
+
wsConnection.onclose = () => {
|
365 |
+
console.log('WebSocket connection closed');
|
366 |
+
// Try to reconnect after a delay if not in offline mode
|
367 |
+
if (!isOfflineMode) {
|
368 |
+
setTimeout(() => {
|
369 |
+
try {
|
370 |
+
setupWebSocket();
|
371 |
+
} catch (e) {
|
372 |
+
console.error("Failed to reconnect WebSocket:", e);
|
373 |
}
|
374 |
+
}, 3000);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
375 |
}
|
376 |
+
};
|
377 |
+
} catch (err) {
|
378 |
+
console.error("WebSocket setup error:", err);
|
379 |
+
throw err;
|
380 |
+
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
381 |
}
|
382 |
|
383 |
// Update connection info in the UI with better error handling
|
|
|
386 |
const hfConnected = await checkHfConnection();
|
387 |
|
388 |
if (!hfConnected) {
|
389 |
+
// If we were online but now offline, update mode
|
390 |
+
if (!isOfflineMode) {
|
391 |
+
isOfflineMode = true;
|
392 |
+
updateStatus('warning', 'Backend unavailable - limited functionality');
|
|
|
|
|
|
|
|
|
|
|
|
|
393 |
}
|
|
|
|
|
|
|
394 |
} else {
|
395 |
+
// If we were offline but now online, update mode
|
396 |
+
if (isOfflineMode) {
|
397 |
+
isOfflineMode = false;
|
398 |
+
|
399 |
+
// Try to reconnect services
|
400 |
try {
|
401 |
+
if (!rtcConnection || rtcConnection.connectionState !== 'connected') {
|
402 |
+
await setupWebRTC();
|
403 |
+
}
|
404 |
+
|
405 |
+
if (!wsConnection || wsConnection.readyState !== WebSocket.OPEN) {
|
406 |
+
setupWebSocket();
|
407 |
+
}
|
408 |
+
|
409 |
+
updateStatus('connected');
|
410 |
} catch (e) {
|
411 |
+
console.warn("Failed to reconnect services:", e);
|
412 |
+
updateStatus('warning', 'Connection partially restored');
|
413 |
+
}
|
414 |
+
} else if (rtcConnection?.connectionState === 'connected' ||
|
415 |
+
rtcConnection?.iceConnectionState === 'connected') {
|
416 |
+
updateStatus('connected');
|
417 |
+
} else {
|
418 |
+
updateStatus('warning', 'Connection unstable');
|
419 |
+
|
420 |
+
// Try to reconnect if needed
|
421 |
+
if (!rtcConnection ||
|
422 |
+
rtcConnection.connectionState === 'failed' ||
|
423 |
+
rtcConnection.connectionState === 'disconnected') {
|
424 |
+
try {
|
425 |
+
await setupWebRTC();
|
426 |
+
} catch (e) {
|
427 |
+
console.warn("Failed to reconnect WebRTC:", e);
|
428 |
+
}
|
429 |
+
}
|
430 |
+
|
431 |
+
if (!wsConnection || wsConnection.readyState !== WebSocket.OPEN) {
|
432 |
+
try {
|
433 |
+
setupWebSocket();
|
434 |
+
} catch (e) {
|
435 |
+
console.warn("Failed to reconnect WebSocket:", e);
|
436 |
+
}
|
437 |
}
|
438 |
}
|
439 |
}
|
440 |
+
} catch (err) {
|
441 |
console.error('Error updating connection info:', err);
|
442 |
// Don't update status here to avoid flickering
|
443 |
}
|
|
|
448 |
const statusText = document.getElementById('status-text');
|
449 |
const statusIcon = document.getElementById('status-icon');
|
450 |
|
451 |
+
if (!statusText || !statusIcon) return;
|
452 |
+
|
453 |
switch(status) {
|
454 |
case 'connected':
|
455 |
statusText.textContent = 'Connected';
|
|
|
514 |
"<i>Conversation cleared. Start speaking again...</i>";
|
515 |
|
516 |
// Then try to update on the backend if available
|
517 |
+
if (!isOfflineMode) {
|
518 |
+
checkHfConnection().then(isConnected => {
|
519 |
+
if (isConnected) {
|
520 |
+
return fetch(`${window.HF_SPACE_URL}/clear`, {
|
521 |
+
method: 'POST'
|
522 |
+
});
|
523 |
+
} else {
|
524 |
+
throw new Error("Backend unavailable");
|
525 |
+
}
|
526 |
+
})
|
527 |
+
.then(resp => resp.json())
|
528 |
+
.then(data => {
|
529 |
+
console.log("Backend conversation cleared successfully");
|
530 |
+
})
|
531 |
+
.catch(err => {
|
532 |
+
console.warn("Backend clear API failed:", err);
|
533 |
+
// No need to update UI again as we already did it above
|
534 |
+
});
|
535 |
+
}
|
536 |
}
|
537 |
|
538 |
// Update settings with better error handling and offline mode support
|
539 |
function updateSettings() {
|
540 |
+
const threshold = document.querySelector('input[data-testid="threshold-slider"]')?.value ||
|
541 |
+
document.getElementById('threshold-slider')?.value;
|
542 |
+
const maxSpeakers = document.querySelector('input[data-testid="speakers-slider"]')?.value ||
|
543 |
+
document.getElementById('speakers-slider')?.value;
|
544 |
+
|
545 |
+
if (!threshold || !maxSpeakers) {
|
546 |
+
console.error("Could not find slider values");
|
547 |
+
return;
|
548 |
+
}
|
549 |
|
550 |
// First update the UI immediately regardless of API success
|
551 |
const statusOutput = document.getElementById('status-output');
|
|
|
565 |
`;
|
566 |
}
|
567 |
|
568 |
+
// Then try to update on the backend if available and not in offline mode
|
569 |
+
if (!isOfflineMode) {
|
570 |
+
checkHfConnection().then(isConnected => {
|
571 |
+
if (isConnected) {
|
572 |
+
return fetch(`${window.HF_SPACE_URL}/settings?threshold=${threshold}&max_speakers=${maxSpeakers}`, {
|
573 |
+
method: 'POST'
|
574 |
+
});
|
575 |
+
} else {
|
576 |
+
throw new Error("Backend unavailable");
|
577 |
+
}
|
578 |
+
})
|
579 |
+
.then(resp => resp.json())
|
580 |
+
.then(data => {
|
581 |
+
console.log("Backend settings updated successfully:", data);
|
582 |
+
})
|
583 |
+
.catch(err => {
|
584 |
+
console.warn("Backend settings update failed:", err);
|
585 |
+
// No need to update UI again as we already did it above
|
586 |
+
});
|
587 |
+
}
|
588 |
}
|
589 |
|
590 |
// Set up event listeners when the DOM is loaded
|
|
|
602 |
// Fallback to aria-label if IDs aren't found
|
603 |
if (!startBtn) startBtn = document.querySelector('button[aria-label="Start Listening"]');
|
604 |
if (!stopBtn) stopBtn = document.querySelector('button[aria-label="Stop"]');
|
605 |
+
if (!clearBtn) stopBtn = document.querySelector('button[aria-label="Clear"]');
|
606 |
if (!updateBtn) updateBtn = document.querySelector('button[aria-label="Update Settings"]');
|
607 |
|
608 |
+
// Fallback to text content as last resort
|
609 |
+
if (!startBtn) startBtn = Array.from(document.querySelectorAll('button')).find(btn => btn.textContent.includes('Start'));
|
610 |
+
if (!stopBtn) stopBtn = Array.from(document.querySelectorAll('button')).find(btn => btn.textContent.includes('Stop'));
|
611 |
+
if (!clearBtn) clearBtn = Array.from(document.querySelectorAll('button')).find(btn => btn.textContent.includes('Clear'));
|
612 |
+
if (!updateBtn) updateBtn = Array.from(document.querySelectorAll('button')).find(btn => btn.textContent.includes('Update'));
|
613 |
+
|
614 |
// Check if all buttons are found
|
615 |
const buttonsFound = startBtn && stopBtn && clearBtn && updateBtn;
|
616 |
|
|
|
640 |
if (!findAndBindButtons()) {
|
641 |
// If not successful, set up a retry mechanism
|
642 |
let retryCount = 0;
|
643 |
+
const maxRetries = 20; // More retries, longer interval
|
644 |
+
const retryInterval = 300; // 300ms between retries
|
645 |
|
646 |
const retryBinding = setInterval(() => {
|
647 |
if (findAndBindButtons() || ++retryCount >= maxRetries) {
|
|
|
668 |
status_output = gr.Markdown(
|
669 |
"""
|
670 |
## System Status
|
671 |
+
Waiting to connect...
|
672 |
|
673 |
*Click Start Listening to begin*
|
674 |
""",
|
|
|
720 |
- 🟠 Speaker 8 (Gold)
|
721 |
""")
|
722 |
|
723 |
+
# Function to get backend status (for periodic updates)
|
|
|
|
|
|
|
724 |
def get_status():
|
725 |
"""API call to get system status - called periodically"""
|
726 |
import requests
|
727 |
try:
|
728 |
+
# Use a short timeout to prevent UI hanging
|
729 |
+
resp = requests.get(f"{HF_SPACE_URL}/status", timeout=2)
|
730 |
if resp.status_code == 200:
|
731 |
+
return resp.json().get('formatted_text', 'No status information')
|
732 |
return "Error getting status"
|
733 |
except Exception as e:
|
734 |
+
return f"Status update unavailable: Backend may be offline"
|
735 |
+
|
736 |
+
# Set up periodic status updates with shorter interval and error handling
|
737 |
+
status_timer = gr.Timer(10) # 10 seconds between updates
|
738 |
+
status_timer.tick(fn=get_status, outputs=status_output)
|
739 |
|
740 |
return demo
|
741 |
|
|
|
748 |
|
749 |
# For standalone testing
|
750 |
if __name__ == "__main__":
|
751 |
+
demo.launch()
|