"use client"; import { useState, useEffect, useRef, useCallback, useMemo } from "react"; import { Button } from "@/components/ui/button"; import { Card } from "@/components/ui/card"; import { Table, TableBody, TableCaption, TableCell, TableHead, TableHeader, TableRow, } from "@/components/ui/table"; import { Input } from "@/components/ui/input"; import { Badge } from "@/components/ui/badge"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue, } from "@/components/ui/select"; import { useToast } from "@/hooks/use-toast"; import { Disc as Record, Download, Upload, PlusCircle, Square, Camera, Trash2, Settings, RefreshCw, X, Edit2, Check, } from "lucide-react"; import { LeRobotDatasetRecorder, LeRobotDatasetRow, NonIndexedLeRobotDatasetRow, LeRobotEpisode } from "@lerobot/web"; import { TeleoperatorEpisodesView } from "./teleoperator-episodes-view"; interface RecorderProps { teleoperators: any[]; robot: any; // eslint-disable-line @typescript-eslint/no-explicit-any onNeedsTeleoperation: () => Promise; videoStreams?: { [key: string]: MediaStream }; } interface RecorderSettings { huggingfaceApiKey: string; cameraConfigs: { [cameraName: string]: { deviceId: string; deviceLabel: string; }; }; } // Storage functions for recorder settings const RECORDER_SETTINGS_KEY = "lerobot-recorder-settings"; function getRecorderSettings(): RecorderSettings { try { const stored = localStorage.getItem(RECORDER_SETTINGS_KEY); if (stored) { return JSON.parse(stored); } } catch (error) { console.warn("Failed to load recorder settings:", error); } return { huggingfaceApiKey: "", cameraConfigs: {}, }; } function saveRecorderSettings(settings: RecorderSettings): void { try { localStorage.setItem(RECORDER_SETTINGS_KEY, JSON.stringify(settings)); } catch (error) { console.warn("Failed to save recorder settings:", error); } } export function Recorder({ teleoperators, robot, onNeedsTeleoperation, }: RecorderProps) { const [isRecording, setIsRecording] = useState(false); const [currentEpisode, setCurrentEpisode] = useState(0); // Use huggingfaceApiKey from recorderSettings instead of separate state const [cameraName, setCameraName] = useState(""); const [additionalCameras, setAdditionalCameras] = useState<{ [key: string]: MediaStream; }>({}); const [availableCameras, setAvailableCameras] = useState( [] ); const [selectedCameraId, setSelectedCameraId] = useState(""); const [previewStream, setPreviewStream] = useState(null); const [isLoadingCameras, setIsLoadingCameras] = useState(false); const [cameraPermissionState, setCameraPermissionState] = useState< "unknown" | "granted" | "denied" >("unknown"); const [showCameraConfig, setShowCameraConfig] = useState(false); const [showConfigure, setShowConfigure] = useState(false); const [recorderSettings, setRecorderSettings] = useState( () => getRecorderSettings() ); const [hasRecordedFrames, setHasRecordedFrames] = useState(false); const [editingCameraName, setEditingCameraName] = useState( null ); const [editingCameraNewName, setEditingCameraNewName] = useState(""); const [huggingfaceApiKey, setHuggingfaceApiKey] = useState("") const recorderRef = useRef(null); const videoRef = useRef(null); const { toast } = useToast(); // Initialize the recorder when teleoperators are available useEffect(() => { if (teleoperators.length > 0) { recorderRef.current = new LeRobotDatasetRecorder( teleoperators, additionalCameras, 30, // fps "Robot teleoperation recording" ); } }, [teleoperators, additionalCameras]); const handleStartRecording = async () => { // If teleoperators aren't available, initialize teleoperation first if (teleoperators.length === 0) { toast({ title: "Initializing...", description: `Setting up robot control for ${robot.robotId || "robot"}`, }); const success = await onNeedsTeleoperation(); if (!success) { toast({ title: "Recording Error", description: "Failed to initialize robot control", variant: "destructive", }); return; } // Wait a moment for the recorder to initialize with new teleoperators await new Promise((resolve) => setTimeout(resolve, 100)); } if (!recorderRef.current) { toast({ title: "Recording Error", description: "Recorder not ready yet. Please try again.", variant: "destructive", }); return; } try { // Set the episode index recorderRef.current.setEpisodeIndex(currentEpisode); recorderRef.current.setTaskIndex(0); // Default task index // Start recording recorderRef.current.startRecording(); setIsRecording(true); setHasRecordedFrames(true); toast({ title: "Recording Started", description: `Episode ${currentEpisode} is now recording`, }); } catch (error) { const errorMessage = error instanceof Error ? error.message : "Failed to start recording"; toast({ title: "Recording Error", description: errorMessage, variant: "destructive", }); } }; const handleStopRecording = async () => { if (!recorderRef.current || !isRecording) { return; } try { const result = await recorderRef.current.stopRecording(); setIsRecording(false); toast({ title: "Recording Stopped", description: `Episode ${currentEpisode} completed with ${result.teleoperatorData.length} frames`, }); } catch (error) { const errorMessage = error instanceof Error ? error.message : "Failed to stop recording"; toast({ title: "Recording Error", description: errorMessage, variant: "destructive", }); } }; const handleNextEpisode = () => { // Make sure we're not recording if (isRecording) { handleStopRecording(); } // Increment episode counter setCurrentEpisode((prev) => prev + 1); toast({ title: "New Episode", description: `Ready to record episode ${currentEpisode + 1}`, }); }; // Reset frames by clearing the recorder data const handleResetFrames = useCallback(() => { if (isRecording) { handleStopRecording(); } if (recorderRef.current) { recorderRef.current.clearRecording(); setHasRecordedFrames(false); toast({ title: "Frames Reset", description: "All recorded frames have been cleared", }); } }, [isRecording, toast]); // Load available cameras const loadAvailableCameras = useCallback( async (isAutoLoad = false) => { if (isLoadingCameras) return; setIsLoadingCameras(true); try { // Check if we already have permission const permission = await navigator.permissions.query({ name: "camera" as PermissionName, }); setCameraPermissionState( permission.state === "granted" ? "granted" : permission.state === "denied" ? "denied" : "unknown" ); let tempStream: MediaStream | null = null; // Try to enumerate devices first (works if we have permission) const devices = await navigator.mediaDevices.enumerateDevices(); const videoDevices = devices.filter( (device) => device.kind === "videoinput" ); // If devices have labels, we already have permission const hasLabels = videoDevices.some((device) => device.label); let finalVideoDevices = videoDevices; if (!hasLabels && videoDevices.length > 0) { // Need to request permission to get device labels tempStream = await navigator.mediaDevices.getUserMedia({ video: true, }); // Re-enumerate to get labels const devicesWithLabels = await navigator.mediaDevices.enumerateDevices(); const videoDevicesWithLabels = devicesWithLabels.filter( (device) => device.kind === "videoinput" ); finalVideoDevices = videoDevicesWithLabels; setAvailableCameras(videoDevicesWithLabels); if (!isAutoLoad) { console.log( `Found ${videoDevicesWithLabels.length} video devices:`, videoDevicesWithLabels.map((d) => d.label || d.deviceId) ); } } else { setAvailableCameras(videoDevices); if (!isAutoLoad) { console.log( `Found ${videoDevices.length} video devices:`, videoDevices.map((d) => d.label || d.deviceId) ); } } // Auto-select and preview first camera if none selected if (finalVideoDevices.length > 0 && !selectedCameraId) { const firstCameraId = finalVideoDevices[0].deviceId; // Stop temp stream since we'll create a fresh one with switchCameraPreview if (tempStream) { tempStream.getTracks().forEach((track) => track.stop()); } setCameraPermissionState("granted"); // Use the same logic as manual camera switching await switchCameraPreview(firstCameraId); } else if (tempStream) { // Stop temp stream if we didn't use it tempStream.getTracks().forEach((track) => track.stop()); } } catch (error) { setCameraPermissionState("denied"); if (!isAutoLoad) { toast({ title: "Camera Error", description: `Failed to load cameras: ${ error instanceof Error ? error.message : String(error) }`, variant: "destructive", }); } } finally { setIsLoadingCameras(false); } }, [selectedCameraId, toast] ); // Switch camera preview const switchCameraPreview = useCallback( async (deviceId: string) => { try { // Stop current preview stream if (previewStream) { previewStream.getTracks().forEach((track) => track.stop()); } // Start new stream with selected camera const newStream = await navigator.mediaDevices.getUserMedia({ video: { deviceId: { exact: deviceId }, width: { ideal: 1280 }, height: { ideal: 720 }, }, }); setPreviewStream(newStream); setSelectedCameraId(deviceId); } catch (error) { toast({ title: "Camera Error", description: `Failed to switch camera: ${ error instanceof Error ? error.message : String(error) }`, variant: "destructive", }); } }, [previewStream, toast] ); // Add a new camera to the recorder const handleAddCamera = useCallback(async () => { if (!cameraName.trim()) { toast({ title: "Camera Error", description: "Please enter a camera name", variant: "destructive", }); return; } if (hasRecordedFrames) { toast({ title: "Camera Error", description: "Cannot add cameras after recording has started", variant: "destructive", }); return; } if (!selectedCameraId) { toast({ title: "Camera Error", description: "Please select a camera first", variant: "destructive", }); return; } try { // Use the current preview stream (already running with correct camera) if (!previewStream) { throw new Error("No camera preview available"); } // Clone the stream for recording (keep preview running) const recordingStream = previewStream.clone(); // Add the new camera to our state setAdditionalCameras((prev) => ({ ...prev, [cameraName]: recordingStream, })); // Save camera configuration to persistent storage const selectedCamera = availableCameras.find( (cam) => cam.deviceId === selectedCameraId ); const newSettings = { ...recorderSettings, cameraConfigs: { ...recorderSettings.cameraConfigs, [cameraName]: { deviceId: selectedCameraId, deviceLabel: selectedCamera?.label || `Camera ${selectedCameraId.slice(0, 8)}...`, }, }, }; setRecorderSettings(newSettings); saveRecorderSettings(newSettings); setCameraName(""); // Clear the input toast({ title: "Camera Added", description: `Camera "${cameraName}" has been added to the recorder`, }); } catch (error) { toast({ title: "Camera Error", description: `Failed to access camera: ${ error instanceof Error ? error.message : String(error) }`, variant: "destructive", }); } }, [ cameraName, hasRecordedFrames, selectedCameraId, previewStream, availableCameras, recorderSettings, toast, ]); // Remove a camera from the recorder const handleRemoveCamera = useCallback( (name: string) => { if (hasRecordedFrames) { toast({ title: "Camera Error", description: "Cannot remove cameras after recording has started", variant: "destructive", }); return; } setAdditionalCameras((prev) => { const newCameras = { ...prev }; if (newCameras[name]) { // Stop the stream tracks newCameras[name].getTracks().forEach((track) => track.stop()); delete newCameras[name]; } return newCameras; }); // Remove camera configuration from persistent storage const newSettings = { ...recorderSettings, cameraConfigs: { ...recorderSettings.cameraConfigs }, }; delete newSettings.cameraConfigs[name]; setRecorderSettings(newSettings); saveRecorderSettings(newSettings); toast({ title: "Camera Removed", description: `Camera "${name}" has been removed`, }); }, [hasRecordedFrames, recorderSettings, toast] ); // Camera name editing functions const handleStartEditingCameraName = (cameraName: string) => { setEditingCameraName(cameraName); setEditingCameraNewName(cameraName); }; const handleConfirmCameraNameEdit = (oldName: string) => { if (editingCameraNewName.trim() && editingCameraNewName !== oldName) { const stream = additionalCameras[oldName]; if (stream) { // Update camera streams setAdditionalCameras((prev) => { const newCameras = { ...prev }; delete newCameras[oldName]; newCameras[editingCameraNewName.trim()] = stream; return newCameras; }); // Update camera configuration in persistent storage const oldConfig = recorderSettings.cameraConfigs[oldName]; if (oldConfig) { const newSettings = { ...recorderSettings, cameraConfigs: { ...recorderSettings.cameraConfigs }, }; delete newSettings.cameraConfigs[oldName]; newSettings.cameraConfigs[editingCameraNewName.trim()] = oldConfig; setRecorderSettings(newSettings); saveRecorderSettings(newSettings); } } } setEditingCameraName(null); setEditingCameraNewName(""); }; const handleCancelCameraNameEdit = () => { setEditingCameraName(null); setEditingCameraNewName(""); }; // Restore cameras from saved configurations const restoreSavedCameras = useCallback(async () => { const savedConfigs = recorderSettings.cameraConfigs; if (!savedConfigs || Object.keys(savedConfigs).length === 0) { return; } for (const [cameraName, config] of Object.entries(savedConfigs)) { try { // Check if this camera is still available const isDeviceAvailable = availableCameras.some( (cam) => cam.deviceId === config.deviceId ); if (!isDeviceAvailable) { console.warn( `Saved camera "${cameraName}" (${config.deviceId}) is no longer available` ); continue; } // Create stream for this saved camera const stream = await navigator.mediaDevices.getUserMedia({ video: { deviceId: { exact: config.deviceId }, width: { ideal: 1280 }, height: { ideal: 720 }, }, }); // Add to additional cameras setAdditionalCameras((prev) => ({ ...prev, [cameraName]: stream, })); } catch (error) { console.error(`Failed to restore camera "${cameraName}":`, error); // Remove invalid configuration const newSettings = { ...recorderSettings, cameraConfigs: { ...recorderSettings.cameraConfigs }, }; delete newSettings.cameraConfigs[cameraName]; setRecorderSettings(newSettings); saveRecorderSettings(newSettings); } } }, [availableCameras, recorderSettings]); // Auto-load cameras on component mount (only once) useEffect(() => { loadAvailableCameras(true); // eslint-disable-next-line react-hooks/exhaustive-deps }, []); // Empty dependency array to run only once // Handle video stream assignment - runs when stream changes OR when settings panel opens useEffect(() => { if (videoRef.current && previewStream) { videoRef.current.srcObject = previewStream; } }, [previewStream, showConfigure]); // Also depend on showConfigure so it re-runs when video element appears // Cleanup preview stream on unmount useEffect(() => { return () => { if (previewStream) { previewStream.getTracks().forEach((track) => track.stop()); } }; }, [previewStream]); // Restore saved cameras when available cameras are loaded useEffect(() => { if (availableCameras.length > 0 && cameraPermissionState === "granted") { restoreSavedCameras(); } }, [availableCameras, cameraPermissionState, restoreSavedCameras]); const handleDownloadZip = async () => { if (!recorderRef.current) { toast({ title: "Download Error", description: "Recorder not initialized", variant: "destructive", }); return; } await recorderRef.current.exportForLeRobot("zip-download"); toast({ title: "Download Started", description: "Your dataset is being downloaded as a ZIP file", }); }; const handleUploadToHuggingFace = async () => { if (!recorderRef.current) { toast({ title: "Upload Error", description: "Recorder not initialized", variant: "destructive", }); return; } if (!recorderSettings.huggingfaceApiKey) { toast({ title: "Upload Error", description: "Please enter your Hugging Face API key in Configure", variant: "destructive", }); return; } try { toast({ title: "Upload Started", description: "Uploading dataset to Hugging Face...", }); // Generate a unique repository name const repoName = `lerobot-recording-${Date.now()}`; const uploader = await recorderRef.current.exportForLeRobot( "huggingface", { repoName, accessToken: recorderSettings.huggingfaceApiKey, } ); uploader.addEventListener("progress", (event: Event) => { console.log(event); }); } catch (error) { const errorMessage = error instanceof Error ? error.message : "Failed to upload to Hugging Face"; toast({ title: "Upload Error", description: errorMessage, variant: "destructive", }); } }; // Helper function to format duration const formatDuration = (seconds: number): string => { const mins = Math.floor(seconds / 60); const secs = Math.floor(seconds % 60); return `${mins.toString().padStart(2, "0")}:${secs .toString() .padStart(2, "0")}`; }; return (

robot movement recorder

dataset recording{" "} interface

{/* Recorder Settings - Toggleable Inline */} {showConfigure && (
{/* Hugging Face Settings */}

Settings

{ const newSettings = { ...recorderSettings, huggingfaceApiKey: e.target.value, }; setRecorderSettings(newSettings); saveRecorderSettings(newSettings); }} type="password" className="bg-black/20 border-white/10" />

Required to upload datasets to Hugging Face Hub

{/* Camera Configuration */}

Camera Setup

{/* Left Column: Camera Selection & Adding */}
{/* Camera Selection and Refresh */} {/* Camera Access Request */} {cameraPermissionState === "unknown" && (

Camera access needed to configure cameras

)} {/* Camera Access Denied */} {cameraPermissionState === "denied" && (

Camera access denied

Please allow camera access in your browser settings and refresh

)} {/* Camera List with Refresh Button */} {cameraPermissionState === "granted" && availableCameras.length > 0 && (
)} {/* Camera Name Input */} {selectedCameraId && (
setCameraName(e.target.value)} className="bg-black/20 border-white/10" disabled={hasRecordedFrames} />

Give this camera a descriptive name for your recording setup

)} {/* Add Camera Button */} {selectedCameraId && (
)}
{/* Right Column: Camera Preview */}
{previewStream ? (
)} {/* Added Camera Previews */} {Object.keys(additionalCameras).length > 0 && (

Active Cameras

{Object.entries(additionalCameras).map(([cameraName, stream]) => (
{editingCameraName === cameraName ? (
setEditingCameraNewName(e.target.value) } className="text-xs h-6 bg-black/20 border-white/10" onKeyDown={(e) => { if (e.key === "Enter") { handleConfirmCameraNameEdit(cameraName); } else if (e.key === "Escape") { handleCancelCameraNameEdit(); } }} autoFocus />
) : ( )}
))}
)} {/* Episode Management & Dataset Actions */}
{/* Camera Configuration */}

Camera Setup

{showCameraConfig && (
{/* Left Column: Camera Preview & Selection */}

Camera Preview

{/* Camera Preview */}
{previewStream ? (
{/* Camera Controls */}
{availableCameras.length > 0 && (
)}
{/* Right Column: Camera Naming & Adding */}

Add to Recorder

setCameraName(e.target.value)} className="bg-black/20 border-white/10" disabled={hasRecordedFrames} />

Give this camera a descriptive name for your recording setup

)} {/* Display added cameras */} {Object.keys(additionalCameras).length > 0 && (

Added Cameras:

{Object.keys(additionalCameras).map((name) => ( {name} ))}
)}
{/* Reset Frames button moved to top bar */}
setHuggingfaceApiKey(e.target.value)} className="flex-1 bg-black/20 border-white/10" type="password" />
); }