commit 1

Dockerfile

@@ -0,0 +1,20 @@
+# Use an official Python runtime as a parent image
+FROM python:3.9
+
+# Set the working directory
+WORKDIR /app
+
+# Copy the requirements file
+COPY requirements.txt .
+
+# Install dependencies
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy the app code
+COPY . .
+
+# Expose port (Flask runs on 5000 by default)
+EXPOSE 5000
+
+# Run the application
+CMD ["python", "app.py"]

Language.py

@@ -0,0 +1,135 @@
+import pickle
+from ultralytics import YOLO
+import cv2
+import mediapipe as mp
+import numpy as np
+
+
+
+model = YOLO('best.pt')
+
+cap = cv2.VideoCapture(0)
+
+mp_hands = mp.solutions.hands
+mp_drawing = mp.solutions.drawing_utils
+mp_drawing_styles = mp.solutions.drawing_styles
+
+hands = mp_hands.Hands(static_image_mode=True, min_detection_confidence=0.3)
+language = ''
+labels_dict = {0: '0', 1: 'B', 2: 'C', 3: 'D', 4: 'E', 5: 'F', 6: 'G', 7: 'H', 8: 'I', 9: 'J', 10: 'K', 11: 'L', 12: 'M', 13: 'N', 14: 'O', 15: 'P', 16: 'Q', 17: 'R', 18: 'S', 19: 'T', 20: 'U', 21: 'V', 22: 'W', 23: 'X', 24: 'Y', 25: 'Z', 26: 'del', 27: 'nothing', 28: 'space'}
+while True:
+
+    data_aux = []
+    x_ = []
+    y_ = []
+    
+
+    ret, frame = cap.read()
+    if not ret:
+        print("Failed to capture frame. Exiting...")
+        break    
+
+    H, W, _ = frame.shape
+
+    frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+
+    results = hands.process(frame_rgb)
+    if results.multi_hand_landmarks:
+        for hand_landmarks in results.multi_hand_landmarks:
+            mp_drawing.draw_landmarks(
+                frame, 
+                hand_landmarks,  
+                mp_hands.HAND_CONNECTIONS,  
+                mp_drawing_styles.get_default_hand_landmarks_style(),
+                mp_drawing_styles.get_default_hand_connections_style())
+
+        for hand_landmarks in results.multi_hand_landmarks:
+            for i in range(len(hand_landmarks.landmark)):
+                x = hand_landmarks.landmark[i].x
+                y = hand_landmarks.landmark[i].y
+
+                x_.append(x)
+                y_.append(y)
+
+            for i in range(len(hand_landmarks.landmark)):
+                x = hand_landmarks.landmark[i].x
+                y = hand_landmarks.landmark[i].y
+                data_aux.append(x - min(x_))
+                data_aux.append(y - min(y_))
+
+        x1 = int(min(x_) * W) - 10
+        y1 = int(min(y_) * H) - 10
+
+        x2 = int(max(x_) * W) - 10
+        y2 = int(max(y_) * H) - 10
+       
+        prediction = model.predict(frame, conf=0.25, iou=0.45)
+        names_dict = prediction[0].names
+        probs = prediction[0].probs.data.numpy()
+        detected_gesture = names_dict[np.argmax(probs)]
+        print(names_dict[np.argmax(probs)])
+        print("Gesture:", detected_gesture)
+        if detected_gesture == 'A':
+            language = 'Arabic'
+        elif detected_gesture == 'B':
+            language = 'Bengali'
+        elif detected_gesture == 'C':
+            language = 'Chinese'
+        elif detected_gesture == 'D':
+            language = 'Dutch'
+        elif detected_gesture == 'E':
+            language = 'English'
+        elif detected_gesture == 'F':
+            language = 'French'
+        elif detected_gesture == 'G':
+            language = 'German'
+        elif detected_gesture == 'H':
+            language = 'Hindi'
+        elif detected_gesture == 'I':
+            language = 'Italian'
+        elif detected_gesture == 'J':
+            language = 'Japanese'
+        elif detected_gesture == 'K':
+            language = 'Korean'
+        elif detected_gesture == 'L':
+            language = 'Latin'
+        elif detected_gesture == 'M':
+            language = 'Malay'
+        elif detected_gesture == 'N':
+            language = 'Norwegian'
+        elif detected_gesture == 'O':
+            language = 'Oriya'
+        elif detected_gesture == 'P':
+            language = 'Polish'
+        elif detected_gesture == 'Q':
+            language = 'Quechua'
+        elif detected_gesture == 'R':
+            language = 'Russian'
+        elif detected_gesture == 'S':
+            language = 'Spanish'
+        elif detected_gesture == 'T':
+            language = 'Turkish'
+        elif detected_gesture == 'U':
+            language = 'Urdu'
+        elif detected_gesture == 'V':
+            language = 'Vietnamese'
+        elif detected_gesture == 'W':
+            language = 'Welsh'
+        elif detected_gesture == 'X':
+            language = 'Xhosa'
+        elif detected_gesture == 'Y':
+            language = 'Yoruba'
+        elif detected_gesture == 'Z':
+            language = 'Zulu'
+
+
+        cv2.rectangle(frame, (x1, y1), (x2, y2), (0, 0, 0), 4)
+        cv2.putText(frame, language, (x1, y1 - 10), cv2.FONT_HERSHEY_SIMPLEX, 1.3, (0, 0, 0), 3,
+                    cv2.LINE_AA)
+
+    cv2.imshow('frame', frame)
+    cv2.waitKey(1)
+
+
+cap.release()
+cv2.destroyAllWindows()

README.md

@@ -1,10 +1,13 @@
----
-title: HandyLabel
-emoji: 🚀
-colorFrom: pink
-colorTo: green
-sdk: docker
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# handgesture-recognition-system
+# handgesture-recognition-system
+
+main.py - file to run the system and give camera access to the python.
+
+
+
+
+
+
+
+yolo.ipynb -  the file that is used to train the ASL dataset to get our custom model
+

Web app/app.py

@@ -0,0 +1,664 @@
+import csv
+import time
+from datetime import datetime, timedelta
+from ultralytics import YOLO
+import cv2
+import mediapipe as mp
+import numpy as np
+from flask import Flask, render_template, Response, redirect, url_for, request, send_from_directory, flash
+import os
+import plotly.express as px
+import pandas as pd
+from werkzeug.utils import secure_filename
+import json
+import matplotlib.pyplot as plt
+import uuid
+import random  # Make sure to import the random module
+from datetime import datetime  # Import datetime for timestamp
+import string  # Add this line to use string.ascii_letters and string.digits
+import pyaudio
+import wave
+import whisper
+from transformers import pipeline
+import csv
+import os
+import pandas as pd
+from flask import Flask, render_template, request, redirect, url_for, send_from_directory, flash
+from flask import  jsonify
+from transformers import T5Tokenizer, T5ForConditionalGeneration
+import torch
+from openai import OpenAI
+from io import StringIO
+import re
+from flask import session
+
+
+# Flask app setup
+app = Flask(__name__)
+app.config['UPLOAD_FOLDER'] = 'uploads'
+app.secret_key = 'supersecretkey'
+
+# Custom function to load YOLO model safely
+def safe_load_yolo_model(model_path):
+    try:
+        return YOLO(model_path)
+    except Exception as e:
+        print(f"Failed to load model: {e}")
+        raise
+
+# Load YOLO model
+model_path = 'best.pt'
+model = safe_load_yolo_model(model_path)
+
+
+
+mp_hands = mp.solutions.hands
+mp_drawing = mp.solutions.drawing_utils
+mp_drawing_styles = mp.solutions.drawing_styles
+
+# Load Whisper model for speech-to-text
+whisper_model = whisper.load_model("base")
+
+# Variables to hold CSV data and other states between requests
+original_data = None
+updated_data = None
+csv_filename = None
+
+
+hands = mp_hands.Hands(static_image_mode=True, min_detection_confidence=0.3)
+
+# Initialize variables for tracking gestures
+previous_gesture = None
+gesture_start_time = None
+gesture_data_list = []
+capture_flag = True  # This flag is used to indicate when to capture
+start_recording_time = None  # To record the start time of the session
+
+# Default labels dictionary
+labels_dict = {0: 'fist', 1: 'ok', 2: 'peace', 3: 'stop', 4: 'two up'}
+custom_labels_dict = labels_dict.copy()  # To store custom labels set by user
+
+# Initialize OpenAI client
+client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY"))
+
+# Function to interact with OpenAI's GPT model using streaming
+def get_gpt_instruction_response(instruction, csv_data):
+    messages = [
+        {"role": "system", "content": "You are a helpful assistant that processes CSV files."},
+        {"role": "user", "content": f"Here is a CSV data:\n\n{csv_data}\n\nThe user has requested the following change: {instruction}\n\nPlease process the data accordingly and return the modified CSV."}
+    ]
+    
+    # Stream response from OpenAI API
+    stream = client.chat.completions.create(
+        model="gpt-4o-mini",  # or "gpt-3.5-turbo"
+        messages=messages,
+        stream=True,
+    )
+
+    response = ""
+    for chunk in stream:
+        if chunk.choices[0].delta.content is not None:
+            response += chunk.choices[0].delta.content
+    
+    return response.strip()
+
+
+# Function to read CSV and convert it to string
+def read_csv_to_string(file_path):
+    df = pd.read_csv(file_path)
+    return df.to_csv(index=False)
+
+
+# Function to write modified CSV string to a file
+def write_csv_from_string(csv_string, output_file_path):
+    with open(output_file_path, 'w') as file:
+        file.write(csv_string)
+
+# Function to record audio
+def record_audio(filename, duration=10):
+    chunk = 1024
+    sample_format = pyaudio.paInt16
+    channels = 1
+    fs = 44100
+
+    p = pyaudio.PyAudio()
+
+    print('Recording...')
+    stream = p.open(format=sample_format, channels=channels, rate=fs, frames_per_buffer=chunk, input=True)
+    frames = []
+
+    for _ in range(0, int(fs / chunk * duration)):
+        data = stream.read(chunk)
+        frames.append(data)
+
+    stream.stop_stream()
+    stream.close()
+    p.terminate()
+    print('Finished recording.')
+
+    wf = wave.open(filename, 'wb')
+    wf.setnchannels(channels)
+    wf.setsampwidth(p.get_sample_size(sample_format))
+    wf.setframerate(fs)
+    wf.writeframes(b''.join(frames))
+    wf.close()
+
+# Function to transcribe audio using Whisper
+def transcribe_audio(file_path):
+    result = whisper_model.transcribe(file_path)
+    return result["text"]
+
+
+
+@app.route('/')
+def index():
+    return render_template('index.html')
+
+@app.route('/set_labels', methods=['GET', 'POST'])
+def set_labels():
+    global custom_labels_dict
+    if request.method == 'POST':
+        custom_labels_dict[0] = request.form['label1']
+        custom_labels_dict[1] = request.form['label2']
+        custom_labels_dict[2] = request.form['label3']
+        custom_labels_dict[3] = request.form['label4']
+        custom_labels_dict[4] = request.form['label5']
+        # Remove empty labels
+        custom_labels_dict = {k: v for k, v in custom_labels_dict.items() if v}
+        return redirect(url_for('recognize'))
+    return render_template('set_labels.html')
+
+@app.route('/recognize')
+def recognize():
+    return render_template('recognize.html')
+
+@app.route('/video_feed')
+def video_feed():
+    return Response(generate_frames(), mimetype='multipart/x-mixed-replace; boundary=frame')
+
+def generate_frames():
+    global previous_gesture, gesture_start_time, gesture_data_list, capture_flag, start_recording_time
+
+    # Initialize start recording time
+    start_recording_time = datetime.now()
+
+    cap = cv2.VideoCapture(0)
+
+    while capture_flag:
+        data_aux = []
+        x_ = []
+        y_ = []
+
+        ret, frame = cap.read()
+        if not ret:
+            break
+
+        H, W, _ = frame.shape
+
+        frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+
+        results = hands.process(frame_rgb)
+        if results.multi_hand_landmarks:
+            for hand_landmarks in results.multi_hand_landmarks:
+                mp_drawing.draw_landmarks(
+                    frame,
+                    hand_landmarks,
+                    mp_hands.HAND_CONNECTIONS,
+                    mp_drawing_styles.get_default_hand_landmarks_style(),
+                    mp_drawing_styles.get_default_hand_connections_style())
+
+            for hand_landmarks in results.multi_hand_landmarks:
+                for i in range(len(hand_landmarks.landmark)):
+                    x = hand_landmarks.landmark[i].x
+                    y = hand_landmarks.landmark[i].y
+
+                    x_.append(x)
+                    y_.append(y)
+
+                for i in range(len(hand_landmarks.landmark)):
+                    x = hand_landmarks.landmark[i].x
+                    y = hand_landmarks.landmark[i].y
+                    data_aux.append(x - min(x_))
+                    data_aux.append(y - min(y_))
+
+            x1 = int(min(x_) * W) - 10
+            y1 = int(min(y_) * H) - 10
+
+            x2 = int(max(x_) * W) + 10
+            y2 = int(max(y_) * H) + 10
+
+            prediction = model.predict(frame, conf=0.25, iou=0.45)
+            probs = prediction[0].probs.data.numpy()
+            detected_gesture_index = np.argmax(probs)
+            detected_gesture = custom_labels_dict.get(detected_gesture_index, None)
+
+            if detected_gesture is None:
+                continue
+
+            # Get the current timestamp and calculate relative time from the start
+            current_time = datetime.now()
+            relative_time = current_time - start_recording_time
+
+            # Check if the detected gesture has changed
+            if detected_gesture != previous_gesture:
+                # If the detected gesture has changed, calculate the duration of the previous gesture
+                if previous_gesture is not None:
+                    gesture_end_time = relative_time.total_seconds()
+                    gesture_duration = gesture_end_time - gesture_start_time
+                    # Store the detected gesture, start time, end time, and duration in the list
+                    gesture_data_list.append([previous_gesture, gesture_start_time, gesture_end_time, round(gesture_duration, 2)])
+
+                # Update the previous gesture and its start time
+                previous_gesture = detected_gesture
+                gesture_start_time = relative_time.total_seconds()
+
+            # Draw rectangle around the detected gesture
+            cv2.rectangle(frame, (x1, y1), (x2, y2), (0, 0, 0), 4)
+            cv2.putText(frame, detected_gesture, (x1, y1 - 10), cv2.FONT_HERSHEY_SIMPLEX, 1.3, (0, 255, 0), 3, cv2.LINE_AA)
+
+        ret, buffer = cv2.imencode('.jpg', frame)
+        frame = buffer.tobytes()
+
+        yield (b'--frame\r\n'
+               b'Content-Type: image/jpeg\r\n\r\n' + frame + b'\r\n')
+
+    cap.release()
+ 
+
+@app.route('/upload_csv', methods=['POST'])
+def upload_csv():
+    try:
+        # Handle file upload
+        file = request.files.get('csv_file')
+        if file:
+            file_path = os.path.join(app.config['UPLOAD_FOLDER'], secure_filename(file.filename))
+            file.save(file_path)
+            flash("CSV file uploaded successfully!", "success")
+
+            # Load the uploaded CSV file as original data
+            original_df = pd.read_csv(file_path)
+            original_data = original_df.to_dict('records')
+            columns = original_df.columns.tolist()
+
+            # Store the original data and file path in the session
+            session['original_data'] = original_data
+            session['columns'] = columns
+            session['file_path'] = file_path
+
+        else:
+            flash("Please upload a CSV file.", "warning")
+    except Exception as e:
+        app.logger.error(f"Error in upload_csv route: {e}")
+        flash("An unexpected error occurred. Please check the logs.", "danger")
+
+    return redirect(url_for('edit_csv'))
+
+
+@app.route('/edit_csv', methods=['GET', 'POST'])
+def edit_csv():
+    updated_data = None
+    original_data = session.get('original_data', None)
+    columns = session.get('columns', None)
+
+    if request.method == 'POST':
+        try:
+            # Ensure a file has been uploaded
+            file_path = session.get('file_path')
+            if not file_path:
+                flash("Please upload a CSV file first.", "warning")
+                return redirect(url_for('edit_csv'))
+
+            # Load the CSV data as string for processing
+            csv_data = read_csv_to_string(file_path)
+
+            # Get the instruction from the form
+            instruction = request.form.get('transcription', "").strip()
+            if not instruction:
+                flash("Please provide an instruction.", "warning")
+                return redirect(url_for('edit_csv'))
+
+            # Process the CSV using OpenAI API
+            raw_output = get_gpt_instruction_response(instruction, csv_data)
+
+            # Extract and clean only the CSV part from the GPT output
+            csv_pattern = re.compile(r"(?<=```)([\s\S]*?)(?=```)|([\s\S]*)", re.DOTALL)
+            match = csv_pattern.search(raw_output)
+            if match:
+                csv_content = match.group(1) or match.group(2)
+                csv_content = csv_content.strip()  # Clean up leading/trailing spaces
+            else:
+                raise ValueError("No valid CSV content found in GPT output.")
+
+            # Further cleaning: Remove any lines not starting with valid CSV columns
+            csv_lines = csv_content.splitlines()
+            cleaned_csv_lines = [
+                line for line in csv_lines if ',' in line and not line.startswith("Here is")
+            ]
+            cleaned_csv_content = "\n".join(cleaned_csv_lines)
+
+            # Save the modified CSV to a file
+            modified_file_path = os.path.join(app.config['UPLOAD_FOLDER'], 'modified_gesture_data.csv')
+            with open(modified_file_path, 'w') as f:
+                f.write(cleaned_csv_content)
+
+            # Load the modified data
+            updated_data = pd.read_csv(StringIO(cleaned_csv_content)).to_dict('records')
+
+            # Store the updated data in the session
+            session['updated_data'] = updated_data
+
+        except Exception as e:
+            app.logger.error(f"Error in edit_csv route: {e}")
+            flash("An unexpected error occurred. Please check the logs.", "danger")
+
+    # Load updated data from session if available
+    updated_data = session.get('updated_data', None)
+
+    return render_template('edit_csv.html', original_data=original_data, updated_data=updated_data, columns=columns)
+
+
+
+# Route: Download Modified CSV
+@app.route('/download_csv_updated')
+def download_csv_updated():
+    file_path = os.path.join(app.config['UPLOAD_FOLDER'], 'modified_gesture_data.csv')
+    if not os.path.isfile(file_path):
+        flash("Updated CSV file not found!", "warning")
+        return redirect(url_for('edit_csv'))
+    return send_from_directory(app.config['UPLOAD_FOLDER'], 'modified_gesture_data.csv', as_attachment=True)
+
+
+
+# Process uploaded audio using Whisper
+@app.route('/process_audio', methods=['POST'])
+def process_audio():
+    if 'audio' not in request.files:
+        return jsonify({'error': 'No audio file provided'}), 400
+
+    audio_file = request.files['audio']
+    audio_file_path = 'recorded_audio.wav'
+    audio_file.save(audio_file_path)
+
+    # Transcribe audio using Whisper
+    transcription = transcribe_audio(audio_file_path)
+    return jsonify({'transcription': transcription})
+
+
+
+@app.route('/data_view', methods=['GET'])
+def data_view():
+    csv_file = request.args.get('csv_file', 'static/gesture_data.csv')
+    gesture_data = load_csv_data(csv_file)
+
+    df = pd.DataFrame(gesture_data, columns=['Gesture', 'Start Time', 'End Time', 'Duration'])
+    gesture_counts = df['Gesture'].value_counts().reset_index()
+    gesture_counts.columns = ['Gesture', 'Count']
+    fig = px.pie(gesture_counts, values='Count', names='Gesture', title='Gesture Distribution')
+    html_chart = fig.to_html(full_html=False)
+
+    return render_template('data.html', gesture_data=gesture_data, html_chart=html_chart)
+
+
+import pandas as pd
+from flask import render_template
+
+@app.route('/datadiff')
+def datadiff():
+    # Load original and modified CSV files
+    original_csv_path = os.path.join(app.config['UPLOAD_FOLDER'], 'gesture_data.csv')
+    modified_csv_path = os.path.join(app.config['UPLOAD_FOLDER'], 'modified_gesture_data.csv')
+
+    # Read the CSVs into pandas DataFrames
+    original_csv = pd.read_csv(original_csv_path)
+    modified_csv = pd.read_csv(modified_csv_path)
+
+    # Render the datadiff.html page with the data for comparison
+    return render_template('datadiff.html', original_csv=original_csv, modified_csv=modified_csv)
+
+
+def load_csv_data(file_path):
+    gesture_data = []
+    with open(file_path, 'r') as csvfile:
+        reader = csv.reader(csvfile)
+        next(reader)
+        for row in reader:
+            gesture_data.append(row)
+    return gesture_data
+
+@app.route('/save_data')
+def save_gesture_data():
+    global capture_flag
+    capture_flag = False
+
+    # Ensure gesture data is actually populated
+    print("Saving gesture data:", gesture_data_list)
+
+    # Ensure the static directory exists
+    os.makedirs('static', exist_ok=True)
+
+    # Save data to JSON file in Label Studio-compatible format
+    json_file_path = os.path.join('static', 'gesture_data_labelstudio.json')
+    save_label_studio_json(gesture_data_list, json_file_path)
+
+    # Save data to CSV file for visualization
+    csv_file_path = os.path.join('static', 'gesture_data.csv')
+    save_gesture_csv(gesture_data_list, csv_file_path)
+
+    return redirect(url_for('data'))
+
+import random  # Make sure to import the random module
+import uuid  # Make sure to import uuid for unique IDs
+from datetime import datetime  # Import datetime for timestamp
+
+def generate_alphanumeric_id(length=5):
+    """Generates a random alphanumeric ID."""
+    return ''.join(random.choices(string.ascii_letters + string.digits, k=length))
+
+def save_label_studio_json(gesture_data, file_path):
+    current_time = datetime.utcnow().isoformat() + "Z"
+    
+    # Create a single task with all annotations
+    annotations = {
+        "id": 1,  # Task ID
+        "annotations": [
+            {
+                "id": 1,  # Annotation ID
+                "completed_by": 1,
+                "result": [],
+                "was_cancelled": False,
+                "ground_truth": False,
+                "created_at": current_time,
+                "updated_at": current_time,
+                "draft_created_at": current_time,
+                "lead_time": sum(duration for _, _, _, duration in gesture_data),
+                "prediction": {},
+                "result_count": 0,
+                "unique_id": str(uuid.uuid4()),
+                "import_id": None,
+                "last_action": None,
+                "task": 1,
+                "project": 25,
+                "updated_by": 1,
+                "parent_prediction": None,
+                "parent_annotation": None,
+                "last_created_by": None
+            }
+        ],
+        "file_upload": "1212df4d-HandyLabels.MP4",
+        "drafts": [],
+        "predictions": [],
+        "data": {
+            "video_url": "/data/upload/30/030cca83-Video_1.mp4"
+        },
+        "meta": {},
+        "created_at": current_time,
+        "updated_at": current_time,
+        "inner_id": 1,
+        "total_annotations": 1,
+        "cancelled_annotations": 0,
+        "total_predictions": 0,
+        "comment_count": 0,
+        "unresolved_comment_count": 0,
+        "last_comment_updated_at": None,
+        "project": 25,
+        "updated_by": 1,
+        "comment_authors": []
+    }
+
+    # Add each gesture as an individual result within the annotation
+    for gesture, start_time, end_time, duration in gesture_data:
+        annotation_result = {
+            "original_length": end_time - start_time,
+            "value": {
+                "start": start_time,
+                "end": end_time,
+                "channel": 0,
+                "labels": [gesture]
+            },
+            "id": generate_alphanumeric_id(),  # Generate a unique 5-character alphanumeric ID for each result
+            "from_name": "tricks",
+            "to_name": "audio",
+            "type": "labels",
+            "origin": "manual"
+        }
+        annotations["annotations"][0]["result"].append(annotation_result)
+
+    # Save the consolidated JSON to the file
+    with open(file_path, 'w') as json_file:
+        json.dump([annotations], json_file, indent=2)
+
+    print(f"Label Studio JSON saved to: {file_path}")
+
+
+def save_gesture_csv(gesture_data, file_path):
+    with open(file_path, 'w', newline='') as csvfile:
+        writer = csv.writer(csvfile)
+        writer.writerow(['Gesture', 'Start Time', 'End Time', 'Duration'])
+        for gesture, start_time, end_time, duration in gesture_data:
+            writer.writerow([gesture, start_time, end_time, duration])
+
+@app.route('/data')
+def data():
+    gesture_data = load_csv_data()
+
+    # Convert to DataFrame for easier manipulation
+    df = pd.DataFrame(gesture_data, columns=['Gesture', 'Start Time', 'End Time', 'Duration'])
+
+    # Count occurrences of each gesture
+    gesture_counts = df['Gesture'].value_counts().reset_index()
+    gesture_counts.columns = ['Gesture', 'Count']
+
+    # Create the pie chart using Plotly
+    fig = px.pie(gesture_counts, values='Count', names='Gesture', title='Gesture Distribution')
+
+    # Convert the plotly chart to HTML
+    html_chart = fig.to_html(full_html=False)
+
+    return render_template('data.html', gesture_data=gesture_data, html_chart=html_chart)
+
+def load_csv_data():
+    gesture_data = []
+    with open('static/gesture_data.csv', 'r') as csvfile:
+        reader = csv.reader(csvfile)
+        next(reader)  # Skip the header row
+        for row in reader:
+            gesture_data.append(row)
+    return gesture_data
+
+@app.route('/download_json')
+def download_json():
+    file_path = os.path.join('static', 'gesture_data_labelstudio.json')
+
+    if not os.path.isfile(file_path):
+        return "JSON file not found!", 404
+
+    return send_from_directory('static', 'gesture_data_labelstudio.json', as_attachment=True)
+
+@app.route('/download_csv')
+def download_csv():
+    filename = request.args.get('filename')
+    if filename == 'original':
+        path = os.path.join(app.config['UPLOAD_FOLDER'], 'gesture_data.csv')
+    elif filename == 'updated':
+        path = os.path.join(app.config['UPLOAD_FOLDER'], 'modified_gesture_data.csv')
+    else:
+        flash('Invalid file requested')
+        return redirect(url_for('edit_csv'))
+
+    if not os.path.exists(path):
+        flash('File not found!')
+        return redirect(url_for('edit_csv'))
+
+    return send_from_directory(app.config['UPLOAD_FOLDER'], os.path.basename(path), as_attachment=True)
+
+
+
+# New route to download the modified CSV
+@app.route('/download_csv_modified')
+def download_csv_modified():
+    file_path = os.path.join(app.config['UPLOAD_FOLDER'], 'modified_gesture_data.csv')
+
+    if not os.path.isfile(file_path):
+        return "Modified CSV file not found!", 404
+
+    return send_from_directory(app.config['UPLOAD_FOLDER'], 'modified_gesture_data.csv', as_attachment=True)
+
+
+# Import Data Functionality to Visualize Imported CSV
+@app.route('/import_data', methods=['GET', 'POST'])
+def import_data():
+    if request.method == 'POST':
+        if 'file' not in request.files:
+            flash('No file part')
+            return redirect(request.url)
+        file = request.files['file']
+        if file.filename == '':
+            flash('No selected file')
+            return redirect(request.url)
+        if file:
+            filename = secure_filename(file.filename)
+            file_path = os.path.join(app.config['UPLOAD_FOLDER'], filename)
+            file.save(file_path)
+            return redirect(url_for('visualize_data', file_path=file_path))
+    return render_template('import_data.html')
+
+@app.route('/visualize_data')
+def visualize_data():
+    file_path = request.args.get('file_path')
+
+    if not os.path.exists(file_path):
+        return "The file could not be found.", 404
+
+    return visualize_csv(file_path)
+
+def visualize_csv(file_path):
+    try:
+        # Load gesture data from CSV and process it for visualization
+        data = pd.read_csv(file_path)
+
+        # Check if necessary columns are present
+        required_columns = ['Gesture', 'Start Time', 'End Time', 'Duration']
+        if not set(required_columns).issubset(data.columns):
+            return f"The uploaded CSV must contain the following columns: {required_columns}", 400
+
+        # Extract relevant columns
+        gesture_df = data[required_columns]
+
+        # Generate a pie chart for gesture distribution
+        gesture_counts = gesture_df['Gesture'].value_counts().reset_index()
+        gesture_counts.columns = ['Gesture', 'Count']
+
+        # Create the pie chart using Plotly
+        fig = px.pie(gesture_counts, values='Count', names='Gesture', title='Gesture Distribution')
+
+        # Convert the plotly chart to HTML
+        html_chart = fig.to_html(full_html=False)
+
+        # Render the data.html template with the gesture data and chart
+        return render_template('data.html', gesture_data=gesture_df.to_dict('records'), html_chart=html_chart)
+
+    except Exception as e:
+        return f"An error occurred while processing the file: {str(e)}", 500
+
+if __name__ == '__main__':
+    app.run(debug=True)

Web app/requirements.txt

@@ -0,0 +1,5 @@
+Flask
+opencv-python-headless
+mediapipe
+numpy
+ultralytics

Web app/static/gesture_data.csv

@@ -0,0 +1,30 @@
+Gesture,Start Time,End Time,Duration
+a,2.703457,2.791474,0.09
+c,2.791474,3.177169,0.39
+a,3.177169,3.477513,0.3
+c,3.477513,3.548062,0.07
+a,3.548062,3.834808,0.29
+c,3.834808,4.306889,0.47
+a,4.306889,4.577862,0.27
+d,4.577862,5.560682,0.98
+a,5.560682,6.556111,1.0
+d,6.556111,7.764478,1.21
+a,7.764478,8.422632,0.66
+d,8.422632,8.488039,0.07
+b,8.488039,8.556642,0.07
+d,8.556642,8.630307,0.07
+b,8.630307,9.021776,0.39
+a,9.021776,9.45754,0.44
+b,9.45754,9.521933,0.06
+c,9.521933,9.657519,0.14
+b,9.657519,9.762675,0.11
+c,9.762675,9.821062,0.06
+b,9.821062,9.892455,0.07
+a,9.892455,9.954735,0.06
+c,9.954735,10.020776,0.07
+a,10.020776,10.08684,0.07
+b,10.08684,10.153284,0.07
+a,10.153284,10.221532,0.07
+c,10.221532,10.419941,0.2
+a,10.419941,10.624734,0.2
+c,10.624734,10.691169,0.07

Web app/static/gesture_data_labelstudio.json

@@ -0,0 +1,513 @@
+[
+  {
+    "id": 1,
+    "annotations": [
+      {
+        "id": 1,
+        "completed_by": 1,
+        "result": [
+          {
+            "original_length": 0.08801699999999979,
+            "value": {
+              "start": 2.703457,
+              "end": 2.791474,
+              "channel": 0,
+              "labels": [
+                "a"
+              ]
+            },
+            "id": "KOibD",
+            "from_name": "tricks",
+            "to_name": "audio",
+            "type": "labels",
+            "origin": "manual"
+          },
+          {
+            "original_length": 0.3856950000000001,
+            "value": {
+              "start": 2.791474,
+              "end": 3.177169,
+              "channel": 0,
+              "labels": [
+                "c"
+              ]
+            },
+            "id": "W7eFd",
+            "from_name": "tricks",
+            "to_name": "audio",
+            "type": "labels",
+            "origin": "manual"
+          },
+          {
+            "original_length": 0.30034399999999994,
+            "value": {
+              "start": 3.177169,
+              "end": 3.477513,
+              "channel": 0,
+              "labels": [
+                "a"
+              ]
+            },
+            "id": "kKjBN",
+            "from_name": "tricks",
+            "to_name": "audio",
+            "type": "labels",
+            "origin": "manual"
+          },
+          {
+            "original_length": 0.07054899999999975,
+            "value": {
+              "start": 3.477513,
+              "end": 3.548062,
+              "channel": 0,
+              "labels": [
+                "c"
+              ]
+            },
+            "id": "1vHtz",
+            "from_name": "tricks",
+            "to_name": "audio",
+            "type": "labels",
+            "origin": "manual"
+          },
+          {
+            "original_length": 0.2867460000000004,
+            "value": {
+              "start": 3.548062,
+              "end": 3.834808,
+              "channel": 0,
+              "labels": [
+                "a"
+              ]
+            },
+            "id": "YWcJm",
+            "from_name": "tricks",
+            "to_name": "audio",
+            "type": "labels",
+            "origin": "manual"
+          },
+          {
+            "original_length": 0.47208099999999975,
+            "value": {
+              "start": 3.834808,
+              "end": 4.306889,
+              "channel": 0,
+              "labels": [
+                "c"
+              ]
+            },
+            "id": "02CzC",
+            "from_name": "tricks",
+            "to_name": "audio",
+            "type": "labels",
+            "origin": "manual"
+          },
+          {
+            "original_length": 0.2709729999999997,
+            "value": {
+              "start": 4.306889,
+              "end": 4.577862,
+              "channel": 0,
+              "labels": [
+                "a"
+              ]
+            },
+            "id": "bEN1o",
+            "from_name": "tricks",
+            "to_name": "audio",
+            "type": "labels",
+            "origin": "manual"
+          },
+          {
+            "original_length": 0.9828200000000002,
+            "value": {
+              "start": 4.577862,
+              "end": 5.560682,
+              "channel": 0,
+              "labels": [
+                "d"
+              ]
+            },
+            "id": "mBtVB",
+            "from_name": "tricks",
+            "to_name": "audio",
+            "type": "labels",
+            "origin": "manual"
+          },
+          {
+            "original_length": 0.9954289999999997,
+            "value": {
+              "start": 5.560682,
+              "end": 6.556111,
+              "channel": 0,
+              "labels": [
+                "a"
+              ]
+            },
+            "id": "yiqVI",
+            "from_name": "tricks",
+            "to_name": "audio",
+            "type": "labels",
+            "origin": "manual"
+          },
+          {
+            "original_length": 1.2083670000000009,
+            "value": {
+              "start": 6.556111,
+              "end": 7.764478,
+              "channel": 0,
+              "labels": [
+                "d"
+              ]
+            },
+            "id": "WbZ1U",
+            "from_name": "tricks",
+            "to_name": "audio",
+            "type": "labels",
+            "origin": "manual"
+          },
+          {
+            "original_length": 0.6581539999999997,
+            "value": {
+              "start": 7.764478,
+              "end": 8.422632,
+              "channel": 0,
+              "labels": [
+                "a"
+              ]
+            },
+            "id": "3eHhx",
+            "from_name": "tricks",
+            "to_name": "audio",
+            "type": "labels",
+            "origin": "manual"
+          },
+          {
+            "original_length": 0.06540700000000044,
+            "value": {
+              "start": 8.422632,
+              "end": 8.488039,
+              "channel": 0,
+              "labels": [
+                "d"
+              ]
+            },
+            "id": "Ohri7",
+            "from_name": "tricks",
+            "to_name": "audio",
+            "type": "labels",
+            "origin": "manual"
+          },
+          {
+            "original_length": 0.06860299999999953,
+            "value": {
+              "start": 8.488039,
+              "end": 8.556642,
+              "channel": 0,
+              "labels": [
+                "b"
+              ]
+            },
+            "id": "OvB28",
+            "from_name": "tricks",
+            "to_name": "audio",
+            "type": "labels",
+            "origin": "manual"
+          },
+          {
+            "original_length": 0.07366500000000009,
+            "value": {
+              "start": 8.556642,
+              "end": 8.630307,
+              "channel": 0,
+              "labels": [
+                "d"
+              ]
+            },
+            "id": "N2Jwo",
+            "from_name": "tricks",
+            "to_name": "audio",
+            "type": "labels",
+            "origin": "manual"
+          },
+          {
+            "original_length": 0.39146899999999896,
+            "value": {
+              "start": 8.630307,
+              "end": 9.021776,
+              "channel": 0,
+              "labels": [
+                "b"
+              ]
+            },
+            "id": "RHjHA",
+            "from_name": "tricks",
+            "to_name": "audio",
+            "type": "labels",
+            "origin": "manual"
+          },
+          {
+            "original_length": 0.4357640000000007,
+            "value": {
+              "start": 9.021776,
+              "end": 9.45754,
+              "channel": 0,
+              "labels": [
+                "a"
+              ]
+            },
+            "id": "IBLPf",
+            "from_name": "tricks",
+            "to_name": "audio",
+            "type": "labels",
+            "origin": "manual"
+          },
+          {
+            "original_length": 0.06439300000000081,
+            "value": {
+              "start": 9.45754,
+              "end": 9.521933,
+              "channel": 0,
+              "labels": [
+                "b"
+              ]
+            },
+            "id": "S4wb0",
+            "from_name": "tricks",
+            "to_name": "audio",
+            "type": "labels",
+            "origin": "manual"
+          },
+          {
+            "original_length": 0.13558599999999998,
+            "value": {
+              "start": 9.521933,
+              "end": 9.657519,
+              "channel": 0,
+              "labels": [
+                "c"
+              ]
+            },
+            "id": "djRDv",
+            "from_name": "tricks",
+            "to_name": "audio",
+            "type": "labels",
+            "origin": "manual"
+          },
+          {
+            "original_length": 0.10515599999999914,
+            "value": {
+              "start": 9.657519,
+              "end": 9.762675,
+              "channel": 0,
+              "labels": [
+                "b"
+              ]
+            },
+            "id": "ZEXyc",
+            "from_name": "tricks",
+            "to_name": "audio",
+            "type": "labels",
+            "origin": "manual"
+          },
+          {
+            "original_length": 0.058386999999999745,
+            "value": {
+              "start": 9.762675,
+              "end": 9.821062,
+              "channel": 0,
+              "labels": [
+                "c"
+              ]
+            },
+            "id": "bCt3N",
+            "from_name": "tricks",
+            "to_name": "audio",
+            "type": "labels",
+            "origin": "manual"
+          },
+          {
+            "original_length": 0.07139300000000048,
+            "value": {
+              "start": 9.821062,
+              "end": 9.892455,
+              "channel": 0,
+              "labels": [
+                "b"
+              ]
+            },
+            "id": "UKpDs",
+            "from_name": "tricks",
+            "to_name": "audio",
+            "type": "labels",
+            "origin": "manual"
+          },
+          {
+            "original_length": 0.06227999999999945,
+            "value": {
+              "start": 9.892455,
+              "end": 9.954735,
+              "channel": 0,
+              "labels": [
+                "a"
+              ]
+            },
+            "id": "4MMP5",
+            "from_name": "tricks",
+            "to_name": "audio",
+            "type": "labels",
+            "origin": "manual"
+          },
+          {
+            "original_length": 0.06604100000000024,
+            "value": {
+              "start": 9.954735,
+              "end": 10.020776,
+              "channel": 0,
+              "labels": [
+                "c"
+              ]
+            },
+            "id": "lWUrb",
+            "from_name": "tricks",
+            "to_name": "audio",
+            "type": "labels",
+            "origin": "manual"
+          },
+          {
+            "original_length": 0.06606400000000079,
+            "value": {
+              "start": 10.020776,
+              "end": 10.08684,
+              "channel": 0,
+              "labels": [
+                "a"
+              ]
+            },
+            "id": "o59rB",
+            "from_name": "tricks",
+            "to_name": "audio",
+            "type": "labels",
+            "origin": "manual"
+          },
+          {
+            "original_length": 0.06644399999999884,
+            "value": {
+              "start": 10.08684,
+              "end": 10.153284,
+              "channel": 0,
+              "labels": [
+                "b"
+              ]
+            },
+            "id": "pUPWY",
+            "from_name": "tricks",
+            "to_name": "audio",
+            "type": "labels",
+            "origin": "manual"
+          },
+          {
+            "original_length": 0.06824800000000053,
+            "value": {
+              "start": 10.153284,
+              "end": 10.221532,
+              "channel": 0,
+              "labels": [
+                "a"
+              ]
+            },
+            "id": "WilaI",
+            "from_name": "tricks",
+            "to_name": "audio",
+            "type": "labels",
+            "origin": "manual"
+          },
+          {
+            "original_length": 0.19840899999999984,
+            "value": {
+              "start": 10.221532,
+              "end": 10.419941,
+              "channel": 0,
+              "labels": [
+                "c"
+              ]
+            },
+            "id": "YBJjW",
+            "from_name": "tricks",
+            "to_name": "audio",
+            "type": "labels",
+            "origin": "manual"
+          },
+          {
+            "original_length": 0.20479300000000045,
+            "value": {
+              "start": 10.419941,
+              "end": 10.624734,
+              "channel": 0,
+              "labels": [
+                "a"
+              ]
+            },
+            "id": "cU0uW",
+            "from_name": "tricks",
+            "to_name": "audio",
+            "type": "labels",
+            "origin": "manual"
+          },
+          {
+            "original_length": 0.06643500000000024,
+            "value": {
+              "start": 10.624734,
+              "end": 10.691169,
+              "channel": 0,
+              "labels": [
+                "c"
+              ]
+            },
+            "id": "mceJf",
+            "from_name": "tricks",
+            "to_name": "audio",
+            "type": "labels",
+            "origin": "manual"
+          }
+        ],
+        "was_cancelled": false,
+        "ground_truth": false,
+        "created_at": "2024-11-09T18:25:42.902443Z",
+        "updated_at": "2024-11-09T18:25:42.902443Z",
+        "draft_created_at": "2024-11-09T18:25:42.902443Z",
+        "lead_time": 8.020000000000001,
+        "prediction": {},
+        "result_count": 0,
+        "unique_id": "f2f0c0dc-d8fa-4eb6-ac07-ba75ee6b3d61",
+        "import_id": null,
+        "last_action": null,
+        "task": 1,
+        "project": 25,
+        "updated_by": 1,
+        "parent_prediction": null,
+        "parent_annotation": null,
+        "last_created_by": null
+      }
+    ],
+    "file_upload": "1212df4d-HandyLabels.MP4",
+    "drafts": [],
+    "predictions": [],
+    "data": {
+      "video_url": "/data/upload/30/030cca83-Video_1.mp4"
+    },
+    "meta": {},
+    "created_at": "2024-11-09T18:25:42.902443Z",
+    "updated_at": "2024-11-09T18:25:42.902443Z",
+    "inner_id": 1,
+    "total_annotations": 1,
+    "cancelled_annotations": 0,
+    "total_predictions": 0,
+    "comment_count": 0,
+    "unresolved_comment_count": 0,
+    "last_comment_updated_at": null,
+    "project": 25,
+    "updated_by": 1,
+    "comment_authors": []
+  }
+]

Web app/templates/compare_data.html

@@ -0,0 +1,62 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Compare CSV Data</title>
+    <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.5.2/css/bootstrap.min.css">
+</head>
+
+<body>
+    <div class="container mt-5">
+        <h2 class="mb-4">Compare Original and Updated CSV</h2>
+        <div class="row">
+            <div class="col-md-6">
+                <h3>Original Data</h3>
+                <table class="table table-bordered">
+                    <thead>
+                        <tr>
+                            {% for column in columns %}
+                            <th>{{ column }}</th>
+                            {% endfor %}
+                        </tr>
+                    </thead>
+                    <tbody>
+                        {% for row in original_data %}
+                        <tr>
+                            {% for column in columns %}
+                            <td>{{ row[column] }}</td>
+                            {% endfor %}
+                        </tr>
+                        {% endfor %}
+                    </tbody>
+                </table>
+            </div>
+            <div class="col-md-6">
+                <h3>Updated Data</h3>
+                <table class="table table-bordered">
+                    <thead>
+                        <tr>
+                            {% for column in columns %}
+                            <th>{{ column }}</th>
+                            {% endfor %}
+                        </tr>
+                    </thead>
+                    <tbody>
+                        {% for row in updated_data %}
+                        <tr>
+                            {% for column in columns %}
+                            <td>{{ row[column] }}</td>
+                            {% endfor %}
+                        </tr>
+                        {% endfor %}
+                    </tbody>
+                </table>
+            </div>
+        </div>
+        <a href="{{ url_for('download_csv_updated') }}" class="btn btn-primary mt-4">Download Updated CSV</a>
+    </div>
+</body>
+
+</html>

Web app/templates/csv_file.html

@@ -0,0 +1,66 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>CSV File Viewer</title>
+    <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.5.2/css/bootstrap.min.css">
+</head>
+<body>
+    <div class="container mt-5">
+        <h2 class="mb-4">Upload and View CSV File</h2>
+        <div class="form-group">
+            <input type="file" id="fileInput" class="form-control-file" accept=".csv">
+        </div>
+        <table id="csvTable" class="table table-bordered mt-4">
+            <thead>
+                <tr id="tableHeader"></tr>
+            </thead>
+            <tbody id="tableBody"></tbody>
+        </table>
+    </div>
+
+    <script>
+        document.getElementById('fileInput').addEventListener('change', function(event) {
+            const file = event.target.files[0];
+            if (file && file.type.match('text/csv')) {
+                const reader = new FileReader();
+                reader.onload = function(e) {
+                    const text = e.target.result;
+                    displayCSVData(text);
+                };
+                reader.readAsText(file);
+            } else {
+                alert('Please upload a valid CSV file.');
+            }
+        });
+
+        function displayCSVData(csvText) {
+            const rows = csvText.split('\n');
+            const headerRow = rows[0].split(',');
+            const headerElement = document.getElementById('tableHeader');
+            headerElement.innerHTML = '';
+            headerRow.forEach(header => {
+                const th = document.createElement('th');
+                th.textContent = header.trim();
+                headerElement.appendChild(th);
+            });
+
+            const bodyElement = document.getElementById('tableBody');
+            bodyElement.innerHTML = '';
+            rows.slice(1).forEach(row => {
+                if (row.trim() !== '') {
+                    const rowElement = document.createElement('tr');
+                    const cells = row.split(',');
+                    cells.forEach(cell => {
+                        const td = document.createElement('td');
+                        td.textContent = cell.trim();
+                        rowElement.appendChild(td);
+                    });
+                    bodyElement.appendChild(rowElement);
+                }
+            });
+        }
+    </script>
+</body>
+</html>

Web app/templates/data.html

@@ -0,0 +1,99 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Gesture Data</title>
+    <style>
+        body {
+            font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
+            padding: 20px;
+            background-color: #fafafa;
+            color: #333;
+        }
+        .container {
+            display: grid;
+            grid-template-columns: repeat(auto-fit, minmax(300px, 1fr));
+            gap: 20px;
+            margin-top: 20px;
+        }
+        .table-container, .chart-container {
+            border: 1px solid #ddd;
+            padding: 20px;
+            border-radius: 8px;
+            background-color: #ffffff;
+            box-shadow: 0 2px 5px rgba(0,0,0,0.1);
+        }
+        table {
+            width: 100%;
+            border-collapse: collapse;
+        }
+        th, td {
+            padding: 10px;
+            border: 1px solid #ddd;
+            text-align: left;
+        }
+        th {
+            background-color: #f8f8f8;
+            font-weight: 600;
+        }
+        tr:nth-child(even) {
+            background-color: #f9f9f9;
+        }
+        tr:hover {
+            background-color: #f1f1f1;
+        }
+        a {
+            display: inline-block;
+            margin-top: 20px;
+            padding: 10px 15px;
+            background-color: #4CAF50;
+            color: white;
+            text-decoration: none;
+            border-radius: 5px;
+            font-weight: bold;
+        }
+        a:hover {
+            background-color: #45a049;
+        }
+        h1, h2 {
+            color: #333;
+        }
+    </style>
+</head>
+<body>
+    <h1>Gesture Data Overview</h1>
+    <div class="container">
+        <div class="table-container">
+            <h2>Gesture Data Table</h2>
+            <table>
+                <thead>
+                    <tr>
+                        <th>Gesture</th>
+                        <th>Start Time</th>
+                        <th>End Time</th>
+                        <th>Duration</th>
+                    </tr>
+                </thead>
+                <tbody>
+                    {% for row in gesture_data %}
+                    <tr>
+                        <td>{{ row[0] }}</td>
+                        <td>{{ row[1] }}</td>
+                        <td>{{ row[2] }}</td>
+                        <td>{{ row[3] }}</td>
+                    </tr>
+                    {% endfor %}
+                </tbody>
+            </table>
+        </div>
+        <div class="chart-container">
+            <h2>Gesture Distribution Chart</h2>
+            {{ html_chart|safe }}
+        </div>
+    </div>
+    <a href="{{ url_for('download_csv') }}">Download Captured Gesture Data CSV</a>
+    <a href="{{ url_for('download_json') }}">Download Captured Gesture Data JSON</a>
+    
+</body>
+</html>

Web app/templates/data1.html

@@ -0,0 +1,147 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Edit CSV via Speech - Gesture Data</title>
+    <style>
+        body {
+            font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
+            padding: 20px;
+            background-color: #fafafa;
+            color: #333;
+        }
+        .container {
+            display: grid;
+            grid-template-columns: 1fr 1fr;
+            gap: 20px;
+            margin-top: 20px;
+        }
+        .table-container, .chart-container {
+            border: 1px solid #ddd;
+            padding: 20px;
+            border-radius: 8px;
+            background-color: #ffffff;
+            box-shadow: 0 2px 5px rgba(0,0,0,0.1);
+        }
+        table {
+            width: 100%;
+            border-collapse: collapse;
+        }
+        th, td {
+            padding: 10px;
+            border: 1px solid #ddd;
+            text-align: left;
+        }
+        th {
+            background-color: #f8f8f8;
+            font-weight: 600;
+        }
+        tr:nth-child(even) {
+            background-color: #f9f9f9;
+        }
+        tr:hover {
+            background-color: #f1f1f1;
+        }
+        a, button {
+            display: inline-block;
+            margin-top: 20px;
+            padding: 10px 15px;
+            background-color: #4CAF50;
+            color: white;
+            text-decoration: none;
+            border-radius: 5px;
+            font-weight: bold;
+            cursor: pointer;
+        }
+        a:hover, button:hover {
+            background-color: #45a049;
+        }
+        h1, h2 {
+            color: #333;
+        }
+    </style>
+</head>
+<body>
+    <h1>Edit CSV via Speech - Gesture Data Overview</h1>
+
+    <!-- CSV Upload Section -->
+    <form method="POST" enctype="multipart/form-data">
+        <div>
+            <input type="file" name="csv_file" accept=".csv" onchange="this.form.submit()">
+        </div>
+
+        <!-- Recording Section -->
+        <div>
+            <label for="transcription">Transcription:</label>
+            <textarea id="transcription" name="transcription" rows="4" cols="50" readonly>{{ transcription }}</textarea>
+        </div>
+
+        <div>
+            <button type="submit" name="record">🎤 Record</button>
+            <button type="submit" name="clear">Clear</button>
+        </div>
+    </form>
+
+    <!-- Data Comparison Section -->
+    {% if original_data is not none and updated_data is not none %}
+    <div class="container">
+        <!-- Original Data Table -->
+        <div class="table-container">
+            <h2>Original Gesture Data</h2>
+            <table>
+                <thead>
+                    <tr>
+                        {% for col in original_data.columns %}
+                            <th>{{ col }}</th>
+                        {% endfor %}
+                    </tr>
+                </thead>
+                <tbody>
+                    {% for row in original_data.itertuples() %}
+                    <tr>
+                        {% for col in row[1:] %}
+                            <td>{{ col }}</td>
+                        {% endfor %}
+                    </tr>
+                    {% endfor %}
+                </tbody>
+            </table>
+        </div>
+
+        <!-- Updated Data Table -->
+        <div class="table-container">
+            <h2>Updated Gesture Data</h2>
+            <table>
+                <thead>
+                    <tr>
+                        {% for col in updated_data.columns %}
+                            <th>{{ col }}</th>
+                        {% endfor %}
+                    </tr>
+                </thead>
+                <tbody>
+                    {% for row in updated_data.itertuples() %}
+                    <tr>
+                        {% for col in row[1:] %}
+                            <td>{{ col }}</td>
+                        {% endfor %}
+                    </tr>
+                    {% endfor %}
+                </tbody>
+            </table>
+        </div>
+    </div>
+
+    <!-- Download Buttons -->
+    <div>
+        <form method="GET" action="{{ url_for('download_csv', filename='original') }}">
+            <button type="submit">Download Original CSV</button>
+        </form>
+        <form method="GET" action="{{ url_for('download_csv', filename='updated') }}">
+            <button type="submit">Download Updated CSV</button>
+        </form>
+    </div>
+    {% endif %}
+</body>
+</html>

Web app/templates/edit_csv.html

@@ -0,0 +1,183 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>CSV File Editor</title>
+    <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.5.2/css/bootstrap.min.css">
+    <style>
+        .container {
+            margin-top: 20px;
+        }
+
+        .btn {
+            margin-top: 20px;
+        }
+
+        .form-control {
+            width: 100%;
+        }
+
+        .data-section {
+            margin-top: 40px;
+        }
+
+        .table-container {
+            overflow-x: auto;
+        }
+
+        .table-wrapper {
+            display: flex;
+            justify-content: space-between;
+            gap: 20px;
+        }
+
+        .table-wrapper table {
+            width: 100%;
+        }
+    </style>
+</head>
+
+<body>
+    <div class="container">
+        <h2 class="mb-4">Upload, View, and Edit CSV File</h2>
+
+        <!-- CSV Upload Section -->
+        <form method="POST" action="/upload_csv" enctype="multipart/form-data">
+            <div class="form-group">
+                <label for="csvFile">Upload CSV File:</label>
+                <input type="file" name="csv_file" id="csvFile" class="form-control-file" accept=".csv" required>
+            
+            <button type="submit" class="btn btn-primary">Upload CSV</button>
+        </form>
+
+        <!-- Instruction Section -->
+        <form method="POST" action="/edit_csv">
+            <!-- Buttons for Speech Recording -->
+            <div class="mt-4">
+                <button type="button" id="recordButton" class="btn btn-danger">🎤 Record</button>
+                <button type="button" id="stopButton" class="btn btn-secondary" disabled>⏹ Stop</button>
+            </div>
+            <div class="form-group mt-4">
+                <label for="transcription">Instruction:</label>
+                <textarea name="transcription" id="transcription" class="form-control" rows="3" placeholder="Provide instructions for CSV manipulation..."></textarea>
+            </div>
+            <button type="submit" class="btn btn-success">Process Instruction</button>
+        </form>
+
+
+        <!-- Original and Updated Data Section -->
+        <div class="data-section">
+            <h3>Data Comparison</h3>
+            <div class="table-wrapper">
+                <!-- Original Data -->
+                {% if original_data %}
+                <div>
+                    <h4>Original Data</h4>
+                    <div class="table-container">
+                        <table class="table table-bordered">
+                            <thead>
+                                <tr>
+                                    {% for column in columns %}
+                                    <th>{{ column }}</th>
+                                    {% endfor %}
+                                </tr>
+                            </thead>
+                            <tbody>
+                                {% for row in original_data %}
+                                <tr>
+                                    {% for column in columns %}
+                                    <td>{{ row[column] }}</td>
+                                    {% endfor %}
+                                </tr>
+                                {% endfor %}
+                            </tbody>
+                        </table>
+                    </div>
+                </div>
+                {% endif %}
+
+                <!-- Updated Data -->
+                {% if updated_data %}
+                <div>
+                    <h4>Updated Data</h4>
+                    <div class="table-container">
+                        <table class="table table-bordered">
+                            <thead>
+                                <tr>
+                                    {% for column in columns %}
+                                    <th>{{ column }}</th>
+                                    {% endfor %}
+                                </tr>
+                            </thead>
+                            <tbody>
+                                {% for row in updated_data %}
+                                <tr>
+                                    {% for column in columns %}
+                                    <td>{{ row[column] }}</td>
+                                    {% endfor %}
+                                </tr>
+                                {% endfor %}
+                            </tbody>
+                        </table>
+                    </div>
+                </div>
+                {% endif %}
+            </div>
+        </div>
+    </div>
+    <script>
+        let mediaRecorder;
+        let audioChunks = [];
+
+        document.getElementById('recordButton').addEventListener('click', function () {
+            navigator.mediaDevices.getUserMedia({ audio: true })
+                .then(stream => {
+                    mediaRecorder = new MediaRecorder(stream);
+                    mediaRecorder.start();
+
+                    audioChunks = [];
+                    mediaRecorder.ondataavailable = event => {
+                        audioChunks.push(event.data);
+                    };
+
+                    mediaRecorder.onstart = () => {
+                        document.getElementById('recordButton').disabled = true;
+                        document.getElementById('stopButton').disabled = false;
+                    };
+
+                    mediaRecorder.onstop = () => {
+                        document.getElementById('recordButton').disabled = false;
+                        document.getElementById('stopButton').disabled = true;
+
+                        const audioBlob = new Blob(audioChunks, { type: 'audio/wav' });
+                        const formData = new FormData();
+                        formData.append('audio', audioBlob, 'recording.wav');
+
+                        // Send the audio blob to the backend for transcription
+                        fetch('/process_audio', {
+                            method: 'POST',
+                            body: formData
+                        })
+                        .then(response => response.json())
+                        .then(data => {
+                            document.getElementById('transcription').value = data.transcription;
+                        })
+                        .catch(error => {
+                            console.error('Error:', error);
+                        });
+                    };
+                })
+                .catch(error => {
+                    console.error('Error accessing microphone:', error);
+                });
+        });
+
+        document.getElementById('stopButton').addEventListener('click', function () {
+            mediaRecorder.stop();
+        });
+    </script>
+</body>
+
+</html>

Web app/templates/import_data.html

@@ -0,0 +1,86 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Import Gesture Data</title>
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0-beta3/css/all.min.css">
+    <link href="https://fonts.googleapis.com/css2?family=Roboto:wght@400;700&display=swap" rel="stylesheet">
+    <style>
+        body {
+            font-family: 'Roboto', sans-serif;
+            background-color: #f4f4f9;
+            margin: 0;
+            padding: 0;
+            display: flex;
+            align-items: center;
+            justify-content: center;
+            min-height: 100vh;
+        }
+        .container {
+            background: #ffffff;
+            box-shadow: 0px 4px 15px rgba(0, 0, 0, 0.1);
+            padding: 30px;
+            border-radius: 10px;
+            text-align: center;
+            max-width: 500px;
+            width: 100%;
+        }
+        h1 {
+            color: #333;
+            margin-bottom: 20px;
+        }
+        form {
+            margin-bottom: 20px;
+        }
+        input[type="file"] {
+            padding: 10px;
+            font-size: 16px;
+            margin-bottom: 20px;
+            border: 1px solid #ccc;
+            border-radius: 5px;
+            width: 100%;
+            box-sizing: border-box;
+        }
+        button {
+            background-color: #4CAF50;
+            color: white;
+            padding: 10px 20px;
+            border: none;
+            border-radius: 5px;
+            font-size: 16px;
+            cursor: pointer;
+            transition: background-color 0.3s ease;
+        }
+        button:hover {
+            background-color: #4CAF50;
+        }
+        .back-link {
+            display: inline-block;
+            margin-top: 20px;
+            color: #4CAF50;
+            text-decoration: none;
+            font-weight: 700;
+            transition: color 0.3s ease;
+        }
+        .back-link:hover {
+            color: #4CAF50;
+        }
+        .fa-upload {
+            margin-right: 8px;
+        }
+    </style>
+</head>
+<body>
+    <div class="container">
+        <h1><i class="fas fa-file-upload"></i> Import Gesture Data</h1>
+        <p>Upload your gesture data CSV file to visualize the results.</p>
+        <form action="{{ url_for('import_data') }}" method="POST" enctype="multipart/form-data">
+            <input type="file" name="file" accept=".csv" required>
+            <button type="submit"><i class="fas fa-upload"></i> Upload and Visualize</button>
+        </form>
+        <a href="{{ url_for('index') }}" class="back-link"><i class="fas fa-arrow-left"></i> Back to Home</a>
+    </div>
+</body>
+</html>
+''

Web app/templates/import_data1.html

@@ -0,0 +1,78 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Import Gesture Data</title>
+    <style>
+        body {
+            font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
+            padding: 20px;
+            background-color: #fafafa;
+            color: #333;
+        }
+        .container {
+            max-width: 600px;
+            margin: 0 auto;
+            border: 1px solid #ddd;
+            padding: 20px;
+            border-radius: 8px;
+            background-color: #ffffff;
+            box-shadow: 0 2px 5px rgba(0,0,0,0.1);
+            text-align: center;
+        }
+        h1 {
+            color: #333;
+        }
+        form {
+            margin-top: 20px;
+        }
+        input[type="file"] {
+            padding: 8px;
+            margin-bottom: 20px;
+            font-size: 16px;
+            border: 1px solid #ddd;
+            border-radius: 5px;
+            width: calc(100% - 16px);
+            box-sizing: border-box;
+        }
+        button {
+            padding: 10px 20px;
+            font-size: 16px;
+            background-color: #4CAF50;
+            color: white;
+            border: none;
+            border-radius: 5px;
+            cursor: pointer;
+            font-weight: bold;
+            transition: background-color 0.3s ease;
+        }
+        button:hover {
+            background-color: #45a049;
+        }
+        a {
+            display: inline-block;
+            margin-top: 20px;
+            padding: 10px 15px;
+            background-color: #4CAF50;
+            color: white;
+            text-decoration: none;
+            border-radius: 5px;
+            font-weight: bold;
+        }
+        a:hover {
+            background-color: #45a049;
+        }
+    </style>
+</head>
+<body>
+    <div class="container">
+        <h1>Import Gesture Data from CSV</h1>
+        <form action="{{ url_for('import_data') }}" method="POST" enctype="multipart/form-data">
+            <input type="file" name="file" accept=".csv" required>
+            <button type="submit">Upload and Visualize</button>
+        </form>
+        <a href="{{ url_for('index') }}">Back to Home</a>
+    </div>
+</body>
+</html>

Web app/templates/index.html

@@ -0,0 +1,56 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <title>HandyLabels: Realtime Hand Gesture Recognition</title>
+    <style>
+        body {
+            font-family: 'Arial', sans-serif;
+            background-color: #f4f4f9;
+            color: #333;
+            text-align: center;
+            padding: 20px;
+        }
+
+        h1 {
+            color: #4A90E2;
+            font-size: 70px; /* Larger font size for the main title */
+            margin-bottom: 5px; /* Reduces space below the main title */
+        }
+
+        h2 {
+            color: #333;
+            font-size: 24px; /* Smaller font size for the subtitle */
+            margin-top: 0;
+            margin-bottom: 50px; /* Reduces space above the subtitle */
+        }
+
+        a {
+            display: inline-block;
+            background-color: #4CAF50;
+            color: white;
+            padding: 10px 20px;
+            margin: 20px;
+            text-decoration: none;
+            font-size: 16px;
+            border-radius: 5px;
+            transition: background-color 0.3s;
+        }
+
+        a:hover {
+            background-color: #45a049;
+        }
+
+    </style>
+</head>
+<body>
+    <h1>HandyLabels</h1>
+    <h2>Realtime Hand Gesture Recognition as Annotation Tool</h2>
+    <a href="{{ url_for('set_labels') }}">Set Custom Labels</a>
+    <a href="{{ url_for('import_data') }}">Import Data from Label Studio</a>
+    <a href="{{ url_for('edit_csv') }}" class="btn btn-primary">Edit File with LLM</a>
+
+
+
+</body>
+</html>

Web app/templates/recognize.html

@@ -0,0 +1,63 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <title>Recognize Gestures</title>
+    <style>
+        body {
+            font-family: Arial, sans-serif;
+            display: flex;
+            flex-direction: column;
+            align-items: center;
+            justify-content: center;
+            height: 100vh;
+            margin: 0;
+        }
+        
+        h1 {
+            text-align: center;
+            margin-bottom: 20px;
+        }
+
+        .video-container {
+            display: flex;
+            justify-content: center;
+            width: 100%;
+            max-width: 720px;
+        }
+
+        img {
+            border: 1px solid #ddd; /* Gray border */
+            border-radius: 4px;  /* Rounded border */
+            padding: 5px; /* Some padding */
+            width: 100%; /* Set width to 100% of the container */
+            max-width: 720px; /* Maximum width */
+        }
+
+        a.button {
+            background-color: #4CAF50; /* Green */
+            border: none;
+            color: white;
+            padding: 10px 20px;
+            text-align: center;
+            text-decoration: none;
+            display: inline-block;
+            font-size: 16px;
+            margin: 20px 0;
+            cursor: pointer;
+            border-radius: 5px;
+        }
+
+    </style>
+</head>
+<body>
+    <h1>Gesture Recognition Application</h1>
+    
+    <div class="video-container">
+        <img src="{{ url_for('video_feed') }}" alt="Video Stream">
+    </div>
+
+    <a href="{{ url_for('save_gesture_data') }}" class="button">Save Data</a>
+
+</body>
+</html>

Web app/templates/set_labels.html

@@ -0,0 +1,133 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Set Custom Labels</title>
+    <style>
+        body {
+            font-family: 'Arial', sans-serif;
+            background-color: #f4f4f9;
+            color: #333;
+            text-align: center;
+            padding: 20px;
+        }
+        
+        h1 {
+            color: #4A90E2;
+            margin-bottom: 20px;
+        }
+
+        .container {
+            background-color: #fff;
+            padding: 20px;
+            border-radius: 8px;
+            box-shadow: 0 0 10px rgba(0,0,0,0.1);
+            margin: auto;
+            width: 500px;
+            display: flex;
+            flex-direction: column;
+            gap: 2px;
+        }
+
+        .gesture-container {
+            display: flex;
+            flex-direction: column;
+            align-items: flex-start;
+            gap: 10px;
+        }
+
+        .input-img-container {
+            display: flex;
+            align-items: center;
+            justify-content: space-between;
+            width: 100%;
+        }
+
+        .input-img-container img {
+            width: 100px;
+            height: auto;
+            border-radius: 8px;
+        }
+        
+        label {
+            text-align: left;
+            font-size: 16px;
+            font-weight: bold;
+            flex: 1;
+            white-space: nowrap;
+        }
+
+        input[type="text"] {
+            padding: 8px;
+            border: 2px solid #ccc;
+            border-radius: 4px;
+            width: calc(100% - 120px); /* Subtract image width and padding */
+            margin-right: 10px;
+        }
+
+        button {
+            background-color: #4CAF50;
+            color: white;
+            padding: 10px 15px;
+            border: none;
+            border-radius: 4px;
+            cursor: pointer;
+            font-size: 16px;
+            transition: background-color 0.3s;
+        }
+
+        button:hover {
+            background-color: #45a049;
+        }
+    </style>
+</head>
+<body>
+    <h1>Set Custom Labels for Gestures</h1>
+    <form method="POST" action="{{ url_for('set_labels') }}">
+        <div class="container">
+            <div class="gesture-container">
+                <label for="label1">Label for Gesture 1 (Fist):</label>
+                <div class="input-img-container">
+                    <input type="text" id="label1" name="label1" placeholder="Enter label for Gesture 1" required>
+                    <img src="{{ url_for('static', filename='images/fist.png') }}" alt="Fist Gesture">
+                </div>
+            </div>
+            
+            <div class="gesture-container">
+                <label for="label2">Label for Gesture 2 (OK):</label>
+                <div class="input-img-container">
+                    <input type="text" id="label2" name="label2" placeholder="Enter label for Gesture 2" required>
+                    <img src="{{ url_for('static', filename='images/ok.png') }}" alt="OK Gesture">
+                </div>
+            </div>
+            
+            <div class="gesture-container">
+                <label for="label3">Label for Gesture 3 (Peace):</label>
+                <div class="input-img-container">
+                    <input type="text" id="label3" name="label3" placeholder="Enter label for Gesture 3" required>
+                    <img src="{{ url_for('static', filename='images/peace.png') }}" alt="Peace Gesture">
+                </div>
+            </div>
+            
+            <div class="gesture-container">
+                <label for="label4">Label for Gesture 4 (Stop):</label>
+                <div class="input-img-container">
+                    <input type="text" id="label4" name="label4" placeholder="Enter label for Gesture 4" required>
+                    <img src="{{ url_for('static', filename='images/stop.png') }}" alt="Stop Gesture">
+                </div>
+            </div>
+            
+            <div class="gesture-container">
+                <label for="label5">Label for Gesture 5 (Two Up):</label>
+                <div class="input-img-container">
+                    <input type="text" id="label5" name="label5" placeholder="Enter label for Gesture 5" required>
+                    <img src="{{ url_for('static', filename='images/two_up.png') }}" alt="Two Up Gesture">
+                </div>
+            </div>
+            
+            <button type="submit">Save Labels</button>
+        </div>
+    </form>
+</body>
+</html>

Web app/templates/speech copy.html

@@ -0,0 +1,74 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.5.2/css/bootstrap.min.css">
+    <link rel="stylesheet" href="{{ url_for('static', filename='styles.css') }}">
+    <title>Edit CSV via Speech</title>
+</head>
+<body>
+    <!-- Navigation Bar -->
+    <nav class="navbar navbar-expand-lg navbar-dark bg-dark">
+        <a class="navbar-brand" href="{{ url_for('index') }}">Gesture Recognition</a>
+        <button class="navbar-toggler" type="button" data-toggle="collapse" data-target="#navbarNav" aria-controls="navbarNav" aria-expanded="false" aria-label="Toggle navigation">
+            <span class="navbar-toggler-icon"></span>
+        </button>
+        <div class="collapse navbar-collapse" id="navbarNav">
+            <ul class="navbar-nav ml-auto">
+                <li class="nav-item">
+                    <a class="nav-link" href="{{ url_for('index') }}">Home</a>
+                </li>
+                <li class="nav-item">
+                    <a class="nav-link" href="{{ url_for('recognize') }}">Recognize Gestures</a>
+                </li>
+                <li class="nav-item">
+                    <a class="nav-link" href="{{ url_for('import_data') }}">Import Data</a>
+                </li>
+                <li class="nav-item">
+                    <a class="nav-link active" href="{{ url_for('edit_csv') }}">Edit CSV via Speech</a>
+                </li>
+            </ul>
+        </div>
+    </nav>
+
+    <!-- Main Container -->
+    <div class="container mt-5">
+        <div class="card">
+            <div class="card-header text-center bg-primary text-white">
+                <h2>Edit CSV via Speech</h2>
+            </div>
+            <div class="card-body">
+                <form method="POST" enctype="multipart/form-data">
+                    <!-- Record Button -->
+                    <div class="form-group text-center">
+                        <button class="btn btn-danger btn-lg" type="submit" name="record">🎤 Record</button>
+                    </div>
+
+                    <!-- Transcription Text Area -->
+                    <div class="form-group">
+                        <label for="transcription">Transcription:</label>
+                        <textarea class="form-control" id="transcription" name="transcription" rows="4" readonly>{{ transcription }}</textarea>
+                    </div>
+
+                    <!-- File Upload -->
+                    <div class="form-group">
+                        <label for="csv_file">Upload CSV File:</label>
+                        <input type="file" class="form-control-file" id="csv_file" name="csv_file" accept=".csv">
+                    </div>
+
+                    <!-- Action Buttons -->
+                    <div class="form-group text-center">
+                        <button class="btn btn-success mr-2" type="submit" name="process">Process</button>
+                        <button class="btn btn-warning" type="submit" name="clear">Clear</button>
+                    </div>
+                </form>
+            </div>
+        </div>
+    </div>
+
+    <!-- Optional JavaScript -->
+    <script src="https://code.jquery.com/jquery-3.5.1.slim.min.js"></script>
+    <script src="https://cdn.jsdelivr.net/npm/bootstrap@4.5.2/dist/js/bootstrap.bundle.min.js"></script>
+</body>
+</html>

Web app/templates/speech.html

@@ -0,0 +1,100 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Edit CSV via Speech</title>
+    <link rel="stylesheet" href="{{ url_for('static', filename='styles.css') }}">
+</head>
+<body>
+    <h1>Edit CSV via Speech</h1>
+
+    <form method="POST" enctype="multipart/form-data">
+        <!-- Recording Button -->
+        <div>
+            <button type="submit" name="record">🎤 Record</button>
+        </div>
+
+        <!-- Display Transcription -->
+        <div>
+            <label for="transcription">Transcription:</label>
+            <textarea id="transcription" name="transcription" rows="4" cols="50" readonly>{{ transcription }}</textarea>
+        </div>
+
+        <!-- Upload CSV File -->
+        <div>
+            <input type="file" name="csv_file" accept=".csv">
+        </div>
+
+        <!-- Process and Clear Buttons -->
+        <div>
+            <button type="submit" name="process">Process</button>
+            <button type="submit" name="clear">Clear</button>
+        </div>
+    </form>
+
+    <!-- Data Comparison Section -->
+    {% if original_data and updated_data %}
+    <div class="data-diff-section">
+        <h2>Original vs Updated Data</h2>
+        <div class="data-comparison">
+            <!-- Original Data Table -->
+            <div class="data-table">
+                <h3>Original Data</h3>
+                <table>
+                    <thead>
+                        <tr>
+                            {% for col in original_data.columns %}
+                                <th>{{ col }}</th>
+                            {% endfor %}
+                        </tr>
+                    </thead>
+                    <tbody>
+                        {% for row in original_data.itertuples() %}
+                        <tr>
+                            {% for col in row[1:] %}
+                                <td>{{ col }}</td>
+                            {% endfor %}
+                        </tr>
+                        {% endfor %}
+                    </tbody>
+                </table>
+            </div>
+
+            <!-- Updated Data Table -->
+            <div class="data-table">
+                <h3>Updated Data</h3>
+                <table>
+                    <thead>
+                        <tr>
+                            {% for col in updated_data.columns %}
+                                <th>{{ col }}</th>
+                            {% endfor %}
+                        </tr>
+                    </thead>
+                    <tbody>
+                        {% for row in updated_data.itertuples() %}
+                        <tr>
+                            {% for col in row[1:] %}
+                                <td>{{ col }}</td>
+                            {% endfor %}
+                        </tr>
+                        {% endfor %}
+                    </tbody>
+                </table>
+            </div>
+        </div>
+    </div>
+
+    <!-- Download Buttons -->
+    <div>
+        <form method="GET" action="{{ url_for('download_csv', filename='original') }}">
+            <button type="submit">Download Original CSV</button>
+        </form>
+        <form method="GET" action="{{ url_for('download_csv', filename='updated') }}">
+            <button type="submit">Download Updated CSV</button>
+        </form>
+    </div>
+    {% endif %}
+</body>
+</html>

action.py

@@ -0,0 +1,130 @@
+import csv
+import time
+from datetime import datetime
+from ultralytics import YOLO
+import cv2
+import mediapipe as mp
+import numpy as np
+
+model = YOLO('best_5.pt')
+
+cap = cv2.VideoCapture(0)
+
+mp_hands = mp.solutions.hands
+mp_drawing = mp.solutions.drawing_utils
+mp_drawing_styles = mp.solutions.drawing_styles
+
+hands = mp_hands.Hands(static_image_mode=True, min_detection_confidence=0.3)
+labels_dict = {0: 'A', 1: 'B', 2: 'C', 3: 'H'}
+
+# File to save gesture data
+csv_file = 'gesture_data.csv'
+
+# Initialize variables for tracking gestures
+previous_gesture = None
+gesture_start_time = None
+
+# List to store gesture data for CSV
+gesture_data_list = []
+
+while True:
+    data_aux = []
+    x_ = []
+    y_ = []
+
+    ret, frame = cap.read()
+    if not ret:
+        print("Failed to capture frame. Exiting...")
+        break
+
+    H, W, _ = frame.shape
+
+    frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+
+    results = hands.process(frame_rgb)
+    if results.multi_hand_landmarks:
+        for hand_landmarks in results.multi_hand_landmarks:
+            mp_drawing.draw_landmarks(
+                frame,
+                hand_landmarks,
+                mp_hands.HAND_CONNECTIONS,
+                mp_drawing_styles.get_default_hand_landmarks_style(),
+                mp_drawing_styles.get_default_hand_connections_style())
+
+        for hand_landmarks in results.multi_hand_landmarks:
+            for i in range(len(hand_landmarks.landmark)):
+                x = hand_landmarks.landmark[i].x
+                y = hand_landmarks.landmark[i].y
+
+                x_.append(x)
+                y_.append(y)
+
+            for i in range(len(hand_landmarks.landmark)):
+                x = hand_landmarks.landmark[i].x
+                y = hand_landmarks.landmark[i].y
+                data_aux.append(x - min(x_))
+                data_aux.append(y - min(y_))
+
+        x1 = int(min(x_) * W) - 10
+        y1 = int(min(y_) * H) - 10
+
+        x2 = int(max(x_) * W) + 10
+        y2 = int(max(y_) * H) + 10
+
+        prediction = model.predict(frame, conf=0.25, iou=0.45)
+        names_dict = prediction[0].names
+        probs = prediction[0].probs.data.numpy()
+        detected_gesture = names_dict[np.argmax(probs)]
+        print("Gesture:", detected_gesture)
+
+        if detected_gesture == 'A':
+            language = 'Arabic'
+        elif detected_gesture == 'B':
+            language = 'Bengali'
+        elif detected_gesture == 'C':
+            language = 'Chinese'
+        elif detected_gesture == 'H':
+            language = 'Hindi'
+
+        # Get the current timestamp
+        current_time = time.time()
+
+        # Check if the detected gesture has changed
+        if detected_gesture != previous_gesture:
+            # If the detected gesture has changed, calculate the duration of the previous gesture
+            if previous_gesture is not None:
+                gesture_end_time = current_time
+                gesture_duration = gesture_end_time - gesture_start_time
+                # Store the detected gesture, start time, end time, and duration in the list
+                gesture_data_list.append([previous_gesture,
+                                          datetime.fromtimestamp(gesture_start_time).strftime('%H:%M:%S.%f'),
+                                          datetime.fromtimestamp(gesture_end_time).strftime('%H:%M:%S.%f'),
+                                          round(gesture_duration, 2)])
+
+            # Update the previous gesture and its start time
+            previous_gesture = detected_gesture
+            gesture_start_time = current_time
+
+        # Calculate the duration of the current gesture
+        gesture_duration = current_time - gesture_start_time
+
+        # Draw rectangle around the detected gesture
+        cv2.rectangle(frame, (x1, y1), (x2, y2), (0, 0, 0), 4)
+        cv2.putText(frame, detected_gesture, (x1, y1 - 10), cv2.FONT_HERSHEY_SIMPLEX, 1.3, (0, 255, 0), 3, cv2.LINE_AA)
+
+    # Display the frame
+    cv2.imshow('frame', frame)
+
+    # Check if 'q' key is pressed to stop the program
+    if cv2.waitKey(1) & 0xFF == ord('q'):
+        # Save data to CSV file
+        with open(csv_file, 'w', newline='') as csvfile:
+            writer = csv.writer(csvfile)
+            writer.writerow(['Gesture', 'Start Time', 'End Time', 'Duration'])
+            for gesture, start_time, end_time, duration in gesture_data_list:
+                writer.writerow([gesture, start_time, end_time, duration])
+        break
+
+# Release resources
+cap.release()
+cv2.destroyAllWindows()

best.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9480d18aae80ec7c7a4c3c3e1a85bfd674d3d061e608baf2b2da401318b68d2a
+size 5897040

main.py

@@ -0,0 +1,127 @@
+import csv
+import pickle
+import time
+from ultralytics import YOLO
+import cv2
+import mediapipe as mp
+import numpy as np
+
+model = YOLO('best_5.pt')
+
+cap = cv2.VideoCapture(0)
+
+mp_hands = mp.solutions.hands
+mp_drawing = mp.solutions.drawing_utils
+mp_drawing_styles = mp.solutions.drawing_styles
+
+hands = mp_hands.Hands(static_image_mode=True, min_detection_confidence=0.3)
+language = ''
+labels_dict = {0: 'A', 1: 'B', 2: 'C', 3: 'H'}
+
+# Dictionary to store the start time of each gesture
+gesture_start_time = {gesture: None for gesture in labels_dict.values()}
+previous_gesture = None
+
+# File to save gesture data
+csv_file = 'gesture_data.csv'
+
+# Dictionary to store detected gestures and their durations
+gesture_duration_dict = {}
+
+while True:
+    data_aux = []
+    x_ = []
+    y_ = []
+
+    ret, frame = cap.read()
+    if not ret:
+        print("Failed to capture frame. Exiting...")
+        break
+
+    H, W, _ = frame.shape
+
+    frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+
+    results = hands.process(frame_rgb)
+    if results.multi_hand_landmarks:
+        for hand_landmarks in results.multi_hand_landmarks:
+            mp_drawing.draw_landmarks(
+                frame,
+                hand_landmarks,
+                mp_hands.HAND_CONNECTIONS,
+                mp_drawing_styles.get_default_hand_landmarks_style(),
+                mp_drawing_styles.get_default_hand_connections_style())
+
+        for hand_landmarks in results.multi_hand_landmarks:
+            for i in range(len(hand_landmarks.landmark)):
+                x = hand_landmarks.landmark[i].x
+                y = hand_landmarks.landmark[i].y
+
+                x_.append(x)
+                y_.append(y)
+
+            for i in range(len(hand_landmarks.landmark)):
+                x = hand_landmarks.landmark[i].x
+                y = hand_landmarks.landmark[i].y
+                data_aux.append(x - min(x_))
+                data_aux.append(y - min(y_))
+
+        x1 = int(min(x_) * W) - 10
+        y1 = int(min(y_) * H) - 10
+
+        x2 = int(max(x_) * W) - 10
+        y2 = int(max(y_) * H) - 10
+
+        prediction = model.predict(frame, conf=0.25, iou=0.45)
+        names_dict = prediction[0].names
+        probs = prediction[0].probs.data.numpy()
+        detected_gesture = names_dict[np.argmax(probs)]
+        print(names_dict[np.argmax(probs)])
+        print("Gesture:", detected_gesture)
+        if detected_gesture == 'A':
+            language = 'Arabic'
+        elif detected_gesture == 'B':
+            language = 'Bengali'
+        elif detected_gesture == 'C':
+            language = 'Chinese'
+        elif detected_gesture == 'H':
+            language = 'Hindi'
+
+        # Get the current timestamp
+        current_time = time.time()
+
+        # Check if the detected gesture has changed
+        if detected_gesture != previous_gesture:
+            # If the detected gesture has changed, calculate the duration of the previous gesture
+            if previous_gesture is not None:
+                gesture_duration = current_time - gesture_start_time[previous_gesture]
+                # Store the detected gesture and its duration in the dictionary
+                gesture_duration_dict[previous_gesture] = gesture_duration
+
+            # Update the previous gesture and its start time
+            previous_gesture = detected_gesture
+            gesture_start_time[detected_gesture] = current_time
+
+        # Calculate the duration of the current gesture
+        gesture_duration = current_time - gesture_start_time[detected_gesture]
+
+        cv2.rectangle(frame, (x1, y1), (x2, y2), (0, 0, 0), 4)
+        cv2.putText(frame, detected_gesture, (x1, y1 - 10), cv2.FONT_HERSHEY_SIMPLEX, 1.3, (0, 255, 0), 3,
+                    cv2.LINE_AA)
+    # Display the frame
+    cv2.imshow('frame', frame)
+
+    # Check if 'q' key is pressed to stop the program
+    if cv2.waitKey(1) & 0xFF == ord('q'):
+        # Save data to CSV file
+        with open(csv_file, 'w', newline='') as csvfile:
+            writer = csv.writer(csvfile)
+            writer.writerow(['Gesture', 'Duration'])
+            for gesture, duration in gesture_duration_dict.items():
+                writer.writerow([gesture, duration])
+        break
+
+# Release resources
+cap.release()
+cv2.destroyAllWindows()
+

output.csv

@@ -0,0 +1,9 @@
+Gesture,Start Time,End Time,Duration
+B,15:52:42.804,15:52:43.391,0.59
+A,15:52:43.391,15:52:43.457,0.07
+B,15:52:43.457,15:52:43.523,0.07
+A,15:52:43.523,15:52:43.590,0.07
+B,15:52:43.590,15:52:43.727,0.14
+C,15:52:43.727,15:52:43.990,0.26
+A,15:52:43.990,15:52:44.524,0.53
+C,15:52:44.524,15:52:44.591,0.07

requirements.txt

@@ -0,0 +1,84 @@
+absl-py==2.1.0
+annotated-types==0.7.0
+anyio==4.9.0
+attrs==25.3.0
+blinker==1.9.0
+certifi==2025.1.31
+cffi==1.17.1
+charset-normalizer==3.4.1
+click==8.1.8
+contourpy==1.3.0
+cycler==0.12.1
+DateTime==5.5
+distro==1.9.0
+exceptiongroup==1.2.2
+filelock==3.18.0
+Flask==3.1.0
+flatbuffers==25.2.10
+fonttools==4.56.0
+fsspec==2025.3.0
+h11==0.14.0
+httpcore==1.0.7
+httpx==0.28.1
+huggingface-hub==0.29.3
+idna==3.10
+importlib_metadata==8.6.1
+importlib_resources==6.5.2
+itsdangerous==2.2.0
+jax==0.4.30
+jaxlib==0.4.30
+Jinja2==3.1.6
+jiter==0.9.0
+kiwisolver==1.4.7
+MarkupSafe==3.0.2
+matplotlib==3.9.4
+mediapipe==0.10.21
+ml_dtypes==0.5.1
+mpmath==1.3.0
+narwhals==1.31.0
+networkx==3.2.1
+numpy==2.0.2
+openai==1.68.0
+opencv-contrib-python==4.11.0.86
+opencv-python==4.11.0.86
+opt_einsum==3.4.0
+packaging==24.2
+pandas==2.2.3
+pillow==11.1.0
+plotly==6.0.1
+protobuf==4.25.6
+psutil==7.0.0
+py-cpuinfo==9.0.0
+PyAudio==0.2.14
+pycparser==2.22
+pydantic==2.10.6
+pydantic_core==2.27.2
+pyparsing==3.2.1
+python-dateutil==2.9.0.post0
+pytz==2025.1
+PyYAML==6.0.2
+regex==2024.11.6
+requests==2.32.3
+safetensors==0.5.3
+scipy==1.13.1
+seaborn==0.13.2
+sentencepiece==0.2.0
+six==1.17.0
+sniffio==1.3.1
+sounddevice==0.5.1
+sympy==1.13.1
+tokenizers==0.21.1
+torch==2.6.0
+torchvision==0.21.0
+tqdm==4.67.1
+transformers==4.49.0
+typing_extensions==4.12.2
+tzdata==2025.1
+ultralytics==8.3.94
+ultralytics-thop==2.0.14
+urllib3==2.3.0
+Wave==0.0.2
+Werkzeug==3.1.3
+whisper==1.1.10
+zipp==3.21.0
+zope.interface==7.2