Upload 3 files

app.py

@@ -0,0 +1,75 @@
+from fastapi import FastAPI, File, UploadFile
+from fastapi.responses import JSONResponse
+from models.model_wav2vec import Wav2VecIntent
+from huggingface_hub import hf_hub_download
+import torch
+import soundfile as sf
+import gradio as gr
+import numpy as np
+import librosa
+
+app = FastAPI()
+
+# Download model from Hugging Face
+MODEL_PATH = hf_hub_download(repo_id="avi292423/speech-intent-recognition-project", filename="wav2vec_best_model.pt")
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+label_map = {
+    "activate_lamp": 0, "activate_lights": 1, "activate_lights_bedroom": 2, "activate_lights_kitchen": 3,
+    "activate_lights_washroom": 4, "activate_music": 5, "bring_juice": 6, "bring_newspaper": 7,
+    "bring_shoes": 8, "bring_socks": 9, "change_language_Chinese": 10, "change_language_English": 11,
+    "change_language_German": 12, "change_language_Korean": 13, "change_language_none": 14,
+    "deactivate_lamp": 15, "deactivate_lights": 16, "deactivate_lights_bedroom": 17, "deactivate_lights_kitchen": 18,
+    "deactivate_lights_washroom": 19, "deactivate_music": 20, "decrease_heat": 21, "decrease_heat_bedroom": 22,
+    "decrease_heat_kitchen": 23, "decrease_heat_washroom": 24, "decrease_volume": 25, "increase_heat": 26,
+    "increase_heat_bedroom": 27, "increase_heat_kitchen": 28, "increase_heat_washroom": 29, "increase_volume": 30
+}
+index_to_label = {v: k for k, v in label_map.items()}
+
+num_classes = 31
+pretrained_model = "facebook/wav2vec2-base"  # Use base for less RAM, or keep large if needed
+model = Wav2VecIntent(num_classes=num_classes, pretrained_model=pretrained_model).to(device)
+state_dict = torch.load(MODEL_PATH, map_location=device)
+model.load_state_dict(state_dict)
+model.eval()
+
+@app.post("/predict")
+async def predict(file: UploadFile = File(...)):
+    audio_bytes = await file.read()
+    with open("temp.wav", "wb") as f:
+        f.write(audio_bytes)
+    audio, sample_rate = sf.read("temp.wav")
+    if sample_rate != 16000:
+        return JSONResponse({"error": "Audio must have a sample rate of 16kHz."}, status_code=400)
+    waveform = torch.tensor(audio, dtype=torch.float32).unsqueeze(0).to(device)
+    with torch.no_grad():
+        output = model(waveform)
+        predicted_class = torch.argmax(output, dim=1).item()
+        predicted_label = index_to_label.get(predicted_class, "Unknown Class")
+    return {"prediction": predicted_label}
+
+def predict_intent(audio):
+    if audio is None:
+        return "No audio provided."
+    sr, y = audio
+    if sr != 16000:
+        # Resample to 16kHz
+        y = librosa.resample(y.astype(float), orig_sr=sr, target_sr=16000)
+        sr = 16000
+    waveform = torch.tensor(y, dtype=torch.float32).unsqueeze(0).to(device)
+    with torch.no_grad():
+        output = model(waveform)
+        predicted_class = torch.argmax(output, dim=1).item()
+        predicted_label = index_to_label.get(predicted_class, "Unknown Class")
+    return predicted_label
+
+demo = gr.Interface(
+    fn=predict_intent,
+    inputs=gr.Audio(source="microphone", type="numpy", label="Record or Upload Audio (16kHz WAV)"),
+    outputs=gr.Textbox(label="Predicted Intent"),
+    title="Speech Intent Recognition",
+    description="Record or upload a 16kHz WAV audio file to predict the intent."
+)
+
+if __name__ == "__main__":
+    demo.launch()

models/model_wav2vec.py

@@ -0,0 +1,48 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from transformers import Wav2Vec2Model
+
+class Wav2VecIntent(nn.Module):
+    def __init__(self, num_classes=31, pretrained_model="facebook/wav2vec2-large"):
+        super().__init__()
+        # Load pretrained wav2vec model
+        self.wav2vec = Wav2Vec2Model.from_pretrained(pretrained_model)
+        
+        # Get hidden size from model config
+        hidden_size = self.wav2vec.config.hidden_size
+        
+        # Add layer normalization
+        self.layer_norm = nn.LayerNorm(hidden_size)
+        
+        # Add attention mechanism
+        self.attention = nn.Linear(hidden_size, 1)
+        
+        # Add dropout for regularization   
+        self.dropout = nn.Dropout(p=0.5)
+        
+        # Classification head
+        self.fc = nn.Linear(hidden_size, num_classes)
+    
+    def forward(self, input_values, attention_mask=None):
+        # Get wav2vec features
+        outputs = self.wav2vec(
+            input_values,
+            attention_mask=attention_mask,
+            return_dict=True
+        )
+        hidden_states = outputs.last_hidden_state  # [batch, sequence, hidden]
+        
+        # Apply layer normalization
+        hidden_states = self.layer_norm(hidden_states)
+        
+        # Apply attention
+        attn_weights = F.softmax(self.attention(hidden_states), dim=1)
+        x = torch.sum(hidden_states * attn_weights, dim=1)  # Weighted sum
+        
+        # Apply dropout
+        x = self.dropout(x)
+        
+        # Final classification
+        x = self.fc(x)
+        return x

requirements.txt

@@ -0,0 +1,10 @@
+fastapi
+torch
+soundfile
+huggingface_hub
+transformers
+gunicorn
+numpy
+uvicorn
+gradio
+librosa