Update app.py

app.py

@@ -9,6 +9,7 @@ import cv2 as cv
 import numpy as np
 import math
 import torch
+from torch import nn
 from PIL import Image
 from feat import Detector
 from feat.utils import FEAT_EMOTION_COLUMNS
@@ -22,7 +23,6 @@ get_resource_path = _get_resource_path
 os.environ["TWILIO_ACCOUNT_SID"] = "ACf1e76f3fd6e9cbca940decc4ed443c20"
 os.environ["TWILIO_AUTH_TOKEN"] = "5cadf5cc7120dd995f11b3dc57e46d52"
 
-
 def get_ice_servers():
     try:
         account_sid = os.environ["TWILIO_ACCOUNT_SID"]
@@ -37,6 +37,28 @@ def get_ice_servers():
 
     return token.ice_servers
 
+class MyNeuralNetwork(nn.Module):
+    def __init__(self, layers, dropout):
+        super().__init__()
+        self.net = nn.Sequential(
+            nn.Linear(70, layers[0]),
+            nn.LeakyReLU(),
+            nn.Dropout(p = dropout[0]),
+            nn.Linear(layers[0], layers[1]),
+            nn.LeakyReLU(),
+            nn.Dropout(p = dropout[1]),
+            nn.Linear(layers[1], layers[2]),
+            nn.LeakyReLU(),
+            nn.Dropout(p = dropout[2]),
+            nn.Linear(layers[2], layers[3]),
+            nn.LeakyReLU(),
+            nn.Dropout(p = dropout[3]),
+            nn.Linear(layers[3], 7),
+        )
+    
+    def forward(self, inputs):
+        return self.net(inputs)
+
 def eye_aspect_ratio(eye):
 
 	A = math.dist(eye[1], eye[5])
@@ -77,15 +99,18 @@ def proc_image(img, detector):
     is_eye_open = [detect_eyes(face, img, 0.20) for face in detected_landmarks[0]]
     eye_dict = {True: "eyes open", False: "eyes closed"}
 
-    detected_emotions = detector.detect_emotions(img, detected_faces, detected_landmarks)
-    assert len(detected_emotions[0]) == faces_detected, "Number of faces and emotions are mismatched!"
-
-    em = detected_emotions[0]
-    em_labels = em.argmax(axis=1)
-
-
+    device = (
+        "cuda"
+        if torch.cuda.is_available()
+        else "cpu"
+    )
 
-    for face, has_open_eyes, label in zip(detected_faces[0], (eye_dict[eyes] for eyes in is_eye_open), em_labels):
+    emo_model = torch.load("acc_96.8", map_location=device)
+    features = [torch.tensor(np.array(extract_features(*object)).astype(np.float32)).to(device) for object in zip(detected_landmarks[0], detected_faces[0])]
+    detected_emotions = [emo_model(facefeat).softmax(dim=0).argmax(dim=0).to("cpu") for facefeat in features]
+    assert len(detected_emotions) == faces_detected, "Number of faces and emotions are mismatched!"
+    
+    for face, has_open_eyes, label in zip(detected_faces[0], (eye_dict[eyes] for eyes in is_eye_open), detected_emotions):
         (x0, y0, x1, y1, p) = face
         res_scale = img.shape[0]/704
         cv.rectangle(img, (int(x0), int(y0)), (int(x1), int(y1)), color = (0, 0, 255), thickness = 3)
@@ -94,8 +119,9 @@ def proc_image(img, detector):
         cv.putText(img, has_open_eyes, (int(x0)-10, int(y0)-10), fontFace = 0, color = (0, 255, 0), thickness = 2, fontScale = res_scale)
     return img
     
-def extract_feat():
-    return [1,2,3,4,5]
+def extract_features(landmarks, face):
+    features = [math.dist(landmarks[33], landmark) for landmark in landmarks] + [face[2] - face[0], face[3] - face[1]]
+    return features
     
 def image_processing(frame):
     return proc_image(img, detector) if recog else img