app added

.gitignore

@@ -0,0 +1,10 @@
+venv/
+credentials.json
+cmake*
+__pycache__/
+.inference*
+encodings.pkl
+labels.pkl
+model.pkl
+dlib
+flagged

README.md

@@ -10,4 +10,35 @@ pinned: false
 license: mit
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Face Recognition ML pipeline
+This project is composed by three different components: feature pipeline, model retraining and inference pipeline. The pipeline diagram can be seen below. 
+![Interactive UI for inference (1)](https://user-images.githubusercontent.com/48623568/211796472-e0f8f9de-2b70-4ebb-9cff-bf8a6386f3cb.jpg)
+
+## Feature pipeline
+
+This pipeline is hosted on a GCP (Google Cloud Platform) instance and performs the following tasks:
+1. Receive an image from the user (from a Gradio UI).
+2. Create multiple different versions of the images by applying filters, rotating it, brighten it, etc...
+3. For each of the resulting images, generate an embedding using the [Histogram of Gradients (HOG)](https://en.wikipedia.org/wiki/Histogram_of_oriented_gradients) method
+4. Upload the original image and the embeddings to a GCP bucket. 
+5. Trigger model retraining.
+
+## Model retraining
+
+Once the training pipeline is called, the embeddings are taken from the bucket, and the training of the KNN classifier is performed. The model is then stored on the local device as our inference pipeline uses the same GCP instance to run the inference. 
+
+## Inference pipeline
+
+Our inference pipeline is in real-time, hence the Gradio UI shows the live webcam with all the predictions on each face. Note: This is not a very well trained model, and hence there would be several false positives. 
+
+
+## Gradio public links:
+- The app to upload an image is provided [here](https://1d5d09cf-5ef3-47f6.gradio.live/)
+- The app to run the real-time inference is [here](https://c3c30239-c028-48a3.gradio.live/)
+## Project structure
+
+- The dataset is stored in `./dataset/<first name of the person>`.
+- The feature pipeline creates a 128d vector for each image in the dataset folder along with some augmentations. Once this is done, the encodings are stored in the file `./encodings.pkl`.
+- The training pipeline then makes use of the encoding file and trains a knn classifier and stores the model in the file `./model.pkl`. The training file also stores the labels (before using a label encoder) in the file 'labels.pkl'.
+- The inference pipeline makes use of all these files and runs the inference in the gradio UI.
+

app.py

@@ -0,0 +1,69 @@
+import os
+import random
+import gradio as gr
+
+from google.cloud import storage
+from google.oauth2 import service_account
+
+from feature_pipeline import preprocess
+from training_pipline import train
+
+use_bucket = True
+
+creds = service_account.Credentials.from_service_account_info({
+    "type": "service_account",
+    "project_id": "scalable-ml-lab-2",
+    "private_key_id": os.environ.get("GCP_PRIVATE_KEY_ID"),
+    "private_key": os.environ.get("GCP_PRIVATE_KEY"),
+    "client_email": os.environ.get("GCP_CLIENT_EMAIL"),
+    "client_id": os.environ.get("GCP_CLIENT_ID"),
+    "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+    "token_uri": "https://oauth2.googleapis.com/token",
+    "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+    "client_x509_cert_url": f"https://www.googleapis.com/robot/v1/metadata/x509/613127572368-compute%40developer.gserviceaccount.com"
+})
+client = storage.Client(credentials=creds)
+
+def upload_to_bucket(client, bucket_name, file_name, object_name):
+    try:
+        bucket = client.bucket(bucket_name)
+        blob = bucket.blob(object_name)
+        blob.upload_from_filename(file_name)
+
+        return True
+    except Exception as e:
+        print(e)
+        return False
+
+
+def image_upload(_, img, name):
+    file_id = random.randint(0, 1000000)
+    
+    if os.path.exists(f"./dataset/{name}") == False:
+        os.mkdir(f"./dataset/{name}")
+    
+    img.save(f"dataset/{name}/{name}_{file_id}.jpg")
+
+    embeddings_filename = preprocess()
+
+    if use_bucket:
+        upload_to_bucket(client, "bucket-faces", f"./dataset/{name}/{name}_{file_id}.jpg", f"{name}/{name}_{file_id}.jpg")
+        upload_to_bucket(client, "bucket-embeddings", embeddings_filename, embeddings_filename)
+
+    train(client, "bucket-embeddings", embeddings_filename)
+
+    return "Model retrained!"
+        
+
+gr.Interface(
+    image_upload, 
+    [
+        gr.Markdown("""
+        # Hello!! Enter your first name and upload one picture of your face.
+        ## The face recognition model will be retrained with the knowledge you gave it of your face.
+        """),
+        gr.Webcam(source="webcam", type="pil", label="Upload a beautiful picture of yourself."),
+        gr.Textbox(placeholder="write your name here...", label="Your name.")
+    ], 
+    outputs="text"
+    ).launch(server_name="0.0.0.0", share=True)

create_bucket.py

@@ -0,0 +1,16 @@
+from google.cloud import storage
+from google.oauth2 import service_account
+
+creds = service_account.Credentials.from_service_account_file("credentials.json")
+client = storage.Client(credentials=creds)
+
+def create_bucket(bucket_name):
+    try:
+        bucket = client.create_bucket(bucket_name)
+        print(f"Bucket {bucket.name} created")
+        return True
+    except Exception as e:
+        print(e)
+        return False
+
+create_bucket("bucket-embeddings")

feature_pipeline.py

@@ -0,0 +1,81 @@
+import time
+import imutils
+from imutils import paths
+import face_recognition
+import pickle
+import cv2
+import os
+import argparse
+
+def augment_image(file_path):
+    """
+    Augment the image by rotating it by 10, -10, 30, -30 degrees, flipping it horizontally and increasing/decreasing the brightness
+
+    params:
+        file_path: path to the image file
+    
+    returns:
+        list of 8 augmented images
+    """
+    image = cv2.imread(file_path)
+    images = []
+    
+    # convert the input image from BGR to RGB
+    rgb_image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+
+    # add the original image to the list of images
+    images.append(rgb_image)
+    # rotate the image by 10 degrees and add it to the list of images
+    images.append(imutils.rotate_bound(rgb_image, 10))
+    # rotate the image by -10 degrees and add it to the list of images
+    images.append(imutils.rotate_bound(rgb_image, -10))
+    # rotate the image by 30 degrees and add it to the list of images
+    images.append(imutils.rotate_bound(rgb_image, 30))
+    # rotate the image by -30 degrees and add it to the list of images
+    images.append(imutils.rotate_bound(rgb_image, -30))
+    # flip the image horizontally and add it to the list of images
+    images.append(cv2.flip(rgb_image, 1))
+    #increase the brightness of the image and add it to the list of images
+    images.append(cv2.convertScaleAbs(rgb_image, alpha=1.5, beta=0))
+    #decrease the brightness of the image and add it to the list of images
+    images.append(cv2.convertScaleAbs(rgb_image, alpha=0.5, beta=0))
+
+    return images
+
+
+def preprocess():
+    dataset = './dataset' # path to input dataset directory
+    encodings_file = './encodings.pkl' #path to output encodings file
+
+    imagePaths = list(paths.list_images(dataset))
+    known_encodings = []
+    known_names = []
+    s = time.time()
+    print(imagePaths)
+    for i, imagePath in enumerate(imagePaths):
+        images = augment_image(imagePath)
+        name = imagePath.split(os.path.sep)[-2] 
+        print(f"processing image [{name}] {i+1}/{len(imagePaths)}")
+    
+        for l in images:
+            boxes = face_recognition.face_locations(l, model='hog')
+            encodings = face_recognition.face_encodings(l, boxes)
+
+            # creating the training set
+            for encoding in encodings:
+                known_encodings.append(encoding)
+                known_names.append(name)
+
+    e = time.time()
+    print(f"Encoding dataset took: {(e-s)} seconds")
+    data = {"encodings": known_encodings, "names": known_names}
+    f = open(encodings_file, "wb")
+    f.write(pickle.dumps(data))
+    f.close()
+
+    return encodings_file
+
+    # replace the above few lines with the script to upload this to hopsworks
+
+if __name__ == '__main__':
+    preprocess()

inference-pipeline.py

@@ -0,0 +1,71 @@
+# import the necessary packages
+import face_recognition
+import pickle
+import argparse
+import cv2
+from sklearn.neighbors import KNeighborsClassifier
+import gradio as gr
+import numpy as np
+
+
+def inf(_, image):
+    # input_image = 'multiple.jpg'
+    # encodings_file = 'encodings.pkl'
+    detection_method = 'hog'
+    classifier_model_file = 'model.pkl'
+    labels_file = 'labels.pkl'
+    # data = pickle.loads(open(encodings_file, "rb").read())
+
+    # image = cv2.imread(input_image)
+    rgb_image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+
+    boxes = face_recognition.face_locations(rgb_image, model=detection_method)
+    encodings = face_recognition.face_encodings(rgb_image, boxes)
+    print(f'Found {len(boxes)} faces')
+    
+    if len(boxes) == 0:
+        return image
+    # initialize the list of names for each face detected
+    names = []
+
+    # # load the model from disk
+    model = pickle.loads(open(classifier_model_file, "rb").read())
+    # predict on the new encodings and display the probabilities
+    y_pred = model.predict_proba(encodings)
+    # print(y_pred)
+    max_y = y_pred.max(axis=1)
+    print(max_y)
+    labels = pickle.loads(open(labels_file, "rb").read())
+    for id, m in enumerate(max_y):
+        
+        if m >= 0.8:
+            names.append(labels[y_pred.argmax(axis=1)[id]])
+        else:
+            names.append('Unknown')
+
+
+    # loop over the recognized faces
+    for ((top, right, bottom, left), name) in zip(boxes, names):
+        # draw the predicted face name on the image
+        cv2.rectangle(image, (left, top), (right, bottom), (0, 255, 0), 2)
+        y = top - 15 if top - 15 > 15 else top + 15
+        cv2.putText(image, name, (left, y), cv2.FONT_HERSHEY_SIMPLEX,
+                    0.75, (0, 255, 0), 2)
+
+    # save the resulting image
+    # cv2.imwrite('output.jpg', image)
+    return image
+
+
+demo = gr.Interface(    
+    inf,
+    [
+    gr.Markdown("""
+    ## Welcome to the face recognition software !!!
+    """),
+    gr.Image(source="webcam", streaming=True)], 
+    "image",
+    live=True
+)
+demo.launch(server_name="0.0.0.0", share=True)
+

requirements.txt

@@ -0,0 +1,11 @@
+Click
+dlib
+face-recognition
+face-recognition-models
+imutils
+numpy
+Pillow
+opencv-python
+scikit-learn
+pydrive
+gdown

training_pipline.py

@@ -0,0 +1,46 @@
+import pickle
+import argparse
+import cv2
+import numpy as np
+from sklearn.svm import SVC
+from sklearn.preprocessing import LabelEncoder
+from sklearn.neighbors import KNeighborsClassifier
+
+
+
+def train(storage_client, bucket_name, embeddings_file):
+
+    bucket = storage_client.get_bucket(bucket_name)
+    blob = bucket.blob(embeddings_file)
+    blob.download_to_filename(embeddings_file)
+
+    data = pickle.loads(open(embeddings_file, "rb").read())
+
+    # train a classification model on these embeddings
+    # use the model to make predictions on the test data
+
+    X = data['encodings']
+    y_raw = data['names']
+
+    le = LabelEncoder()
+    y = le.fit_transform(y_raw)
+    print(le.classes_)
+    # save the labels in a file 
+    f = open('labels.pkl', "wb")
+    f.write(pickle.dumps(le.classes_))
+    f.close()
+
+    model = KNeighborsClassifier(n_neighbors=3)
+    model.fit(X, y)
+
+    accuracy = model.score(X, y)
+    print(f'Accuracy: {accuracy}')
+
+    #save the model to disk
+    f = open('model.pkl', "wb")
+    f.write(pickle.dumps(model))
+    f.close()
+
+
+if __name__ == '__main__':
+    train()