init commit

.gitattributes

@@ -0,0 +1,30 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bin.* filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zstandard filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+dog.jpg filter=lfs diff=lfs merge=lfs -text
+germany.jpg filter=lfs diff=lfs merge=lfs -text
+rocket.jpg filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,2 @@
+venv/
+transformers/

README.md

@@ -0,0 +1,11 @@
+---
+title: FLAVA Zero Shot Image Classification
+emoji: 💻
+colorFrom: blue
+colorTo: gray
+sdk: gradio
+app_file: app.py
+pinned: false
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces#reference

app.py

@@ -0,0 +1,35 @@
+import gradio as gr
+from transformers import FlavaModel, BertTokenizer, FlavaFeatureExtractor
+import numpy as np
+from PIL import Image
+import torch
+
+
+images="dog.jpg"
+model = FlavaModel.from_pretrained("facebook/flava-full")
+fe = FlavaFeatureExtractor.from_pretrained("facebook/flava-full")
+tokenizer = BertTokenizer.from_pretrained("facebook/flava-full")
+
+def shot(image, labels_text):
+    PIL_image = Image.fromarray(np.uint8(image)).convert('RGB')
+    labels = labels_text.split(",")
+    label_with_template = [f"This is a photo of a {label}" for label in labels]
+    image_input = fe([PIL_image], return_tensors="pt")
+    text_inputs = tokenizer(label_with_template, padding="max_length", return_tensors="pt")
+
+    image_embeddings = model.get_image_features(**image_input)[:, 0, :]
+    text_embeddings = model.get_text_features(**text_inputs)[:, 0, :]
+    similarities = list(torch.nn.functional.softmax((text_embeddings @ image_embeddings.T).squeeze(0), dim=0))
+    return {label: similarities[idx].item() for idx, label in enumerate(labels)}
+    
+iface = gr.Interface(shot, 
+                    ["image", "text"], 
+                    "label", 
+                    examples=[["dog.jpg", "dog,cat,bird"], 
+                              ["germany.jpg", "germany,belgium,colombia"],
+                              ["rocket.jpg", "car,rocket,train"]
+                            ],
+                    description="Add a picture and a list of labels separated by commas",
+                    title="FLAVA Zero-shot Image Classification")
+
+iface.launch()

requirements.txt

@@ -0,0 +1,2 @@
+transformers
+torch