Update evaluate.py

evaluate.py

@@ -1,19 +1,37 @@
+from transformers import AutoTokenizer, AutoModel
+import torch
 from datasets import load_dataset
-from sentence_transformers import SentenceTransformer, util
+from sklearn.metrics.pairwise import cosine_similarity
+import numpy as np
 
 def evaluate_model(model_name):
     try:
-        model = SentenceTransformer(model_name)
-        dataset = load_dataset("arshiaafshani/persian-natural-fluently", split="train[:10]")
+        tokenizer = AutoTokenizer.from_pretrained(model_name)
+        model = AutoModel.from_pretrained(model_name)
+        model.eval()
+        model.to("cuda" if torch.cuda.is_available() else "cpu")
+    except:
+        return None
+
+    dataset = load_dataset("arshiaafshani/persian-natural-fluently", split="train[:10]")
+    embeddings1, embeddings2 = [], []
+    try:
+        for item in dataset:
+            inputs1 = tokenizer(item["instruction"], return_tensors="pt", truncation=True, padding=True)
+            inputs2 = tokenizer(item["output"], return_tensors="pt", truncation=True, padding=True)
 
-        scores = []
-        for row in dataset:
-            emb1 = model.encode(row["instruction"], convert_to_tensor=True)
-            emb2 = model.encode(row["output"], convert_to_tensor=True)
-            sim_score = float(util.cos_sim(emb1, emb2)[0])
-            scores.append(sim_score)
+            with torch.no_grad():
+                embed1 = model(**inputs1).last_hidden_state[:, 0, :]
+                embed2 = model(**inputs2).last_hidden_state[:, 0, :]
 
-        return sum(scores) / len(scores)
+            embeddings1.append(embed1.squeeze().numpy())
+            embeddings2.append(embed2.squeeze().numpy())
+
+        sims = [cosine_similarity([e1], [e2])[0][0] for e1, e2 in zip(embeddings1, embeddings2)]
+        labels = [item["similarity_score"] for item in dataset]
+
+        corr = np.corrcoef(sims, labels)[0, 1]
+        return float(corr)
     except Exception as e:
         print(f"Evaluation failed: {e}")
-        return None
+        return None