Update cross_encoder_reranking_train.py

cross_encoder_reranking_train.py

@@ -37,8 +37,17 @@ def cluster_and_rank(texts, threshold=0.75):
         return texts 
 
     embeddings = embed_text_list(texts)
-    clustering = AgglomerativeClustering(n_clusters=None, distance_threshold=1-threshold, metric = "cosine", linkage='average')
-    labels = clustering.fit_predict(embeddings)
+    similarity_matrix = np.dot(embeddings, np.transpose(embeddings))
+    # Convert similarity to distance (larger dot product = more similar => smaller distance)
+    distance_matrix = -similarity_matrix  # negative dot product for distance-like behavior
+
+    clustering = AgglomerativeClustering(
+        n_clusters=None,
+        distance_threshold=-threshold,  # lower threshold = tighter clusters
+        affinity='precomputed',
+        linkage='average'
+    )
+    labels = clustering.fit_predict(distance_matrix)
     
     clustered_texts = {}
     for label, text in zip(labels, texts):