Update app.py

app.py

@@ -1,6 +1,5 @@
 import pandas as pd
 import gradio as gr
-import numpy as np
 from retriever import get_relevant_passages
 from reranker import rerank
 
@@ -85,19 +84,15 @@ def recommend(query):
         # Print some debug info
         print(f"Processing query: {query[:50]}...")
         
-        # Get relevant passages with increased diversity (more candidates)
-        top_k_df = get_relevant_passages(query, df_clean, top_k=30)
+        # Get relevant passages
+        top_k_df = get_relevant_passages(query, df_clean, top_k=20)
         
         # Debug: Check if we got any results
         print(f"Retrieved {len(top_k_df)} assessments")
         
         if top_k_df.empty:
             return {"error": "No matching assessments found"}
-        
-        # Remove duplicates by URL - keep the first occurrence (highest ranked)
-        top_k_df = top_k_df.drop_duplicates(subset=['url'])
-        print(f"After deduplication: {len(top_k_df)} unique assessments")
-        
+            
         # Convert test_type to list if it's not already
         top_k_df['test_type'] = top_k_df['test_type'].apply(
             lambda x: x if isinstance(x, list) else 
@@ -108,15 +103,6 @@ def recommend(query):
         top_k_df['duration'] = top_k_df['duration'].fillna(-1).astype(int)
         top_k_df.loc[top_k_df['duration'] == -1, 'duration'] = None
         
-        # Add a diverse set of assessment types - ensure we have different test types represented
-        test_types = set()
-        for _, row in top_k_df.iterrows():
-            if isinstance(row['test_type'], list):
-                for test_type in row['test_type']:
-                    test_types.add(test_type)
-        
-        print(f"Found assessments covering {len(test_types)} different test types")
-        
         # Convert DataFrame to list of dictionaries
         candidates = top_k_df.to_dict(orient="records")
         
@@ -130,22 +116,10 @@ def recommend(query):
         # Get recommendations
         result = rerank(query, candidates)
         
-        # Post-process result to ensure no duplicates
+        # Post-process result
         if 'recommended_assessments' in result:
-            recommendations = result['recommended_assessments']
-            
-            # Deduplicate by URL
-            unique_urls = set()
-            unique_recommendations = []
-            
-            for rec in recommendations:
-                if rec['url'] not in unique_urls:
-                    unique_urls.add(rec['url'])
-                    unique_recommendations.append(rec)
-            
-            # Validate URLs
-            result['recommended_assessments'] = validate_and_fix_urls(unique_recommendations)
-            print(f"Returning {len(result['recommended_assessments'])} unique recommended assessments")
+            result['recommended_assessments'] = validate_and_fix_urls(result['recommended_assessments'])
+            print(f"Returning {len(result['recommended_assessments'])} recommended assessments")
             
         return result
     except Exception as e: