Upload 3 files

README (1).md

@@ -0,0 +1,26 @@
+---
+title: Financial RAG
+emoji: 📊
+colorFrom: blue
+colorTo: green
+sdk: gradio
+sdk_version: "4.11.0"
+app_file: app.py
+pinned: false
+---
+
+# 📊 Financial RAG Model
+
+This is a Retrieval-Augmented Generation (RAG) model for answering financial queries based on company financial statements.
+
+## 🛠 How to Use
+1. **Upload a Financial PDF** (e.g., balance sheet, income statement).
+2. **Ask a Financial Question** related to the document.
+3. **Get an AI-generated response** based on relevant financial data.
+
+## 🚀 Built With
+- **FAISS & BM25** for document retrieval.
+- **Google Gemini API** for answer generation.
+- **Gradio** for the web interface.
+
+🔗 **Try it out in the Hugging Face Space!**

app (1).py

@@ -0,0 +1,204 @@
+import os
+import re
+import faiss
+import numpy as np
+import requests
+import pdfplumber
+import spacy
+from sentence_transformers import SentenceTransformer, CrossEncoder
+from rank_bm25 import BM25Okapi
+import gradio as gr
+
+
+# ✅ Load Models
+spacy.cli.download("en_core_web_sm")
+nlp = spacy.load("en_core_web_sm")
+embed_model = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
+cross_encoder = CrossEncoder("cross-encoder/ms-marco-MiniLM-L-12-v2")
+
+# ✅ Load API Key from Hugging Face Secrets
+GEMINI_API_KEY = os.getenv("GEMINI_API_KEY")
+
+if not GEMINI_API_KEY:
+    raise ValueError("🚨 Please set the Google API Key in Hugging Face Secrets!")
+
+GEMINI_API_URL = "https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash:generateContent"
+
+# ✅ Financial Keywords for Filtering
+FINANCIAL_KEYWORDS = [
+    "revenue", "profit", "loss", "balance sheet", "cash flow", 
+    "earnings", "expenses", "investment", "financial", "liability", 
+    "assets", "equity", "debt", "capital", "tax", "dividends", 
+    "reserves", "net income", "operating income"
+]
+
+# ✅ Global Variables for FAISS & BM25
+bm25, chunk_texts, faiss_index = None, [], None
+
+
+# 🔹 1. Extract and Clean Text from PDF
+def extract_text_from_pdf(pdf_path):
+    text = ""
+    with pdfplumber.open(pdf_path) as pdf:
+        for page in pdf.pages:
+            extracted = page.extract_text()
+            if extracted:
+                text += extracted + "\n"
+    return clean_text(text)
+
+
+# 🔹 2. Clean Extracted Text
+def clean_text(text):
+    text = re.sub(r"https?://\S+", "", text)  # Remove URLs
+    text = re.sub(r"^\d{2}/\d{2}/\d{4}.*$", "", text, flags=re.MULTILINE)  # Remove timestamps
+    text = re.sub(r"(?i)this data can be easily copy pasted.*?", "", text, flags=re.MULTILINE)  # Remove metadata
+    text = re.sub(r"(?i)moneycontrol.com.*?", "", text, flags=re.MULTILINE)  # Remove source attribution
+    text = re.sub(r"(\n\s*)+", "\n", text)  # Remove extra blank lines
+    return text.strip()
+
+
+# 🔹 3. Chunking Extracted Text
+def chunk_text(text, max_tokens=64):
+    doc = nlp(text)
+    sentences = [sent.text for sent in doc.sents]
+    
+    chunks, current_chunk = [], []
+    token_count = 0
+    
+    for sentence in sentences:
+        tokens = sentence.split()
+        if token_count + len(tokens) > max_tokens:
+            chunks.append(" ".join(current_chunk))
+            current_chunk = []
+            token_count = 0
+        current_chunk.append(sentence)
+        token_count += len(tokens)
+    
+    if current_chunk:
+        chunks.append(" ".join(current_chunk))
+    
+    return chunks
+
+
+# 🔹 4. Store Chunks in FAISS & BM25
+def store_in_faiss(chunks):
+    global bm25, chunk_texts, faiss_index
+    embeddings = embed_model.encode(chunks, convert_to_numpy=True)
+    
+    # Create FAISS index
+    faiss_index = faiss.IndexFlatL2(embeddings.shape[1])
+    faiss_index.add(embeddings)
+    
+    chunk_texts = chunks
+    bm25 = BM25Okapi([chunk.split() for chunk in chunks])
+    return faiss_index
+
+
+# 🔹 5. Retrieve Chunks using BM25 with Scores
+def retrieve_bm25(query, top_k=2):
+    tokenized_query = query.split()
+    scores = bm25.get_scores(tokenized_query)
+    top_indices = np.argsort(scores)[-top_k:][::-1]  # Get top indices
+    
+    # Normalize BM25 scores
+    min_score, max_score = np.min(scores), np.max(scores)
+    normalized_scores = [(scores[i] - min_score) / (max_score - min_score) if max_score != min_score else 1 for i in top_indices]
+
+    retrieved_chunks = [(chunk_texts[i], normalized_scores[idx]) for idx, i in enumerate(top_indices)]
+    return retrieved_chunks
+
+
+# 🔹 6. Generate Response Using Google Gemini
+def refine_with_gemini(query, retrieved_text):
+    if not retrieved_text.strip():
+        return "❌ No relevant financial data found for your query."
+    
+    payload = {
+        "contents": [{
+            "parts": [{
+                "text": f"You are an expert financial analyst. Based on the provided data, extract only the relevant financial details related to the query: '{query}' and present them in a clear format.\n\nData:\n{retrieved_text}"
+            }]
+        }]
+    }
+    
+    try:
+        response = requests.post(
+            f"{GEMINI_API_URL}?key={GEMINI_API_KEY}", 
+            json=payload, headers={"Content-Type": "application/json"}
+        )
+        response_json = response.json()
+        
+        if response.status_code != 200:
+            print("🚨 Gemini API Error Response:", response_json)
+            return f"⚠️ Gemini API Error: {response_json.get('error', {}).get('message', 'Unknown error')}"
+        
+        print("✅ Gemini API Response:", response_json)
+        return response_json.get("candidates", [{}])[0].get("content", {}).get("parts", [{}])[0].get("text", "⚠️ Error generating response.")
+    
+    except Exception as e:
+        print("🚨 Exception in Gemini API Call:", str(e))
+        return "⚠️ Gemini API Exception: Unable to fetch response."
+
+
+# 🔹 7. Final Retrieval Function with Confidence Score
+def retrieve_and_generate_secure(query):
+    print("🔍 Query Received:", query)
+    if bm25 is None or not chunk_texts:
+        return "❌ No PDF data loaded. Please upload a PDF first."
+
+    bm25_results = retrieve_bm25(query)
+    if not bm25_results:
+        return "❌ No relevant financial data found for your query."
+
+    # Extract text and confidence scores
+    retrieved_texts, bm25_confidences = zip(*bm25_results)
+
+    # Average BM25 Confidence Score
+    avg_bm25_confidence = sum(bm25_confidences) / len(bm25_confidences)
+
+    # Get FAISS Similarity Score
+    query_embedding = embed_model.encode([query])
+    D, I = faiss_index.search(query_embedding, 1)  # Top-1 FAISS retrieval
+    faiss_confidence = 1 / (1 + D[0][0]) if D[0][0] != 0 else 1  # Convert distance to similarity
+
+    # Combine Confidence Scores (Weighted Average)
+    final_confidence = (0.6 * avg_bm25_confidence) + (0.4 * faiss_confidence)
+
+    # Generate Final Answer
+    final_answer = refine_with_gemini(query, "\n".join(retrieved_texts))
+
+    return f"💬 Answer: {final_answer}\n\n🔹 Confidence Score: {round(final_confidence * 100, 2)}%"
+
+
+# 🔹 8. Load PDF and Process Data
+def process_uploaded_pdf(pdf_file):
+    global faiss_index
+    text = extract_text_from_pdf(pdf_file.name)
+    chunks = chunk_text(text)
+    faiss_index = store_in_faiss(chunks)
+    return "✅ PDF Processed Successfully! Now you can ask financial questions."
+
+
+# 🔹 9. Build Gradio UI
+with gr.Blocks() as app:
+    gr.Markdown("# 📊 Financial RAG Model")
+    gr.Markdown("Upload a company financial report PDF and ask relevant financial questions.")
+    
+    with gr.Row():
+        pdf_input = gr.File(label="📂 Upload Financial PDF", type="filepath")
+        process_button = gr.Button("📜 Process PDF")
+
+    status_output = gr.Textbox(label="Processing Status", interactive=False)
+
+    with gr.Row():
+        query_input = gr.Textbox(label="❓ Ask a financial question")
+        answer_output = gr.Textbox(label="💬 Answer", interactive=False)
+
+    query_button = gr.Button("🔍 Get Answer")
+
+    # Events
+    process_button.click(process_uploaded_pdf, inputs=pdf_input, outputs=status_output)
+    query_button.click(retrieve_and_generate_secure, inputs=query_input, outputs=answer_output)
+
+# 🔹 10. Launch UI
+app.launch()

requirements.txt

@@ -0,0 +1,10 @@
+gradio
+faiss-cpu
+numpy
+scipy
+sentence-transformers
+torch
+spacy
+pdfplumber
+rank-bm25
+requests