Clean version for HF Space

.streamlit/secrets.toml.txt

@@ -0,0 +1 @@
+GOOGLE_API_KEY= "AIzaSyBIfGJRoet_wzzYXIiWXxStkIigEOzSR2o"

README.md

@@ -0,0 +1,12 @@
+---
+title: ScientificChatbot
+emoji: 🏃
+colorFrom: gray
+colorTo: purple
+sdk: streamlit
+sdk_version: 1.44.1
+app_file: app.py
+pinned: false
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

evaluation.py

@@ -0,0 +1,116 @@
+import os
+import io
+import pandas as pd
+from datasets import load_dataset
+from dotenv import load_dotenv
+from nltk.translate.bleu_score import corpus_bleu
+from rouge_score import rouge_scorer
+from bert_score import score
+from modtran_gemini import (
+    handle_user_query,
+    initialize_chatbot_agent,
+    get_uploaded_text,
+    get_text_chunks,
+    get_vectorstore,
+    set_global_vectorstore
+)
+import string
+import re
+
+load_dotenv()
+
+# Load SQuAD dataset for benchmark
+ds = load_dataset("squad", split="validation[:100]")
+
+# Extract context, questions, and answers
+contexts = [item["context"] for item in ds]
+questions = [item["question"] for item in ds]
+answers = [item["answers"]["text"][0] for item in ds]
+
+# Create dataframe
+df = pd.DataFrame({"context": contexts, "question": questions, "answer": answers})
+
+# Save contexts to disk to simulate file uploads
+os.makedirs("squad_contexts", exist_ok=True)
+for i, context in enumerate(df["context"].unique()):
+    with open(f"squad_contexts/context_{i}.txt", "w", encoding="utf-8") as f:
+        f.write(context)
+
+# Simulate file uploads (Streamlit-like file objects)
+uploaded_files = []
+for filename in os.listdir("squad_contexts"):
+    if filename.endswith(".txt"):
+        with open(os.path.join("squad_contexts", filename), "rb") as f:
+            file_obj = io.BytesIO(f.read())
+            file_obj.name = filename
+            uploaded_files.append(file_obj)
+
+# Initialize vectorstore and agent
+raw_text = get_uploaded_text(uploaded_files)
+text_chunks = get_text_chunks(raw_text)
+vectorstore = get_vectorstore(text_chunks)
+set_global_vectorstore(vectorstore)
+
+agent = initialize_chatbot_agent()
+
+# Run chatbot predictions
+df["chatbot_answer"] = df["question"].apply(lambda q: handle_user_query(q, agent))
+
+# BLEU Evaluation
+references = [[ans.split()] for ans in df["answer"]]
+hypotheses = [pred.split() for pred in df["chatbot_answer"]]
+bleu_score = corpus_bleu(references, hypotheses)
+
+# ROUGE Evaluation
+scorer = rouge_scorer.RougeScorer(['rouge1', 'rougeL'], use_stemmer=True)
+rouge_scores = [scorer.score(ref, hyp) for ref, hyp in zip(df["answer"], df["chatbot_answer"])]
+rouge1 = sum(score["rouge1"].fmeasure for score in rouge_scores) / len(rouge_scores)
+rougeL = sum(score["rougeL"].fmeasure for score in rouge_scores) / len(rouge_scores)
+
+# BERTScore Evaluation
+P, R, F1 = score(df["chatbot_answer"].tolist(), df["answer"].tolist(),model_type='distilbert-base-uncased', batch_size=4,lang="en", verbose=True)
+mean_precision = P.mean().item()
+mean_recall = R.mean().item()
+mean_f1 = F1.mean().item()
+
+# SQuAD F1 Evaluation
+def normalize_answer(s):
+    def remove_articles(text):
+        return re.sub(r'\b(a|an|the)\b', ' ', text)
+
+    def white_space_fix(text):
+        return ' '.join(text.split())
+
+    def remove_punctuation(text):
+        return ''.join(ch for ch in text if ch not in set(string.punctuation))
+
+    def lowercase(text):
+        return text.lower()
+
+    return white_space_fix(remove_articles(remove_punctuation(lowercase(s))))
+
+def compute_f1(a_gold, a_pred):
+    gold_toks = normalize_answer(a_gold).split()
+    pred_toks = normalize_answer(a_pred).split()
+    common = set(gold_toks) & set(pred_toks)
+
+    if len(common) == 0:
+        return 0.0
+
+    precision = len(common) / len(pred_toks)
+    recall = len(common) / len(gold_toks)
+    return (2 * precision * recall) / (precision + recall)
+
+df["squad_f1_score"] = df.apply(lambda row: compute_f1(row["answer"], row["chatbot_answer"]), axis=1)
+mean_squad_f1 = df["squad_f1_score"].mean()
+
+# Print evaluation results
+print(f"BLEU Score: {bleu_score:.3f}")
+print(f"ROUGE-1: {rouge1:.3f}")
+print(f"ROUGE-L: {rougeL:.3f}")
+# BERTScore Evaluation
+P, R, F1 = score(df["chatbot_answer"].tolist(), df["answer"].tolist(), lang="en")
+df["BERTScore_F1"] = F1.numpy()
+print(f"Mean BERTScore F1: {F1.mean().item():.3f}")
+# Save evaluation results
+df.to_csv('evaluation_results.csv', index=False)

evaluation_modtran.py

@@ -0,0 +1,146 @@
+import os, io, re
+import pandas as pd
+from sklearn.metrics import accuracy_score
+from bert_score import score as bert_score
+import google.generativeai as genai
+
+from modtran_gemini import (
+    handle_user_query,
+    initialize_chatbot_agent,
+    get_uploaded_text,
+    get_text_chunks,
+    get_vectorstore,
+    set_global_vectorstore,
+    self_reasoning,
+    faiss_search_with_keywords,
+    faiss_search_with_reasoning
+)
+
+from langchain_openai import ChatOpenAI
+
+class GeminiLLM:
+    def __init__(self, model_name="models/gemini-1.5-pro-latest", api_key=None):
+        api_key = api_key or os.getenv("GOOGLE_API_KEY")
+        if not api_key:
+            raise ValueError("Missing GOOGLE_API_KEY")
+        genai.configure(api_key=api_key)
+        self.model = genai.GenerativeModel(model_name)
+
+    def predict(self, prompt: str) -> str:
+        response = self.model.generate_content(prompt)
+        return response.text.strip()
+
+# Load CSV dataset (ensure columns are 'question', 'answer' with no extra spaces)
+df = pd.read_csv("modtran_dataset.csv")
+df.columns = df.columns.str.strip()  # Strip whitespace from column names
+
+# Load the MODTRAN user manual
+with open("MODTRAN 6 User's Manual.pdf", "rb") as f:
+    file_obj = io.BytesIO(f.read())
+    file_obj.name = "MODTRAN 6 User's Manual.pdf"
+    uploaded_files = [file_obj]
+
+# Document processing
+raw_text = get_uploaded_text(uploaded_files)
+text_chunks = get_text_chunks(raw_text)
+vectorstore = get_vectorstore(text_chunks)
+set_global_vectorstore(vectorstore)
+llm = GeminiLLM()
+
+# Direct retrieval + answer generation
+def direct_llm_rag_response(question):
+    from modtran_gemini import vectorstore_global  
+    if vectorstore_global is None:
+        raise ValueError("Vectorstore is not initialized.")
+
+    # Retrieve relevant documents
+    retriever = vectorstore_global.as_retriever(search_kwargs={"k": 20})
+    docs = retriever.get_relevant_documents(question)
+
+    # Build a simple prompt with raw context
+    context = "\n\n".join([doc.page_content for doc in docs])
+
+    prompt = f"""
+    You are an AI assistant that analyzes the context provided to answer the user's query comprehensively and clearly. 
+    Answer in a concise, factual way using the terminology from the context. Avoid extra explanation unless explicitly asked.
+    If asked for the page number,YOU MUST mention the page number. 
+    ### Example 1:
+    **Question:** What is the purpose of the MODTRAN GUI?
+    **Context:**
+    [Page 10 of the docuemnt] The MODTRAN GUI helps users set parameters and visualize the model's output.
+    **Answer:** The MODTRAN GUI assists users in parameter setup and output visualization. You can find the answer at Page 10 of the document provided.
+
+    ### Example 2:
+    **Question:** How do you run MODTRAN on Linux? Answer with page number. 
+    **Context:**
+    [Page 15 of the docuemnt] On Linux systems, MODTRAN can be run using the `mod6c` binary via terminal.
+    **Answer:** Use the `mod6c` binary via terminal. (Page 15)
+
+    ### Now answer:
+    **Question:** {question}
+    **Context:**
+    {context}
+
+    **Answer:**
+    """
+    return llm.predict(prompt)
+
+# Predict answers
+df["predicted"] = df["question"].apply(direct_llm_rag_response)
+
+# Clean up answers
+true_answers = df["answer"].str.lower().str.strip()
+pred_answers = df["predicted"].str.lower().str.strip()
+
+# Normalize answers
+def normalize_text(s):
+    s = s.lower()
+    s = re.sub(r'\b(a|an|the)\b', ' ', s)
+    s = re.sub(r'[^a-z0-9]', ' ', s)
+    return ' '.join(s.split())
+
+normalized_preds = [normalize_text(p) for p in pred_answers]
+normalized_refs = [normalize_text(r) for r in true_answers]
+
+# Token-level F1
+def compute_f1(pred, ref):
+    pred_tokens = pred.split()
+    ref_tokens = ref.split()
+    common = set(pred_tokens) & set(ref_tokens)
+    if not common:
+        return 0.0
+    precision = len(common) / len(pred_tokens)
+    recall = len(common) / len(ref_tokens)
+    return 2 * precision * recall / (precision + recall)
+
+
+def manual_tool_routing(question):
+    if "how" in question.lower():
+        context = faiss_search_with_reasoning(question)
+    else:
+        context = faiss_search_with_keywords(question)
+    return self_reasoning(question, context)
+
+# Create predictions using different strategies
+df["agent_predicted"] = df["question"].apply(manual_tool_routing)
+df["keyword_predicted"] = df["question"].apply(faiss_search_with_keywords)
+df["reasoning_predicted"] = df["question"].apply(faiss_search_with_reasoning)
+
+refs = df["answer"].str.lower().str.strip()
+
+for col in ["agent_predicted", "keyword_predicted", "reasoning_predicted"]:
+    preds = df[col].str.lower().str.strip()
+    normalized_preds = [normalize_text(p) for p in preds]
+    normalized_refs = [normalize_text(r) for r in refs]
+
+    em = sum([int(p == r) for p, r in zip(normalized_preds, normalized_refs)]) / len(refs)
+    f1 = sum([compute_f1(p, r) for p, r in zip(normalized_preds, normalized_refs)]) / len(refs)
+    P, R, F1_bert = bert_score(preds.tolist(), refs.tolist(), lang="en", verbose=True)
+    bert_f1 = F1_bert.mean().item()
+
+    print(f"\n🔹 Evaluation for: {col}")
+    print(f" - Exact Match: {em:.3f}")
+    print(f" - F1 Score: {f1:.3f}")
+    print(f" - BERTScore F1: {bert_f1:.3f}")
+
+    df[f"{col}_bert_f1"] = F1_bert.numpy()

evaluation_qasper.py

@@ -0,0 +1,75 @@
+from datasets import load_dataset
+import pandas as pd
+import os, io
+from bert_score import score
+from sklearn.metrics import accuracy_score, f1_score
+from modtran import (
+    handle_user_query,
+    initialize_chatbot_agent,
+    get_uploaded_text,
+    get_text_chunks,
+    get_vectorstore,
+    set_global_vectorstore,
+)
+
+# Load SciQ
+ds = load_dataset("sciq", split="validation[:100]")
+
+# Extract supports as context, question, and correct answer
+contexts = [item["support"] for item in ds]
+questions = [item["question"] for item in ds]
+answers = [item["correct_answer"] for item in ds]
+predictions = []
+
+# Create dataframe
+df = pd.DataFrame({"context": contexts, "question": questions, "answer": answers})
+
+# Save contexts to disk to simulate file uploads
+os.makedirs("sciq_contexts", exist_ok=True)
+for i, context in enumerate(df["context"].unique()):
+    with open(f"sciq_contexts/context_{i}.txt", "w", encoding="utf-8") as f:
+        f.write(context)
+
+# Simulate file uploads
+uploaded_files = []
+for filename in os.listdir("sciq_contexts"):
+    if filename.endswith(".txt"):
+        with open(os.path.join("sciq_contexts", filename), "rb") as f:
+            file_obj = io.BytesIO(f.read())
+            file_obj.name = filename
+            uploaded_files.append(file_obj)
+
+print("Total uploaded files:", len(uploaded_files))
+
+# Vectorstore pipeline
+raw_text = get_uploaded_text(uploaded_files)
+text_chunks = get_text_chunks(raw_text)
+vectorstore = get_vectorstore(text_chunks)
+set_global_vectorstore(vectorstore)
+
+# Initialize chatbot agent
+agent = initialize_chatbot_agent()
+
+
+# Predict answers
+df["chatbot_answer"] = df["question"].apply(lambda q: handle_user_query(q, agent))
+
+# BERTScore Evaluation
+P, R, F1 = score(df["chatbot_answer"].tolist(), df["answer"].tolist(), lang="en")
+df["BERTScore_F1"] = F1.numpy()
+
+print(f"Mean BERTScore F1: {F1.mean().item():.3f}")
+for q in questions:
+    pred = handle_user_query(q, agent)
+    predictions.append(pred)
+
+# Compute Accuracy
+acc = accuracy_score(answers, predictions)
+print(f"Accuracy: {acc:.3f}")
+
+# Compute F1 (macro average - good for open-ended QA)
+f1 = f1_score(answers, predictions, average='macro')
+print(f"F1 Score (macro): {f1:.3f}")
+
+# Save results
+df.to_csv("sciq_evaluation_results.csv", index=False)

evaluation_results.csv

@@ -0,0 +1,101 @@
+context,question,answer,chatbot_answer,squad_f1_score,BERTScore_F1
+"Super Bowl 50 was an American football game to determine the champion of the National Football League (NFL) for the 2015 season. The American Football Conference (AFC) champion Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24–10 to earn their third Super Bowl title. The game was played on February 7, 2016, at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California. As this was the 50th Super Bowl, the league emphasized the ""golden anniversary"" with various gold-themed initiatives, as well as temporarily suspending the tradition of naming each Super Bowl game with Roman numerals (under which the game would have been known as ""Super Bowl L""), so that the logo could prominently feature the Arabic numerals 50.",Which NFL team represented the AFC at Super Bowl 50?,Denver Broncos,The Denver Broncos represented the AFC at Super Bowl 50.,0.4,0.8831762
+"Super Bowl 50 was an American football game to determine the champion of the National Football League (NFL) for the 2015 season. The American Football Conference (AFC) champion Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24–10 to earn their third Super Bowl title. The game was played on February 7, 2016, at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California. As this was the 50th Super Bowl, the league emphasized the ""golden anniversary"" with various gold-themed initiatives, as well as temporarily suspending the tradition of naming each Super Bowl game with Roman numerals (under which the game would have been known as ""Super Bowl L""), so that the logo could prominently feature the Arabic numerals 50.",Which NFL team represented the NFC at Super Bowl 50?,Carolina Panthers,The Carolina Panthers represented the NFC at Super Bowl 50.,0.4,0.88892263
+"Super Bowl 50 was an American football game to determine the champion of the National Football League (NFL) for the 2015 season. The American Football Conference (AFC) champion Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24–10 to earn their third Super Bowl title. The game was played on February 7, 2016, at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California. As this was the 50th Super Bowl, the league emphasized the ""golden anniversary"" with various gold-themed initiatives, as well as temporarily suspending the tradition of naming each Super Bowl game with Roman numerals (under which the game would have been known as ""Super Bowl L""), so that the logo could prominently feature the Arabic numerals 50.",Where did Super Bowl 50 take place?,"Santa Clara, California","Super Bowl 50 took place at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California.",0.3,0.86463374
+"Super Bowl 50 was an American football game to determine the champion of the National Football League (NFL) for the 2015 season. The American Football Conference (AFC) champion Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24–10 to earn their third Super Bowl title. The game was played on February 7, 2016, at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California. As this was the 50th Super Bowl, the league emphasized the ""golden anniversary"" with various gold-themed initiatives, as well as temporarily suspending the tradition of naming each Super Bowl game with Roman numerals (under which the game would have been known as ""Super Bowl L""), so that the logo could prominently feature the Arabic numerals 50.",Which NFL team won Super Bowl 50?,Denver Broncos,The Denver Broncos won Super Bowl 50.,0.5,0.8863907
+"Super Bowl 50 was an American football game to determine the champion of the National Football League (NFL) for the 2015 season. The American Football Conference (AFC) champion Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24–10 to earn their third Super Bowl title. The game was played on February 7, 2016, at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California. As this was the 50th Super Bowl, the league emphasized the ""golden anniversary"" with various gold-themed initiatives, as well as temporarily suspending the tradition of naming each Super Bowl game with Roman numerals (under which the game would have been known as ""Super Bowl L""), so that the logo could prominently feature the Arabic numerals 50.",What color was used to emphasize the 50th anniversary of the Super Bowl?,gold,Gold was used to emphasize the 50th anniversary of the Super Bowl.,0.18181818181818182,0.8246054
+"Super Bowl 50 was an American football game to determine the champion of the National Football League (NFL) for the 2015 season. The American Football Conference (AFC) champion Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24–10 to earn their third Super Bowl title. The game was played on February 7, 2016, at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California. As this was the 50th Super Bowl, the league emphasized the ""golden anniversary"" with various gold-themed initiatives, as well as temporarily suspending the tradition of naming each Super Bowl game with Roman numerals (under which the game would have been known as ""Super Bowl L""), so that the logo could prominently feature the Arabic numerals 50.",What was the theme of Super Bowl 50?,"""golden anniversary""","The theme of Super Bowl 50 was the ""golden anniversary"".",0.4,0.88012564
+"Super Bowl 50 was an American football game to determine the champion of the National Football League (NFL) for the 2015 season. The American Football Conference (AFC) champion Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24–10 to earn their third Super Bowl title. The game was played on February 7, 2016, at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California. As this was the 50th Super Bowl, the league emphasized the ""golden anniversary"" with various gold-themed initiatives, as well as temporarily suspending the tradition of naming each Super Bowl game with Roman numerals (under which the game would have been known as ""Super Bowl L""), so that the logo could prominently feature the Arabic numerals 50.",What day was the game played on?,"February 7, 2016",The question lacks specific details about which game is being referred to. Please provide more information.,0.0,0.8093692
+"Super Bowl 50 was an American football game to determine the champion of the National Football League (NFL) for the 2015 season. The American Football Conference (AFC) champion Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24–10 to earn their third Super Bowl title. The game was played on February 7, 2016, at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California. As this was the 50th Super Bowl, the league emphasized the ""golden anniversary"" with various gold-themed initiatives, as well as temporarily suspending the tradition of naming each Super Bowl game with Roman numerals (under which the game would have been known as ""Super Bowl L""), so that the logo could prominently feature the Arabic numerals 50.",What is the AFC short for?,American Football Conference,"In the context of the National Football League (NFL), AFC stands for American Football Conference.",0.375,0.8462678
+"Super Bowl 50 was an American football game to determine the champion of the National Football League (NFL) for the 2015 season. The American Football Conference (AFC) champion Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24–10 to earn their third Super Bowl title. The game was played on February 7, 2016, at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California. As this was the 50th Super Bowl, the league emphasized the ""golden anniversary"" with various gold-themed initiatives, as well as temporarily suspending the tradition of naming each Super Bowl game with Roman numerals (under which the game would have been known as ""Super Bowl L""), so that the logo could prominently feature the Arabic numerals 50.",What was the theme of Super Bowl 50?,"""golden anniversary""","The theme of Super Bowl 50 was the ""golden anniversary"".",0.4,0.88012564
+"Super Bowl 50 was an American football game to determine the champion of the National Football League (NFL) for the 2015 season. The American Football Conference (AFC) champion Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24–10 to earn their third Super Bowl title. The game was played on February 7, 2016, at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California. As this was the 50th Super Bowl, the league emphasized the ""golden anniversary"" with various gold-themed initiatives, as well as temporarily suspending the tradition of naming each Super Bowl game with Roman numerals (under which the game would have been known as ""Super Bowl L""), so that the logo could prominently feature the Arabic numerals 50.",What does AFC stand for?,American Football Conference,"In the context of the NFL, AFC stands for American Football Conference.",0.4615384615384615,0.85463834
+"Super Bowl 50 was an American football game to determine the champion of the National Football League (NFL) for the 2015 season. The American Football Conference (AFC) champion Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24–10 to earn their third Super Bowl title. The game was played on February 7, 2016, at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California. As this was the 50th Super Bowl, the league emphasized the ""golden anniversary"" with various gold-themed initiatives, as well as temporarily suspending the tradition of naming each Super Bowl game with Roman numerals (under which the game would have been known as ""Super Bowl L""), so that the logo could prominently feature the Arabic numerals 50.",What day was the Super Bowl played on?,"February 7, 2016","I'm sorry, I am unable to provide the date of the most recent Super Bowl at this time.",0.0,0.8131411
+"Super Bowl 50 was an American football game to determine the champion of the National Football League (NFL) for the 2015 season. The American Football Conference (AFC) champion Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24–10 to earn their third Super Bowl title. The game was played on February 7, 2016, at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California. As this was the 50th Super Bowl, the league emphasized the ""golden anniversary"" with various gold-themed initiatives, as well as temporarily suspending the tradition of naming each Super Bowl game with Roman numerals (under which the game would have been known as ""Super Bowl L""), so that the logo could prominently feature the Arabic numerals 50.",Who won Super Bowl 50?,Denver Broncos,The Denver Broncos won Super Bowl 50.,0.5,0.8863907
+"Super Bowl 50 was an American football game to determine the champion of the National Football League (NFL) for the 2015 season. The American Football Conference (AFC) champion Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24–10 to earn their third Super Bowl title. The game was played on February 7, 2016, at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California. As this was the 50th Super Bowl, the league emphasized the ""golden anniversary"" with various gold-themed initiatives, as well as temporarily suspending the tradition of naming each Super Bowl game with Roman numerals (under which the game would have been known as ""Super Bowl L""), so that the logo could prominently feature the Arabic numerals 50.",What venue did Super Bowl 50 take place in?,Levi's Stadium,"Super Bowl 50 took place at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California.",0.21052631578947367,0.815798
+"Super Bowl 50 was an American football game to determine the champion of the National Football League (NFL) for the 2015 season. The American Football Conference (AFC) champion Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24–10 to earn their third Super Bowl title. The game was played on February 7, 2016, at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California. As this was the 50th Super Bowl, the league emphasized the ""golden anniversary"" with various gold-themed initiatives, as well as temporarily suspending the tradition of naming each Super Bowl game with Roman numerals (under which the game would have been known as ""Super Bowl L""), so that the logo could prominently feature the Arabic numerals 50.",What city did Super Bowl 50 take place in?,Santa Clara,"Super Bowl 50 took place in the San Francisco Bay Area at Santa Clara, California.",0.25,0.8431589
+"Super Bowl 50 was an American football game to determine the champion of the National Football League (NFL) for the 2015 season. The American Football Conference (AFC) champion Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24–10 to earn their third Super Bowl title. The game was played on February 7, 2016, at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California. As this was the 50th Super Bowl, the league emphasized the ""golden anniversary"" with various gold-themed initiatives, as well as temporarily suspending the tradition of naming each Super Bowl game with Roman numerals (under which the game would have been known as ""Super Bowl L""), so that the logo could prominently feature the Arabic numerals 50.","If Roman numerals were used, what would Super Bowl 50 have been called?",Super Bowl L,"If Roman numerals were used, Super Bowl 50 would have been called ""Super Bowl L"".",0.33333333333333337,0.840755
+"Super Bowl 50 was an American football game to determine the champion of the National Football League (NFL) for the 2015 season. The American Football Conference (AFC) champion Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24–10 to earn their third Super Bowl title. The game was played on February 7, 2016, at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California. As this was the 50th Super Bowl, the league emphasized the ""golden anniversary"" with various gold-themed initiatives, as well as temporarily suspending the tradition of naming each Super Bowl game with Roman numerals (under which the game would have been known as ""Super Bowl L""), so that the logo could prominently feature the Arabic numerals 50.",Super Bowl 50 decided the NFL champion for what season?,2015,Super Bowl 50 decided the NFL champion for the 2015 season.,0.19999999999999998,0.8175393
+"Super Bowl 50 was an American football game to determine the champion of the National Football League (NFL) for the 2015 season. The American Football Conference (AFC) champion Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24–10 to earn their third Super Bowl title. The game was played on February 7, 2016, at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California. As this was the 50th Super Bowl, the league emphasized the ""golden anniversary"" with various gold-themed initiatives, as well as temporarily suspending the tradition of naming each Super Bowl game with Roman numerals (under which the game would have been known as ""Super Bowl L""), so that the logo could prominently feature the Arabic numerals 50.",What year did the Denver Broncos secure a Super Bowl title for the third time?,2015,The Denver Broncos secured their third Super Bowl title in the year 2016.,0.0,0.8254469
+"Super Bowl 50 was an American football game to determine the champion of the National Football League (NFL) for the 2015 season. The American Football Conference (AFC) champion Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24–10 to earn their third Super Bowl title. The game was played on February 7, 2016, at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California. As this was the 50th Super Bowl, the league emphasized the ""golden anniversary"" with various gold-themed initiatives, as well as temporarily suspending the tradition of naming each Super Bowl game with Roman numerals (under which the game would have been known as ""Super Bowl L""), so that the logo could prominently feature the Arabic numerals 50.",What city did Super Bowl 50 take place in?,Santa Clara,"Super Bowl 50 took place in the San Francisco Bay Area at Santa Clara, California.",0.25,0.8431589
+"Super Bowl 50 was an American football game to determine the champion of the National Football League (NFL) for the 2015 season. The American Football Conference (AFC) champion Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24–10 to earn their third Super Bowl title. The game was played on February 7, 2016, at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California. As this was the 50th Super Bowl, the league emphasized the ""golden anniversary"" with various gold-themed initiatives, as well as temporarily suspending the tradition of naming each Super Bowl game with Roman numerals (under which the game would have been known as ""Super Bowl L""), so that the logo could prominently feature the Arabic numerals 50.",What stadium did Super Bowl 50 take place in?,Levi's Stadium,"Super Bowl 50 took place at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California.",0.21052631578947367,0.815798
+"Super Bowl 50 was an American football game to determine the champion of the National Football League (NFL) for the 2015 season. The American Football Conference (AFC) champion Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24–10 to earn their third Super Bowl title. The game was played on February 7, 2016, at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California. As this was the 50th Super Bowl, the league emphasized the ""golden anniversary"" with various gold-themed initiatives, as well as temporarily suspending the tradition of naming each Super Bowl game with Roman numerals (under which the game would have been known as ""Super Bowl L""), so that the logo could prominently feature the Arabic numerals 50.",What was the final score of Super Bowl 50? ,24–10,"The final score of Super Bowl 50 was Denver Broncos 24, Carolina Panthers 10.",0.0,0.8035014
+"Super Bowl 50 was an American football game to determine the champion of the National Football League (NFL) for the 2015 season. The American Football Conference (AFC) champion Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24–10 to earn their third Super Bowl title. The game was played on February 7, 2016, at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California. As this was the 50th Super Bowl, the league emphasized the ""golden anniversary"" with various gold-themed initiatives, as well as temporarily suspending the tradition of naming each Super Bowl game with Roman numerals (under which the game would have been known as ""Super Bowl L""), so that the logo could prominently feature the Arabic numerals 50.","What month, day and year did Super Bowl 50 take place? ","February 7, 2016","Super Bowl 50 took place on February 7, 2016.",0.5,0.87301385
+"Super Bowl 50 was an American football game to determine the champion of the National Football League (NFL) for the 2015 season. The American Football Conference (AFC) champion Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24–10 to earn their third Super Bowl title. The game was played on February 7, 2016, at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California. As this was the 50th Super Bowl, the league emphasized the ""golden anniversary"" with various gold-themed initiatives, as well as temporarily suspending the tradition of naming each Super Bowl game with Roman numerals (under which the game would have been known as ""Super Bowl L""), so that the logo could prominently feature the Arabic numerals 50.",What year was Super Bowl 50?,2015,Super Bowl 50 was in the year 2016.,0.0,0.8231886
+"Super Bowl 50 was an American football game to determine the champion of the National Football League (NFL) for the 2015 season. The American Football Conference (AFC) champion Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24–10 to earn their third Super Bowl title. The game was played on February 7, 2016, at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California. As this was the 50th Super Bowl, the league emphasized the ""golden anniversary"" with various gold-themed initiatives, as well as temporarily suspending the tradition of naming each Super Bowl game with Roman numerals (under which the game would have been known as ""Super Bowl L""), so that the logo could prominently feature the Arabic numerals 50.",What team was the AFC champion?,Denver Broncos,The AFC champion for the 2015 season was the Denver Broncos. Could you please specify the year you are interested in?,0.21052631578947367,0.8456418
+"Super Bowl 50 was an American football game to determine the champion of the National Football League (NFL) for the 2015 season. The American Football Conference (AFC) champion Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24–10 to earn their third Super Bowl title. The game was played on February 7, 2016, at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California. As this was the 50th Super Bowl, the league emphasized the ""golden anniversary"" with various gold-themed initiatives, as well as temporarily suspending the tradition of naming each Super Bowl game with Roman numerals (under which the game would have been known as ""Super Bowl L""), so that the logo could prominently feature the Arabic numerals 50.",What team was the NFC champion?,Carolina Panthers,"I'm sorry, I don't have the most recent information on the NFC champion.",0.0,0.8142391
+"Super Bowl 50 was an American football game to determine the champion of the National Football League (NFL) for the 2015 season. The American Football Conference (AFC) champion Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24–10 to earn their third Super Bowl title. The game was played on February 7, 2016, at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California. As this was the 50th Super Bowl, the league emphasized the ""golden anniversary"" with various gold-themed initiatives, as well as temporarily suspending the tradition of naming each Super Bowl game with Roman numerals (under which the game would have been known as ""Super Bowl L""), so that the logo could prominently feature the Arabic numerals 50.",Who won Super Bowl 50?,Denver Broncos,The Denver Broncos won Super Bowl 50.,0.5,0.8863907
+"Super Bowl 50 was an American football game to determine the champion of the National Football League (NFL) for the 2015 season. The American Football Conference (AFC) champion Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24–10 to earn their third Super Bowl title. The game was played on February 7, 2016, at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California. As this was the 50th Super Bowl, the league emphasized the ""golden anniversary"" with various gold-themed initiatives, as well as temporarily suspending the tradition of naming each Super Bowl game with Roman numerals (under which the game would have been known as ""Super Bowl L""), so that the logo could prominently feature the Arabic numerals 50.",Super Bowl 50 determined the NFL champion for what season?,2015,Super Bowl 50 determined the NFL champion for the 2015 season.,0.19999999999999998,0.81693447
+"Super Bowl 50 was an American football game to determine the champion of the National Football League (NFL) for the 2015 season. The American Football Conference (AFC) champion Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24–10 to earn their third Super Bowl title. The game was played on February 7, 2016, at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California. As this was the 50th Super Bowl, the league emphasized the ""golden anniversary"" with various gold-themed initiatives, as well as temporarily suspending the tradition of naming each Super Bowl game with Roman numerals (under which the game would have been known as ""Super Bowl L""), so that the logo could prominently feature the Arabic numerals 50.",Which team won Super Bowl 50.,Denver Broncos,The Denver Broncos won Super Bowl 50.,0.5,0.8863907
+"Super Bowl 50 was an American football game to determine the champion of the National Football League (NFL) for the 2015 season. The American Football Conference (AFC) champion Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24–10 to earn their third Super Bowl title. The game was played on February 7, 2016, at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California. As this was the 50th Super Bowl, the league emphasized the ""golden anniversary"" with various gold-themed initiatives, as well as temporarily suspending the tradition of naming each Super Bowl game with Roman numerals (under which the game would have been known as ""Super Bowl L""), so that the logo could prominently feature the Arabic numerals 50.",Where was Super Bowl 50 held?,"Santa Clara, California.","Super Bowl 50 was held at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California.",0.3,0.8843575
+"Super Bowl 50 was an American football game to determine the champion of the National Football League (NFL) for the 2015 season. The American Football Conference (AFC) champion Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24–10 to earn their third Super Bowl title. The game was played on February 7, 2016, at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California. As this was the 50th Super Bowl, the league emphasized the ""golden anniversary"" with various gold-themed initiatives, as well as temporarily suspending the tradition of naming each Super Bowl game with Roman numerals (under which the game would have been known as ""Super Bowl L""), so that the logo could prominently feature the Arabic numerals 50.",The name of the NFL championship game is?,Super Bowl,The name of the NFL championship game is the Super Bowl.,0.4,0.8565423
+"Super Bowl 50 was an American football game to determine the champion of the National Football League (NFL) for the 2015 season. The American Football Conference (AFC) champion Denver Broncos defeated the National Football Conference (NFC) champion Carolina Panthers 24–10 to earn their third Super Bowl title. The game was played on February 7, 2016, at Levi's Stadium in the San Francisco Bay Area at Santa Clara, California. As this was the 50th Super Bowl, the league emphasized the ""golden anniversary"" with various gold-themed initiatives, as well as temporarily suspending the tradition of naming each Super Bowl game with Roman numerals (under which the game would have been known as ""Super Bowl L""), so that the logo could prominently feature the Arabic numerals 50.",What 2015 NFL team one the AFC playoff?,Denver Broncos,The Denver Broncos won the AFC playoff in 2015.,0.4444444444444445,0.88645095
+"The Panthers finished the regular season with a 15–1 record, and quarterback Cam Newton was named the NFL Most Valuable Player (MVP). They defeated the Arizona Cardinals 49–15 in the NFC Championship Game and advanced to their second Super Bowl appearance since the franchise was founded in 1995. The Broncos finished the regular season with a 12–4 record, and denied the New England Patriots a chance to defend their title from Super Bowl XLIX by defeating them 20–18 in the AFC Championship Game. They joined the Patriots, Dallas Cowboys, and Pittsburgh Steelers as one of four teams that have made eight appearances in the Super Bowl.",Which Carolina Panthers player was named Most Valuable Player?,Cam Newton,Cam Newton was named the NFL Most Valuable Player while playing for the Carolina Panthers.,0.2666666666666667,0.8782701
+"The Panthers finished the regular season with a 15–1 record, and quarterback Cam Newton was named the NFL Most Valuable Player (MVP). They defeated the Arizona Cardinals 49–15 in the NFC Championship Game and advanced to their second Super Bowl appearance since the franchise was founded in 1995. The Broncos finished the regular season with a 12–4 record, and denied the New England Patriots a chance to defend their title from Super Bowl XLIX by defeating them 20–18 in the AFC Championship Game. They joined the Patriots, Dallas Cowboys, and Pittsburgh Steelers as one of four teams that have made eight appearances in the Super Bowl.",How many appearances have the Denver Broncos made in the Super Bowl?,8,The Denver Broncos have made eight appearances in the Super Bowl.,0.0,0.8227126
+"The Panthers finished the regular season with a 15–1 record, and quarterback Cam Newton was named the NFL Most Valuable Player (MVP). They defeated the Arizona Cardinals 49–15 in the NFC Championship Game and advanced to their second Super Bowl appearance since the franchise was founded in 1995. The Broncos finished the regular season with a 12–4 record, and denied the New England Patriots a chance to defend their title from Super Bowl XLIX by defeating them 20–18 in the AFC Championship Game. They joined the Patriots, Dallas Cowboys, and Pittsburgh Steelers as one of four teams that have made eight appearances in the Super Bowl.",What year was the Carolina Panthers franchise founded?,1995,The Carolina Panthers franchise was founded in 1995.,0.25,0.8233834
+"The Panthers finished the regular season with a 15–1 record, and quarterback Cam Newton was named the NFL Most Valuable Player (MVP). They defeated the Arizona Cardinals 49–15 in the NFC Championship Game and advanced to their second Super Bowl appearance since the franchise was founded in 1995. The Broncos finished the regular season with a 12–4 record, and denied the New England Patriots a chance to defend their title from Super Bowl XLIX by defeating them 20–18 in the AFC Championship Game. They joined the Patriots, Dallas Cowboys, and Pittsburgh Steelers as one of four teams that have made eight appearances in the Super Bowl.",What team did the Panthers defeat?,Arizona Cardinals,The Panthers defeated the Arizona Cardinals.,0.6666666666666666,0.90900004
+"The Panthers finished the regular season with a 15–1 record, and quarterback Cam Newton was named the NFL Most Valuable Player (MVP). They defeated the Arizona Cardinals 49–15 in the NFC Championship Game and advanced to their second Super Bowl appearance since the franchise was founded in 1995. The Broncos finished the regular season with a 12–4 record, and denied the New England Patriots a chance to defend their title from Super Bowl XLIX by defeating them 20–18 in the AFC Championship Game. They joined the Patriots, Dallas Cowboys, and Pittsburgh Steelers as one of four teams that have made eight appearances in the Super Bowl.",Who did the Broncos prevent from going to the Super Bowl?,New England Patriots,The Broncos prevented the New England Patriots from going to the Super Bowl.,0.4615384615384615,0.8878024
+"The Panthers finished the regular season with a 15–1 record, and quarterback Cam Newton was named the NFL Most Valuable Player (MVP). They defeated the Arizona Cardinals 49–15 in the NFC Championship Game and advanced to their second Super Bowl appearance since the franchise was founded in 1995. The Broncos finished the regular season with a 12–4 record, and denied the New England Patriots a chance to defend their title from Super Bowl XLIX by defeating them 20–18 in the AFC Championship Game. They joined the Patriots, Dallas Cowboys, and Pittsburgh Steelers as one of four teams that have made eight appearances in the Super Bowl.",Who did the Panthers beat in the NFC Championship Game?,Arizona Cardinals,The Panthers beat the Arizona Cardinals in the NFC Championship Game.,0.4,0.8863175
+"The Panthers finished the regular season with a 15–1 record, and quarterback Cam Newton was named the NFL Most Valuable Player (MVP). They defeated the Arizona Cardinals 49–15 in the NFC Championship Game and advanced to their second Super Bowl appearance since the franchise was founded in 1995. The Broncos finished the regular season with a 12–4 record, and denied the New England Patriots a chance to defend their title from Super Bowl XLIX by defeating them 20–18 in the AFC Championship Game. They joined the Patriots, Dallas Cowboys, and Pittsburgh Steelers as one of four teams that have made eight appearances in the Super Bowl.",Who lost to the Broncos in the AFC Championship?,New England Patriots,The New England Patriots lost to the Broncos in the AFC Championship.,0.5,0.9091872
+"The Panthers finished the regular season with a 15–1 record, and quarterback Cam Newton was named the NFL Most Valuable Player (MVP). They defeated the Arizona Cardinals 49–15 in the NFC Championship Game and advanced to their second Super Bowl appearance since the franchise was founded in 1995. The Broncos finished the regular season with a 12–4 record, and denied the New England Patriots a chance to defend their title from Super Bowl XLIX by defeating them 20–18 in the AFC Championship Game. They joined the Patriots, Dallas Cowboys, and Pittsburgh Steelers as one of four teams that have made eight appearances in the Super Bowl.",Who were the defending Super Bowl champions?,New England Patriots,The Denver Broncos were the defending Super Bowl champions.,0.0,0.8717444
+"The Panthers finished the regular season with a 15–1 record, and quarterback Cam Newton was named the NFL Most Valuable Player (MVP). They defeated the Arizona Cardinals 49–15 in the NFC Championship Game and advanced to their second Super Bowl appearance since the franchise was founded in 1995. The Broncos finished the regular season with a 12–4 record, and denied the New England Patriots a chance to defend their title from Super Bowl XLIX by defeating them 20–18 in the AFC Championship Game. They joined the Patriots, Dallas Cowboys, and Pittsburgh Steelers as one of four teams that have made eight appearances in the Super Bowl.",How many teams have been in the Super Bowl eight times?,four,"Four teams have been in the Super Bowl eight times. These are the Denver Broncos, New England Patriots, Dallas Cowboys, and Pittsburgh Steelers.",0.0909090909090909,0.79814947
+"The Panthers finished the regular season with a 15–1 record, and quarterback Cam Newton was named the NFL Most Valuable Player (MVP). They defeated the Arizona Cardinals 49–15 in the NFC Championship Game and advanced to their second Super Bowl appearance since the franchise was founded in 1995. The Broncos finished the regular season with a 12–4 record, and denied the New England Patriots a chance to defend their title from Super Bowl XLIX by defeating them 20–18 in the AFC Championship Game. They joined the Patriots, Dallas Cowboys, and Pittsburgh Steelers as one of four teams that have made eight appearances in the Super Bowl.",Who was this season's NFL MVP?,Cam Newton,The MVP of the most recent NFL season mentioned in the context was Cam Newton.,0.2857142857142857,0.86420196
+"The Panthers finished the regular season with a 15–1 record, and quarterback Cam Newton was named the NFL Most Valuable Player (MVP). They defeated the Arizona Cardinals 49–15 in the NFC Championship Game and advanced to their second Super Bowl appearance since the franchise was founded in 1995. The Broncos finished the regular season with a 12–4 record, and denied the New England Patriots a chance to defend their title from Super Bowl XLIX by defeating them 20–18 in the AFC Championship Game. They joined the Patriots, Dallas Cowboys, and Pittsburgh Steelers as one of four teams that have made eight appearances in the Super Bowl.",What was the win/loss ratio in 2015 for the Carolina Panthers during their regular season?,15–1,The Carolina Panthers had a win/loss ratio of 15-1 during their 2015 regular season.,0.0,0.83852804
+"The Panthers finished the regular season with a 15–1 record, and quarterback Cam Newton was named the NFL Most Valuable Player (MVP). They defeated the Arizona Cardinals 49–15 in the NFC Championship Game and advanced to their second Super Bowl appearance since the franchise was founded in 1995. The Broncos finished the regular season with a 12–4 record, and denied the New England Patriots a chance to defend their title from Super Bowl XLIX by defeating them 20–18 in the AFC Championship Game. They joined the Patriots, Dallas Cowboys, and Pittsburgh Steelers as one of four teams that have made eight appearances in the Super Bowl.",Which Carolina Panthers team member was picked as the team's MVP in 2015? ,Cam Newton,Cam Newton was picked as the Carolina Panthers' team MVP in 2015.,0.3076923076923077,0.8891575
+"The Panthers finished the regular season with a 15–1 record, and quarterback Cam Newton was named the NFL Most Valuable Player (MVP). They defeated the Arizona Cardinals 49–15 in the NFC Championship Game and advanced to their second Super Bowl appearance since the franchise was founded in 1995. The Broncos finished the regular season with a 12–4 record, and denied the New England Patriots a chance to defend their title from Super Bowl XLIX by defeating them 20–18 in the AFC Championship Game. They joined the Patriots, Dallas Cowboys, and Pittsburgh Steelers as one of four teams that have made eight appearances in the Super Bowl.",What were the win/loss game stats for the Denver Bronco's regular season in 2015?,12–4,The Denver Broncos finished the regular season with a 12–4 record in 2015.,0.18181818181818182,0.83720595
+"The Panthers finished the regular season with a 15–1 record, and quarterback Cam Newton was named the NFL Most Valuable Player (MVP). They defeated the Arizona Cardinals 49–15 in the NFC Championship Game and advanced to their second Super Bowl appearance since the franchise was founded in 1995. The Broncos finished the regular season with a 12–4 record, and denied the New England Patriots a chance to defend their title from Super Bowl XLIX by defeating them 20–18 in the AFC Championship Game. They joined the Patriots, Dallas Cowboys, and Pittsburgh Steelers as one of four teams that have made eight appearances in the Super Bowl.",How many teams have played in the Super Bowl eight times?,4,"The teams that have played in the Super Bowl eight times are the Patriots, Dallas Cowboys, Pittsburgh Steelers, and the Denver Broncos.",0.0,0.801293
+"The Panthers finished the regular season with a 15–1 record, and quarterback Cam Newton was named the NFL Most Valuable Player (MVP). They defeated the Arizona Cardinals 49–15 in the NFC Championship Game and advanced to their second Super Bowl appearance since the franchise was founded in 1995. The Broncos finished the regular season with a 12–4 record, and denied the New England Patriots a chance to defend their title from Super Bowl XLIX by defeating them 20–18 in the AFC Championship Game. They joined the Patriots, Dallas Cowboys, and Pittsburgh Steelers as one of four teams that have made eight appearances in the Super Bowl.",Which team did not get a chance to defend their Super Bowl XLIX win in Super Bowl 50?,New England Patriots,The New England Patriots.,1.0,0.9574103
+"The Panthers finished the regular season with a 15–1 record, and quarterback Cam Newton was named the NFL Most Valuable Player (MVP). They defeated the Arizona Cardinals 49–15 in the NFC Championship Game and advanced to their second Super Bowl appearance since the franchise was founded in 1995. The Broncos finished the regular season with a 12–4 record, and denied the New England Patriots a chance to defend their title from Super Bowl XLIX by defeating them 20–18 in the AFC Championship Game. They joined the Patriots, Dallas Cowboys, and Pittsburgh Steelers as one of four teams that have made eight appearances in the Super Bowl.",Who is the quarterback for the Panthers?,Cam Newton,"I'm sorry, I couldn't find the current information about the quarterback for the Carolina Panthers.",0.0,0.81419134
+"The Panthers finished the regular season with a 15–1 record, and quarterback Cam Newton was named the NFL Most Valuable Player (MVP). They defeated the Arizona Cardinals 49–15 in the NFC Championship Game and advanced to their second Super Bowl appearance since the franchise was founded in 1995. The Broncos finished the regular season with a 12–4 record, and denied the New England Patriots a chance to defend their title from Super Bowl XLIX by defeating them 20–18 in the AFC Championship Game. They joined the Patriots, Dallas Cowboys, and Pittsburgh Steelers as one of four teams that have made eight appearances in the Super Bowl.",Who did Carolina beat in the NFC championship game?,Arizona Cardinals,Carolina beat the Arizona Cardinals in the NFC championship game.,0.4,0.8821087
+"The Panthers finished the regular season with a 15–1 record, and quarterback Cam Newton was named the NFL Most Valuable Player (MVP). They defeated the Arizona Cardinals 49–15 in the NFC Championship Game and advanced to their second Super Bowl appearance since the franchise was founded in 1995. The Broncos finished the regular season with a 12–4 record, and denied the New England Patriots a chance to defend their title from Super Bowl XLIX by defeating them 20–18 in the AFC Championship Game. They joined the Patriots, Dallas Cowboys, and Pittsburgh Steelers as one of four teams that have made eight appearances in the Super Bowl.",How many times have the Panthers been in the Super Bowl?,2,The Panthers have been in the Super Bowl twice.,0.0,0.83318734
+"The Panthers finished the regular season with a 15–1 record, and quarterback Cam Newton was named the NFL Most Valuable Player (MVP). They defeated the Arizona Cardinals 49–15 in the NFC Championship Game and advanced to their second Super Bowl appearance since the franchise was founded in 1995. The Broncos finished the regular season with a 12–4 record, and denied the New England Patriots a chance to defend their title from Super Bowl XLIX by defeating them 20–18 in the AFC Championship Game. They joined the Patriots, Dallas Cowboys, and Pittsburgh Steelers as one of four teams that have made eight appearances in the Super Bowl.",Who did Denver beat in the AFC championship?,New England Patriots,The Denver Broncos beat the New England Patriots in the AFC championship.,0.5,0.89412516
+"The Panthers finished the regular season with a 15–1 record, and quarterback Cam Newton was named the NFL Most Valuable Player (MVP). They defeated the Arizona Cardinals 49–15 in the NFC Championship Game and advanced to their second Super Bowl appearance since the franchise was founded in 1995. The Broncos finished the regular season with a 12–4 record, and denied the New England Patriots a chance to defend their title from Super Bowl XLIX by defeating them 20–18 in the AFC Championship Game. They joined the Patriots, Dallas Cowboys, and Pittsburgh Steelers as one of four teams that have made eight appearances in the Super Bowl.",Who was the Most Valuable Player for the 2015 NFL season?,Cam Newton,The Most Valuable Player for the 2015 NFL season was Cam Newton.,0.33333333333333337,0.862852
+"The Panthers finished the regular season with a 15–1 record, and quarterback Cam Newton was named the NFL Most Valuable Player (MVP). They defeated the Arizona Cardinals 49–15 in the NFC Championship Game and advanced to their second Super Bowl appearance since the franchise was founded in 1995. The Broncos finished the regular season with a 12–4 record, and denied the New England Patriots a chance to defend their title from Super Bowl XLIX by defeating them 20–18 in the AFC Championship Game. They joined the Patriots, Dallas Cowboys, and Pittsburgh Steelers as one of four teams that have made eight appearances in the Super Bowl.",Who did Denver beat in the 2015 AFC Championship game?,New England Patriots,Denver beat the New England Patriots in the 2015 AFC Championship game.,0.4615384615384615,0.8849049
+"The Panthers finished the regular season with a 15–1 record, and quarterback Cam Newton was named the NFL Most Valuable Player (MVP). They defeated the Arizona Cardinals 49–15 in the NFC Championship Game and advanced to their second Super Bowl appearance since the franchise was founded in 1995. The Broncos finished the regular season with a 12–4 record, and denied the New England Patriots a chance to defend their title from Super Bowl XLIX by defeating them 20–18 in the AFC Championship Game. They joined the Patriots, Dallas Cowboys, and Pittsburgh Steelers as one of four teams that have made eight appearances in the Super Bowl.",Who did the Carolina Panthers beat in the 2015 NFC Championship game?,Arizona Cardinals,The Carolina Panthers beat the Arizona Cardinals in the 2015 NFC Championship game.,0.33333333333333337,0.8737707
+"The Panthers finished the regular season with a 15–1 record, and quarterback Cam Newton was named the NFL Most Valuable Player (MVP). They defeated the Arizona Cardinals 49–15 in the NFC Championship Game and advanced to their second Super Bowl appearance since the franchise was founded in 1995. The Broncos finished the regular season with a 12–4 record, and denied the New England Patriots a chance to defend their title from Super Bowl XLIX by defeating them 20–18 in the AFC Championship Game. They joined the Patriots, Dallas Cowboys, and Pittsburgh Steelers as one of four teams that have made eight appearances in the Super Bowl.",Who was the 2015 NFL MVP?,Cam Newton,The 2015 NFL MVP was Cam Newton.,0.5,0.8813283
+"The Panthers finished the regular season with a 15–1 record, and quarterback Cam Newton was named the NFL Most Valuable Player (MVP). They defeated the Arizona Cardinals 49–15 in the NFC Championship Game and advanced to their second Super Bowl appearance since the franchise was founded in 1995. The Broncos finished the regular season with a 12–4 record, and denied the New England Patriots a chance to defend their title from Super Bowl XLIX by defeating them 20–18 in the AFC Championship Game. They joined the Patriots, Dallas Cowboys, and Pittsburgh Steelers as one of four teams that have made eight appearances in the Super Bowl.",Who did the Panthers beat to become the NFC champs?,Arizona Cardinals,The Panthers beat the Arizona Cardinals to become the NFC champs.,0.4,0.876848
+"The Panthers finished the regular season with a 15–1 record, and quarterback Cam Newton was named the NFL Most Valuable Player (MVP). They defeated the Arizona Cardinals 49–15 in the NFC Championship Game and advanced to their second Super Bowl appearance since the franchise was founded in 1995. The Broncos finished the regular season with a 12–4 record, and denied the New England Patriots a chance to defend their title from Super Bowl XLIX by defeating them 20–18 in the AFC Championship Game. They joined the Patriots, Dallas Cowboys, and Pittsburgh Steelers as one of four teams that have made eight appearances in the Super Bowl.",What year did the Carolina Panthers form?,1995.,The Carolina Panthers were formed in 1995.,0.2857142857142857,0.87928814
+"The Broncos took an early lead in Super Bowl 50 and never trailed. Newton was limited by Denver's defense, which sacked him seven times and forced him into three turnovers, including a fumble which they recovered for a touchdown. Denver linebacker Von Miller was named Super Bowl MVP, recording five solo tackles, 2½ sacks, and two forced fumbles.",Who was the Super Bowl 50 MVP?,Von Miller,The Super Bowl 50 MVP was Von Miller.,0.4444444444444445,0.85164344
+"The Broncos took an early lead in Super Bowl 50 and never trailed. Newton was limited by Denver's defense, which sacked him seven times and forced him into three turnovers, including a fumble which they recovered for a touchdown. Denver linebacker Von Miller was named Super Bowl MVP, recording five solo tackles, 2½ sacks, and two forced fumbles.",How many fumbles did Von Miller force in Super Bowl 50?,2,Von Miller forced two fumbles in Super Bowl 50.,0.0,0.8216288
+"The Broncos took an early lead in Super Bowl 50 and never trailed. Newton was limited by Denver's defense, which sacked him seven times and forced him into three turnovers, including a fumble which they recovered for a touchdown. Denver linebacker Von Miller was named Super Bowl MVP, recording five solo tackles, 2½ sacks, and two forced fumbles.",Which team held the scoring lead throughout the entire game?,Broncos,The Denver Broncos.,0.6666666666666666,0.8619311
+"The Broncos took an early lead in Super Bowl 50 and never trailed. Newton was limited by Denver's defense, which sacked him seven times and forced him into three turnovers, including a fumble which they recovered for a touchdown. Denver linebacker Von Miller was named Super Bowl MVP, recording five solo tackles, 2½ sacks, and two forced fumbles.",Which Denver linebacker was named Super Bowl MVP?,linebacker Von Miller,The Denver linebacker who was named Super Bowl MVP is Von Miller.,0.42857142857142855,0.88970226
+"The Broncos took an early lead in Super Bowl 50 and never trailed. Newton was limited by Denver's defense, which sacked him seven times and forced him into three turnovers, including a fumble which they recovered for a touchdown. Denver linebacker Von Miller was named Super Bowl MVP, recording five solo tackles, 2½ sacks, and two forced fumbles.",How many solo tackles did Von Miller make at Super Bowl 50?,five solo tackles,Von Miller made five solo tackles at Super Bowl 50.,0.4615384615384615,0.8914581
+"The Broncos took an early lead in Super Bowl 50 and never trailed. Newton was limited by Denver's defense, which sacked him seven times and forced him into three turnovers, including a fumble which they recovered for a touchdown. Denver linebacker Von Miller was named Super Bowl MVP, recording five solo tackles, 2½ sacks, and two forced fumbles.",Who was limited by Denver's defense?,Newton was limited by Denver's defense,Cam Newton was limited by Denver's defense.,0.923076923076923,0.95191014
+"The Broncos took an early lead in Super Bowl 50 and never trailed. Newton was limited by Denver's defense, which sacked him seven times and forced him into three turnovers, including a fumble which they recovered for a touchdown. Denver linebacker Von Miller was named Super Bowl MVP, recording five solo tackles, 2½ sacks, and two forced fumbles.",How many times was Cam Newton sacked?,seven,"I'm sorry, I couldn't find the specific information on how many times Cam Newton was sacked in his career.",0.0,0.80530393
+"The Broncos took an early lead in Super Bowl 50 and never trailed. Newton was limited by Denver's defense, which sacked him seven times and forced him into three turnovers, including a fumble which they recovered for a touchdown. Denver linebacker Von Miller was named Super Bowl MVP, recording five solo tackles, 2½ sacks, and two forced fumbles.",Who won the Super Bowl MVP?,Von Miller,"I'm sorry, I am unable to provide the most recent Super Bowl MVP at this time.",0.0,0.7907247
+"The Broncos took an early lead in Super Bowl 50 and never trailed. Newton was limited by Denver's defense, which sacked him seven times and forced him into three turnovers, including a fumble which they recovered for a touchdown. Denver linebacker Von Miller was named Super Bowl MVP, recording five solo tackles, 2½ sacks, and two forced fumbles.",How many turnovers did Cam Newton have?,three,"I'm sorry, I couldn't find the specific number of total turnovers Cam Newton has had in his career.",0.0,0.8055697
+"The Broncos took an early lead in Super Bowl 50 and never trailed. Newton was limited by Denver's defense, which sacked him seven times and forced him into three turnovers, including a fumble which they recovered for a touchdown. Denver linebacker Von Miller was named Super Bowl MVP, recording five solo tackles, 2½ sacks, and two forced fumbles.",How many fumbles did Von Miller force?,two,"I'm sorry, I couldn't find the exact number of career forced fumbles by Von Miller.",0.0,0.80948853
+"The Broncos took an early lead in Super Bowl 50 and never trailed. Newton was limited by Denver's defense, which sacked him seven times and forced him into three turnovers, including a fumble which they recovered for a touchdown. Denver linebacker Von Miller was named Super Bowl MVP, recording five solo tackles, 2½ sacks, and two forced fumbles.",Who was given the esteemed status of MVP for Super Bowl 50?,Von Miller,The MVP for Super Bowl 50 was Denver Broncos linebacker Von Miller.,0.3076923076923077,0.84020406
+"The Broncos took an early lead in Super Bowl 50 and never trailed. Newton was limited by Denver's defense, which sacked him seven times and forced him into three turnovers, including a fumble which they recovered for a touchdown. Denver linebacker Von Miller was named Super Bowl MVP, recording five solo tackles, 2½ sacks, and two forced fumbles.",What position does Von Miller play for the Denver Broncos?,linebacker,Von Miller plays the position of linebacker for the Denver Broncos.,0.19999999999999998,0.8042575
+"The Broncos took an early lead in Super Bowl 50 and never trailed. Newton was limited by Denver's defense, which sacked him seven times and forced him into three turnovers, including a fumble which they recovered for a touchdown. Denver linebacker Von Miller was named Super Bowl MVP, recording five solo tackles, 2½ sacks, and two forced fumbles.",What was the number of solo tackles that Von Miller had in Super Bowl 50?,5,Von Miller had five solo tackles in Super Bowl 50.,0.0,0.8285005
+"The Broncos took an early lead in Super Bowl 50 and never trailed. Newton was limited by Denver's defense, which sacked him seven times and forced him into three turnovers, including a fumble which they recovered for a touchdown. Denver linebacker Von Miller was named Super Bowl MVP, recording five solo tackles, 2½ sacks, and two forced fumbles.",How many forced fumbles did Von Miller have during the Super Bowl 50 game?,2,Von Miller had two forced fumbles during the Super Bowl 50 game.,0.0,0.8272358
+"The Broncos took an early lead in Super Bowl 50 and never trailed. Newton was limited by Denver's defense, which sacked him seven times and forced him into three turnovers, including a fumble which they recovered for a touchdown. Denver linebacker Von Miller was named Super Bowl MVP, recording five solo tackles, 2½ sacks, and two forced fumbles.",Who won the MVP for the Super Bowl?,Von Miller,"I'm sorry, I am unable to provide the most recent Super Bowl MVP at this time.",0.0,0.7907247
+"The Broncos took an early lead in Super Bowl 50 and never trailed. Newton was limited by Denver's defense, which sacked him seven times and forced him into three turnovers, including a fumble which they recovered for a touchdown. Denver linebacker Von Miller was named Super Bowl MVP, recording five solo tackles, 2½ sacks, and two forced fumbles.",How many tackles did Von Miller get during the game?,5,Von Miller made five tackles during the game.,0.0,0.83889985
+"The Broncos took an early lead in Super Bowl 50 and never trailed. Newton was limited by Denver's defense, which sacked him seven times and forced him into three turnovers, including a fumble which they recovered for a touchdown. Denver linebacker Von Miller was named Super Bowl MVP, recording five solo tackles, 2½ sacks, and two forced fumbles.",How many times was Cam Newton sacked in Super Bowl 50?,seven,Cam Newton was sacked seven times in Super Bowl 50.,0.18181818181818182,0.8137235
+"The Broncos took an early lead in Super Bowl 50 and never trailed. Newton was limited by Denver's defense, which sacked him seven times and forced him into three turnovers, including a fumble which they recovered for a touchdown. Denver linebacker Von Miller was named Super Bowl MVP, recording five solo tackles, 2½ sacks, and two forced fumbles.",How many times did the Denver defense force Newton into turnovers?,three,The Denver defense forced Newton into three turnovers.,0.25,0.82676935
+"The Broncos took an early lead in Super Bowl 50 and never trailed. Newton was limited by Denver's defense, which sacked him seven times and forced him into three turnovers, including a fumble which they recovered for a touchdown. Denver linebacker Von Miller was named Super Bowl MVP, recording five solo tackles, 2½ sacks, and two forced fumbles.",Which Newton turnover resulted in seven points for Denver?,a fumble,The Newton turnover that resulted in seven points for Denver was a fumble that Denver recovered for a touchdown during Super Bowl 50.,0.09523809523809523,0.8525294
+"The Broncos took an early lead in Super Bowl 50 and never trailed. Newton was limited by Denver's defense, which sacked him seven times and forced him into three turnovers, including a fumble which they recovered for a touchdown. Denver linebacker Von Miller was named Super Bowl MVP, recording five solo tackles, 2½ sacks, and two forced fumbles.",Who was the Most Valuable Player of Super Bowl 50?,Von Miller,The Most Valuable Player of Super Bowl 50 was Denver linebacker Von Miller.,0.2857142857142857,0.82736576
+"The Broncos took an early lead in Super Bowl 50 and never trailed. Newton was limited by Denver's defense, which sacked him seven times and forced him into three turnovers, including a fumble which they recovered for a touchdown. Denver linebacker Von Miller was named Super Bowl MVP, recording five solo tackles, 2½ sacks, and two forced fumbles.",What position does Von Miller play?,linebacker,Von Miller plays the position of linebacker.,0.2857142857142857,0.8274767
+"The Broncos took an early lead in Super Bowl 50 and never trailed. Newton was limited by Denver's defense, which sacked him seven times and forced him into three turnovers, including a fumble which they recovered for a touchdown. Denver linebacker Von Miller was named Super Bowl MVP, recording five solo tackles, 2½ sacks, and two forced fumbles.",How many times was the Panthers' quarterback sacked?,seven,"Cam Newton, the Panthers' quarterback, was sacked seven times in Super Bowl 50.",0.15384615384615385,0.813856
+"The Broncos took an early lead in Super Bowl 50 and never trailed. Newton was limited by Denver's defense, which sacked him seven times and forced him into three turnovers, including a fumble which they recovered for a touchdown. Denver linebacker Von Miller was named Super Bowl MVP, recording five solo tackles, 2½ sacks, and two forced fumbles.",How many times did the Broncos cause turnovers in the game?,three,The Broncos caused three turnovers in the game.,0.2857142857142857,0.8297505
+"The Broncos took an early lead in Super Bowl 50 and never trailed. Newton was limited by Denver's defense, which sacked him seven times and forced him into three turnovers, including a fumble which they recovered for a touchdown. Denver linebacker Von Miller was named Super Bowl MVP, recording five solo tackles, 2½ sacks, and two forced fumbles.",What Denver player caused two fumbles for the Panthers?,Von Miller,Von Miller was the Denver player who caused two fumbles for the Panthers.,0.3076923076923077,0.8870876
+"The Broncos took an early lead in Super Bowl 50 and never trailed. Newton was limited by Denver's defense, which sacked him seven times and forced him into three turnovers, including a fumble which they recovered for a touchdown. Denver linebacker Von Miller was named Super Bowl MVP, recording five solo tackles, 2½ sacks, and two forced fumbles.",How many tackles did Von Miller accomlish by himself in the game?,five,Von Miller accomplished five solo tackles in the game.,0.2222222222222222,0.8373196
+"CBS broadcast Super Bowl 50 in the U.S., and charged an average of $5 million for a 30-second commercial during the game. The Super Bowl 50 halftime show was headlined by the British rock group Coldplay with special guest performers Beyoncé and Bruno Mars, who headlined the Super Bowl XLVII and Super Bowl XLVIII halftime shows, respectively. It was the third-most watched U.S. broadcast ever.",Which network broadcasted Super Bowl 50 in the U.S.?,CBS,CBS broadcasted Super Bowl 50 in the U.S.,0.25,0.83527106
+"CBS broadcast Super Bowl 50 in the U.S., and charged an average of $5 million for a 30-second commercial during the game. The Super Bowl 50 halftime show was headlined by the British rock group Coldplay with special guest performers Beyoncé and Bruno Mars, who headlined the Super Bowl XLVII and Super Bowl XLVIII halftime shows, respectively. It was the third-most watched U.S. broadcast ever.",What was the average cost for a 30 second commercial during Super Bowl 50?,$5 million,The average cost for a 30 second commercial during Super Bowl 50 was $5 million.,0.2666666666666667,0.8531522
+"CBS broadcast Super Bowl 50 in the U.S., and charged an average of $5 million for a 30-second commercial during the game. The Super Bowl 50 halftime show was headlined by the British rock group Coldplay with special guest performers Beyoncé and Bruno Mars, who headlined the Super Bowl XLVII and Super Bowl XLVIII halftime shows, respectively. It was the third-most watched U.S. broadcast ever.",Which group headlined the Super Bowl 50 halftime show?,Coldplay,The British rock group Coldplay headlined the Super Bowl 50 halftime show.,0.18181818181818182,0.8169567
+"CBS broadcast Super Bowl 50 in the U.S., and charged an average of $5 million for a 30-second commercial during the game. The Super Bowl 50 halftime show was headlined by the British rock group Coldplay with special guest performers Beyoncé and Bruno Mars, who headlined the Super Bowl XLVII and Super Bowl XLVIII halftime shows, respectively. It was the third-most watched U.S. broadcast ever.",Which performers joined the headliner during the Super Bowl 50 halftime show?,Beyoncé and Bruno Mars,The performers who joined the headliner during the Super Bowl 50 halftime show were Beyoncé and Bruno Mars.,0.4210526315789474,0.88569283
+"CBS broadcast Super Bowl 50 in the U.S., and charged an average of $5 million for a 30-second commercial during the game. The Super Bowl 50 halftime show was headlined by the British rock group Coldplay with special guest performers Beyoncé and Bruno Mars, who headlined the Super Bowl XLVII and Super Bowl XLVIII halftime shows, respectively. It was the third-most watched U.S. broadcast ever.",At which Super Bowl did Beyonce headline the halftime show?,Super Bowl XLVII,Beyonce headlined the halftime show at Super Bowl XLVII.,0.5454545454545454,0.90231645
+"CBS broadcast Super Bowl 50 in the U.S., and charged an average of $5 million for a 30-second commercial during the game. The Super Bowl 50 halftime show was headlined by the British rock group Coldplay with special guest performers Beyoncé and Bruno Mars, who headlined the Super Bowl XLVII and Super Bowl XLVIII halftime shows, respectively. It was the third-most watched U.S. broadcast ever.",Who was the broadcaster for Super Bowl 50 in the United States?,CBS,CBS was the broadcaster for Super Bowl 50 in the United States.,0.18181818181818182,0.81726897
+"CBS broadcast Super Bowl 50 in the U.S., and charged an average of $5 million for a 30-second commercial during the game. The Super Bowl 50 halftime show was headlined by the British rock group Coldplay with special guest performers Beyoncé and Bruno Mars, who headlined the Super Bowl XLVII and Super Bowl XLVIII halftime shows, respectively. It was the third-most watched U.S. broadcast ever.",What was the average cost of a 30-second commercial?,$5 million,"I'm sorry, I couldn't find the specific information on the average cost of a 30-second TV commercial in general. However, for high-profile events like the Super Bowl, the cost can go up to $5 million for a 30-second slot. The cost can vary greatly depending on the time, channel, and program during which the commercial is aired.",0.07999999999999999,0.82230425
+"CBS broadcast Super Bowl 50 in the U.S., and charged an average of $5 million for a 30-second commercial during the game. The Super Bowl 50 halftime show was headlined by the British rock group Coldplay with special guest performers Beyoncé and Bruno Mars, who headlined the Super Bowl XLVII and Super Bowl XLVIII halftime shows, respectively. It was the third-most watched U.S. broadcast ever.",What halftime performer previously headlined Super Bowl XLVII?,Beyoncé,Beyoncé.,1.0,0.98789775
+"CBS broadcast Super Bowl 50 in the U.S., and charged an average of $5 million for a 30-second commercial during the game. The Super Bowl 50 halftime show was headlined by the British rock group Coldplay with special guest performers Beyoncé and Bruno Mars, who headlined the Super Bowl XLVII and Super Bowl XLVIII halftime shows, respectively. It was the third-most watched U.S. broadcast ever.",What halftime performer previously headlined Super Bowl XLVIII?,Bruno Mars,Bruno Mars.,1.0,0.98733324
+"CBS broadcast Super Bowl 50 in the U.S., and charged an average of $5 million for a 30-second commercial during the game. The Super Bowl 50 halftime show was headlined by the British rock group Coldplay with special guest performers Beyoncé and Bruno Mars, who headlined the Super Bowl XLVII and Super Bowl XLVIII halftime shows, respectively. It was the third-most watched U.S. broadcast ever.",Who was the main performer at this year's halftime show?,Coldplay,The most recent Super Bowl halftime show performer in the context provided was the British rock group Coldplay with special guest performers Beyoncé and Bruno Mars.,0.08333333333333333,0.8075878
+"CBS broadcast Super Bowl 50 in the U.S., and charged an average of $5 million for a 30-second commercial during the game. The Super Bowl 50 halftime show was headlined by the British rock group Coldplay with special guest performers Beyoncé and Bruno Mars, who headlined the Super Bowl XLVII and Super Bowl XLVIII halftime shows, respectively. It was the third-most watched U.S. broadcast ever.",Which network broadcasted the 50th Super Bowl game? ,CBS,CBS broadcasted the 50th Super Bowl game in the U.S.,0.2222222222222222,0.82476676
+"CBS broadcast Super Bowl 50 in the U.S., and charged an average of $5 million for a 30-second commercial during the game. The Super Bowl 50 halftime show was headlined by the British rock group Coldplay with special guest performers Beyoncé and Bruno Mars, who headlined the Super Bowl XLVII and Super Bowl XLVIII halftime shows, respectively. It was the third-most watched U.S. broadcast ever.",What was the average cost for a TV ad lasting 30 seconds during Super Bowl 50?,$5 million,The average cost for a 30-second TV ad during Super Bowl 50 was $5 million.,0.2666666666666667,0.8501834
+"CBS broadcast Super Bowl 50 in the U.S., and charged an average of $5 million for a 30-second commercial during the game. The Super Bowl 50 halftime show was headlined by the British rock group Coldplay with special guest performers Beyoncé and Bruno Mars, who headlined the Super Bowl XLVII and Super Bowl XLVIII halftime shows, respectively. It was the third-most watched U.S. broadcast ever.",Who was the male singer who performed as a special guest during Super Bowl 50?,Bruno Mars,The male singer who performed as a special guest during Super Bowl 50 was Bruno Mars.,0.25,0.8465806
+"CBS broadcast Super Bowl 50 in the U.S., and charged an average of $5 million for a 30-second commercial during the game. The Super Bowl 50 halftime show was headlined by the British rock group Coldplay with special guest performers Beyoncé and Bruno Mars, who headlined the Super Bowl XLVII and Super Bowl XLVIII halftime shows, respectively. It was the third-most watched U.S. broadcast ever.",What ranking does the Super Bowl 50 halftime show have on the list of most watched TV broadcasts?,third,The Super Bowl 50 halftime show ranks third on the list of most watched TV broadcasts in the U.S.,0.11764705882352941,0.8186487
+"CBS broadcast Super Bowl 50 in the U.S., and charged an average of $5 million for a 30-second commercial during the game. The Super Bowl 50 halftime show was headlined by the British rock group Coldplay with special guest performers Beyoncé and Bruno Mars, who headlined the Super Bowl XLVII and Super Bowl XLVIII halftime shows, respectively. It was the third-most watched U.S. broadcast ever.",What station aired the Super Bowl?,CBS,CBS aired the Super Bowl.,0.4,0.84404844
+"CBS broadcast Super Bowl 50 in the U.S., and charged an average of $5 million for a 30-second commercial during the game. The Super Bowl 50 halftime show was headlined by the British rock group Coldplay with special guest performers Beyoncé and Bruno Mars, who headlined the Super Bowl XLVII and Super Bowl XLVIII halftime shows, respectively. It was the third-most watched U.S. broadcast ever.",How much money did a 1/2 minute commercial cost?,$5 million,The cost of a 1/2 minute commercial can vary greatly depending on the platform and event during which it is aired. Could you please provide more details?,0.0,0.8010742
+"CBS broadcast Super Bowl 50 in the U.S., and charged an average of $5 million for a 30-second commercial during the game. The Super Bowl 50 halftime show was headlined by the British rock group Coldplay with special guest performers Beyoncé and Bruno Mars, who headlined the Super Bowl XLVII and Super Bowl XLVIII halftime shows, respectively. It was the third-most watched U.S. broadcast ever.",What band headlined half-time during Super Bowl 50?,Coldplay,The band that headlined the half-time show during Super Bowl 50 was the British rock group Coldplay.,0.13333333333333333,0.8181728
+"CBS broadcast Super Bowl 50 in the U.S., and charged an average of $5 million for a 30-second commercial during the game. The Super Bowl 50 halftime show was headlined by the British rock group Coldplay with special guest performers Beyoncé and Bruno Mars, who headlined the Super Bowl XLVII and Super Bowl XLVIII halftime shows, respectively. It was the third-most watched U.S. broadcast ever.",What two artists came out with Coldplay during the half-time show?,Beyoncé and Bruno Mars,The two artists that came out with Coldplay during the half-time show were Beyoncé and Bruno Mars.,0.4210526315789474,0.887571
+"CBS broadcast Super Bowl 50 in the U.S., and charged an average of $5 million for a 30-second commercial during the game. The Super Bowl 50 halftime show was headlined by the British rock group Coldplay with special guest performers Beyoncé and Bruno Mars, who headlined the Super Bowl XLVII and Super Bowl XLVIII halftime shows, respectively. It was the third-most watched U.S. broadcast ever.",Who broadcast the Super Bowl on TV?,CBS,"I'm sorry, I couldn't find the specific information on which network broadcasted the Super Bowl in 2021.",0.0,0.8034885
+"CBS broadcast Super Bowl 50 in the U.S., and charged an average of $5 million for a 30-second commercial during the game. The Super Bowl 50 halftime show was headlined by the British rock group Coldplay with special guest performers Beyoncé and Bruno Mars, who headlined the Super Bowl XLVII and Super Bowl XLVIII halftime shows, respectively. It was the third-most watched U.S. broadcast ever.",Who headlined the halftime show for Super Bowl 50?,Coldplay,The Super Bowl 50 halftime show was headlined by the British rock group Coldplay with special guest performers Beyoncé and Bruno Mars.,0.09523809523809523,0.80412585

model.py

@@ -0,0 +1,9 @@
+import os
+import google.generativeai as genai
+
+genai.configure(api_key="AIzaSyBIfGJRoet_wzzYXIiWXxStkIigEOzSR2o")
+
+models = genai.list_models()
+
+for model in models:
+    print(f"Name: {model.name} | Methods: {model.supported_generation_methods}")

modtran.py

@@ -0,0 +1,222 @@
+import streamlit as st
+import os
+import subprocess
+import pdfplumber
+from lxml import etree
+from bs4 import BeautifulSoup
+from PyPDF2 import PdfReader
+from langchain_community.vectorstores import FAISS
+from langchain_community.embeddings import OpenAIEmbeddings
+from langchain_community.embeddings import HuggingFaceBgeEmbeddings
+from langchain_openai import ChatOpenAI
+from langchain.agents import initialize_agent, AgentType
+from langchain.agents import Tool
+from langchain.memory import ConversationBufferMemory
+from langchain.text_splitter import CharacterTextSplitter
+from dotenv import load_dotenv
+from keybert import KeyBERT
+from sentence_transformers import CrossEncoder
+reranker = CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2')
+
+# Initialize global variable
+vectorstore_global = None
+
+# Load OpenAI API key
+def load_environment():
+    load_dotenv()
+    
+# PDF to XML Conversion
+def convert_pdf_to_xml(pdf_file, xml_path):
+    os.makedirs("temp", exist_ok=True)
+    pdf_path = os.path.join("temp", pdf_file.name)
+    with open(pdf_path, 'wb') as f:
+        f.write(pdf_file.getbuffer())
+    subprocess.run(["pdftohtml", "-xml", pdf_path, xml_path], check=True)
+    return xml_path
+
+# Extract text from XML
+def extract_text_from_xml(xml_path, document_name):
+    from lxml import etree
+    tree = etree.parse(xml_path)
+    text_chunks = []
+    for page in tree.xpath("//page"):
+        page_num = int(page.get("number", 0))
+        texts = [text.text for text in page.xpath('.//text') if text.text]
+        combined_text = '\n'.join(texts)
+        text_chunks.append({"text": combined_text, "page": page_num, "document": document_name})
+    return text_chunks
+
+# Process uploaded files
+def get_uploaded_text(uploaded_files):
+    raw_text = []
+    print(f"Total uploaded files: {len(uploaded_files)}")
+    for uploaded_file in uploaded_files:
+        document_name = uploaded_file.name
+        if document_name.endswith(".pdf"):
+            xml_path = os.path.join("temp", document_name.replace(".pdf", ".xml"))
+            text_chunks = extract_text_from_xml(convert_pdf_to_xml(uploaded_file, xml_path), document_name)
+            raw_text.extend(text_chunks)
+        elif uploaded_file.name.endswith((".html", ".htm")):
+            soup = BeautifulSoup(uploaded_file.getvalue(), 'lxml')
+            raw_text.append({"text": soup.get_text(), "page": None, "document": document_name})
+        elif uploaded_file.name.endswith((".txt")):
+            content = uploaded_file.getvalue().decode("utf-8") 
+            raw_text.append({"text": content, "page": None, "document": document_name})
+    return raw_text
+
+# Text Chunking
+def get_text_chunks(raw_text):
+    splitter = CharacterTextSplitter(separator='\n', chunk_size=500, chunk_overlap=100)
+    final_chunks = []
+    for chunk in raw_text:
+        for split_text in splitter.split_text(chunk["text"]):
+            final_chunks.append({"text": split_text, "page": chunk["page"], "document": chunk["document"]})
+    return final_chunks
+
+# Vectorstore Initialization
+def get_vectorstore(text_chunks):
+    if not text_chunks:
+        raise ValueError("text_chunks is empty. Cannot initialize FAISS vectorstore.")
+
+    #model_name = "BAAI/bge-large-en-v1.5"
+    #encode_kwargs = {'normalize_embeddings': True}
+    embeddings = OpenAIEmbeddings(model="text-embedding-3-large")
+    texts = [chunk["text"] for chunk in text_chunks]
+    metadatas = [{"page": chunk["page"], "document": chunk["document"]} for chunk in text_chunks]
+
+    return FAISS.from_texts(texts, embedding=embeddings, metadatas=metadatas)
+
+def set_global_vectorstore(vectorstore):
+    global vectorstore_global
+    vectorstore_global = vectorstore
+
+kw_model = KeyBERT()
+
+def faiss_search_with_keywords(query):
+    global vectorstore_global
+    if vectorstore_global is None:
+        raise ValueError("FAISS vectorstore is not initialized.")
+    
+    # Extract keywords from the query
+    keywords = kw_model.extract_keywords(query, keyphrase_ngram_range=(1,2), stop_words='english', top_n=5)
+    refined_query = " ".join([keyword[0] for keyword in keywords])
+
+    retriever = vectorstore_global.as_retriever(search_kwargs={"k": 13})
+    docs = retriever.get_relevant_documents(refined_query)
+    
+    return '\n\n'.join([f"[Page {doc.metadata.get('page', 'Unknown')}] {doc.page_content}" for doc in docs])
+
+def self_reasoning(query, context):
+    llm = ChatOpenAI(model="gpt-4", temperature=0.3)
+    reasoning_prompt = f"""
+    You are an AI assistant that analyzes the context provided to answer the user's query comprehensively and clearly. 
+    Answer in a concise, factual way using the terminology from the context. Avoid extra explanation unless explicitly asked.
+    If asked for the page number, mention that the provided page number is for the document, not the labeled page number. 
+    ### Example 1:
+    **Question:** What is the purpose of the MODTRAN GUI?
+    **Context:**
+    [Page 10 of the docuemnt] The MODTRAN GUI helps users set parameters and visualize the model's output.
+    **Answer:** The MODTRAN GUI assists users in parameter setup and output visualization. You can find the answer at Page 10 of the document provided.
+
+    ### Example 2:
+    **Question:** How do you run MODTRAN on Linux?
+    **Context:**
+    [Page 15 of the docuemnt] On Linux systems, MODTRAN can be run using the `mod6c` binary via terminal.
+    **Answer:** Use the `mod6c` binary via terminal. (Page 15)
+
+    ### Now answer:
+    **Question:** {query}
+    **Context:**
+    {context}
+
+    **Answer:**
+    """
+    response = llm.predict(reasoning_prompt)
+    return response
+
+def faiss_search_with_reasoning(query):
+    global vectorstore_global
+    if vectorstore_global is None:
+        raise ValueError("FAISS vectorstore is not initialized.")
+    
+    retriever = vectorstore_global.as_retriever(search_kwargs={"k": 13})
+    docs = retriever.get_relevant_documents(query)
+    
+    # Rerank using cross-encoder
+    #pairs = [(query, doc.page_content) for doc in docs]
+    #scores = reranker.predict(pairs)
+    #reranked_docs = sorted(zip(scores, docs), key=lambda x: x[0], reverse=True)
+    #top_docs = [doc for _, doc in reranked_docs[:5]]
+
+    context = '\n\n'.join([f"[Page {doc.metadata.get('page', 'Unknown')}] {doc.page_content.strip()}" for doc in docs])
+    return self_reasoning(query, context)
+
+
+faiss_keyword_tool = Tool(
+    name="FAISS Keyword Search",
+    func=faiss_search_with_keywords,
+    description="Searches FAISS with a keyword-based approach to retrieve context."
+)
+
+faiss_reasoning_tool = Tool(
+    name="FAISS Reasoning Search",
+    func=faiss_search_with_reasoning,
+    description="Searches FAISS with detailed reasoning to retrieve context."
+)
+
+# Agent Initialization
+def initialize_chatbot_agent():
+    llm = ChatOpenAI(model="gpt-4", temperature=0.3)
+    memory = ConversationBufferMemory(memory_key="chat_history", return_messages=True)
+    tools = [faiss_keyword_tool, faiss_reasoning_tool]
+    agent = initialize_agent(
+        tools=tools, 
+        llm=llm, 
+        agent=AgentType.ZERO_SHOT_REACT_DESCRIPTION, 
+        memory=memory,
+        verbose=False,
+        handle_parsing_errors=True)
+    return agent
+
+# Query Handler
+def handle_user_query(query, agent):
+    response = agent.run(query)
+    return response
+
+# Main Streamlit App
+def main():
+    global vectorstore_global
+    load_environment()
+
+    if "agent" not in st.session_state:
+        st.session_state.agent = None
+    if "chat_history" not in st.session_state:
+        st.session_state.chat_history = []
+
+    st.header("Chat with MODTRAN Documents :satellite:")
+    user_question = st.text_input("Ask a question about your uploaded files:")
+
+    with st.sidebar:
+        uploaded_files = st.file_uploader("Upload PDF, HTML, or MODTRAN output files:", accept_multiple_files=True)
+        if st.button("Process") and uploaded_files:
+            with st.spinner("Processing..."):
+                raw_text = get_uploaded_text(uploaded_files)
+                print(f"Total text chunks: {len(raw_text)}")
+                if raw_text:
+                    print("Example chunk:", raw_text[0])
+                text_chunks = get_text_chunks(raw_text)
+                vectorstore_global = get_vectorstore(text_chunks)
+                st.session_state.agent = initialize_chatbot_agent()
+                st.success("Files processed successfully!")
+
+    if st.session_state.agent and user_question:
+        response = handle_user_query(user_question, st.session_state.agent)
+        st.session_state.chat_history.append({"user": user_question, "bot": response})
+
+    for chat in st.session_state.chat_history:
+        st.write(f"**You:** {chat['user']}")
+        st.write(f"**Bot:** {chat['bot']}")
+
+if __name__ == "__main__":
+    load_environment()
+    main()

modtran_dataset.csv

@@ -0,0 +1,31 @@
+question,answer 
+What atmospheric models are included with MODTRAN?,"Tropical, mid-latitude summer, mid-latitude winter, sub-arctic summer, sub-arctic winter, U.S. standard"
+Can you tell me the page number where I can find this-What atmospheric models are included with MODTRAN?,Page 19
+How do I set the atmospheric water vapor content to the standard atmosphere?,Set H2OSTR to 1.0
+Can you tell me the page number where I can find this-How do I set the atmospheric water vapor content to the standard atmosphere?,Page 52
+How do I set the surface temperature to 279 K?,Set TPTEMP to 279
+Can you tell me the page number where I can find this-How do I set the surface temperature to 279 K?,Page 156
+"If I want to specify the amount of oxygen in the atmosphere, what field is relevant?",PROF_O2
+"Can you tell me the page number where I can find this-If I want to specify the amount of oxygen in the atmosphere, what field is relevant?",Page 56
+How do I set the the aerosol visibility to urban?,Set field [MODTRANINPUT][AEROSOLS][IHAZE] to AER_URBAN
+Can you tell me the page number where I can find this-How do I set the the aerosol visibility to urban?,Page 60
+"In the configuration file, how do I set surface reflectivity to 60%.",Set field [MODTRANINPUT][SURFACE][SURREF] to 0.6
+"Can you tell me the page number where I can find this-In the configuration file, how do I set surface reflectivity to 60%.","Page 37, 81"
+How do you call the MODTRAN GUI from the command line?,mod6gui
+Can you tell me the page number where I can find this-How do you call the MODTRAN GUI from the command line?,Page 15
+"If you want to run MODTRAN from the command line using an input json file, what is the command?",mod6c_cons
+"Can you tell me the page number where I can find this-If you want to run MODTRAN from the command line using an input json file, what is the command?","Page 13, 14"
+What file includes the full radiance data?,tp7
+Can you tell me the page number where I can find this-What file includes the full radiance data?,Page 88
+What file includes the radiance data convolved with a user-supplied SRF?,chn
+Can you tell me the page number where I can find this- What file includes the radiance data convolved with a user-supplied SRF?,Page 11
+What range of solar zenith angle are allowed?,0 to 180 
+Can you tell me the page number where I can find this-What range of solar zenith angle are allowed?,Page 78
+"How to set a surface ground temperature, distinct from the surface air temperature, for lines-of-sight not terminating at the ground?",Set SURFACE keyword SURREF to either ?LAMBER? or ?BRDF?; set SURFACE keyword NSURF to 2 and set SURFACE keyword AATEMP to the desired surface ground temperature in degrees Kelvin.
+"Can you tell me the page number where I can find this-How to set a surface ground temperature, distinct from the surface air temperature, for lines-of-sight not terminating at the ground?",Page 156
+"If I want to specify the amount of nitrous oxide in the atmosphere, what field is relevant?",PROF_N2O
+"Can you tell me the page number where I can find this-If I want to specify the amount of oxygen in the atmosphere, what field is relevant?",Page 56
+What is the meaning of CHKRES Warning? ,DV is being increased from 3.000000 to 15.000000 CM-1. The output step size cannot be smaller than the band model resolution
+Can you tell me the page number where I can find this- What is the meaning of CHKRES Warning? ,Page 96
+What is the keyword used for JSONObject for spectral bandpass and related inputs?,SPECTRAL
+Can you tell me the page number where I can find this - What is the keyword used for JSONObject for spectral bandpass and related inputs?,Page 45

modtran_eval_results.csv

@@ -0,0 +1,38 @@
+question,answer,predicted,BERTScore_F1
+What atmospheric models are included with MODTRAN?,"Tropical, mid-latitude summer, mid-latitude winter, sub-arctic summer, sub-arctic winter, U.S. standard","MODTRAN provides 6 built-in model atmospheres: Tropical, Mid-latitude summer, Mid-latitude winter, Sub-arctic summer, Sub-arctic winter, and US Standard 1976.  Users can also enter their own profiles. (Page 89)",0.89045244
+Can you tell me the page number where I can find this-What atmospheric models are included with MODTRAN?,Page 19,"Tropical, Mid-latitude summer, Mid-latitude winter, Sub-arctic summer, Sub-arctic winter, and US Standard 1976 are included with MODTRAN.  (Page 89)",0.8110852
+How do I set the atmospheric water vapor content to the standard atmosphere?,Set H2OSTR to 1.0,"Select ""US Standard 1976"" from the ""Model"" dropdown menu in the Atmosphere tab.",0.82688975
+Can you tell me the page number where I can find this-How do I set the atmospheric water vapor content to the standard atmosphere?,Page 52,"To set the atmospheric water vapor content to the standard atmosphere, select one of the six built-in model atmospheres, which include the ""US Standard 1976,"" using the ""Model"" options in the Atmosphere tab (Page 11).  You can also create user-defined profiles (Page 11). Another option is to use the 'MODEL' keyword within the 'ATMOSPHERE' JSONObject (Page 28).",0.7879687
+How do I set the surface temperature to 279 K?,Set TPTEMP to 279,"Uncheck the ""Default Temperature"" box in the Surfaces tab and enter 279 in the adjacent box. Ensure the units are set to Kelvin.",0.82670957
+Can you tell me the page number where I can find this-How do I set the surface temperature to 279 K?,Page 156,"Remove the check on the **Default** box following **Temperature**. Then enter the desired ground temperature (279 K), using radio buttons to choose between Kelvin and Celsius units. (Page 21)",0.8073552
+"If I want to specify the amount of oxygen in the atmosphere, what field is relevant?",PROF_O2,"To specify the amount of oxygen, use the `O2STR` keyword within the `ATMOSPHERE` JSONObject.  If using a user-defined profile (`MODEL` = `ATM_USER_ALT_PROFILE`), oxygen values are specified within the `""PROFILES""` array using `""TYPE"":""PROF_OXYGEN""`. (Pages 41, 47-48, 89, 91)",0.8142033
+"Can you tell me the page number where I can find this-If I want to specify the amount of oxygen in the atmosphere, what field is relevant?",Page 56,"Any of the molecular profiles can be scaled.  A scale factor of 0.0000 is interpreted as no scaling, equivalent to 1.0000. (Page 11)",0.82349986
+How do I set the the aerosol visibility to urban?,Set field [MODTRANINPUT][AEROSOLS][IHAZE] to AER_URBAN,"Select ""Urban"" from the *Model* drop-down menu in the *Aerosols* section of the *Clouds & Aerosols* window. (Page 13)",0.8157179
+Can you tell me the page number where I can find this-How do I set the the aerosol visibility to urban?,Page 60,"Page 13: MODTRAN includes 9 built-in aerosol models, one of which is ""Urban"".  You can select it from the *Model* drop-down menu in the *Aerosols* section of the *Clouds & Aerosols* Window.",0.82025063
+"In the configuration file, how do I set surface reflectivity to 60%.",Set field [MODTRANINPUT][SURFACE][SURREF] to 0.6,"To set a constant surface reflectivity to 60%, set `SURFTYPE` to `REFL_CONSTANT` and enter `0.6` in the `REFLECTANCE` box.  Additionally, page 21 describes setting constant reflectance, and page 35 shows a configuration snippet with `SURFTYPE` and reflectivity values.",0.8203288
+"Can you tell me the page number where I can find this-In the configuration file, how do I set surface reflectivity to 60%.","Page 37, 81","To set the surface reflectivity to 60%, set the value of the  `SURREF` keyword to 0.6 when `SURFTYPE` is set to `REFL_CONSTANT`. (Page 23)",0.7944811
+How do you call the MODTRAN GUI from the command line?,mod6gui,"To run the MODTRAN GUI in Windows, click the ""MODTRAN6 GUI"" shortcut. For Linux and Mac, run the script `<path>/bin/linux/mod6gui` or `<path>/bin/macos/mod6gui` respectively, where `<path>` is the MODTRAN6 directory. (Page 6)",0.8081023
+Can you tell me the page number where I can find this-How do you call the MODTRAN GUI from the command line?,Page 15,"For Linux and Mac users, the GUI is initialized by running a script: `<...>/bin/linux/mod6gui` or `<...>/bin/macos/mod6gui` where `<...>` is the full or relative path to the MODTRAN6 directory. (Page 6)",0.77987325
+"If you want to run MODTRAN from the command line using an input json file, what is the command?",mod6c_cons,The provided text describes how to run MODTRAN using the GUI or by converting .tp5 files to .json files. It does not contain the command to run MODTRAN from the command line using a .json input file.,0.7959351
+"Can you tell me the page number where I can find this-If you want to run MODTRAN from the command line using an input json file, what is the command?","Page 13, 14","To run MODTRAN from the command line using an input JSON file, see instructions mentioned on Page 27.",0.80603164
+What file includes the full radiance data?,tp7,"The files containing full radiance data are denoted by `.r_k` (r_kdis.dat) and `_b.r_k` (r_kdis.bin).  (Page 4.7)  Additionally, spectral radiance data can be found in `.tp7` (tape7),`_b.tp7` (tape7b), `.plt` (pltout), `_b.plt` (pltoutb), and within the ""FLUXES"" JSONObject when multiple scattering calculations are performed. (Pages 4.3, 4.5, and 81)",0.7743452
+Can you tell me the page number where I can find this-What file includes the full radiance data?,Page 88,"The files with the full radiance data are specified as ""*.r_k"" and ""*_b.r_k"".  (Page 4.7)",0.8054742
+What file includes the radiance data convolved with a user-supplied SRF?,chn,The file `<ROOTNAME>_scan.csv` includes the radiance data convolved with a user-supplied SRF. (Page 71),0.7839959
+Can you tell me the page number where I can find this- What file includes the radiance data convolved with a user-supplied SRF?,Page 11,r_kdis.dat (Page 4.7),0.83215964
+What range of solar zenith angle are allowed?,0 to 180 ,The allowed solar zenith angles range from 0° to 180°. (Page 55),0.83336747
+Can you tell me the page number where I can find this-What range of solar zenith angle are allowed?,Page 78,"Solar zenith angles are defined between 0° and 180°.  This information can be found across multiple pages, including those discussing geometry inputs and solar scattering angles, such as page 55 (where specific examples of 30° are given).  There is no single page that states the allowed range.",0.79898494
+"How to set a surface ground temperature, distinct from the surface air temperature, for lines-of-sight not terminating at the ground?",Set SURFACE keyword SURREF to either ?LAMBER? or ?BRDF?; set SURFACE keyword NSURF to 2 and set SURFACE keyword AATEMP to the desired surface ground temperature in degrees Kelvin.,"To set a surface ground temperature different from the surface air temperature for lines-of-sight not terminating at the ground:
+
+1. Set the `SURFACE` keyword `LAMBER` or `BRDF`.
+2. Set the `SURFACE` keyword `NSURF` to 2.
+3. Set the `SURFACE` keyword `AATEMP` to the desired ground temperature in Kelvin.
+
+
+If a temperature is specified using the ""Temperature"" box in the GUI and the ""Default Temperature"" box is unchecked, grey-body surface emission will be included even if the LOS doesn't intersect the ground.",0.87613213
+"Can you tell me the page number where I can find this-How to set a surface ground temperature, distinct from the surface air temperature, for lines-of-sight not terminating at the ground?",Page 156,"To define the ground surface temperature for lines-of-sight not terminating at the ground, set SURFACE keyword LAMBER or BRDF; set SURFACE keyword NSURF to 2; and set SURFACE keyword AATEMP to the desired temperature in Kelvin. (Page 98)",0.79768395
+"If I want to specify the amount of nitrous oxide in the atmosphere, what field is relevant?",PROF_N2O,The `PROF_N2O` or `PROF_NITROUS_OXIDE` field is relevant for specifying the amount of nitrous oxide. (Page 47),0.8499733
+"Can you tell me the page number where I can find this-If I want to specify the amount of oxygen in the atmosphere, what field is relevant?",Page 56,"Any of the molecular profiles can be scaled, including O3.  A scale factor of 0.0000 is interpreted as no scaling. If you want to zero out the O3 profile, use the drop-down boxes to replace ""Scaled"" with column density units of either g/cm^2 or atm-cm. (Page 11)",0.80292183
+What is the meaning of CHKRES Warning? ,DV is being increased from 3.000000 to 15.000000 CM-1. The output step size cannot be smaller than the band model resolution,"CHKRES warnings are generated when spectral response functions do not drop to zero at their endpoints. MODTRAN models the responses as zero outside the spectral range, generating the warning.  Also, super refractive conditions, where the line of sight turns downward, can cause a CHKRES warning, but only for non-vertical paths. It is important to note that warnings generally do not indicate faulty calculations. (Pages 38, 46)",0.82513857
+Can you tell me the page number where I can find this- What is the meaning of CHKRES Warning? ,Page 96,"The meaning of CHKRES warning is explained on page 46.  It occurs when the output spectral step size exceeds the spectral resolution, resulting in gaps in spectral coverage.",0.81771094
+What is the keyword used for JSONObject for spectral bandpass and related inputs?,SPECTRAL,SPECTRAL (Page 30),0.94234455
+Can you tell me the page number where I can find this - What is the keyword used for JSONObject for spectral bandpass and related inputs?,Page 45,"SPECTRAL is the keyword used for JSONObject for spectral bandpass and related inputs. (Page 30, 31, 78)",0.8083677

modtran_gemini.py

@@ -0,0 +1,271 @@
+import streamlit as st
+import os
+import subprocess
+import pdfplumber
+from lxml import etree
+from bs4 import BeautifulSoup
+from PyPDF2 import PdfReader
+from langchain_community.vectorstores import FAISS
+from langchain.embeddings.base import Embeddings
+from langchain_openai import ChatOpenAI
+from langchain.agents import initialize_agent, AgentType
+from langchain.agents import Tool
+from langchain.memory import ConversationBufferMemory
+from langchain.text_splitter import CharacterTextSplitter
+from dotenv import load_dotenv
+from keybert import KeyBERT
+from sentence_transformers import CrossEncoder
+import google.generativeai as genai
+from typing import List
+from langchain_core.language_models import BaseLanguageModel
+
+import google.generativeai as genai
+
+
+class GeminiLLM(BaseLanguageModel):
+    def __init__(self, model_name="models/gemini-1.5-pro-latest", api_key=None):
+        self.api_key = api_key or st.secrets["GOOGLE_API_KEY"]
+        if not self.api_key:
+            raise ValueError("GOOGLE_API_KEY not found in Streamlit secrets.")
+        genai.configure(api_key=self.api_key)
+        self.model = genai.GenerativeModel(model_name)
+
+    def _call(self, prompt, stop=None):
+        response = self.model.generate_content(prompt)
+        return response.text
+
+    @property
+    def _llm_type(self):
+        return "custom_gemini"
+
+class GeminiEmbeddings(Embeddings):
+    def __init__(self, model_name="models/embedding-001", api_key=None):
+        api_key = "AIzaSyBIfGJRoet_wzzYXIiWXxStkIigEOzSR2o"
+        if not api_key:
+            raise ValueError("GOOGLE_API_KEY not found in environment variables.")
+        os.environ["GOOGLE_API_KEY"] = api_key
+        genai.configure(api_key=api_key)
+        self.model_name = model_name
+
+    def embed_documents(self, texts: List[str]) -> List[List[float]]:
+        return [
+            genai.embed_content(
+                model=self.model_name,
+                content=text,
+                task_type="retrieval_document"
+            )["embedding"]
+            for text in texts
+        ]
+
+    def embed_query(self, text: str) -> List[float]:
+        return genai.embed_content(
+            model=self.model_name,
+            content=text,
+            task_type="retrieval_query"
+        )["embedding"]
+
+
+class GeminiLLM:
+    def __init__(self, model_name="models/gemini-1.5-pro-latest", api_key=None):
+        api_key = api_key or os.getenv("GOOGLE_API_KEY")
+        if not api_key:
+            raise ValueError("Missing GOOGLE_API_KEY")
+        genai.configure(api_key=api_key)
+        self.model = genai.GenerativeModel(model_name)
+
+    def predict(self, prompt: str) -> str:
+        response = self.model.generate_content(prompt)
+        return response.text.strip()
+    
+reranker = CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2')
+
+vectorstore_global = None
+
+def load_environment():
+    load_dotenv()
+    genai.configure(api_key=os.getenv("GOOGLE_API_KEY"))
+
+
+def convert_pdf_to_xml(pdf_file, xml_path):
+    os.makedirs("temp", exist_ok=True)
+    pdf_path = os.path.join("temp", pdf_file.name)
+    with open(pdf_path, 'wb') as f:
+        f.write(pdf_file.getbuffer())
+    subprocess.run(["pdftohtml", "-xml", pdf_path, xml_path], check=True)
+    return xml_path
+
+def extract_text_from_xml(xml_path, document_name):
+    tree = etree.parse(xml_path)
+    text_chunks = []
+    for page in tree.xpath("//page"):
+        page_num = int(page.get("number", 0))
+        texts = [text.text for text in page.xpath('.//text') if text.text]
+        combined_text = '\n'.join(texts)
+        text_chunks.append({"text": combined_text, "page": page_num, "document": document_name})
+    return text_chunks
+
+def get_uploaded_text(uploaded_files):
+    raw_text = []
+    for uploaded_file in uploaded_files:
+        document_name = uploaded_file.name
+        if document_name.endswith(".pdf"):
+            xml_path = os.path.join("temp", document_name.replace(".pdf", ".xml"))
+            text_chunks = extract_text_from_xml(convert_pdf_to_xml(uploaded_file, xml_path), document_name)
+            raw_text.extend(text_chunks)
+        elif uploaded_file.name.endswith((".html", ".htm")):
+            soup = BeautifulSoup(uploaded_file.getvalue(), 'lxml')
+            raw_text.append({"text": soup.get_text(), "page": None, "document": document_name})
+        elif uploaded_file.name.endswith((".txt")):
+            content = uploaded_file.getvalue().decode("utf-8")
+            raw_text.append({"text": content, "page": None, "document": document_name})
+    return raw_text
+
+def get_text_chunks(raw_text):
+    splitter = CharacterTextSplitter(separator='\n', chunk_size=500, chunk_overlap=100)
+    final_chunks = []
+    for chunk in raw_text:
+        for split_text in splitter.split_text(chunk["text"]):
+            final_chunks.append({"text": split_text, "page": chunk["page"], "document": chunk["document"]})
+    return final_chunks
+
+def get_vectorstore(text_chunks):
+    if not text_chunks:
+        raise ValueError("text_chunks is empty. Cannot initialize FAISS vectorstore.")
+
+    embeddings = GeminiEmbeddings()
+    texts = [chunk["text"] for chunk in text_chunks]
+    metadatas = [{"page": chunk["page"], "document": chunk["document"]} for chunk in text_chunks]
+    
+    return FAISS.from_texts(texts, embedding=embeddings, metadatas=metadatas)
+
+def set_global_vectorstore(vectorstore):
+    global vectorstore_global
+    vectorstore_global = vectorstore
+
+kw_model = KeyBERT()
+
+def self_reasoning(query, context):
+    llm = GeminiLLM()
+    reasoning_prompt = f"""
+    You are an AI assistant that analyzes the context provided to answer the user's query comprehensively and clearly. 
+    Answer in a concise, factual way using the terminology from the context. Avoid extra explanation unless explicitly asked.
+    YOU MUST mention the page number. 
+    ### Example 1:
+    **Question:** What is the purpose of the MODTRAN GUI?
+    **Context:**
+    [Page 10 of the docuemnt] The MODTRAN GUI helps users set parameters and visualize the model's output.
+    **Answer:** The MODTRAN GUI assists users in parameter setup and output visualization. You can find the answer at Page 10 of the document provided.
+
+    ### Example 2:
+    **Question:** How do you run MODTRAN on Linux? Answer with page number. 
+    **Context:**
+    [Page 15 of the docuemnt] On Linux systems, MODTRAN can be run using the `mod6c` binary via terminal.
+    **Answer:** Use the `mod6c` binary via terminal. (Page 15 of the document)
+
+    ### Now answer:
+    **Question:** {query}
+    **Context:**
+    {context}
+
+    **Answer:**
+    """
+    return llm.predict(reasoning_prompt)
+
+def faiss_search_with_keywords(query):
+    global vectorstore_global
+    if vectorstore_global is None:
+        raise ValueError("FAISS vectorstore is not initialized.")
+    keywords = kw_model.extract_keywords(query, keyphrase_ngram_range=(1,2), stop_words='english', top_n=5)
+    refined_query = " ".join([keyword[0] for keyword in keywords])
+    retriever = vectorstore_global.as_retriever(search_kwargs={"k": 13})
+    docs = retriever.get_relevant_documents(refined_query)
+    context= '\n\n'.join([f"[Page {doc.metadata.get('page', 'Unknown')}] {doc.page_content}" for doc in docs])
+    return self_reasoning(query, context)
+
+def faiss_search_with_reasoning(query):
+    global vectorstore_global
+    if vectorstore_global is None:
+        raise ValueError("FAISS vectorstore is not initialized.")
+    retriever = vectorstore_global.as_retriever(search_kwargs={"k": 13})
+    docs = retriever.get_relevant_documents(query)
+    pairs = [(query, doc.page_content) for doc in docs]
+    scores = reranker.predict(pairs)
+    reranked_docs = sorted(zip(scores, docs), key=lambda x: x[0], reverse=True)
+    top_docs = [doc for _, doc in reranked_docs[:5]]
+    context = '\n\n'.join([f"[Page {doc.metadata.get('page', 'Unknown')}] {doc.page_content.strip()}" for doc in top_docs])
+    return self_reasoning(query, context)
+
+faiss_keyword_tool = Tool(
+    name="FAISS Keyword Search",
+    func=faiss_search_with_keywords,
+    description="Searches FAISS with a keyword-based approach to retrieve context."
+)
+
+faiss_reasoning_tool = Tool(
+    name="FAISS Reasoning Search",
+    func=faiss_search_with_reasoning,
+    description="Searches FAISS with detailed reasoning to retrieve context."
+)
+
+def initialize_chatbot_agent():
+    llm = GeminiLLM()  # <-- Gemini instead of OpenAI
+    memory = ConversationBufferMemory(memory_key="chat_history", return_messages=True)
+    tools = [faiss_keyword_tool, faiss_reasoning_tool]
+    agent = initialize_agent(
+        tools=tools,
+        llm=llm,
+        agent=AgentType.ZERO_SHOT_REACT_DESCRIPTION,
+        memory=memory,
+        verbose=False,
+        handle_parsing_errors=True
+    )
+    return agent
+
+def handle_user_query(query):
+    # Same routing logic as in evaluation.py
+    global vectorstore_global
+    if vectorstore_global is None:
+        raise ValueError("Vectorstore is not initialized.")
+    
+    if "how" in query.lower():
+        context = faiss_search_with_reasoning(query)
+    else:
+        context = faiss_search_with_keywords(query)
+    return self_reasoning(query, context)
+def main():
+    load_environment()
+
+    if "chat_ready" not in st.session_state:
+        st.session_state.chat_ready = False
+    if "chat_history" not in st.session_state:
+        st.session_state.chat_history = []
+    if "vectorstore" not in st.session_state:
+        st.session_state.vectorstore = None
+
+    st.header("Chat with MODTRAN Documents :satellite:")
+    user_question = st.text_input("Ask a question about your uploaded files:")
+
+    with st.sidebar:
+        uploaded_files = st.file_uploader("Upload PDF, HTML, or MODTRAN output files:", accept_multiple_files=True)
+        if st.button("Process") and uploaded_files:
+            with st.spinner("Processing..."):
+                raw_text = get_uploaded_text(uploaded_files)
+                text_chunks = get_text_chunks(raw_text)
+                st.session_state.vectorstore = get_vectorstore(text_chunks)  
+                set_global_vectorstore(st.session_state.vectorstore)         
+                st.session_state.chat_ready = True
+                st.success("Files processed successfully!")
+
+    if st.session_state.chat_ready and user_question:
+        # Restore the global vectorstore reference
+        set_global_vectorstore(st.session_state.vectorstore)
+        response = handle_user_query(user_question)
+        st.session_state.chat_history.append({"user": user_question, "bot": response})
+
+    for chat in st.session_state.chat_history:
+        st.write(f"**You:** {chat['user']}")
+        st.write(f"**Bot:** {chat['bot']}")
+
+if __name__ == "__main__":
+    load_environment()
+    main()

modtran_groq.py

@@ -0,0 +1,236 @@
+import streamlit as st
+import os
+import subprocess
+import pdfplumber
+from lxml import etree
+from bs4 import BeautifulSoup
+from PyPDF2 import PdfReader
+from langchain_community.vectorstores import FAISS
+from langchain_community.embeddings import OpenAIEmbeddings
+from langchain_community.embeddings import HuggingFaceBgeEmbeddings
+from langchain_openai import ChatOpenAI
+from langchain.agents import initialize_agent, AgentType
+from langchain.agents import Tool
+from langchain.memory import ConversationBufferMemory
+from langchain.text_splitter import CharacterTextSplitter
+from dotenv import load_dotenv
+from keybert import KeyBERT
+from sentence_transformers import CrossEncoder
+from langchain_groq import ChatGroq
+import sys
+import asyncio
+
+# Windows fix for asyncio compatibility
+if sys.platform.startswith('win') and sys.version_info >= (3, 8):
+    asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
+    
+reranker = CrossEncoder('cross-encoder/ms-marco-MiniLM-L-6-v2')
+
+# Initialize global variable
+vectorstore_global = None
+
+# Load OpenAI API key
+def load_environment():
+    load_dotenv()
+    
+# PDF to XML Conversion
+def convert_pdf_to_xml(pdf_file, xml_path):
+    os.makedirs("temp", exist_ok=True)
+    pdf_path = os.path.join("temp", pdf_file.name)
+    with open(pdf_path, 'wb') as f:
+        f.write(pdf_file.getbuffer())
+    subprocess.run(["pdftohtml", "-xml", pdf_path, xml_path], check=True)
+    return xml_path
+
+# Extract text from XML
+def extract_text_from_xml(xml_path, document_name):
+    from lxml import etree
+    tree = etree.parse(xml_path)
+    text_chunks = []
+    for page in tree.xpath("//page"):
+        page_num = int(page.get("number", 0))
+        texts = [text.text for text in page.xpath('.//text') if text.text]
+        combined_text = '\n'.join(texts)
+        text_chunks.append({"text": combined_text, "page": page_num, "document": document_name})
+    return text_chunks
+
+# Process uploaded files
+def get_uploaded_text(uploaded_files):
+    raw_text = []
+    print(f"Total uploaded files: {len(uploaded_files)}")
+    for uploaded_file in uploaded_files:
+        document_name = uploaded_file.name
+        if document_name.endswith(".pdf"):
+            xml_path = os.path.join("temp", document_name.replace(".pdf", ".xml"))
+            text_chunks = extract_text_from_xml(convert_pdf_to_xml(uploaded_file, xml_path), document_name)
+            raw_text.extend(text_chunks)
+        elif uploaded_file.name.endswith((".html", ".htm")):
+            soup = BeautifulSoup(uploaded_file.getvalue(), 'lxml')
+            raw_text.append({"text": soup.get_text(), "page": None, "document": document_name})
+        elif uploaded_file.name.endswith((".txt")):
+            content = uploaded_file.getvalue().decode("utf-8") 
+            raw_text.append({"text": content, "page": None, "document": document_name})
+    return raw_text
+
+# Text Chunking
+def get_text_chunks(raw_text):
+    splitter = CharacterTextSplitter(separator='\n', chunk_size=500, chunk_overlap=100)
+    final_chunks = []
+    for chunk in raw_text:
+        for split_text in splitter.split_text(chunk["text"]):
+            final_chunks.append({"text": split_text, "page": chunk["page"], "document": chunk["document"]})
+    return final_chunks
+
+# Vectorstore Initialization
+def get_vectorstore(text_chunks):
+    if not text_chunks:
+        raise ValueError("text_chunks is empty. Cannot initialize FAISS vectorstore.")
+
+    #model_name = "BAAI/bge-large-en-v1.5"
+    #encode_kwargs = {'normalize_embeddings': True}
+    #embeddings = OpenAIEmbeddings(model="text-embedding-3-large")
+    embeddings = HuggingFaceBgeEmbeddings(
+    model_name="BAAI/bge-base-en-v1.5",
+    encode_kwargs={"normalize_embeddings": True}
+    )
+    texts = [chunk["text"] for chunk in text_chunks]
+    metadatas = [{"page": chunk["page"], "document": chunk["document"]} for chunk in text_chunks]
+
+    return FAISS.from_texts(texts, embedding=embeddings, metadatas=metadatas)
+
+def set_global_vectorstore(vectorstore):
+    global vectorstore_global
+    vectorstore_global = vectorstore
+
+kw_model = KeyBERT()
+
+def faiss_search_with_keywords(query):
+    global vectorstore_global
+    if vectorstore_global is None:
+        raise ValueError("FAISS vectorstore is not initialized.")
+    
+    # Extract keywords from the query
+    keywords = kw_model.extract_keywords(query, keyphrase_ngram_range=(1,2), stop_words='english', top_n=5)
+    refined_query = " ".join([keyword[0] for keyword in keywords])
+
+    retriever = vectorstore_global.as_retriever(search_kwargs={"k": 13})
+    docs = retriever.get_relevant_documents(refined_query)
+    
+    return '\n\n'.join([f"[Page {doc.metadata.get('page', 'Unknown')}] {doc.page_content}" for doc in docs])
+
+def self_reasoning(query, context):
+    llm = ChatOpenAI(model="gpt-4", temperature=0.3)
+    #llm = ChatGroq(temperature=0.3, model_name="llama3-8b-8192")
+    reasoning_prompt = f"""
+    You are an AI assistant that analyzes the context provided to answer the user's query comprehensively and clearly. 
+    Answer in a concise, factual way using the terminology from the context. Avoid extra explanation unless explicitly asked.
+    If asked for the page number,YOU MUST mention the page number. 
+    ### Example 1:
+    **Question:** What is the purpose of the MODTRAN GUI?
+    **Context:**
+    [Page 10 of the docuemnt] The MODTRAN GUI helps users set parameters and visualize the model's output.
+    **Answer:** The MODTRAN GUI assists users in parameter setup and output visualization. You can find the answer at Page 10 of the document provided.
+
+    ### Example 2:
+    **Question:** How do you run MODTRAN on Linux? Answer with page number. 
+    **Context:**
+    [Page 15 of the docuemnt] On Linux systems, MODTRAN can be run using the `mod6c` binary via terminal.
+    **Answer:** Use the `mod6c` binary via terminal. (Page 15)
+
+    ### Now answer:
+    **Question:** {query}
+    **Context:**
+    {context}
+
+    **Answer:**
+    """
+    response = llm.predict(reasoning_prompt)
+    return response
+
+def faiss_search_with_reasoning(query):
+    global vectorstore_global
+    if vectorstore_global is None:
+        raise ValueError("FAISS vectorstore is not initialized.")
+    
+    retriever = vectorstore_global.as_retriever(search_kwargs={"k": 13})
+    docs = retriever.get_relevant_documents(query)
+    
+    # Rerank using cross-encoder
+    #pairs = [(query, doc.page_content) for doc in docs]
+    #scores = reranker.predict(pairs)
+    #reranked_docs = sorted(zip(scores, docs), key=lambda x: x[0], reverse=True)
+    #top_docs = [doc for _, doc in reranked_docs[:5]]
+
+    context = '\n\n'.join([f"[Page {doc.metadata.get('page', 'Unknown')}] {doc.page_content.strip()}" for doc in docs])
+    return self_reasoning(query, context)
+
+
+faiss_keyword_tool = Tool(
+    name="FAISS Keyword Search",
+    func=faiss_search_with_keywords,
+    description="Searches FAISS with a keyword-based approach to retrieve context."
+)
+
+faiss_reasoning_tool = Tool(
+    name="FAISS Reasoning Search",
+    func=faiss_search_with_reasoning,
+    description="Searches FAISS with detailed reasoning to retrieve context."
+)
+
+# Agent Initialization
+def initialize_chatbot_agent():
+    llm = ChatOpenAI(model="gpt-4", temperature=0.3)
+    #llm = ChatGroq(temperature=0.3, model_name="llama3-8b-8192")
+    memory = ConversationBufferMemory(memory_key="chat_history", return_messages=True)
+    tools = [faiss_keyword_tool, faiss_reasoning_tool]
+    agent = initialize_agent(
+        tools=tools, 
+        llm=llm, 
+        agent=AgentType.ZERO_SHOT_REACT_DESCRIPTION, 
+        memory=memory,
+        verbose=False,
+        handle_parsing_errors=True)
+    return agent
+
+# Query Handler
+def handle_user_query(query, agent):
+    response = agent.run(query)
+    return response
+
+# Main Streamlit App
+def main():
+    global vectorstore_global
+    load_environment()
+
+    if "agent" not in st.session_state:
+        st.session_state.agent = None
+    if "chat_history" not in st.session_state:
+        st.session_state.chat_history = []
+
+    st.header("Chat with MODTRAN Documents :satellite:")
+    user_question = st.text_input("Ask a question about your uploaded files:")
+
+    with st.sidebar:
+        uploaded_files = st.file_uploader("Upload PDF, HTML, or MODTRAN output files:", accept_multiple_files=True)
+        if st.button("Process") and uploaded_files:
+            with st.spinner("Processing..."):
+                raw_text = get_uploaded_text(uploaded_files)
+                print(f"Total text chunks: {len(raw_text)}")
+                if raw_text:
+                    print("Example chunk:", raw_text[0])
+                text_chunks = get_text_chunks(raw_text)
+                vectorstore_global = get_vectorstore(text_chunks)
+                st.session_state.agent = initialize_chatbot_agent()
+                st.success("Files processed successfully!")
+
+    if st.session_state.agent and user_question:
+        response = handle_user_query(user_question, st.session_state.agent)
+        st.session_state.chat_history.append({"user": user_question, "bot": response})
+
+    for chat in st.session_state.chat_history:
+        st.write(f"**You:** {chat['user']}")
+        st.write(f"**Bot:** {chat['bot']}")
+
+if __name__ == "__main__":
+    load_environment()
+    main()

requirements.txt

@@ -0,0 +1,17 @@
+streamlit
+pandas
+scikit-learn
+pdfplumber
+PyPDF2
+beautifulsoup4
+lxml
+python-dotenv
+keybert
+sentence-transformers
+langchain
+langchain-community
+langchain-openai
+google-generativeai
+transformers
+torch
+bert-score