Update app.py

app.py

@@ -18,16 +18,6 @@ from huggingface_hub import InferenceClient
 import inspect
 import logging
 import shutil
-from sentence_transformers import CrossEncoder
-from datetime import datetime
-from dateutil import parser as date_parser
-from sklearn.feature_extraction.text import TfidfVectorizer
-from sklearn.metrics.pairwise import cosine_similarity
-from trafilatura import fetch_url, extract
-import json
-from requests.exceptions import RequestException
-
-logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 
 
 # Set up basic configuration for logging
@@ -281,66 +271,10 @@ def generate_chunked_response(prompt, model, max_tokens=10000, num_calls=3, temp
     print(f"Final clean response: {final_response[:100]}...")
     return final_response
 
-class SimpleDDGSearch:
-    def search(self, query: str, num_results: int = 5):
-        results = []
-        with DDGS() as ddgs:
-            for r in ddgs.text(query, region='wt-wt', safesearch='off', max_results=num_results * 2):  # Request more results than needed
-                results.append(r["href"])
-                if len(results) >= num_results:
-                    break
-        return results
-
-class TrafilaturaWebCrawler:
-    def get_website_content_from_url(self, url: str) -> str:
-        try:
-            downloaded = fetch_url(url)
-            if downloaded is None:
-                raise RequestException(f"Failed to fetch content from URL: {url}")
-            
-            result = extract(downloaded, output_format='json', include_comments=False, with_metadata=True, url=url)
-            if result:
-                result_dict = json.loads(result)
-                title = result_dict.get('title', 'No title found')
-                content = result_dict.get('text', 'No content extracted')
-                
-                if content == 'No content extracted':
-                    content = extract(downloaded, include_comments=False)
-                
-                return f'=========== Website Title: {title} ===========\n\n=========== Website URL: {url} ===========\n\n=========== Website Content ===========\n\n{content}\n\n=========== Website Content End ===========\n\n'
-            else:
-                raise ValueError(f"No content extracted from URL: {url}")
-        except Exception as e:
-            logging.error(f"An error occurred while processing {url}: {str(e)}")
-            return None
-
-def search_and_crawl(query: str, num_results: int = 10):
-    searcher = SimpleDDGSearch()
-    search_results = searcher.search(query, num_results=num_results * 2)  # Request more results than needed
-    
-    crawler = TrafilaturaWebCrawler()
-    output = ""
-    successful_crawls = 0
-    
-    for url in search_results:
-        if successful_crawls >= num_results:
-            break
-        
-        content = crawler.get_website_content_from_url(url)
-        if content:
-            output += f"Results for URL {successful_crawls + 1}: {url}\n\n"
-            output += content + "\n"
-            output += "------------------------------------------------------------\n\n"
-            successful_crawls += 1
-    
-    if successful_crawls == 0:
-        logging.warning(f"No successful crawls for query: {query}")
-        return "No results could be fetched for the given query."
-    
-    return output
-
 def duckduckgo_search(query):
-    return search_and_crawl(query, num_results=10)
+    with DDGS() as ddgs:
+        results = ddgs.text(query, max_results=5)
+    return results
 
 class CitingSources(BaseModel):
     sources: List[str] = Field(
@@ -373,19 +307,17 @@ def retry_last_response(history, use_web_search, model, temperature, num_calls):
     
     return chatbot_interface(last_user_msg, history, use_web_search, model, temperature, num_calls)
 
-def respond(message, history, model, temperature, num_calls, use_web_search, selected_docs):
+from duckduckgo_search import DDGS
+
+def respond(message, history, use_web_search, model, temperature, num_calls, selected_docs):
     logging.info(f"User Query: {message}")
-    logging.info(f"Model Used: {model}")
     logging.info(f"Search Type: {'Web Search' if use_web_search else 'PDF Search'}")
-
     logging.info(f"Selected Documents: {selected_docs}")
 
     try:
         if use_web_search:
             for main_content, sources in get_response_with_search(message, model, num_calls=num_calls, temperature=temperature):
                 response = f"{main_content}\n\n{sources}"
-                first_line = response.split('\n')[0] if response else ''
-#                logging.info(f"Generated Response (first line): {first_line}")
                 yield response
         else:
             embed = get_embeddings()
@@ -393,7 +325,6 @@ def respond(message, history, model, temperature, num_calls, use_web_search, sel
                 database = FAISS.load_local("faiss_database", embed, allow_dangerous_deserialization=True)
                 retriever = database.as_retriever(search_kwargs={"k": 20})
                 
-                # Filter relevant documents based on user selection
                 all_relevant_docs = retriever.get_relevant_documents(message)
                 relevant_docs = [doc for doc in all_relevant_docs if doc.metadata["source"] in selected_docs]
                 
@@ -407,26 +338,21 @@ def respond(message, history, model, temperature, num_calls, use_web_search, sel
                 yield "No documents available. Please upload PDF documents to answer questions."
                 return
             
-            if model == "@cf/meta/llama-3.1-8b-instruct":
-                # Use Cloudflare API
-                for partial_response in get_response_from_cloudflare(prompt="", context=context_str, query=message, num_calls=num_calls, temperature=temperature, search_type="pdf"):
-                    first_line = partial_response.split('\n')[0] if partial_response else ''
-#                   logging.info(f"Generated Response (first line): {first_line}")
-                    yield partial_response
-            else:
-                # Use Hugging Face API
-                for partial_response in get_response_from_pdf(message, model, selected_docs, num_calls=num_calls, temperature=temperature):
-                    first_line = partial_response.split('\n')[0] if partial_response else ''
-#                    logging.info(f"Generated Response (first line): {first_line}")
-                    yield partial_response
+            prompt = f"""Using the following context from the PDF documents:
+{context_str}
+Write a detailed and complete response that answers the following user question: '{message}'"""
+
+            try:
+                response = DDGS().chat(prompt, model="llama-3-70b")
+                yield response
+            except Exception as e:
+                logging.error(f"Error with DuckDuckGo chat API: {str(e)}")
+                logging.info("Falling back to Hugging Face API")
+                yield from get_response_from_pdf(message, model, selected_docs, num_calls=num_calls, temperature=temperature)
+
     except Exception as e:
-        logging.error(f"Error with {model}: {str(e)}")
-        if "microsoft/Phi-3-mini-4k-instruct" in model:
-            logging.info("Falling back to Mistral model due to Phi-3 error")
-            fallback_model = "mistralai/Mistral-7B-Instruct-v0.3"
-            yield from respond(message, history, fallback_model, temperature, num_calls, use_web_search, selected_docs)
-        else:
-            yield f"An error occurred with the {model} model: {str(e)}. Please try again or select a different model."
+        logging.error(f"Error: {str(e)}")
+        yield f"An error occurred: {str(e)}. Please try again or select a different model."
 
 logging.basicConfig(level=logging.DEBUG)
 
@@ -485,119 +411,51 @@ After writing the document, please provide a list of sources used in your respon
     if not full_response:
         yield "I apologize, but I couldn't generate a response at this time. Please try again later."
 
-def rank_results(query, results):
-    # Sort by date, most recent first
-    results.sort(key=lambda x: x['date'], reverse=True)
-    
-    # Calculate relevance scores
-    vectorizer = TfidfVectorizer().fit_transform([query] + [f"{r['title']} {r['body']}" for r in results])
-    relevance_scores = cosine_similarity(vectorizer[0:1], vectorizer[1:])[0]
-    
-    # Combine date priority and relevance score
-    for i, result in enumerate(results):
-        days_old = (datetime.now() - result['date']).days
-        date_score = 1 / (days_old + 1)  # Newer articles get higher scores
-        result['combined_score'] = (date_score + relevance_scores[i]) / 2
-    
-    # Sort by combined score and return top 3
-    return sorted(results, key=lambda x: x['combined_score'], reverse=True)[:3]
-
 def create_web_search_vectors(search_results):
     embed = get_embeddings()
     
     documents = []
     for result in search_results:
         if 'body' in result:
-            content = f"{result['title']}\n{result['body']}\nSource: {result['href']}\nDate: {result['date']}"
-            documents.append(Document(page_content=content, metadata={"source": result['href'], "date": result['date']}))
+            content = f"{result['title']}\n{result['body']}\nSource: {result['href']}"
+            documents.append(Document(page_content=content, metadata={"source": result['href']}))
     
     return FAISS.from_documents(documents, embed)
 
 def get_response_with_search(query, model, num_calls=3, temperature=0.2):
     search_results = duckduckgo_search(query)
+    context = "\n".join([f"{result['title']}\n{result['body']}" for result in search_results])
     
-    if not search_results:
-        yield "No web search results available. Please try again.", ""
-        return
-    
-    accumulated_response = ""
-
-    # Split the search results into separate documents
-    documents = search_results.split("------------------------------------------------------------")
-    
-    for i, doc in enumerate(documents, 1):
-        if not doc.strip():  # Skip empty documents
-            continue
-        
-        # Extract title, URL, and content from the document
-        title_start = doc.find("Website Title:") + len("Website Title:")
-        title_end = doc.find("===========", title_start)
-        title = doc[title_start:title_end].strip()
-        
-        url_start = doc.find("Website URL:") + len("Website URL:")
-        url_end = doc.find("===========", url_start)
-        source = doc[url_start:url_end].strip()
-        
-        content_start = doc.find("Website Content") + len("Website Content ===========")
-        content_end = doc.find("Website Content End")
-        context = doc[content_start:content_end].strip()
-        
-        prompt = f"""Using the following context from a web search result:
+    prompt = f"""Using the following context from web search results:
 {context}
-This information is from the website titled: {title}
-You are an expert AI assistant. Write a detailed and complete research article that fulfills the following user request: '{query}'
-Base your summary strictly on the information from this source. Only include information that is directly supported by the given content.
-If any part of the information cannot be verified from this source, clearly state that it could not be confirmed."""
-        
-        if model == "@cf/meta/llama-3.1-8b-instruct":
-            # Use Cloudflare API
-            source_response = ""
-            for response in get_response_from_cloudflare(prompt="", context=context, query=query, num_calls=1, temperature=temperature, search_type="web"):
-                source_response += response
-            accumulated_response += f"Source {i} ({source}):\n\n{source_response}\n\n"
-            yield accumulated_response, ""
-        else:
-            # Use Hugging Face API
-            client = InferenceClient(model, token=huggingface_token)
-            
-            source_response = ""
-            for message in client.chat_completion(
-                messages=[{"role": "user", "content": prompt}],
-                max_tokens=10000,
-                temperature=temperature,
-                stream=True,
-            ):
-                if message.choices and message.choices[0].delta and message.choices[0].delta.content:
-                    chunk = message.choices[0].delta.content
-                    source_response += chunk
-            accumulated_response += f"Source {i} ({source}):\n\n{source_response}\n\n"
-            yield accumulated_response, ""
-    
-    # Generate an overall summary after processing all sources
-    overall_prompt = f"""Based on the summaries you've generated for each source: '{accumulated_response}', provide a concise overall summary that addresses the user's query: '{query}'
-Highlight any conflicting information or gaps in the available data."""
+You are an expert AI assistant, write a detailed and complete research document that fulfills the following user request: '{query}'
+Base your entire response strictly on the information retrieved from trusted sources. Importantly, only include information that is directly supported by the retrieved content. 
+If any part of the information cannot be verified from the given sources, clearly state that it could not be confirmed.
+After writing the document, please provide a list of sources used in your response."""
+
+    try:
+        response = DDGS().chat(prompt, model="llama-3-70b")
+        yield response, ""
+    except Exception as e:
+        logging.error(f"Error with DuckDuckGo chat API: {str(e)}")
+        logging.info("Falling back to Hugging Face API")
+        yield from get_response_from_huggingface(prompt, model, num_calls, temperature)
+
+def get_response_from_huggingface(prompt, model, num_calls=3, temperature=0.2):
+    client = InferenceClient(model, token=huggingface_token)
     
-    if model == "@cf/meta/llama-3.1-8b-instruct":
-        # Use Cloudflare API for overall summary
-        overall_response = ""
-        for response in get_response_from_cloudflare(prompt="", context="", query=overall_prompt, num_calls=1, temperature=temperature, search_type="web"):
-            overall_response += response
-        accumulated_response += f"Overall Summary:\n\n{overall_response}\n\n"
-        yield accumulated_response, ""
-    else:
-        # Use Hugging Face API for overall summary
-        overall_summary = ""
+    main_content = ""
+    for i in range(num_calls):
         for message in client.chat_completion(
-            messages=[{"role": "user", "content": overall_prompt}],
+            messages=[{"role": "user", "content": prompt}],
             max_tokens=10000,
             temperature=temperature,
             stream=True,
         ):
             if message.choices and message.choices[0].delta and message.choices[0].delta.content:
                 chunk = message.choices[0].delta.content
-                overall_summary += chunk
-        accumulated_response += f"Overall Summary:\n\n{overall_summary}\n\n"
-        yield accumulated_response, ""
+                main_content += chunk
+                yield main_content, ""  # Yield partial main content without sources
 
 def get_response_from_pdf(query, model, selected_docs, num_calls=3, temperature=0.2):
     logging.info(f"Entering get_response_from_pdf with query: {query}, model: {model}, selected_docs: {selected_docs}")
@@ -611,7 +469,6 @@ def get_response_from_pdf(query, model, selected_docs, num_calls=3, temperature=
         yield "No documents available. Please upload PDF documents to answer questions."
         return
 
-    # Pre-filter the documents
     filtered_docs = []
     for doc_id, doc in database.docstore._dict.items():
         if isinstance(doc, Document) and doc.metadata.get("source") in selected_docs:
@@ -624,7 +481,6 @@ def get_response_from_pdf(query, model, selected_docs, num_calls=3, temperature=
         yield "No relevant information found in the selected documents. Please try selecting different documents or rephrasing your query."
         return
 
-    # Create a new FAISS index with only the selected documents
     filtered_db = FAISS.from_documents(filtered_docs, embed)
     
     retriever = filtered_db.as_retriever(search_kwargs={"k": 10})
@@ -632,40 +488,20 @@ def get_response_from_pdf(query, model, selected_docs, num_calls=3, temperature=
     relevant_docs = retriever.get_relevant_documents(query)
     logging.info(f"Number of relevant documents retrieved: {len(relevant_docs)}")
 
-    for doc in relevant_docs:
-        logging.info(f"Document source: {doc.metadata['source']}")
-        logging.info(f"Document content preview: {doc.page_content[:100]}...")  # Log first 100 characters of each document
-
     context_str = "\n".join([doc.page_content for doc in relevant_docs])
     logging.info(f"Total context length: {len(context_str)}")
 
-    if model == "@cf/meta/llama-3.1-8b-instruct":
-        logging.info("Using Cloudflare API")
-        # Use Cloudflare API with the retrieved context
-        for response in get_response_from_cloudflare(prompt="", context=context_str, query=query, num_calls=num_calls, temperature=temperature, search_type="pdf"):
-            yield response
-    else:
-        logging.info("Using Hugging Face API")
-        # Use Hugging Face API
-        prompt = f"""Using the following context from the PDF documents:
+    prompt = f"""Using the following context from the PDF documents:
 {context_str}
 Write a detailed and complete response that answers the following user question: '{query}'"""
-        
-        client = InferenceClient(model, token=huggingface_token)
-        
-        response = ""
-        for i in range(num_calls):
-            logging.info(f"API call {i+1}/{num_calls}")
-            for message in client.chat_completion(
-                messages=[{"role": "user", "content": prompt}],
-                max_tokens=10000,
-                temperature=temperature,
-                stream=True,
-            ):
-                if message.choices and message.choices[0].delta and message.choices[0].delta.content:
-                    chunk = message.choices[0].delta.content
-                    response += chunk
-                    yield response  # Yield partial response
+
+    try:
+        response = DDGS().chat(prompt, model="llama-3-70b")
+        yield response
+    except Exception as e:
+        logging.error(f"Error with DuckDuckGo chat API: {str(e)}")
+        logging.info("Falling back to Hugging Face API")
+        yield from get_response_from_huggingface(prompt, model, num_calls, temperature)
         
         logging.info("Finished generating response")
 
@@ -807,4 +643,4 @@ with demo:
     )
 
 if __name__ == "__main__":
-    demo.launch(share=True)
+    demo.launch(share=True)