Update app.py

app.py

@@ -25,6 +25,9 @@ from sklearn.feature_extraction.text import TfidfVectorizer
 from sklearn.metrics.pairwise import cosine_similarity
 from trafilatura import fetch_url, extract
 import json
+from requests.exceptions import RequestException
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 
 
 # Set up basic configuration for logging
@@ -280,15 +283,20 @@ def generate_chunked_response(prompt, model, max_tokens=10000, num_calls=3, temp
 
 class SimpleDDGSearch:
     def search(self, query: str, num_results: int = 5):
-        results = DDGS().text(query, region='wt-wt', safesearch='off', max_results=num_results)
-        return [res["href"] for res in results]
+        results = []
+        with DDGS() as ddgs:
+            for r in ddgs.text(query, region='wt-wt', safesearch='off', max_results=num_results * 2):  # Request more results than needed
+                results.append(r["href"])
+                if len(results) >= num_results:
+                    break
+        return results
 
 class TrafilaturaWebCrawler:
     def get_website_content_from_url(self, url: str) -> str:
         try:
             downloaded = fetch_url(url)
             if downloaded is None:
-                return f"Failed to fetch content from URL: {url}"
+                raise RequestException(f"Failed to fetch content from URL: {url}")
             
             result = extract(downloaded, output_format='json', include_comments=False, with_metadata=True, url=url)
             if result:
@@ -301,21 +309,33 @@ class TrafilaturaWebCrawler:
                 
                 return f'=========== Website Title: {title} ===========\n\n=========== Website URL: {url} ===========\n\n=========== Website Content ===========\n\n{content}\n\n=========== Website Content End ===========\n\n'
             else:
-                return f"No content extracted from URL: {url}"
+                raise ValueError(f"No content extracted from URL: {url}")
         except Exception as e:
-            return f"An error occurred while processing {url}: {str(e)}"
+            logging.error(f"An error occurred while processing {url}: {str(e)}")
+            return None
 
 def search_and_crawl(query: str, num_results: int = 10):
     searcher = SimpleDDGSearch()
-    search_results = searcher.search(query, num_results=num_results)
+    search_results = searcher.search(query, num_results=num_results * 2)  # Request more results than needed
     
     crawler = TrafilaturaWebCrawler()
     output = ""
+    successful_crawls = 0
     
-    for i, url in enumerate(search_results):
-        output += f"Results for URL {i+1}: {url}\n\n"
-        output += crawler.get_website_content_from_url(url) + "\n"
-        output += "------------------------------------------------------------\n\n"
+    for url in search_results:
+        if successful_crawls >= num_results:
+            break
+        
+        content = crawler.get_website_content_from_url(url)
+        if content:
+            output += f"Results for URL {successful_crawls + 1}: {url}\n\n"
+            output += content + "\n"
+            output += "------------------------------------------------------------\n\n"
+            successful_crawls += 1
+    
+    if successful_crawls == 0:
+        logging.warning(f"No successful crawls for query: {query}")
+        return "No results could be fetched for the given query."
     
     return output