Update app.py

app.py

@@ -4,6 +4,8 @@ import requests
 import pytz
 import yaml
 from tools.final_answer import FinalAnswerTool
+from playwright.sync_api import sync_playwright
+import time
 from bs4 import BeautifulSoup
 import pandas as pd
 import time
@@ -27,45 +29,49 @@ def scrape_drug_reviews(drug:str)-> str: #it's import to specify the return type
         return f"Error fetching reviews for the target drug you provided: '{drug}'"
 
 
-def scrape_drugs_com_reviews(drug_name, max_pages=5, sleep_time=2):
+def scrape_drugs_com_reviews(drug_name, max_pages=3, delay=2):
     """
     Scrapes user reviews from Drugs.com for a given drug.
     """
     base_url = f"https://www.drugs.com/comments/{drug_name}/"
-    reviews = []
-
-    for page in range(1, max_pages + 1):
-        url = base_url if page == 1 else f"{base_url}?page={page}"
-        response = requests.get(url, headers={'User-Agent': 'Mozilla/5.0'})
-
-        if response.status_code != 200:
-            print(f"Failed to fetch page {page}")
-            break
-
-        soup = BeautifulSoup(response.text, "html.parser")
-        review_blocks = soup.select('.user-comment')
-
-        if not review_blocks:
-            break
-
-        for block in review_blocks:
-            review_text = block.select_one('.user-comment-text')
-            condition = block.select_one('.drug-condition')
-            rating = block.select_one('.rating-score')
-            date = block.select_one('.comment-date')
-
-            reviews.append({
-                "condition": condition.get_text(strip=True) if condition else None,
-                "rating": rating.get_text(strip=True) if rating else None,
-                "review": review_text.get_text(strip=True) if review_text else None,
-                "date": date.get_text(strip=True) if date else None,
-                "source": url
-            })
-
-        print(f"[✓] Page {page} scraped.")
-        time.sleep(sleep_time)
-
-    return pd.DataFrame(reviews)
+    all_reviews = []
+
+    with sync_playwright() as p:
+        browser = p.chromium.launch(headless=True)
+        page = browser.new_page()
+
+        for page_num in range(1, max_pages + 1):
+            url = base_url if page_num == 1 else f"{base_url}?page={page_num}"
+            print(f"Scraping: {url}")
+            page.goto(url, timeout=60000)
+            time.sleep(delay)  # Give page some time to load
+
+            html = page.content()
+            soup = BeautifulSoup(html, 'html.parser')
+            review_blocks = soup.select('.user-comment')
+
+            if not review_blocks:
+                print("No reviews found on this page.")
+                break
+
+            for block in review_blocks:
+                review_text = block.select_one('.user-comment-text')
+                condition = block.select_one('.drug-condition')
+                rating = block.select_one('.rating-score')
+                date = block.select_one('.comment-date')
+
+                all_reviews.append({
+                    "condition": condition.get_text(strip=True) if condition else None,
+                    "rating": rating.get_text(strip=True) if rating else None,
+                    "review": review_text.get_text(strip=True) if review_text else None,
+                    "date": date.get_text(strip=True) if date else None,
+                    "source": url
+                })
+
+            time.sleep(delay)
+
+        browser.close()
+    return pd.DataFrame(all_reviews)