Update main.py

main.py

@@ -1,8 +1,8 @@
 import os
 import re
 from dotenv import load_dotenv
-from fastapi import FastAPI, HTTPException, Request, Depends, Security
-from fastapi.responses import StreamingResponse, HTMLResponse, JSONResponse, FileResponse
+from fastapi import FastAPI, HTTPException, Request, Depends, Security, Query
+from fastapi.responses import StreamingResponse, HTMLResponse, JSONResponse, FileResponse,  PlainTextResponse
 from fastapi.security import APIKeyHeader
 from pydantic import BaseModel
 import httpx
@@ -401,6 +401,24 @@ async def dynamic_ai_page(request: Request):
         html_content = ' '.join(html_content.split(' ')[1:])
     
     return HTMLResponse(content=html_content)
+
+
+    ######################################
+
+@app.get("/scraper", response_class=PlainTextResponse)
+def scrape_site(url: str = Query(..., description="URL to scrape")):
+    try:
+        # Try cloudscraper first
+        scraper = cloudscraper.create_scraper()
+        response = scraper.get(url)
+        if response.status_code == 200 and len(response.text.strip()) > 0:
+            return response.text
+    except Exception as e:
+        print(f"Cloudscraper failed: {e}")
+        return "Cloudscraper failed."
+
+
+    #######################################
     
 @app.get("/playground", response_class=HTMLResponse)
 async def playground():