改成可以多層爬取

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1 @@
+myenv/

README.md

@@ -0,0 +1,13 @@
+---
+title: Web Scraper & Q&A Chatbot with RAG
+emoji: 🏃
+colorFrom: blue
+colorTo: yellow
+sdk: streamlit
+sdk_version: 1.43.1
+app_file: app.py
+pinned: false
+short_description: Scrape, store, and query web data using RAG and AI chat.
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,131 @@
+import streamlit as st
+import requests
+import re
+import urllib.parse
+from bs4 import BeautifulSoup
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain.docstore.document import Document
+import chromadb
+from sentence_transformers import SentenceTransformer
+import google.generativeai as genai
+
+# Page configuration
+st.set_page_config(layout="wide")
+
+# Initialize Gemini API
+genai.configure(api_key="AIzaSyAxUd2tS-qj9C7frYuHRsv92tziXHgIvLo")
+
+# Initialize ChromaDB
+CHROMA_PATH = "chroma_db"
+chroma_client = chromadb.PersistentClient(path=CHROMA_PATH)
+
+# Initialize session state
+if 'scraped' not in st.session_state:
+    st.session_state.scraped = False
+if 'collection_name' not in st.session_state:
+    st.session_state.collection_name = "default_collection"
+if 'chat_history' not in st.session_state:
+    st.session_state.chat_history = []
+
+# Initialize embedding model
+embedding_model = SentenceTransformer("all-MiniLM-L6-v2")
+
+def clean_text(text):
+    return re.sub(r'\s+', ' ', re.sub(r'http\S+', '', text)).strip()
+
+def split_content_into_chunks(content):
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200, length_function=len)
+    return text_splitter.split_documents([Document(page_content=content)])
+
+def add_chunks_to_db(chunks, collection_name):
+    collection = chroma_client.get_or_create_collection(name=collection_name)
+    documents = [chunk.page_content for chunk in chunks]
+    embeddings = embedding_model.encode(documents, convert_to_list=True)
+    collection.upsert(documents=documents, ids=[f"ID{i}" for i in range(len(chunks))], embeddings=embeddings)
+
+def scrape_text(url, max_depth=1, same_domain=True):
+    visited = set()
+    base_domain = urllib.parse.urlparse(url).netloc
+
+    def _scrape(u, depth):
+        if depth > max_depth or u in visited:
+            return
+        visited.add(u)
+        try:
+            response = requests.get(u)
+            response.raise_for_status()
+            soup = BeautifulSoup(response.text, 'html.parser')
+            
+            text = clean_text(soup.get_text())
+            chunks = split_content_into_chunks(text)
+            add_chunks_to_db(chunks, st.session_state.collection_name)
+            
+            # 遞迴爬取新連結
+            if depth < max_depth:
+                for link in soup.find_all('a', href=True):
+                    next_url = urllib.parse.urljoin(u, link['href'])
+                    next_domain = urllib.parse.urlparse(next_url).netloc
+                    if same_domain and next_domain != base_domain:
+                        continue
+                    if next_url.startswith('mailto:') or next_url.startswith('javascript:'):
+                        continue
+                    _scrape(next_url, depth + 1)
+        except requests.exceptions.RequestException:
+            pass  # 忽略單一頁面錯誤
+
+    _scrape(url, 1)
+    st.session_state.scraped = True
+    return "Scraping and processing complete. You can now ask questions!"
+
+def ask_question(query, collection_name):
+    collection = chroma_client.get_or_create_collection(name=collection_name)
+    query_embedding = embedding_model.encode(query, convert_to_list=True)
+    results = collection.query(query_embeddings=[query_embedding], n_results=2)
+    top_chunks = results.get("documents", [[]])[0]
+    
+    system_prompt = f"""
+    You are a helpful assistant. Answer only from the provided context. 
+    If you lack information, say: "I don't have enough information to answer that question."
+    Context:
+    {str(top_chunks)}
+    """
+    
+    model = genai.GenerativeModel('gemini-2.0-flash')
+    response = model.generate_content(system_prompt + "\nUser Query: " + query)
+    return response.text
+
+# Sidebar
+with st.sidebar:
+    st.header("Database Management")
+    if st.button("Clear Chat History"):
+        st.session_state.chat_history = []
+        st.rerun()
+    
+    st.header("Step 1: Scrape a Website")
+    url = st.text_input("Enter URL:")
+    max_depth = st.selectbox("Recursion Depth (層數)", options=[1,2,3,4,5], index=0, help="選擇要遞迴爬幾層，預設1層")
+    same_domain = st.checkbox("只允許同網域遞迴", value=True, help="預設只爬同一網域的連結")
+    if url and st.button("Scrape & Process"):
+        with st.spinner("Scraping..."):
+            st.success(scrape_text(url, max_depth=max_depth, same_domain=same_domain))
+
+# Main content
+st.title("Web Scraper & Q&A Chatbot")
+if st.session_state.scraped:
+    st.subheader("Step 2: Ask Questions")
+    for message in st.session_state.chat_history:
+        with st.chat_message(message["role"]):
+            st.write(message["content"])
+    
+    user_query = st.chat_input("Ask your question here")
+    if user_query:
+        st.session_state.chat_history.append({"role": "user", "content": user_query})
+        with st.spinner("Searching..."):
+            answer = ask_question(user_query, st.session_state.collection_name)
+        st.session_state.chat_history.append({"role": "assistant", "content": answer})
+        
+        # Limit chat history to 6 messages
+        st.session_state.chat_history = st.session_state.chat_history[-6:]
+        st.rerun()
+else:
+    st.info("Please scrape a website first.")

requirements.txt

@@ -0,0 +1,7 @@
+requests
+beautifulsoup4
+langchain
+chromadb
+sentence-transformers
+google-generativeai
+streamlit