Spaces:
Sleeping
Sleeping
File size: 1,552 Bytes
e01283e 7ab7204 e01283e 6688095 e01283e 7ab7204 e01283e |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
import os
from dotenv import load_dotenv
from langchain.chat_models import ChatOpenAI
from langchain.document_loaders import PyPDFLoader
from langchain.text_splitter import CharacterTextSplitter
from langchain.vectorstores import FAISS
from langchain.embeddings import HuggingFaceEmbeddings
import serpapi
load_dotenv()
os.environ["HF_HOME"] = "/tmp/huggingface"
os.environ["TRANSFORMERS_CACHE"] = "/tmp/huggingface/transformers"
os.environ["HF_DATASETS_CACHE"] = "/tmp/huggingface/datasets"
# LLM (Groq + LLaMA3)
llm = ChatOpenAI(
model="llama3-8b-8192",
openai_api_base="https://api.groq.com/openai/v1",
openai_api_key=os.environ["GROQ_API_KEY"]
)
# Embeddings (HuggingFace)
embeddings = HuggingFaceEmbeddings(model_name="all-MiniLM-L6-v2")
# Load PDFs and create FAISS vectorstore
def load_vectorstore(pdf_dir="pdfs/"):
docs = []
for file in os.listdir(pdf_dir):
if file.endswith(".pdf"):
loader = PyPDFLoader(os.path.join(pdf_dir, file))
docs.extend(loader.load())
splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
chunks = splitter.split_documents(docs)
return FAISS.from_documents(chunks, embedding=embeddings)
# Custom Web Search tool using SerpAPI
def search_tool(query: str):
client = serpapi.Client(api_key=os.getenv("SERPAPI_API_KEY"))
search = client.search({
"engine": "google",
"q": query,
})
results = dict(search)
return results["organic_results"][0]["snippet"] # Return the snippet or any part of the result
|