rm bm25 & fix docstore kwarg

langchain-streamlit-demo/app.py

@@ -17,9 +17,8 @@ from streamlit_feedback import streamlit_feedback
 from defaults import default_values
 
 from llm_resources import (
-    get_runnable, 
-    get_llm, 
-    get_texts_and_retriever, 
+    get_runnable,
+    get_llm,
     get_texts_and_multiretriever,
     StreamHandler,
 )

langchain-streamlit-demo/llm_resources.py

@@ -11,7 +11,7 @@ from langchain.chat_models import (
 )
 from langchain.document_loaders import PyPDFLoader
 from langchain.embeddings import AzureOpenAIEmbeddings, OpenAIEmbeddings
-from langchain.retrievers import BM25Retriever, EnsembleRetriever
+from langchain.retrievers import EnsembleRetriever
 from langchain.schema import Document, BaseRetriever
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain.vectorstores import FAISS
@@ -116,48 +116,6 @@ def get_llm(
     return None
 
 
-def get_texts_and_retriever(
-    uploaded_file_bytes: bytes,
-    openai_api_key: str,
-    chunk_size: int = DEFAULT_CHUNK_SIZE,
-    chunk_overlap: int = DEFAULT_CHUNK_OVERLAP,
-    k: int = DEFAULT_RETRIEVER_K,
-    azure_kwargs: Optional[Dict[str, str]] = None,
-    use_azure: bool = False,
-) -> Tuple[List[Document], BaseRetriever]:
-    with NamedTemporaryFile() as temp_file:
-        temp_file.write(uploaded_file_bytes)
-        temp_file.seek(0)
-
-        loader = PyPDFLoader(temp_file.name)
-        documents = loader.load()
-        text_splitter = RecursiveCharacterTextSplitter(
-            chunk_size=chunk_size,
-            chunk_overlap=chunk_overlap,
-        )
-        texts = text_splitter.split_documents(documents)
-        embeddings_kwargs = {"openai_api_key": openai_api_key}
-        if use_azure and azure_kwargs:
-            azure_kwargs["azure_endpoint"] = azure_kwargs.pop("openai_api_base")
-            embeddings_kwargs.update(azure_kwargs)
-            embeddings = AzureOpenAIEmbeddings(**embeddings_kwargs)
-        else:
-            embeddings = OpenAIEmbeddings(**embeddings_kwargs)
-
-        bm25_retriever = BM25Retriever.from_documents(texts)
-        bm25_retriever.k = k
-
-        faiss_vectorstore = FAISS.from_documents(texts, embeddings)
-        faiss_retriever = faiss_vectorstore.as_retriever(search_kwargs={"k": k})
-
-        ensemble_retriever = EnsembleRetriever(
-            retrievers=[bm25_retriever, faiss_retriever],
-            weights=[0.5, 0.5],
-        )
-
-        return texts, ensemble_retriever
-
-
 def get_texts_and_multiretriever(
     uploaded_file_bytes: bytes,
     openai_api_key: str,
@@ -204,7 +162,7 @@ def get_texts_and_multiretriever(
         multivectorstore = FAISS.from_documents(sub_texts, embeddings)
         multivector_retriever = MultiVectorRetriever(
             vectorstore=multivectorstore,
-            base_store=store,
+            docstore=store,
             id_key=id_key,
         )
         multivector_retriever.docstore.mset(list(zip(text_ids, texts)))

requirements.txt

@@ -7,7 +7,6 @@ openai==1.3.8
 pillow>=10.0.1 # not directly required, pinned by Snyk to avoid a vulnerability
 pyarrow>=14.0.1 # not directly required, pinned by Snyk to avoid a vulnerability
 pypdf==3.17.2
-rank_bm25==0.2.2
 streamlit==1.29.0
 streamlit-feedback==0.1.3
 tiktoken==0.5.2