Update app.py

app.py

@@ -1,5 +1,8 @@
 import streamlit as st
 import os
+import shutil
+import schedule
+import time
 import pickle
 from langchain.prompts import ChatPromptTemplate
 from langchain.text_splitter import RecursiveCharacterTextSplitter
@@ -14,6 +17,7 @@ from langchain_core.output_parsers import StrOutputParser
 from langchain_core.runnables import RunnableLambda
 from datetime import date
 import time
+import subprocess
 llm_list = ['Mistral-7B-Instruct-v0.2','Mixtral-8x7B-Instruct-v0.1']
 blablador_base = "https://helmholtz-blablador.fz-juelich.de:8000/v1"
 # Environment variables
@@ -21,6 +25,67 @@ os.environ['LANGCHAIN_TRACING_V2'] = 'true'
 os.environ['LANGCHAIN_ENDPOINT'] = 'https://api.smith.langchain.com'
 os.environ['LANGCHAIN_API_KEY'] = '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'
 
+# Function to update your retriever
+# Function to update your retriever
+def update_retriever():
+    # Define the directory and file paths
+    directory_path = "ohw_proj_chorma_db"
+    file_path = "ohw_proj_chorma_db.pcl"
+
+    # Remove the directory and its contents if it exists
+    if os.path.exists(directory_path):
+        shutil.rmtree(directory_path)
+        st.write(f"Directory '{directory_path}' and its contents were removed successfully.")
+
+    # Remove the file if it exists
+    if os.path.exists(file_path):
+        os.remove(file_path)
+        st.write(f"File '{file_path}' was removed successfully.")
+
+    # Run the first Python script
+    try:
+        subprocess.run(["python", "scrape_github.py"], check=True)
+        st.write("GitHub repos downloaded")
+    except subprocess.CalledProcessError as e:
+        st.error(f"Error running scrape_github.py: {e}")
+
+    # Run the second Python script
+    try:
+        subprocess.run(["python", "create_retriever.py"], check=True)
+        st.write("Retriever updated")
+    except subprocess.CalledProcessError as e:
+        st.error(f"Error running create_retriever.py: {e}")
+
+    # Additional logic to update your retriever after running the scripts
+    st.write("Retriever updated!")
+
+# Function to run the scheduler
+def run_scheduler():
+    while True:
+        schedule.run_pending()
+        time.sleep(1)  # Check every second
+
+# Schedule the retriever update every 24 hours
+schedule.every(24).hours.do(update_retriever)
+
+# Run the scheduler in a separate thread to not block the main thread
+scheduler_thread = threading.Thread(target=run_scheduler)
+scheduler_thread.start()
+if os.path.exists(directory_path):
+    st.write('retriever loaded')
+else:
+    try:
+        subprocess.run(["python", "scrape_github.py"], check=True)
+        st.write("GitHub repos downloaded")
+    except subprocess.CalledProcessError as e:
+        st.error(f"Error running scrape_github.py: {e}")
+
+    # Run the second Python script
+    try:
+        subprocess.run(["python", "create_retriever.py"], check=True)
+        st.write("Retriever updated")
+    except subprocess.CalledProcessError as e:
+        st.error(f"Error running create_retriever.py: {e}")
 def load_from_pickle(filename):
     with open(filename, "rb") as file:
         return pickle.load(file)