Moving HF spaces to HF model

.devcontainer/Dockerfile

@@ -0,0 +1,7 @@
+FROM mcr.microsoft.com/devcontainers/python:1-3.12-bullseye
+
+RUN pip install --no-cache-dir python-dotenv==1.0.1 langchain==0.3.10 langchain-community==0.3.10 \
+qdrant-client==1.12.1 gradio==5.8.0 pymongo==4.10.1 requests==2.32.3 bs4==0.0.2 ipykernel==6.29.5 ipython==8.27.0 \
+ipywidgets==8.1.5 jupyter==1.1.1 jupyter-client==8.6.2 jupyter-console==6.6.3 jupyter-core==5.7.2 jupyter-server==2.14.2 \
+jupyter-events==0.10.0 jupyter-lsp==2.2.5 jupyter-server-terminals==0.5.3 jupyterlab==4.2.5 jupyterlab-pygments==0.3.0 \
+jupyterlab-quarto==0.3.5 jupyterlab-server==2.27.3 jupyterlab-widgets==3.0.13 langchain-openai==0.2.11 clearml==1.16.4

.devcontainer/devcontainer.json

@@ -0,0 +1,37 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/python
+{
+	"name": "Python 3",
+	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+	"build": {
+        // Path is relative to the devcontainer.json file.
+        "dockerfile": "Dockerfile"
+    },
+
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	// "features": {},
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	// "postCreateCommand": "pip3 install --user -r requirements.txt",
+
+	// Configure tool-specific properties.
+	"customizations": {
+		// Install jupyter, mongodb, and docker
+		"vscode": {
+			"extensions": [
+				"ms-toolsai.jupyter",
+				"mongodb.mongodb-vscode",
+				"ms-azuretools.vscode-docker"
+			],
+			"settings": {
+
+			}
+		}
+	},
+
+	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+	"remoteUser": "root"
+}

project/.env.example

@@ -0,0 +1,38 @@
+# MongoDB database
+# Change DATABASE_HOST if you run mongodb from the web
+# https://www.mongodb.com/resources/products/fundamentals/mongodb-cluster-setup
+DATABASE_HOST="mongodb://host.docker.internal/27017"
+
+# Qdrant vector database
+# USE_QDRANT_CLOUD="False" if you setup the qdrant docker container(docker compose)
+# USE_QDRANT_CLOUD="True" if you run qdrant from the web
+USE_QDRANT_CLOUD="False"
+# Change QDRANT_CLOUD_URL and fill out QDRANT_APIKEY if you run qdrant from the web
+# https://qdrant.tech/documentation/cloud/create-cluster/
+QDRANT_CLOUD_URL="host.docker.internal:6333"
+QDRANT_APIKEY=your_qdrant_apikey
+
+
+# Ollama
+# USE_DOCKER="True" if you setup the ollama docker container(docker compose)
+# USE_DOCKER="False" if you run ollama serve from command line
+USE_DOCKER="True"
+
+# ClearML(optional for running files in ClearML folder)
+# https://clear.ml/docs/latest/docs/clearml_serving/clearml_serving_setup/
+CLEARML_WEB_HOST=your_clearml_web_host(link)
+CLEARML_API_HOST=your_clearml_api_host(link)
+CLEARML_FILES_HOST=your_clearml_files_host(link)
+CLEARML_API_ACCESS_KEY=your_clearml_api_access_key(str)
+CLEARML_API_SECRET_KEY=your_clearml_api_secret_key(str)
+
+# --- OpenAI is not used for this project(optional), but a function to pull the model is given. ---
+
+# OpenAI API Config(unused)
+# https://platform.openai.com/api-keys
+OPENAI_MODEL_ID=gpt-4o-mini
+OPENAI_API_KEY=your_openai_api_key
+
+# Huggingface API Config(unused)
+# https://huggingface.co/docs/hub/en/security-tokens
+HUGGINGFACE_ACCESS_TOKEN=your_huggingface_access_token

project/.gitignore

@@ -0,0 +1,2 @@
+.gradio
+.env

project/ClearML/DataCollectionPipeline.py

@@ -0,0 +1,173 @@
+# See README for more info on how the DataCollectionPipeline works
+# The ETL pipeline is part of the DataCollectionPipeline
+# Remove the time.sleep(1) line if you are sure you won't get blocked from a webpage for requesting too often
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import time
+
+import pymongo
+import requests
+from bs4 import BeautifulSoup
+from clearml import PipelineDecorator
+from dotenv import load_dotenv
+
+# Setup ClearML
+try:
+    load_dotenv(override=True)
+except Exception:
+    load_dotenv(sys.path[1] + "/.env", override=True)
+CLEARML_WEB_HOST = os.getenv("CLEARML_WEB_HOST")
+CLEARML_API_HOST = os.getenv("CLEARML_API_HOST")
+CLEARML_FILES_HOST = os.getenv("CLEARML_FILES_HOST")
+CLEARML_API_ACCESS_KEY = os.getenv("CLEARML_API_ACCESS_KEY")
+CLEARML_API_SECRET_KEY = os.getenv("CLEARML_API_SECRETKEY")
+
+# Input into the Data Collection Pipeline is a list of links to domains
+links = [
+    "https://www.ros.org/",
+    "https://docs.nav2.org/",
+    "https://moveit.ai/",
+    "https://gazebosim.org/home",
+    "https://github.com/ros2/ros2",
+    "https://github.com/ros-navigation/navigation2",
+    "https://github.com/moveit/moveit2",
+    "https://github.com/gazebosim/gazebo-classic",
+]
+links = ["https://www.ros.org/", "https://github.com/ros2/ros2"]
+
+
+# ETL pipeline
+@PipelineDecorator.component(cache=False, return_values=["documents, codes"])
+def ETL_Pipeline(links):
+    # Create a mongoDB connection to check for duplicates before inserting
+    try:
+        load_dotenv(override=True)
+    except Exception:
+        load_dotenv(sys.path[1] + "/.env", override=True)
+    DATABASE_HOST = os.getenv("DATABASE_HOST")
+    mongoHost = pymongo.MongoClient(DATABASE_HOST)
+    mongoDatabase = mongoHost["twin"]
+
+    # Extract data from links and their subdirectories(using crawlers)
+    documents = []
+    codes = []
+    for link in links:
+        # Web scraper/crawler for github links
+        if "https://github.com" in link:
+            # Do not revisit a link already in the database
+            mongoCollection = mongoDatabase["Github"]
+            result = mongoCollection.find_one({"link": link})
+            if result is None:
+                # Modified GithubCrawler from LLM-Engineer for scraping github
+                local_temp = tempfile.mkdtemp()
+                try:
+                    os.chdir(local_temp)
+                    subprocess.run(["git", "clone", link])
+                    repo_path = os.path.join(local_temp, os.listdir(local_temp)[0])
+                    tree = {}
+                    for root, _, files in os.walk(repo_path):
+                        dir = root.replace(repo_path, "").lstrip("/")
+                        if dir.startswith((".git", ".toml", ".lock", ".png")):
+                            continue
+                        for file in files:
+                            if file.endswith((".git", ".toml", ".lock", ".png")):
+                                continue
+                            file_path = os.path.join(dir, file)
+                            with open(
+                                os.path.join(root, file), "r", errors="ignore"
+                            ) as f:
+                                tree[file_path] = f.read().replace(" ", "")
+                except Exception:
+                    print(f"Error scrapping {link}")
+                finally:
+                    shutil.rmtree(local_temp)
+                    # Correct the link
+                    r = requests.get(link)
+                    soup = BeautifulSoup(r.content, "html.parser")
+                    # Find the file path to any of the files in the repository
+                    link_element = soup.find("a", attrs={"class": "Link--primary"})
+                    path = link_element.get("href")
+                    path = path.rsplit("/", 1)[0]
+                    # Push all the subdirectories to mongo
+                    for subdirectory in tree:
+                        text = tree[subdirectory]
+                        # Transform the data
+                        # Get rid of repeating \n characters and spaces
+                        text = text.replace("\t", " ")
+                        text = text.replace("\n", " ")
+                        text_len = len(text)
+                        for i in range(text_len):
+                            while (
+                                i + 1 < text_len
+                                and text[i] == " "
+                                and text[i + 1] == " "
+                            ):
+                                text = text[:i] + text[i + 1 :]
+                                text_len -= 1
+                        codes.append(
+                            {
+                                "link": "https://github.com"
+                                + path
+                                + "/"
+                                + subdirectory,
+                                "type": "Github",
+                                "content": text,
+                            }
+                        )
+        # Web scraper/crawler for other links(Documents)
+        else:
+            # Do not revisit a link already in the database
+            mongoCollection = mongoDatabase["Document"]
+            result = mongoCollection.find_one({"link": link})
+            if result is None:
+                # Get all text in the website
+                r = requests.get(link)
+                soup = BeautifulSoup(r.content, "html.parser")
+                soup.find_all(["p", "h1", "h2", "h3", "h4", "h5", "h6"])
+                text = soup.get_text()
+                # Transform the data
+                # Get rid of repeating \n characters and spaces
+                text = text.replace("\t", " ")
+                text = text.replace("\n", " ")
+                text_len = len(text)
+                for i in range(text_len):
+                    while i + 1 < text_len and text[i] == " " and text[i + 1] == " ":
+                        text = text[:i] + text[i + 1 :]
+                        text_len -= 1
+                documents.append({"link": link, "type": "Document", "content": text})
+                # Also crawl through all subdirectorys in the link(related links)
+                soup = BeautifulSoup(r.content, "html.parser")
+                subdirectories = [a.get("href") for a in soup.find_all("a")]
+                for subdirectory in subdirectories:
+                    if (
+                        subdirectory is not None
+                        and mongoCollection.find_one({"link": link + subdirectory})
+                        is not None
+                    ):
+                        links.append(link + subdirectory)
+        # Avoid spamming sites
+        time.sleep(1)
+    # Each document has a link, type(github or other), and content(text)
+    mongoCollection = mongoDatabase["Document"]
+    mongoCollection.insert_many(documents)
+    mongoCollection = mongoDatabase["Github"]
+    mongoCollection.insert_many(codes)
+    return documents, codes
+
+
+# Allow ClearML to monitor and run the ETL pipeline
+@PipelineDecorator.pipeline(
+    name="Data Collection Pipeline",
+    project="RAG LLM",
+    version="0.2",
+)
+def main():
+    return ETL_Pipeline(links)
+
+
+if __name__ == "__main__":
+    PipelineDecorator.run_locally()
+    main()

project/ClearML/FeaturePipeline.py

@@ -0,0 +1,168 @@
+# See README for more info on how the FeaturePipeline works
+# The Ingestion pipeline is part of the FeaturePipeline
+# Make sure to ollama serve before running!
+import os
+import sys
+
+import pymongo
+from clearml import PipelineDecorator
+from dotenv import load_dotenv
+from langchain_community.embeddings import OllamaEmbeddings
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from qdrant_client import QdrantClient
+from qdrant_client.http.models import Distance, PointStruct, VectorParams
+
+# Setup ClearML
+try:
+    load_dotenv(override=True)
+except Exception:
+    load_dotenv(sys.path[1] + "/.env", override=True)
+CLEARML_WEB_HOST = os.getenv("CLEARML_WEB_HOST")
+CLEARML_API_HOST = os.getenv("CLEARML_API_HOST")
+CLEARML_FILES_HOST = os.getenv("CLEARML_FILES_HOST")
+CLEARML_API_ACCESS_KEY = os.getenv("CLEARML_API_ACCESS_KEY")
+CLEARML_API_SECRET_KEY = os.getenv("CLEARML_API_SECRETKEY")
+
+
+@PipelineDecorator.component(cache=False, return_values=["links, resultTypes, texts"])
+def retreiveDocuments():
+    links = []
+    resultTypes = []
+    texts = []
+    # Create a mongoDB connection
+    try:
+        load_dotenv(override=True)
+    except Exception:
+        load_dotenv(sys.path[1] + "/.env", override=True)
+    DATABASE_HOST = os.getenv("DATABASE_HOST")
+    mongoHost = pymongo.MongoClient(DATABASE_HOST)
+    mongoDatabase = mongoHost["twin"]
+    collections = mongoDatabase.list_collection_names()
+    for collection in collections:
+        mongoCollection = mongoDatabase[collection]
+        results = mongoCollection.find()
+        for result in results:
+            # For each document, split it into chunks
+            links.append(result["link"])
+            resultTypes.append(result["type"])
+            texts.append(result["content"])
+    return links, resultTypes, texts
+
+
+@PipelineDecorator.component(cache=False, return_values=["cleanTexts"])
+def cleanDocuments(texts):
+    cleanTexts = []
+    for text in texts:
+        cleanTexts.append("".join(char for char in text if 32 <= ord(char) <= 126))
+    return cleanTexts
+
+
+@PipelineDecorator.component(cache=False, return_values=["chunks", "chunkNums"])
+def chunkDocuments(texts):
+    chunks = []
+    chunkNums = []
+    text_splitter = RecursiveCharacterTextSplitter(
+        chunk_size=500,
+        chunk_overlap=20,
+        length_function=len,
+        is_separator_regex=False,
+    )
+    for text in texts:
+        textChunks = text_splitter.split_text(text)
+        chunkNum = 0
+        for chunk in textChunks:
+            chunks.append(text_splitter.split_text(chunk))
+            chunkNums.append(chunkNum)
+            chunkNum += 1
+    return chunks, chunkNums
+
+
+@PipelineDecorator.component(cache=False, return_values=["embeddings"])
+def embedChunks(chunks):
+    embeddings = []
+    # Setup the text embedder
+    MODEL = "llama3.2"
+    try:
+        load_dotenv(override=True)
+    except Exception:
+        load_dotenv(sys.path[1] + "/.env", override=True)
+    USE_DOCKER = os.getenv("USE_DOCKER")
+    if USE_DOCKER == "True":
+        embeddingsModel = OllamaEmbeddings(model=MODEL, base_url="http://host.docker.internal:11434")
+    else:
+        embeddingsModel = OllamaEmbeddings(model=MODEL)
+    for chunk in chunks:
+        embeddings.append(embeddingsModel.embed_query(chunk))
+    return embeddings
+
+
+@PipelineDecorator.component(cache=False)
+def storeEmbeddings(embeddings, links, resultTypes, chunks, chunkNums):
+    # Create a qdrant connection
+    try:
+        load_dotenv(override=True)
+    except Exception:
+        load_dotenv(sys.path[1] + "/.env", override=True)
+    USE_QDRANT_CLOUD = os.getenv("USE_QDRANT_CLOUD")
+    QDRANT_CLOUD_URL = os.getenv("QDRANT_CLOUD_URL")
+    QDRANT_APIKEY = os.getenv("QDRANT_APIKEY")
+    if USE_QDRANT_CLOUD:
+        qClient = QdrantClient(url=QDRANT_CLOUD_URL, api_key=QDRANT_APIKEY)
+    else:
+        qClient = QdrantClient(url=QDRANT_CLOUD_URL)
+
+    # Create qdrant collections to store embeddings
+    if not qClient.collection_exists("Github"):
+        qClient.create_collection(
+            collection_name="Github",
+            vectors_config=VectorParams(size=3072, distance=Distance.COSINE),
+        )
+    if not qClient.collection_exists("Document"):
+        qClient.create_collection(
+            collection_name="Document",
+            vectors_config=VectorParams(size=3072, distance=Distance.COSINE),
+        )
+    chunkIndex = -1
+    documentIndex = -1
+    for chunkNum in chunkNums:
+        chunkIndex += 1
+        if chunkNum == 0:
+            documentIndex += 1
+        # Store all documents from each MongoDB collection into qdrant
+        # Create embeddings for each chunk, of length 2048 using the embedding model
+        # Store the embedding along with some metadata into the Qdrant vector database
+        qClient.upsert(
+            collection_name=resultTypes[documentIndex],
+            wait=True,
+            points=[
+                PointStruct(
+                    id=chunkIndex,
+                    vector=embeddings[chunkIndex],
+                    payload={
+                        "link": links[documentIndex],
+                        "type": resultTypes[documentIndex],
+                        "chunk": chunkNum,
+                        "text": chunks[chunkIndex],
+                    },
+                )
+            ],
+        )
+
+
+# Ingestion Pipeline
+@PipelineDecorator.pipeline(
+    name="Feature Pipeline",
+    project="RAG LLM",
+    version="0.2",
+)
+def main():
+    links, resultTypes, texts = retreiveDocuments()
+    texts = cleanDocuments(texts)
+    chunks, chunkNums = chunkDocuments(texts)
+    embeddings = embedChunks(chunks)
+    storeEmbeddings(embeddings, links, resultTypes, chunks, chunkNums)
+
+
+if __name__ == "__main__":
+    PipelineDecorator.run_locally()
+    main()

project/ClearML/InferencePipeline.py

@@ -0,0 +1,214 @@
+# See README for more info on how the DataCollectionPipeline works
+# The retrieval pipeline is part of the DataCollectionPipeline
+import os
+import sys
+from operator import itemgetter
+
+from clearml import PipelineDecorator
+from dotenv import load_dotenv
+from langchain.prompts import PromptTemplate
+from langchain_community.embeddings import OllamaEmbeddings
+from langchain_community.llms import Ollama
+from qdrant_client import QdrantClient
+
+# Setup ClearML
+try:
+    load_dotenv(override=True)
+except Exception:
+    load_dotenv(sys.path[1] + "/.env", override=True)
+CLEARML_WEB_HOST = os.getenv("CLEARML_WEB_HOST")
+CLEARML_API_HOST = os.getenv("CLEARML_API_HOST")
+CLEARML_FILES_HOST = os.getenv("CLEARML_FILES_HOST")
+CLEARML_API_ACCESS_KEY = os.getenv("CLEARML_API_ACCESS_KEY")
+CLEARML_API_SECRET_KEY = os.getenv("CLEARML_API_SECRETKEY")
+
+
+# Query expansion(I only generate one additional prompt for simplicity)
+@PipelineDecorator.component(cache=False, return_values=["newQuery"])
+def queryExpansion(query):
+    # Setup the model
+    MODEL = "llama3.2"
+    try:
+        load_dotenv(override=True)
+    except Exception:
+        load_dotenv(sys.path[1] + "/.env", override=True)
+    USE_DOCKER = os.getenv("USE_DOCKER")
+    if USE_DOCKER == "True":
+        model = Ollama(model=MODEL, base_url="http://host.docker.internal:11434")
+    else:
+        model = Ollama(model=MODEL)
+
+    template = """
+    Rewrite the prompt. The new prompt must offer a different perspective.
+    Do not change the meaning. Output only the rewritten prompt with no introduction.
+        Prompt: {prompt}
+    """
+    prompt = PromptTemplate.from_template(template)
+    chain = {"prompt": itemgetter("prompt")} | prompt | model
+    return chain.invoke({"prompt": query})
+
+
+# Self-querying(The metadata I will be generating determines whether to look through the Qdrant collection containing github code)
+@PipelineDecorator.component(cache=False, return_values=["codingQuestion"])
+def selfQuerying(query):
+    # Setup the model
+    MODEL = "llama3.2"
+    try:
+        load_dotenv(override=True)
+    except Exception:
+        load_dotenv(sys.path[1] + "/.env", override=True)
+    USE_DOCKER = os.getenv("USE_DOCKER")
+    if USE_DOCKER == "True":
+        model = Ollama(model=MODEL, base_url="http://host.docker.internal:11434")
+    else:
+        model = Ollama(model=MODEL)
+
+    template = """
+    You are an AI assistant. You must determine if the prompt requires code as the answer.
+    Output a 1 if it is or a 0 if it is not and nothing else.
+        Prompt: {prompt}
+    """
+    prompt = PromptTemplate.from_template(template)
+    chain = {"prompt": itemgetter("prompt")} | prompt | model
+    return chain.invoke({"prompt": query})
+
+
+# Filtered vector search for each of the N=2 queries after expansion
+@PipelineDecorator.component(cache=False, return_values=["results1, results2"])
+def filteredVectorSearch(query, newQuery, codingQuestion):
+    # Create a qdrant connection
+    try:
+        load_dotenv(override=True)
+    except Exception:
+        load_dotenv(sys.path[1] + "/.env", override=True)
+    USE_QDRANT_CLOUD = os.getenv("USE_QDRANT_CLOUD")
+    QDRANT_CLOUD_URL = os.getenv("QDRANT_CLOUD_URL")
+    QDRANT_APIKEY = os.getenv("QDRANT_APIKEY")
+    if USE_QDRANT_CLOUD=="True":
+        qClient = QdrantClient(url=QDRANT_CLOUD_URL, api_key=QDRANT_APIKEY)
+    else:
+        qClient = QdrantClient(url=QDRANT_CLOUD_URL)
+
+    # Setup the text embedder
+    MODEL = "llama3.2"
+    try:
+        load_dotenv(override=True)
+    except Exception:
+        load_dotenv(sys.path[1] + "/.env", override=True)
+    USE_DOCKER = os.getenv("USE_DOCKER")
+    if USE_DOCKER == "True":
+        embeddingsModel = OllamaEmbeddings(model=MODEL, base_url="http://host.docker.internal:11434")
+    else:
+        embeddingsModel = OllamaEmbeddings(model=MODEL)
+
+    # Search the related collection
+    relatedCollection = "Document"
+    if codingQuestion == "1":
+        relatedCollection = "Github"
+    results1 = qClient.search(
+        collection_name=relatedCollection,
+        query_vector=embeddingsModel.embed_query(query),
+        limit=10,
+    )
+    results2 = qClient.search(
+        collection_name=relatedCollection,
+        query_vector=embeddingsModel.embed_query(newQuery),
+        limit=10,
+    )
+    return results1, results2
+
+
+# Collecting results
+@PipelineDecorator.component(cache=False, return_values=["results"])
+def collectingResults(results1, results2):
+    return results1 + results2
+
+
+# Reranking(Instead of using a CrossEncoder, I will manually compare embeddings)
+@PipelineDecorator.component(cache=False, return_values=["topTexts"])
+def reranking(results):
+    ids = [result.id for result in results]
+    scores = [result.score for result in results]
+    topIds = []
+    topIndexes = []
+    for x in range(3):
+        maxScore = 0
+        maxIndex = 0
+        for i in range(len(ids)):
+            if ids[i] not in topIds and scores[i] > maxScore:
+                maxScore = scores[i]
+                maxIndex = i
+        topIds.append(ids[maxIndex])
+        topIndexes.append(maxIndex)
+    texts = [result.payload["text"] for result in results]
+    topTexts = ""
+    for index in topIndexes:
+        topTexts += texts[index][0]
+    return topTexts
+
+
+# Building prompt
+@PipelineDecorator.component(cache=False, return_values=["prompt"])
+def buildingPrompt(codingQuestion):
+    if codingQuestion == "1":
+        template = """
+        Write code for the following question given the related coding document below.
+
+        Document: {document}
+        Question: {question}
+        """
+        return PromptTemplate.from_template(template)
+    else:
+        template = """
+        Answer the question based on the document below. If you can't answer the question, reply "I don't know"
+
+        Document: {document}
+        Question: {question}
+        """
+        return PromptTemplate.from_template(template)
+
+
+# Obtaining answer
+@PipelineDecorator.component(cache=False, return_values=["answer"])
+def obtainingAnswer(query, prompt, topTexts):
+    # Setup the model
+    MODEL = "llama3.2"
+    try:
+        load_dotenv(override=True)
+    except Exception:
+        load_dotenv(sys.path[1] + "/.env", override=True)
+    USE_DOCKER = os.getenv("USE_DOCKER")
+    if USE_DOCKER == "True":
+        model = Ollama(model=MODEL, base_url="http://host.docker.internal:11434")
+    else:
+        model = Ollama(model=MODEL)
+
+    chain = (
+        {"document": itemgetter("document"), "question": itemgetter("question")}
+        | prompt
+        | model
+    )
+    chain.invoke({"document": topTexts, "question": query})
+
+
+# Inference Pipeline
+@PipelineDecorator.pipeline(
+    name="Inference Pipeline",
+    project="RAG LLM",
+    version="0.1",
+)
+def main():
+    # User query
+    query = "What operating system was ROS written for?"
+    newQuery = queryExpansion(query)
+    codingQuestion = selfQuerying(query)
+    results1, results2 = filteredVectorSearch(query, newQuery, codingQuestion)
+    results = collectingResults(results1, results2)
+    topTexts = reranking(results)
+    prompt = buildingPrompt(codingQuestion)
+    return obtainingAnswer(query, prompt, topTexts)
+
+
+if __name__ == "__main__":
+    PipelineDecorator.run_locally()
+    main()

project/DataCollectionPipeline.ipynb

@@ -0,0 +1,200 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Visiting link:  https://github.com/ros2/ros2\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Cloning into 'ros2'...\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Adding subdirectory: https://github.com/ros2/ros2/tree/rolling/README.md\n",
+      "Adding subdirectory: https://github.com/ros2/ros2/tree/rolling/.gitignore\n",
+      "Adding subdirectory: https://github.com/ros2/ros2/tree/rolling/CODEOWNERS\n",
+      "Adding subdirectory: https://github.com/ros2/ros2/tree/rolling/ros2.repos\n",
+      "Adding subdirectory: https://github.com/ros2/ros2/tree/rolling/src/.gitkeep\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "InsertManyResult([ObjectId('675531b926a728d5b045a2e6'), ObjectId('675531b926a728d5b045a2e7'), ObjectId('675531b926a728d5b045a2e8'), ObjectId('675531b926a728d5b045a2e9'), ObjectId('675531b926a728d5b045a2ea')], acknowledged=True)"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# See README for more info on how the DataCollectionPipeline works\n",
+    "# The ETL pipeline is part of the DataCollectionPipeline\n",
+    "# Remove the time.sleep(1) line if you are sure you won't get blocked from a webpage for requesting too often\n",
+    "import requests\n",
+    "from bs4 import BeautifulSoup\n",
+    "import time\n",
+    "import os\n",
+    "import shutil\n",
+    "import subprocess\n",
+    "import tempfile\n",
+    "from shared import getMongoClient\n",
+    "\n",
+    "# Input into the Data Collection Pipeline is a list of links to domains\n",
+    "links = ['https://www.ros.org/','https://docs.nav2.org/','https://moveit.ai/','https://gazebosim.org/home', 'https://github.com/ros2/ros2', 'https://github.com/ros-navigation/navigation2', 'https://github.com/moveit/moveit2', 'https://github.com/gazebosim/gazebo-classic']\n",
+    "links = ['https://www.ros.org/', 'https://github.com/ros2/ros2']\n",
+    "\n",
+    "# Create a mongoDB connection\n",
+    "mongoHost =  getMongoClient()\n",
+    "mongoDatabase =  mongoHost[\"twin\"]\n",
+    "\n",
+    "# ETL pipeline\n",
+    "# Extract data from links and their subdirectories(using crawlers)\n",
+    "documents = []\n",
+    "codes = []\n",
+    "for link in links:\n",
+    "    # Web scraper/crawler for github links\n",
+    "    if \"https://github.com\" in link:\n",
+    "        # Do not revisit a link already in the database\n",
+    "        mongoCollection = mongoDatabase[\"Github\"]\n",
+    "        result = mongoCollection.find_one({\"link\": link})\n",
+    "        if result is None:\n",
+    "            print(\"Visiting link: \", link)\n",
+    "            # Modified GithubCrawler from LLM-Engineer for scraping github\n",
+    "            local_temp = tempfile.mkdtemp()\n",
+    "            try:\n",
+    "                os.chdir(local_temp)\n",
+    "                subprocess.run([\"git\", \"clone\", link])\n",
+    "                repo_path = os.path.join(local_temp, os.listdir(local_temp)[0])\n",
+    "                tree = {}\n",
+    "                for root, _, files in os.walk(repo_path):\n",
+    "                    dir = root.replace(repo_path, \"\").lstrip(\"/\")\n",
+    "                    if dir.startswith((\".git\", \".toml\", \".lock\", \".png\")):\n",
+    "                        continue\n",
+    "                    for file in files:\n",
+    "                        if file.endswith((\".git\", \".toml\", \".lock\", \".png\")):\n",
+    "                            continue\n",
+    "                        file_path = os.path.join(dir, file)\n",
+    "                        with open(\n",
+    "                            os.path.join(root, file), \"r\", errors=\"ignore\"\n",
+    "                        ) as f:\n",
+    "                            tree[file_path] = f.read().replace(\" \", \"\")\n",
+    "            except Exception:\n",
+    "                print(f\"Error scrapping {link}\")\n",
+    "            finally:\n",
+    "                shutil.rmtree(local_temp)\n",
+    "                # Correct the link\n",
+    "                r = requests.get(link)\n",
+    "                soup = BeautifulSoup(r.content, \"html.parser\")\n",
+    "                # Find the file path to any of the files in the repository\n",
+    "                link_element = soup.find(\"a\", attrs={\"class\": \"Link--primary\"})\n",
+    "                path = link_element.get(\"href\")\n",
+    "                path = path.rsplit(\"/\", 1)[0]\n",
+    "                # Push all the subdirectories to mongo\n",
+    "                for subdirectory in tree:\n",
+    "                    print(\n",
+    "                        f\"Adding subdirectory: https://github.com{path}/{subdirectory}\"\n",
+    "                    )\n",
+    "                    text = tree[subdirectory]\n",
+    "                    # Transform the data\n",
+    "                    # Get rid of repeating \\n characters and spaces\n",
+    "                    text = text.replace(\"\\t\", \" \")\n",
+    "                    text = text.replace(\"\\n\", \" \")\n",
+    "                    text_len = len(text)\n",
+    "                    for i in range(text_len):\n",
+    "                        while i + 1 < text_len and text[i] == \" \" and text[i + 1] == \" \":\n",
+    "                            text = text[:i] + text[i + 1 :]\n",
+    "                            text_len -= 1\n",
+    "                    codes.append(\n",
+    "                        {\n",
+    "                            \"link\": \"https://github.com\"\n",
+    "                            + path\n",
+    "                            + \"/\"\n",
+    "                            + subdirectory,\n",
+    "                            \"type\": \"Github\",\n",
+    "                            \"content\": text,\n",
+    "                        }\n",
+    "                    )\n",
+    "        else:\n",
+    "            print(\"Already visited: \", link)\n",
+    "    # Web scraper/crawler for other links(Documents)\n",
+    "    else:\n",
+    "        # Do not revisit a link already in the database\n",
+    "        mongoCollection = mongoDatabase[\"Document\"]\n",
+    "        result = mongoCollection.find_one({\"link\": link})\n",
+    "        if result is None:\n",
+    "            # Get all text in the website\n",
+    "            r = requests.get(link)\n",
+    "            soup = BeautifulSoup(r.content, \"html.parser\")\n",
+    "            soup.find_all([\"p\", \"h1\", \"h2\", \"h3\", \"h4\", \"h5\", \"h6\"])\n",
+    "            text = soup.get_text()\n",
+    "            # Transform the data\n",
+    "            # Get rid of repeating \\n characters and spaces\n",
+    "            text = text.replace(\"\\t\", \" \")\n",
+    "            text = text.replace(\"\\n\", \" \")\n",
+    "            text_len = len(text)\n",
+    "            for i in range(text_len):\n",
+    "                while i + 1 < text_len and text[i] == \" \" and text[i + 1] == \" \":\n",
+    "                    text = text[:i] + text[i + 1 :]\n",
+    "                    text_len -= 1\n",
+    "            documents.append({\"link\": link, \"type\": \"Document\", \"content\": text})\n",
+    "            # Also crawl through all subdirectorys in the link(related links)\n",
+    "            soup = BeautifulSoup(r.content, \"html.parser\")\n",
+    "            subdirectories = [a.get(\"href\") for a in soup.find_all(\"a\")]\n",
+    "            for subdirectory in subdirectories:\n",
+    "                if (\n",
+    "                    subdirectory is not None\n",
+    "                    and mongoCollection.find_one({\"link\": link + subdirectory})\n",
+    "                    is not None\n",
+    "                ):\n",
+    "                    print(\"Adding subdirectory: \", link + subdirectory)\n",
+    "                    links.append(link + subdirectory)\n",
+    "        else:\n",
+    "            print(\"Already visited: \", link)\n",
+    "    # Avoid spamming sites\n",
+    "    time.sleep(1)\n",
+    "# Each document has a link, type(github or other), and content(text)\n",
+    "# You can go to Tools/mongoTools to view the inserted documents\n",
+    "mongoCollection = mongoDatabase[\"Document\"]\n",
+    "mongoCollection.insert_many(documents)\n",
+    "mongoCollection = mongoDatabase[\"Github\"]\n",
+    "mongoCollection.insert_many(codes)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

project/Dockerfile

@@ -0,0 +1,11 @@
+FROM python:3.11.9
+
+WORKDIR /gradio-app
+COPY . .
+RUN pip install --no-cache-dir python-dotenv==1.0.1 langchain==0.3.10 langchain-community==0.3.10 \
+    qdrant-client==1.12.1 gradio==5.8.0 pymongo==4.10.1 langchain-openai==0.2.11
+    
+EXPOSE 7860
+ENV GRADIO_SERVER_NAME="0.0.0.0"
+
+CMD ["python", "app.py"]

project/FeaturePipeline.ipynb

@@ -0,0 +1,107 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/workspaces/RAG_LLM/project/shared.py:57: LangChainDeprecationWarning: The class `OllamaEmbeddings` was deprecated in LangChain 0.3.1 and will be removed in 1.0.0. An updated version of the class exists in the :class:`~langchain-ollama package and should be used instead. To use it run `pip install -U :class:`~langchain-ollama` and import as `from :class:`~langchain_ollama import OllamaEmbeddings``.\n",
+      "  return OllamaEmbeddings(model=MODEL, base_url=\"http://host.docker.internal:11434\")\n"
+     ]
+    }
+   ],
+   "source": [
+    "# See README for more info on how the FeaturePipeline works\n",
+    "# The Ingestion pipeline is part of the FeaturePipeline\n",
+    "# Make sure to ollama serve before running!\n",
+    "from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
+    "from qdrant_client.http.models import Distance, VectorParams, PointStruct\n",
+    "from shared import getMongoClient, getQdrantClient, getEmbeddingsModel\n",
+    "\n",
+    "# Create a mongoDB connection\n",
+    "mongoHost = getMongoClient()\n",
+    "\n",
+    "# Create a qdrant connection\n",
+    "qClient = getQdrantClient()\n",
+    "\n",
+    "# Create qdrant collections to store embeddings\n",
+    "if not qClient.collection_exists(\"Github\"):\n",
+    "    qClient.create_collection(\n",
+    "        collection_name=\"Github\",\n",
+    "        vectors_config=VectorParams(size=3072, distance=Distance.COSINE),\n",
+    "    )\n",
+    "if not qClient.collection_exists(\"Document\"):\n",
+    "    qClient.create_collection(\n",
+    "        collection_name=\"Document\",\n",
+    "        vectors_config=VectorParams(size=3072, distance=Distance.COSINE),\n",
+    "    )\n",
+    "\n",
+    "# Ingestion Pipeline Setup\n",
+    "# Define a text cleaner\n",
+    "def cleanText(text):\n",
+    "    return ''.join(char for char in text if 32 <= ord(char) <= 126)\n",
+    "\n",
+    "# Setup the text chunker\n",
+    "text_splitter = RecursiveCharacterTextSplitter(\n",
+    "    chunk_size=500,\n",
+    "    chunk_overlap=20,\n",
+    "    length_function=len,\n",
+    "    is_separator_regex=False,\n",
+    ")\n",
+    "\n",
+    "# Setup the text embedder\n",
+    "embeddingsModel = getEmbeddingsModel()\n",
+    "\n",
+    "# Running the ingestion pipeline\n",
+    "# Store all documents from each MongoDB collection into qdrant\n",
+    "mongoDatabase =  mongoHost[\"twin\"]\n",
+    "collections = mongoDatabase.list_collection_names()\n",
+    "for collection in collections:\n",
+    "    mongoCollection = mongoDatabase[collection]\n",
+    "\n",
+    "    documents = mongoCollection.find()\n",
+    "    id = 0\n",
+    "    for document in documents:\n",
+    "        # For each document, split it into chunks\n",
+    "        link = document[\"link\"]\n",
+    "        resultType = document[\"type\"]\n",
+    "        text = document[\"content\"]\n",
+    "        text = cleanText(text)\n",
+    "        chunks = text_splitter.split_text(text)\n",
+    "        chunkNum = 0\n",
+    "        for chunk in chunks:\n",
+    "            # Create embeddings for each chunk, of length 2048 using the embedding model\n",
+    "            embedding = embeddingsModel.embed_query(chunk)\n",
+    "            # Store the embedding along with some metadata into the Qdrant vector database\n",
+    "            qClient.upsert(collection_name=resultType, wait=True, points=[PointStruct(id=id, vector=embedding, payload={\"link\": link, \"type\": resultType, \"chunk\": chunkNum, \"text\": chunk})])\n",
+    "            chunkNum += 1\n",
+    "            id += 1\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

project/InferencePipeline.ipynb

@@ -0,0 +1,181 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/workspaces/RAG_LLM/project/shared.py:57: LangChainDeprecationWarning: The class `OllamaEmbeddings` was deprecated in LangChain 0.3.1 and will be removed in 1.0.0. An updated version of the class exists in the :class:`~langchain-ollama package and should be used instead. To use it run `pip install -U :class:`~langchain-ollama` and import as `from :class:`~langchain_ollama import OllamaEmbeddings``.\n",
+      "  return OllamaEmbeddings(model=MODEL, base_url=\"http://host.docker.internal:11434\")\n",
+      "/workspaces/RAG_LLM/project/shared.py:70: LangChainDeprecationWarning: The class `Ollama` was deprecated in LangChain 0.3.1 and will be removed in 1.0.0. An updated version of the class exists in the :class:`~langchain-ollama package and should be used instead. To use it run `pip install -U :class:`~langchain-ollama` and import as `from :class:`~langchain_ollama import OllamaLLM``.\n",
+      "  return Ollama(model=MODEL, base_url=\"http://host.docker.internal:11434\")\n"
+     ]
+    }
+   ],
+   "source": [
+    "# See README for more info on how the DataCollectionPipeline works\n",
+    "# The retrieval pipeline is part of the DataCollectionPipeline\n",
+    "from shared import getQdrantClient, getEmbeddingsModel, getModel\n",
+    "from langchain_community.llms import Ollama\n",
+    "from langchain.prompts import PromptTemplate\n",
+    "from operator import itemgetter\n",
+    "# Create a qdrant connection\n",
+    "qClient = getQdrantClient()\n",
+    "\n",
+    "# Setup the text embedder\n",
+    "embeddingsModel = getEmbeddingsModel()\n",
+    "\n",
+    "# Setup the model\n",
+    "model = getModel()\n",
+    "\n",
+    "# Retrieval Pipeline\n",
+    "# Retrieve the chunks with the most similar embeddings from Qdrant\n",
+    "def retriever(text, collection):\n",
+    "    results = qClient.search(\n",
+    "        collection_name=collection,\n",
+    "        query_vector = embeddingsModel.embed_query(text),\n",
+    "        limit=10\n",
+    "    )\n",
+    "    return results"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Query expansion:  Create a user-friendly, community-driven guide that provides an alternative to the traditional ROS documentation, focusing on real-world scenarios and practical applications rather than technical specifications and developer guides.\n",
+      "Coding Question?:  1\n",
+      "Related Collection:  Github\n",
+      "Top texts:  #About TheRobotOperatingSystem(ROS)isasetofsoftwarelibrariesandtoolsthathelpyoubuildrobotapplications. Fromdriverstostate-of-the-artalgorithms,andwithpowerfuldevelopertools,ROShaswhatyouneedforyournextroboticsproject. Andit'sallopensource. Fullprojectdetailson[ROS.org](https://ros.org/) #GettingStarted LookingtogetstartedwithROS? Our[installationguideishere](https://www.ros.org/blog/getting-started/).\n",
+      "Link:  https://github.com/ros2/ros2/tree/rolling/README.md\n",
+      "Top texts:  type:git url:https://github.com/ros2/tinyxml2_vendor.git version:rolling ros2/tlsf: type:git url:https://github.com/ros2/tlsf.git version:rolling ros2/unique_identifier_msgs: type:git url:https://github.com/ros2/unique_identifier_msgs.git version:rolling ros2/urdf: type:git url:https://github.com/ros2/urdf.git version:rolling ros2/yaml_cpp_vendor: type:git url:https://github.com/ros2/yaml_cpp_vendor.git version:rolling\n",
+      "Link:  https://github.com/ros2/ros2/tree/rolling/ros2.repos\n",
+      "Top texts:  *[ROSResourceStatusPage](https://status.openrobotics.org/) *[REP-2000](https://ros.org/reps/rep-2000.html):ROS2ReleasesandTargetPlatforms ##ProjectResources *[PurchaseROSSwag](https://spring.ros.org/) *[InformationabouttheROSTrademark](https://www.ros.org/blog/media/) *OnSocialMedia *[OpenRoboticsonLinkedIn](https://www.linkedin.com/company/open-source-robotics-foundation) *[OpenRoboticsonTwitter](https://twitter.com/OpenRoboticsOrg) *[ROS.orgonTwitter](https://twitter.com/ROSOrg)\n",
+      "Link:  https://github.com/ros2/ros2/tree/rolling/README.md\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "\"Here's an example of what the README file for ROS could look like:\\n\\n**Welcome to the Robot Operating System (ROS)**\\n\\nROS is a set of software libraries and tools that help you build robot applications. From driver development to state-of-the-art algorithms, and with powerful development tools, ROS has everything you need for your next robotics project.\\n\\n### Getting Started\\n\\nTo get started with ROS, check out our [installation guide](https://www.ros.org/blog/getting-started/).\\n\\n### What's Included\\n\\nROS includes a range of open-source projects, including:\\n\\n*   **tinyxml2_vendor**: A fork of the tinyxml2 library for parsing XML files.\\n*   **tlsf**: A library for secure communication over TLS (Transport Layer Security).\\n*   **unique_identifier_msgs**: A package for generating unique identifiers for robots and other entities.\\n*   **urdf**: A package for working with URDF (Unified Robot Description Format) files.\\n*   **yaml_cpp_vendor**: A fork of the yaml-cpp library for parsing YAML files.\\n\\n### ROS Releases and Target Platforms\\n\\nFor more information on ROS releases, target platforms, and release notes, check out [REP-2000](https://ros.org/reps/rep-2000.html).\\n\\n### Project Resources\\n\\n*   **ROSSwag**: Purchase ROS-related merchandise from our online store.\\n*   **ROS Trademark Information**: Learn about the ROS trademark.\\n\\n### Get Involved\\n\\nStay up-to-date with the latest news and developments in ROS:\\n\\n*   Follow us on [LinkedIn](https://www.linkedin.com/company/open-source-robotics-foundation)\\n*   Join our Twitter community: [OpenRoboticsOrg](https://twitter.com/OpenRoboticsOrg), [ROSOrg](https://twitter.com/ROSOrg)\\n\\n### License and Contributions\\n\\nROS is an open-source project, licensed under the Apache 2.0 license.\\n\\nWe welcome contributions from the ROS community! If you have any ideas or bug fixes to contribute, check out our [contribution guidelines](https://ros.org/blog/contribute/).\\n\\n**Thank You**\\n\\nThanks for choosing ROS as your platform for robotics development!\\n\\nYou can modify this README file according to your needs and preferences.\""
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# User query\n",
+    "query = \"Can you create a README file for ROS\"\n",
+    "\n",
+    "# Query expansion(I only generate one additional prompt for simplicity)\n",
+    "template = \"\"\"\n",
+    "Rewrite the prompt. The new prompt must offer a different perspective.\n",
+    "Do not change the meaning. Output only the rewritten prompt with no introduction.\n",
+    "    Prompt: {prompt}\n",
+    "\"\"\"\n",
+    "prompt = PromptTemplate.from_template(template)\n",
+    "chain = {\"prompt\": itemgetter(\"prompt\")} | prompt | model\n",
+    "queryExpansion = chain.invoke({\"prompt\": query})\n",
+    "print(\"Query expansion: \", queryExpansion)\n",
+    "\n",
+    "# Self-querying(The metadata I will be generating determines whether to look through the Qdrant collection containing github code)\n",
+    "template = \"\"\"\n",
+    "You are an AI assistant. You must determine if the prompt requires code as the answer.\n",
+    "Output a 1 if it is or a 0 if it is not and nothing else.\n",
+    "    Prompt: {prompt}\n",
+    "\"\"\"\n",
+    "prompt = PromptTemplate.from_template(template)\n",
+    "chain = {\"prompt\": itemgetter(\"prompt\")} | prompt | model\n",
+    "codingQuestion = chain.invoke({\"prompt\": query})\n",
+    "print(\"Coding Question?: \", codingQuestion)\n",
+    "\n",
+    "# Filtered vector search for each of the N queries after expansion\n",
+    "relatedCollection = 'Document'\n",
+    "if (codingQuestion == '1'):\n",
+    "    relatedCollection = 'Github'\n",
+    "print(\"Related Collection: \", relatedCollection)\n",
+    "results1 = retriever(query, relatedCollection)\n",
+    "results2 = retriever(queryExpansion, relatedCollection)\n",
+    "\n",
+    "# Collecting results\n",
+    "results = results1+results2\n",
+    "\n",
+    "# Reranking(Instead of using a CrossEncoder, I will manually compare embeddings)\n",
+    "ids = [result.id for result in results]\n",
+    "scores = [result.score for result in results]\n",
+    "topIds = []\n",
+    "topIndexes = []\n",
+    "for x in range(3):\n",
+    "    maxScore = 0\n",
+    "    maxIndex = 0\n",
+    "    for i in range(len(ids)):\n",
+    "        if ids[i] not in topIds and scores[i] > maxScore:\n",
+    "            maxScore = scores[i]\n",
+    "            maxIndex = i\n",
+    "    topIds.append(ids[maxIndex])\n",
+    "    topIndexes.append(maxIndex)\n",
+    "texts = [result.payload['text'] for result in results]\n",
+    "links = [result.payload['link'] for result in results]\n",
+    "topTexts = ''\n",
+    "for index in topIndexes:\n",
+    "    print(\"Top texts: \", texts[index])\n",
+    "    print(\"Link: \", links[index])\n",
+    "    topTexts += texts[index]\n",
+    "\n",
+    "# Building prompt\n",
+    "if(codingQuestion == '1'):\n",
+    "    template = \"\"\"\n",
+    "    Write code for the following question given the related coding document below.\n",
+    "\n",
+    "    Document: {document}\n",
+    "    Question: {question}\n",
+    "    \"\"\"\n",
+    "    prompt = PromptTemplate.from_template(template)\n",
+    "else:\n",
+    "    template = \"\"\"\n",
+    "    Answer the question based on the document below. If you can't answer the question, reply \"I don't know\"\n",
+    "\n",
+    "    Document: {document}\n",
+    "    Question: {question}\n",
+    "    \"\"\"\n",
+    "    prompt = PromptTemplate.from_template(template)\n",
+    "\n",
+    "# Obtaining answer\n",
+    "chain = {\"document\": itemgetter(\"document\"), \"question\": itemgetter(\"question\")} | prompt | model\n",
+    "chain.invoke({\"document\": topTexts, \"question\": query})"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

project/README.md

@@ -0,0 +1,62 @@
+<h1>Installation:</h1>
+<h3>Docker setup(easy):<h3>
+
+* Clone the repository from huggingface
+* Reopen the repository in a dev container
+* Copy the .env.example into a new .env file in the project folder
+* If you want to run files in the ClearML folder, fill out the ClearML env variables, otherwise no changes needed.
+* Open a shell on the host machine(not the dev container) and navigate to the project folder
+* Run "docker compose up -d"
+* Run "docker exec -it ollama ollama pull llama3.2"
+* Select the python 3.12.7 kernels for the notebooks and run DataCollectionPipeline.ipynb and FeaturePipeline.ipynb(to populate the mongodb and qdrant databases)
+* The app is available on localhost:7860
+
+<h3>Non-Docker(web based) setup:<h3>
+
+If for some reason the docker setup does not work try connecting to mongodb, qdrant, ollama, and gradio from the web:
+* Clone the repository from huggingface or the entire repository from github
+* Reopen the repository in a dev container
+* Copy the .env.example into a new .env file in the project folder
+* Modify the .env file as instructed in the comments(create accounts for each website)
+* Install ollama in the dev container
+  * curl -fsSL https://ollama.com/install.sh | sh
+* Start up ollama
+  * ollama serve
+* Download llama3.2(in a new dev container terminal)
+  * ollama pull llama3.2
+* Select the python 3.12.7 kernels for the notebooks and run DataCollectionPipeline.ipynb and FeaturePipeline.ipynb(to populate the mongodb and qdrant databases)
+* Run app.py and click on the link
+
+<h1>Project infrastructure</h1>
+
+Note some files may have similar code with other files, such as the ClearML files containing ipynb files rewritten in python in order to work in ClearML or gradio containing code from InferencePipeline.ipynb. The ipynb file prints output to help see what is happening.
+
+# app.py
+Sends a query to the inference pipeline to generate an answer. The DataCollectionPipeline.ipynb and FeaturePipeline.ipynb files must be run first to populate the databases.
+
+# Data Collection Pipeline
+The Data Collection pipeline takes as input a list of links to domains. The links are fed into the ETL pipeline which Extracts data from the links using a crawler, Transforms the data into a standardized format, and Loads the extracted data into a NoSQL data warehouse, which in this case is MongoDB. The ETL pipeline uses a different method of extracting and transforming based on the link type. In this project, I classify links as either a github repository or document each with their own crawler and cleaner. This raw data is used by the feature pipeline.
+
+# Feature Pipeline
+The Feature pipeline contains the ingestion pipeline.
+* The ingestion pipeline extracts documents from MongoDB that were stored by the Data Collection Pipeline. It further cleans the data, breaks it into chunks depending on the data category, passes the chunks through an embedding model to generate embeddings, then loads the embeddings plus their metadata into a vector database, which in this case is Qdrant. The embeddings are passed with additional metadata that contains the document link, type, chunk number, and content.
+
+# Training Pipeline
+The training pipeline performs finetuning. I skipped this step since it was not required.
+
+# Inference Pipeline
+The inference pipeline contains the retrieval client/pipeline.
+* The retreival client takes a prompt as input. It uses the same embedding model as the ingestion pipeline in order to create an embedding for the prompt. It then queries the Qdrant database for the 10 closest embeddings using cosine distance and extracts the text chunk stored in the embeddings' metadata. This returns chunks that are related to the prompt.
+* The inference pipeline takes a query as input. It expands the query into N=2 queries using a prompt template, performs self-querying to extract metadata (document type) from the original query, searches the Qdrant for K=10 relevant chunks to each of the N=2 queries plus metadata using the retrieval client, combines the K=10 results from each of the N=2 queries, filters out only the most relevant 3 results, prompts the LLM with the results as context, and generates an answer.
+
+# ClearML
+The ClearML folder contains the notebook (.ipynb) pipeline files rewritten to work with ClearML. It is similar code to the notebooks, however ClearML does not print any output but instead logs all output in website. The website stores the pipelines which take input and produces output stored in artifacts. These are the differences between the notebook(.ipynb) pipeline files and the ClearML pipeline files(.py):
+* The ClearML Data Collection Pipeline works the same way, running the entire ETL pipeline in a single step (I could not split the ETL pipeline into 3 steps (Extract, Transform, Load) since my list of links gets bigger while looping through it(Since I also goes through some links inside of the websites crawled). Breaking it into steps would require more HTTP requests which would greatly slow down the pipeline).
+* The Feature Pipeline breaks down the notebook's loop (from the ingestion pipeline) into 5 stages: retrieve documents, clean documents, chunk documents, embed chunks, and store embeddings.
+* The Inference Pipeline simply puts each step in the notebook's version into a function. These functions are query expansion, self-querying, filtered vector search, collecting results, reranking, building prompt, and obtaining answer.
+
+# Tools
+The tools folder contains code for viewing/deleting what has been stored in MongoDB and Qdrant
+
+# shared.py
+shared.py is in both the project folder and project/Tools folder. It contains functions for setting up the connections with either the docker containers or web services. If you are running into errors connecting to any of the services, consider editing this file or double checking the .env file. Note the ClearML folder hardcodes all functions since it had trouble importing code.

project/Tools/QdrantTools.ipynb

@@ -0,0 +1,192 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "id=0 payload={'link': 'https://www.ros.org/', 'type': 'Document', 'chunk': 0, 'text': 'ROS: Home Why ROS? Getting Started Community Ecosystem ROS - Robot Operating System The Robot Operating System (ROS) is a set of software libraries and tools that help you build robot applications. From drivers to state-of-the-art algorithms, and with powerful developer tools, ROS has what you need for your next robotics project. And it\\'s all open source. What is ROS? ROS Videos \" Install Jazzy Jalisco Jazzy Jalisco is our latest ROS 2 LTS release targeted at the Ubuntu 24.04 (Noble) and'} vector=None shard_key=None order_value=None\n",
+      "id=1 payload={'link': 'https://www.ros.org/', 'type': 'Document', 'chunk': 1, 'text': '24.04 (Noble) and Windows 10, though other systems are supported to varying degrees. Learn More Humble Hawksbill ROS 2 Humble Hawksbill is a slighly older LTS release of ROS 2 targeted at Ubuntu 22.04 (Jammy) and Windows 10. Other systems are supported including tier 3 support for 20.04 for those transitioning from ROS 1. Learn More Support There are several mechanisms in place to support the ROS community, each with its own purpose. Documentation Documentation and tutorials for ROS 2 Stack'} vector=None shard_key=None order_value=None\n",
+      "id=2 payload={'link': 'https://www.ros.org/', 'type': 'Document', 'chunk': 2, 'text': 'for ROS 2 Stack Exchange Ask questions. Get answers. Forums Hear the latest discussions ROS 1 Wiki Legacy documentation and tutorials for ROS 1 Documentation Documentation and tutorials for ROS 2 Robotics Stack Exchange Ask questions.Get answers.All ROS versions Forums Hear the latest discussions ROS 1 Wiki Legacy documentation and tutorials for ROS 1 Recent Updates and Highlights ROSCon 2024 Videos are Now Available See the ROSCon 2024 website for details 11/18/2024 - Katherine Scott The'} vector=None shard_key=None order_value=None\n",
+      "id=3 payload={'link': 'https://www.ros.org/', 'type': 'Document', 'chunk': 3, 'text': 'Katherine Scott The videos from ROSCon 2024 in Odense are now available on the ROSCon Website (see the program), this Vimeo showcase, and in the ROS documentation. The ROSCon website also includes the slides from all the talks at ROSCon. I have also included a list of all the videos below. I want to thank AMD for being our 2024 ROSCon video sponsor, their generous support makes the ROSCon live stream and videos possible. READ MORE Recent ROS Discourse Posts ROS News of the Week 11/22/2024 - ROS'} vector=None shard_key=None order_value=None\n",
+      "id=4 payload={'link': 'https://www.ros.org/', 'type': 'Document', 'chunk': 4, 'text': '11/22/2024 - ROS Discourse Gazebo Classic and Citadel End of Life 12/2/2024 - ROS Discourse ROS 2 driver for Mitsubishi Melfa RV-FR 10/24/2024 ROS Discourse Home Why ROS? Getting Started Community Ecosystem Q&A Forum Packages Wiki Documentation media Q&A Forum Packages ROSCon Wiki documentation discord Brought to you by Open Robotics | licensed under Creative Commons Attributions 3.0 | 2021 Open Robotics'} vector=None shard_key=None order_value=None\n",
+      "Number of document chunks:  5\n",
+      "\n",
+      "Sample document chunk(metadata not the vector): \n",
+      "id=0 payload={'link': 'https://www.ros.org/', 'type': 'Document', 'chunk': 0, 'text': 'ROS: Home Why ROS? Getting Started Community Ecosystem ROS - Robot Operating System The Robot Operating System (ROS) is a set of software libraries and tools that help you build robot applications. From drivers to state-of-the-art algorithms, and with powerful developer tools, ROS has what you need for your next robotics project. And it\\'s all open source. What is ROS? ROS Videos \" Install Jazzy Jalisco Jazzy Jalisco is our latest ROS 2 LTS release targeted at the Ubuntu 24.04 (Noble) and'} vector=None shard_key=None order_value=None \n",
+      "\n",
+      "id=0 payload={'link': 'https://github.com/ros2/ros2/tree/rolling/README.md', 'type': 'Github', 'chunk': 0, 'text': \"#About TheRobotOperatingSystem(ROS)isasetofsoftwarelibrariesandtoolsthathelpyoubuildrobotapplications. Fromdriverstostate-of-the-artalgorithms,andwithpowerfuldevelopertools,ROShaswhatyouneedforyournextroboticsproject. Andit'sallopensource. Fullprojectdetailson[ROS.org](https://ros.org/) #GettingStarted LookingtogetstartedwithROS? Our[installationguideishere](https://www.ros.org/blog/getting-started/).\"} vector=None shard_key=None order_value=None\n",
+      "id=1 payload={'link': 'https://github.com/ros2/ros2/tree/rolling/README.md', 'type': 'Github', 'chunk': 1, 'text': \"Onceyou'veinstalledROSstartbylearningsome[basicconcepts](https://docs.ros.org/en/rolling/Concepts/Basic.html)andtakealookatour[beginnertutorials](https://docs.ros.org/en/rolling/Tutorials/Beginner-CLI-Tools.html). #JointheROSCommunity ##CommunityResources *[ROSDiscussionForum](https://discourse.ros.org/) *[ROSDiscordServer](https://discord.com/servers/open-robotics-1077825543698927656) *[RoboticsStackExchange](https://robotics.stackexchange.com/)(preferredROSsupportforum).\"} vector=None shard_key=None order_value=None\n",
+      "id=2 payload={'link': 'https://github.com/ros2/ros2/tree/rolling/README.md', 'type': 'Github', 'chunk': 2, 'text': '*[OfficialROSVideos](https://vimeo.com/osrfoundation) *[ROSCon](https://roscon.ros.org),ouryearlydeveloperconference. *CiteROS2inacademicworkusing[DOI:10.1126/scirobotics.abm6074](https://www.science.org/doi/10.1126/scirobotics.abm6074) ##DeveloperResources *[ROS2Documentation](https://docs.ros.org/) *[ROSPackageAPIreference](https://docs.ros.org/en/rolling/p/) *[ROSPackageIndex](https://index.ros.org/) *[ROSonDockerHub](https://hub.docker.com/_/ros/)'} vector=None shard_key=None order_value=None\n",
+      "id=3 payload={'link': 'https://github.com/ros2/ros2/tree/rolling/README.md', 'type': 'Github', 'chunk': 3, 'text': '*[ROSResourceStatusPage](https://status.openrobotics.org/) *[REP-2000](https://ros.org/reps/rep-2000.html):ROS2ReleasesandTargetPlatforms ##ProjectResources *[PurchaseROSSwag](https://spring.ros.org/) *[InformationabouttheROSTrademark](https://www.ros.org/blog/media/) *OnSocialMedia *[OpenRoboticsonLinkedIn](https://www.linkedin.com/company/open-source-robotics-foundation) *[OpenRoboticsonTwitter](https://twitter.com/OpenRoboticsOrg) *[ROS.orgonTwitter](https://twitter.com/ROSOrg)'} vector=None shard_key=None order_value=None\n",
+      "id=4 payload={'link': 'https://github.com/ros2/ros2/tree/rolling/README.md', 'type': 'Github', 'chunk': 4, 'text': 'ROSismadepossiblethroughthegeneroussupportofopensourcecontributorsandthenon-profit[OpenSourceRoboticsFoundation(OSRF)](https://www.openrobotics.org/). TaxdeductibledonationstotheOSRFcanbe[madehere.](https://donorbox.org/support-open-robotics?utm_medium=qrcode&utm_source=qrcode)'} vector=None shard_key=None order_value=None\n",
+      "id=5 payload={'link': 'https://github.com/ros2/ros2/tree/rolling/.gitignore', 'type': 'Github', 'chunk': 0, 'text': '#Ignoredefaultnamesforcolconcreatedfolders build install log #Ignoreeverythinginsrcexcepta.gitkeepfile src/* !src/.gitkeep'} vector=None shard_key=None order_value=None\n",
+      "id=6 payload={'link': 'https://github.com/ros2/ros2/tree/rolling/CODEOWNERS', 'type': 'Github', 'chunk': 0, 'text': '#Thisfilewasgeneratedbyhttps://github.com/audrow/update-ros2-repos *@clalancette@codebot'} vector=None shard_key=None order_value=None\n",
+      "id=7 payload={'link': 'https://github.com/ros2/ros2/tree/rolling/ros2.repos', 'type': 'Github', 'chunk': 0, 'text': 'repositories: ament/ament_cmake: type:git url:https://github.com/ament/ament_cmake.git version:rolling ament/ament_index: type:git url:https://github.com/ament/ament_index.git version:rolling ament/ament_lint: type:git url:https://github.com/ament/ament_lint.git version:rolling ament/ament_package: type:git url:https://github.com/ament/ament_package.git version:rolling ament/google_benchmark_vendor: type:git url:https://github.com/ament/google_benchmark_vendor.git version:rolling'} vector=None shard_key=None order_value=None\n",
+      "id=8 payload={'link': 'https://github.com/ros2/ros2/tree/rolling/ros2.repos', 'type': 'Github', 'chunk': 1, 'text': 'version:rolling ament/googletest: type:git url:https://github.com/ament/googletest.git version:rolling ament/uncrustify_vendor: type:git url:https://github.com/ament/uncrustify_vendor.git version:rolling eProsima/Fast-CDR: type:git url:https://github.com/eProsima/Fast-CDR.git version:2.2.x eProsima/Fast-DDS: type:git url:https://github.com/eProsima/Fast-DDS.git version:2.14.x eProsima/foonathan_memory_vendor: type:git url:https://github.com/eProsima/foonathan_memory_vendor.git version:master'} vector=None shard_key=None order_value=None\n",
+      "id=9 payload={'link': 'https://github.com/ros2/ros2/tree/rolling/ros2.repos', 'type': 'Github', 'chunk': 2, 'text': 'version:master eclipse-cyclonedds/cyclonedds: type:git url:https://github.com/eclipse-cyclonedds/cyclonedds.git version:releases/0.10.x eclipse-iceoryx/iceoryx: type:git url:https://github.com/eclipse-iceoryx/iceoryx.git version:release_2.0 gazebo-release/gz_cmake_vendor: type:git url:https://github.com/gazebo-release/gz_cmake_vendor.git version:rolling gazebo-release/gz_math_vendor: type:git url:https://github.com/gazebo-release/gz_math_vendor.git version:rolling'} vector=None shard_key=None order_value=None\n",
+      "\n",
+      "Number of Github chunks:  10\n",
+      "\n",
+      "Sample Github chunk(metadata not the vector): \n",
+      "id=0 payload={'link': 'https://github.com/ros2/ros2/tree/rolling/README.md', 'type': 'Github', 'chunk': 0, 'text': \"#About TheRobotOperatingSystem(ROS)isasetofsoftwarelibrariesandtoolsthathelpyoubuildrobotapplications. Fromdriverstostate-of-the-artalgorithms,andwithpowerfuldevelopertools,ROShaswhatyouneedforyournextroboticsproject. Andit'sallopensource. Fullprojectdetailson[ROS.org](https://ros.org/) #GettingStarted LookingtogetstartedwithROS? Our[installationguideishere](https://www.ros.org/blog/getting-started/).\"} vector=None shard_key=None order_value=None \n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/workspaces/RAG_LLM/project/Tools/shared.py:57: LangChainDeprecationWarning: The class `OllamaEmbeddings` was deprecated in LangChain 0.3.1 and will be removed in 1.0.0. An updated version of the class exists in the :class:`~langchain-ollama package and should be used instead. To use it run `pip install -U :class:`~langchain-ollama` and import as `from :class:`~langchain_ollama import OllamaEmbeddings``.\n",
+      "  return OllamaEmbeddings(model=MODEL, base_url=\"http://host.docker.internal:11434\")\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "Sample search result(n=2): \n",
+      "id=4 version=4 score=0.38799083 payload={'link': 'https://www.ros.org/', 'type': 'Document', 'chunk': 4, 'text': '11/22/2024 - ROS Discourse Gazebo Classic and Citadel End of Life 12/2/2024 - ROS Discourse ROS 2 driver for Mitsubishi Melfa RV-FR 10/24/2024 ROS Discourse Home Why ROS? Getting Started Community Ecosystem Q&A Forum Packages Wiki Documentation media Q&A Forum Packages ROSCon Wiki documentation discord Brought to you by Open Robotics | licensed under Creative Commons Attributions 3.0 | 2021 Open Robotics'} vector=None shard_key=None order_value=None\n",
+      "id=2 version=2 score=0.35047314 payload={'link': 'https://www.ros.org/', 'type': 'Document', 'chunk': 2, 'text': 'for ROS 2 Stack Exchange Ask questions. Get answers. Forums Hear the latest discussions ROS 1 Wiki Legacy documentation and tutorials for ROS 1 Documentation Documentation and tutorials for ROS 2 Robotics Stack Exchange Ask questions.Get answers.All ROS versions Forums Hear the latest discussions ROS 1 Wiki Legacy documentation and tutorials for ROS 1 Recent Updates and Highlights ROSCon 2024 Videos are Now Available See the ROSCon 2024 website for details 11/18/2024 - Katherine Scott The'} vector=None shard_key=None order_value=None\n"
+     ]
+    }
+   ],
+   "source": [
+    "from shared import getQdrantClient, getEmbeddingsModel\n",
+    "qClient = getQdrantClient()\n",
+    "\n",
+    "# Show everything in the Document collection\n",
+    "numDocumentChunks = 0\n",
+    "# Note with_vectors defaults to false, so the vectors are not returned\n",
+    "chunks = qClient.scroll(collection_name='Document')\n",
+    "#print(chunks)\n",
+    "for chunk in chunks[0]:\n",
+    "    # Only display chunks if vector database is small\n",
+    "    print(chunk)\n",
+    "    if numDocumentChunks == 0:\n",
+    "        sampleDocumentChunk = chunk\n",
+    "    numDocumentChunks += 1\n",
+    "print(\"Number of document chunks: \", numDocumentChunks)\n",
+    "if numDocumentChunks > 0:\n",
+    "    print(\"\\nSample document chunk(metadata not the vector): \")\n",
+    "    print(sampleDocumentChunk, '\\n')\n",
+    "\n",
+    "# Show everything in the Github collection\n",
+    "numGithubChunks = 0\n",
+    "chunks = qClient.scroll(collection_name='Github')\n",
+    "#print(chunks)\n",
+    "for chunk in chunks[0]:\n",
+    "    # Only display chunks if vector database is small\n",
+    "    print(chunk)\n",
+    "    if numGithubChunks == 0:\n",
+    "        sampleGithubChunk = chunk\n",
+    "    numGithubChunks += 1\n",
+    "print(\"\\nNumber of Github chunks: \", numGithubChunks)\n",
+    "if numGithubChunks > 0:\n",
+    "    print(\"\\nSample Github chunk(metadata not the vector): \")\n",
+    "    print(sampleGithubChunk, '\\n')\n",
+    "\n",
+    "# Show a sample search\n",
+    "embeddingsModel = getEmbeddingsModel()\n",
+    "results = qClient.search(\n",
+    "    collection_name=\"Document\",\n",
+    "    query_vector = embeddingsModel.embed_query(\"What operating system is ROS made for?\"),\n",
+    "    limit=2\n",
+    ")\n",
+    "print(\"\\nSample search result(n=2): \")\n",
+    "for result in results:\n",
+    "    print(result)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Cosine Similarity for related sentences: 0.7035977848391597\n",
+      "Cosine Similarity for unrelated sentences: 0.3566534327076298\n"
+     ]
+    }
+   ],
+   "source": [
+    "import numpy as np\n",
+    "# How cosine distance works\n",
+    "\n",
+    "embedding1 = embeddingsModel.embed_query(\"What is the weather like?\")\n",
+    "embedding2 = embeddingsModel.embed_query(\"It is raining today.\")\n",
+    "embedding3 = embeddingsModel.embed_query(\"ROS is an open source platform\")\n",
+    "def cosine_similarity(vec1, vec2):\n",
+    "    dot_product = np.dot(vec1, vec2)\n",
+    "    norm_vec1 = np.linalg.norm(vec1)\n",
+    "    norm_vec2 = np.linalg.norm(vec2)\n",
+    "    return dot_product / (norm_vec1 * norm_vec2)\n",
+    "similarity1 = cosine_similarity(embedding1, embedding2)\n",
+    "similarity2 = cosine_similarity(embedding1, embedding3)\n",
+    "print(\"Cosine Similarity for related sentences:\", similarity1)\n",
+    "print(\"Cosine Similarity for unrelated sentences:\", similarity2)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 20,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from qdrant_client.http.models import Distance, VectorParams\n",
+    "# Delete all collections and vectors inside them\n",
+    "qClient.delete_collection(collection_name = \"Document\")\n",
+    "qClient.delete_collection(collection_name = \"Github\")\n",
+    "# Recreate the empty collections\n",
+    "qClient.create_collection(\n",
+    "    collection_name = \"Document\",\n",
+    "    vectors_config=VectorParams(size=3072, distance=Distance.COSINE)\n",
+    ")\n",
+    "qClient.create_collection(\n",
+    "    collection_name = \"Github\",\n",
+    "    vectors_config=VectorParams(size=3072, distance=Distance.COSINE)\n",
+    ")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

project/Tools/mongoTools.ipynb

@@ -0,0 +1,99 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Number of regular documents:  1\n",
+      "Number of github documents:  5\n",
+      "Links crawled:  ['https://www.ros.org/', 'https://github.com/ros2/ros2/tree/rolling/README.md', 'https://github.com/ros2/ros2/tree/rolling/.gitignore', 'https://github.com/ros2/ros2/tree/rolling/CODEOWNERS', 'https://github.com/ros2/ros2/tree/rolling/ros2.repos', 'https://github.com/ros2/ros2/tree/rolling/src/.gitkeep']\n",
+      "Sample regular document:  {'_id': ObjectId('675531b926a728d5b045a2e5'), 'link': 'https://www.ros.org/', 'type': 'Document', 'content': ' ROS: Home Why ROS? Getting Started Community Ecosystem ROS - Robot Operating System The Robot Operating System (ROS) is a set of software libraries and tools that help you build robot applications. From drivers to state-of-the-art algorithms, and with powerful developer tools, ROS has what you need for your next robotics project. And it\\'s all open source. What is ROS? ROS Videos \" Install Jazzy Jalisco Jazzy Jalisco is our latest ROS 2 LTS release targeted at the Ubuntu 24.04 (Noble) and Windows 10, though other systems are supported to varying degrees. Learn More Humble Hawksbill ROS 2 Humble Hawksbill is a slighly older LTS release of ROS 2 targeted at Ubuntu 22.04 (Jammy) and Windows 10. Other systems are supported including tier 3 support for 20.04 for those transitioning from ROS 1. Learn More Support There are several mechanisms in place to support the ROS community, each with its own purpose. Documentation Documentation and tutorials for ROS 2 Stack Exchange Ask questions. Get answers. Forums Hear the latest discussions ROS 1 Wiki Legacy documentation and tutorials for ROS 1 Documentation Documentation and tutorials for ROS 2 Robotics Stack Exchange Ask questions.Get answers.All ROS versions Forums Hear the latest discussions ROS 1 Wiki Legacy documentation and tutorials for ROS 1 Recent Updates and Highlights ROSCon 2024 Videos are Now Available See the ROSCon 2024 website for details 11/18/2024 - Katherine Scott The videos from ROSCon 2024 in Odense are now available on the ROSCon Website (see the program), this Vimeo showcase, and in the ROS documentation. The ROSCon website also includes the slides from all the talks at ROSCon. I have also included a list of all the videos below. I want to thank AMD for being our 2024 ROSCon video sponsor, their generous support makes the ROSCon live stream and videos possible. READ MORE Recent ROS Discourse Posts ROS News of the Week 11/22/2024 - ROS Discourse Gazebo Classic and Citadel End of Life 12/2/2024 - ROS Discourse ROS 2 driver for Mitsubishi Melfa RV-FR 10/24/2024 ROS Discourse Home Why ROS? Getting Started Community Ecosystem Q&A Forum Packages Wiki Documentation media Q&A Forum Packages ROSCon Wiki documentation discord Brought to you by Open Robotics | licensed under Creative Commons Attributions 3.0 | ©2021 Open Robotics '}\n",
+      "Sample github document {'_id': ObjectId('675531b926a728d5b045a2e6'), 'link': 'https://github.com/ros2/ros2/tree/rolling/README.md', 'type': 'Github', 'content': \"#About TheRobotOperatingSystem(ROS)isasetofsoftwarelibrariesandtoolsthathelpyoubuildrobotapplications. Fromdriverstostate-of-the-artalgorithms,andwithpowerfuldevelopertools,ROShaswhatyouneedforyournextroboticsproject. Andit'sallopensource. Fullprojectdetailson[ROS.org](https://ros.org/) #GettingStarted LookingtogetstartedwithROS? Our[installationguideishere](https://www.ros.org/blog/getting-started/). Onceyou'veinstalledROSstartbylearningsome[basicconcepts](https://docs.ros.org/en/rolling/Concepts/Basic.html)andtakealookatour[beginnertutorials](https://docs.ros.org/en/rolling/Tutorials/Beginner-CLI-Tools.html). #JointheROSCommunity ##CommunityResources *[ROSDiscussionForum](https://discourse.ros.org/) *[ROSDiscordServer](https://discord.com/servers/open-robotics-1077825543698927656) *[RoboticsStackExchange](https://robotics.stackexchange.com/)(preferredROSsupportforum). *[OfficialROSVideos](https://vimeo.com/osrfoundation) *[ROSCon](https://roscon.ros.org),ouryearlydeveloperconference. *CiteROS2inacademicworkusing[DOI:10.1126/scirobotics.abm6074](https://www.science.org/doi/10.1126/scirobotics.abm6074) ##DeveloperResources *[ROS2Documentation](https://docs.ros.org/) *[ROSPackageAPIreference](https://docs.ros.org/en/rolling/p/) *[ROSPackageIndex](https://index.ros.org/) *[ROSonDockerHub](https://hub.docker.com/_/ros/) *[ROSResourceStatusPage](https://status.openrobotics.org/) *[REP-2000](https://ros.org/reps/rep-2000.html):ROS2ReleasesandTargetPlatforms ##ProjectResources *[PurchaseROSSwag](https://spring.ros.org/) *[InformationabouttheROSTrademark](https://www.ros.org/blog/media/) *OnSocialMedia *[OpenRoboticsonLinkedIn](https://www.linkedin.com/company/open-source-robotics-foundation) *[OpenRoboticsonTwitter](https://twitter.com/OpenRoboticsOrg) *[ROS.orgonTwitter](https://twitter.com/ROSOrg) ROSismadepossiblethroughthegeneroussupportofopensourcecontributorsandthenon-profit[OpenSourceRoboticsFoundation(OSRF)](https://www.openrobotics.org/). TaxdeductibledonationstotheOSRFcanbe[madehere.](https://donorbox.org/support-open-robotics?utm_medium=qrcode&utm_source=qrcode) \"}\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Shows the state of the mongo database\n",
+    "from shared import getMongoClient\n",
+    "mongoHost = getMongoClient()\n",
+    "mongoDatabase = mongoHost[\"twin\"]\n",
+    "mongoDocumentCollection = mongoDatabase[\"Document\"]\n",
+    "mongoGithubCollection = mongoDatabase[\"Github\"]\n",
+    "documents = mongoDocumentCollection.find()\n",
+    "codes = mongoGithubCollection.find()\n",
+    "numDocuments = 0\n",
+    "numCodes = 0\n",
+    "links = []\n",
+    "for document in documents:\n",
+    "    links.append(document[\"link\"])\n",
+    "    if numDocuments == 0:\n",
+    "        sampleDocument = document\n",
+    "    numDocuments += 1\n",
+    "for code in codes:\n",
+    "    links.append(code[\"link\"])\n",
+    "    if numCodes == 0:\n",
+    "        sampleCode = code\n",
+    "    numCodes += 1\n",
+    "print(\"Number of regular documents: \", numDocuments)\n",
+    "print(\"Number of github documents: \", numCodes)\n",
+    "print(\"Links crawled: \", links)\n",
+    "if (numDocuments > 0):\n",
+    "    print(\"Sample regular document: \", sampleDocument)\n",
+    "else:\n",
+    "    print(\"No documents\")\n",
+    "if (numCodes > 0):\n",
+    "    print(\"Sample github document\", sampleCode)\n",
+    "else:\n",
+    "    print(\"No github documents\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "DeleteResult({'n': 18, 'electionId': ObjectId('7fffffff000000000000016a'), 'opTime': {'ts': Timestamp(1733585625, 33), 't': 362}, 'ok': 1.0, '$clusterTime': {'clusterTime': Timestamp(1733585625, 33), 'signature': {'hash': b'\\xd0\\xa2\\xaf\\x1c?p\\xc5\\xd7\\x9a\\x1e\\x1f\\x15\\x1ews\\xdc\\xab)\\xf2B', 'keyId': 7395232362797203469}}, 'operationTime': Timestamp(1733585625, 33)}, acknowledged=True)"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Delete all files\n",
+    "mongoDocumentCollection.delete_many({})\n",
+    "mongoGithubCollection.delete_many({})"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

project/Tools/shared.py

@@ -0,0 +1,92 @@
+# This file contains shared functions used by multiple files
+import os
+import sys
+
+import pymongo
+from dotenv import load_dotenv
+from langchain_community.embeddings import OllamaEmbeddings
+from langchain_openai.chat_models import ChatOpenAI
+from qdrant_client import QdrantClient
+from langchain_community.llms import Ollama
+
+
+# Unused since usage limit reached since years ago...rip
+def getOpenAiModel():
+    MODEL = "gpt-3.5-turbo"
+    try:
+        load_dotenv(override=True)
+    except Exception:
+        load_dotenv(sys.path[1] + "/.env", override=True)
+    OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
+    return ChatOpenAI(openai_api_key=OPENAI_API_KEY, model=MODEL)
+
+
+# Create a mongoDB connection
+def getMongoClient():
+    try:
+        load_dotenv(override=True)
+    except Exception:
+        load_dotenv(sys.path[1] + "/.env", override=True)
+    DATABASE_HOST = os.getenv("DATABASE_HOST")
+    return pymongo.MongoClient(DATABASE_HOST)
+
+
+# Create a qdrant connection
+def getQdrantClient():
+    try:
+        load_dotenv(override=True)
+    except Exception:
+        load_dotenv(sys.path[1] + "/.env", override=True)
+    USE_QDRANT_CLOUD = os.getenv("USE_QDRANT_CLOUD")
+    QDRANT_CLOUD_URL = os.getenv("QDRANT_CLOUD_URL")
+    QDRANT_APIKEY = os.getenv("QDRANT_APIKEY")
+    if USE_QDRANT_CLOUD:
+        return QdrantClient(url=QDRANT_CLOUD_URL, api_key=QDRANT_APIKEY)
+    else:
+        return QdrantClient(url=QDRANT_CLOUD_URL)
+
+
+# Setup the text embedder
+def getEmbeddingsModel(MODEL="llama3.2"):
+    try:
+        load_dotenv(override=True)
+    except Exception:
+        load_dotenv(sys.path[1] + "/.env", override=True)
+    USE_DOCKER = os.getenv("USE_DOCKER")
+    if USE_DOCKER == "True":
+        return OllamaEmbeddings(model=MODEL, base_url="http://host.docker.internal:11434")
+    else:
+        return OllamaEmbeddings(model=MODEL)
+
+
+# Setup the model
+def getModel(MODEL="llama3.2"):
+    try:
+        load_dotenv(override=True)
+    except Exception:
+        load_dotenv(sys.path[1] + "/.env", override=True)
+    USE_DOCKER = os.getenv("USE_DOCKER")
+    if USE_DOCKER == "True":
+        return Ollama(model=MODEL, base_url="http://host.docker.internal:11434")
+    else:
+        return Ollama(model=MODEL)
+
+
+# Setup clearML
+def setupClearML():
+    try:
+        load_dotenv(override=True)
+    except Exception:
+        load_dotenv(sys.path[1] + "/.env", override=True)
+    CLEARML_WEB_HOST = os.getenv("CLEARML_WEB_HOST")
+    CLEARML_API_HOST = os.getenv("CLEARML_API_HOST")
+    CLEARML_FILES_HOST = os.getenv("CLEARML_FILES_HOST")
+    CLEARML_API_ACCESS_KEY = os.getenv("CLEARML_API_ACCESS_KEY")
+    CLEARML_API_SECRET_KEY = os.getenv("CLEARML_API_SECRETKEY")
+    return (
+        CLEARML_WEB_HOST,
+        CLEARML_API_HOST,
+        CLEARML_FILES_HOST,
+        CLEARML_API_ACCESS_KEY,
+        CLEARML_API_SECRET_KEY,
+    )

project/TrainingPipeline.ipynb

@@ -0,0 +1,34 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Fine-tuning not required"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

project/app.py

@@ -0,0 +1,126 @@
+# Make sure you have run "ollama serve"
+# This is the same code as ClearML
+import os
+import sys
+from operator import itemgetter
+
+import gradio as gr
+from dotenv import load_dotenv
+from langchain.prompts import PromptTemplate
+from langchain_community.embeddings import OllamaEmbeddings
+from langchain_community.llms import Ollama
+from qdrant_client import QdrantClient
+from shared import getModel, getEmbeddingsModel, getQdrantClient
+
+def answer(samplePrompt, useSample, Query):
+    if useSample:
+        query = samplePrompt
+    else:
+        query = Query
+    # Create a qdrant connection
+    qClient = getQdrantClient()
+
+    # Setup the text embedder
+    embeddingsModel = getEmbeddingsModel()
+
+    # Setup the model
+    model = getModel()
+
+    # Retrieval Pipeline
+    # Retrieve the chunks with the most similar embeddings from Qdrant
+    def retriever(text, collection):
+        results = qClient.search(
+            collection_name=collection,
+            query_vector = embeddingsModel.embed_query(text),
+            limit=10
+        )
+        return results
+
+    # Query expansion(I only generate one additional prompt for simplicity)
+    template = """
+    Rewrite the prompt. The new prompt must offer a different perspective.
+    Do not change the meaning. Output only the rewritten prompt with no introduction.
+        Prompt: {prompt}
+    """
+    prompt = PromptTemplate.from_template(template)
+    chain = {"prompt": itemgetter("prompt")} | prompt | model
+    queryExpansion = chain.invoke({"prompt": query})
+
+    # Self-querying(The metadata I will be generating determines whether to look through the Qdrant collection containing github code)
+    template = """
+    You are an AI assistant. You must determine if the prompt requires code as the answer.
+    Output a 1 if it is or a 0 if it is not and nothing else.
+        Prompt: {prompt}
+    """
+    prompt = PromptTemplate.from_template(template)
+    chain = {"prompt": itemgetter("prompt")} | prompt | model
+    codingQuestion = chain.invoke({"prompt": query})
+
+    # Filtered vector search for each of the N queries after expansion
+    relatedCollection = 'Document'
+    if (codingQuestion == '1'):
+        relatedCollection = 'Github'
+    results1 = retriever(query, relatedCollection)
+    results2 = retriever(queryExpansion, relatedCollection)
+
+    # Collecting results
+    results = results1+results2
+
+    # Reranking(Instead of using a CrossEncoder, I will manually compare embeddings)
+    ids = [result.id for result in results]
+    scores = [result.score for result in results]
+    topIds = []
+    topIndexes = []
+    for x in range(3):
+        maxScore = 0
+        maxIndex = 0
+        for i in range(len(ids)):
+            if ids[i] not in topIds and scores[i] > maxScore:
+                maxScore = scores[i]
+                maxIndex = i
+        topIds.append(ids[maxIndex])
+        topIndexes.append(maxIndex)
+    texts = [result.payload['text'] for result in results]
+    links = [result.payload['link'] for result in results]
+    topTexts = ''
+    for index in topIndexes:
+        print("Top texts: ", texts[index])
+        print("Link: ", links[index])
+        topTexts += texts[index]
+
+    # Building prompt
+    if(codingQuestion == '1'):
+        template = """
+        Write code for the following question given the related coding document below.
+
+        Document: {document}
+        Question: {question}
+        """
+        prompt = PromptTemplate.from_template(template)
+    else:
+        template = """
+        Answer the question based on the document below. If you can't answer the question, reply "I don't know"
+
+        Document: {document}
+        Question: {question}
+        """
+        prompt = PromptTemplate.from_template(template)
+
+    # Obtaining answer
+    chain = {"document": itemgetter("document"), "question": itemgetter("question")} | prompt | model
+    return chain.invoke({"document": topTexts, "question": query})
+
+
+demo = gr.Interface(
+    fn=answer,
+    inputs=[
+        gr.Dropdown(
+            ["What is ROS?", "Write me code to move a robot"], label="Sample Prompt"
+        ),
+        "checkbox",
+        "text",
+    ],
+    outputs=["text"],
+)
+
+demo.launch(share=False)

project/docker-compose.yml

@@ -0,0 +1,72 @@
+services:
+  ollama:
+    container_name: ollama
+    image: ollama/ollama:latest
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              capabilities: ["gpu"]
+              count: all
+    ports:
+      - 11434:11434
+    expose:
+      - 11434
+    volumes:
+      - ollama:/root/.ollama
+  gradio:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    ports:
+      - 7860:7860
+    expose:
+      - 7860
+    networks:
+      - net
+    environment:
+      - GRADIO_SERVER_NAME=0.0.0.0
+      - GRADIO_SERVER_PORT=7860
+  mongo:
+    image: mongo:latest
+    ports:
+      - 27017:27017
+    expose:
+      - 27017
+    volumes:
+      - mongo-data:/data/db
+  qdrant:
+    image: qdrant/qdrant:latest
+    restart: always
+    ports:
+      - 6333:6333
+      - 6334:6334
+    expose:
+      - 6333
+      - 6334
+      - 6335
+    configs:
+      - source: qdrant_config
+        target: /qdrant/config/production.yaml
+    volumes:
+      - qdrant-data:/qdrant/storage
+
+
+
+
+configs:
+  qdrant_config:
+    content: |
+      log_level: INFO
+
+volumes:
+  mongo-data:
+    driver: local
+  qdrant-data:
+    driver: local
+  ollama:
+    driver: local
+
+networks:
+  net:

project/shared.py

@@ -0,0 +1,92 @@
+# This file contains shared functions used by multiple files
+import os
+import sys
+
+import pymongo
+from dotenv import load_dotenv
+from langchain_community.embeddings import OllamaEmbeddings
+from langchain_openai.chat_models import ChatOpenAI
+from qdrant_client import QdrantClient
+from langchain_community.llms import Ollama
+
+
+# Unused since usage limit reached since years ago...rip
+def getOpenAiModel():
+    MODEL = "gpt-3.5-turbo"
+    try:
+        load_dotenv(override=True)
+    except Exception:
+        load_dotenv(sys.path[1] + "/.env", override=True)
+    OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
+    return ChatOpenAI(openai_api_key=OPENAI_API_KEY, model=MODEL)
+
+
+# Create a mongoDB connection
+def getMongoClient():
+    try:
+        load_dotenv(override=True)
+    except Exception:
+        load_dotenv(sys.path[1] + "/.env", override=True)
+    DATABASE_HOST = os.getenv("DATABASE_HOST")
+    return pymongo.MongoClient(DATABASE_HOST)
+
+
+# Create a qdrant connection
+def getQdrantClient():
+    try:
+        load_dotenv(override=True)
+    except Exception:
+        load_dotenv(sys.path[1] + "/.env", override=True)
+    USE_QDRANT_CLOUD = os.getenv("USE_QDRANT_CLOUD")
+    QDRANT_CLOUD_URL = os.getenv("QDRANT_CLOUD_URL")
+    QDRANT_APIKEY = os.getenv("QDRANT_APIKEY")
+    if USE_QDRANT_CLOUD=="True":
+        return QdrantClient(url=QDRANT_CLOUD_URL, api_key=QDRANT_APIKEY)
+    else:
+        return QdrantClient(url=QDRANT_CLOUD_URL)
+
+
+# Setup the text embedder
+def getEmbeddingsModel(MODEL="llama3.2"):
+    try:
+        load_dotenv(override=True)
+    except Exception:
+        load_dotenv(sys.path[1] + "/.env", override=True)
+    USE_DOCKER = os.getenv("USE_DOCKER")
+    if USE_DOCKER == "True":
+        return OllamaEmbeddings(model=MODEL, base_url="http://host.docker.internal:11434")
+    else:
+        return OllamaEmbeddings(model=MODEL)
+
+
+# Setup the model
+def getModel(MODEL="llama3.2"):
+    try:
+        load_dotenv(override=True)
+    except Exception:
+        load_dotenv(sys.path[1] + "/.env", override=True)
+    USE_DOCKER = os.getenv("USE_DOCKER")
+    if USE_DOCKER == "True":
+        return Ollama(model=MODEL, base_url="http://host.docker.internal:11434")
+    else:
+        return Ollama(model=MODEL)
+
+
+# Setup clearML
+def setupClearML():
+    try:
+        load_dotenv(override=True)
+    except Exception:
+        load_dotenv(sys.path[1] + "/.env", override=True)
+    CLEARML_WEB_HOST = os.getenv("CLEARML_WEB_HOST")
+    CLEARML_API_HOST = os.getenv("CLEARML_API_HOST")
+    CLEARML_FILES_HOST = os.getenv("CLEARML_FILES_HOST")
+    CLEARML_API_ACCESS_KEY = os.getenv("CLEARML_API_ACCESS_KEY")
+    CLEARML_API_SECRET_KEY = os.getenv("CLEARML_API_SECRETKEY")
+    return (
+        CLEARML_WEB_HOST,
+        CLEARML_API_HOST,
+        CLEARML_FILES_HOST,
+        CLEARML_API_ACCESS_KEY,
+        CLEARML_API_SECRET_KEY,
+    )