first commit to deploy

.flake8

@@ -0,0 +1,8 @@
+[flake8]
+ignore = E203, W503
+# matches black's default
+max-line-length = 88
+docstring-convention = numpy
+per-file-ignores =
+    __init__.py:F401
+exclude = .git, __pycache__, .venv

.gitignore

@@ -0,0 +1,14 @@
+# poetry
+#   Tt is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility,
+#   and is more commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# Environments
+.env
+.venv
+
+__pycache__/
+
+*.sqlite

.pre-commit-config.yaml

@@ -0,0 +1,53 @@
+default_language_version:
+    python: python3.10
+
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v5.0.0
+  hooks:
+    - id: check-json
+      description: This hook checks json files for parseable syntax.
+    - id: check-yaml
+      description: This hook checks yaml files for parseable syntax.
+    - id: check-toml
+      description: This hook checks toml files for parseable syntax.
+    - id: check-ast
+      description: Simply check whether files parse as valid python.
+    - id: check-merge-conflict
+      description: Check for files that contain merge conflict strings.
+    - id: debug-statements
+      description: Check for debugger imports and py37+ `breakpoint()` calls in python source.
+    - id: end-of-file-fixer
+      description: Ensures that a file is either empty, or ends with one newline.
+    - id: name-tests-test
+      description: This verifies that test files are named correctly - test* format (prefix and not suffix)
+      args: ["--pytest-test-first"]
+    - id: trailing-whitespace
+      args: ["--markdown-linebreak-ext=md"]
+      description: This hook trims trailing whitespace.
+
+- repo: https://github.com/asottile/pyupgrade
+  rev: v3.19.1
+  hooks:
+  - id: pyupgrade
+    args: [--py310-plus]
+
+# Pre-commit is installed by poetry alongside black, isort, flake8
+- repo: local
+  hooks:
+  - id: black
+    name: black
+    entry: poetry run black --config pyproject.toml
+    language: system
+    require_serial: true
+    types: [python]
+  - id: isort
+    name: isort
+    entry: poetry run isort --settings-path pyproject.toml
+    language: system
+    types: [python]
+  - id: flake8
+    name: flake8
+    entry: poetry run flake8 --config .flake8
+    language: system
+    types: [python]

README.md

@@ -10,4 +10,4 @@ pinned: false
 short_description: RAG chatbot trained on my master thesis.
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+This is a private demo using my own OpenAI API key. Please use responsibly.

app.py

@@ -0,0 +1,74 @@
+import gradio as gr
+
+from mythesis_chatbot.rag_setup import (
+    SupportedRags,
+    automerging_retrieval_setup,
+    basic_rag_setup,
+    sentence_window_retrieval_setup,
+)
+
+input_file = "./data/Master_Thesis.pdf"
+save_dir = "./data/indices/"
+
+automerging_engine = automerging_retrieval_setup(
+    input_file=input_file,
+    save_dir=save_dir,
+    llm_openai_model="gpt-4o-mini",
+    embed_model="BAAI/bge-small-en-v1.5",
+    chunk_sizes=[2048, 512, 128],
+    similarity_top_k=6,
+    rerank_model="cross-encoder/ms-marco-MiniLM-L-2-v2",
+    rerank_top_n=2,
+)
+
+sentence_window_engine = sentence_window_retrieval_setup(
+    input_file=input_file,
+    save_dir=save_dir,
+    llm_openai_model="gpt-4o-mini",
+    embed_model="BAAI/bge-small-en-v1.5",
+    sentence_window_size=3,
+    similarity_top_k=6,
+    rerank_model="cross-encoder/ms-marco-MiniLM-L-2-v2",
+    rerank_top_n=2,
+)
+
+basic_engine = basic_rag_setup(
+    input_file=input_file,
+    save_dir=save_dir,
+    llm_openai_model="gpt-4o-mini",
+    embed_model="BAAI/bge-small-en-v1.5",
+    similarity_top_k=6,
+    rerank_model="cross-encoder/ms-marco-MiniLM-L-2-v2",
+    rerank_top_n=2,
+)
+
+
+def chat_bot(query: str, rag_mode: SupportedRags) -> str:
+    if rag_mode == "basic":
+        return basic_engine.query(query).response
+    if rag_mode == "auto-merging retrieval":
+        return automerging_engine.query(query).response
+    if rag_mode == "sentence window retrieval":
+        return sentence_window_engine.query(query).response
+
+
+default_message = (
+    "Ask a about a topic that is discussed in my master thesis."
+    "E.g., what is epistemic uncertainty?"
+)
+
+gradio_app = gr.Interface(
+    fn=chat_bot,
+    inputs=[
+        gr.Textbox(placeholder=default_message),
+        gr.Dropdown(
+            choices=["basic", "sentence window retrieval", "auto-merging retrieval"],
+            label="RAG mode",
+            value="basic",
+        ),
+    ],
+    outputs=["text"],
+)
+
+if __name__ == "__main__":
+    gradio_app.launch()

data/indices/auto_merging/d5c92a9b2f/graph_store.json

@@ -0,0 +1 @@
+{"graph_dict": {}}

data/indices/auto_merging/d5c92a9b2f/image__vector_store.json

@@ -0,0 +1 @@
+{"embedding_dict": {}, "text_id_to_ref_doc_id": {}, "metadata_dict": {}}

data/indices/auto_merging/d5c92a9b2f/index_store.json

@@ -0,0 +1 @@
+{"index_store/data": {"01d4fc3b-cde7-4254-a776-0340613d6542": {"__type__": "vector_store", "__data__": "{\"index_id\": \"01d4fc3b-cde7-4254-a776-0340613d6542\", \"summary\": null, \"nodes_dict\": {\"127eed05-f4c0-492c-aebb-fe8ea02ca1e5\": \"127eed05-f4c0-492c-aebb-fe8ea02ca1e5\", \"2935ee7a-e93e-49cf-95c5-78bc8dff3e9a\": \"2935ee7a-e93e-49cf-95c5-78bc8dff3e9a\", \"4a4dbbb4-76fe-4fdd-a535-6b639d06fed1\": \"4a4dbbb4-76fe-4fdd-a535-6b639d06fed1\", \"4149f79f-033e-4947-89f4-d0f132e26664\": \"4149f79f-033e-4947-89f4-d0f132e26664\", \"70d2a7d7-8312-47b7-8e75-cdac5ba2e41c\": \"70d2a7d7-8312-47b7-8e75-cdac5ba2e41c\", \"6ba0ab8c-7d35-464d-b1a7-3e51a3305397\": \"6ba0ab8c-7d35-464d-b1a7-3e51a3305397\", \"f48df62c-d67c-4992-a458-b8b82f24a21e\": \"f48df62c-d67c-4992-a458-b8b82f24a21e\", \"d9b7d187-5420-4a21-ba94-b677e41b31c5\": \"d9b7d187-5420-4a21-ba94-b677e41b31c5\", \"d18aaf46-78ba-4ac4-b2d8-04877df01f04\": \"d18aaf46-78ba-4ac4-b2d8-04877df01f04\", \"823db52f-3c7c-49f2-b06d-6c592db7ef42\": \"823db52f-3c7c-49f2-b06d-6c592db7ef42\", \"774f5af1-b593-47eb-9d6a-354c538e08f3\": \"774f5af1-b593-47eb-9d6a-354c538e08f3\", \"b0a8bf13-67ba-44ed-b2f8-ba822e7faadb\": \"b0a8bf13-67ba-44ed-b2f8-ba822e7faadb\", \"e2dfa4a4-b378-4b62-84f7-bef1ec7ceb0f\": \"e2dfa4a4-b378-4b62-84f7-bef1ec7ceb0f\", \"63f8f248-245f-4955-bedd-a2866caaed7a\": \"63f8f248-245f-4955-bedd-a2866caaed7a\", \"fe153069-6018-4026-9d8a-b4685d64ed38\": \"fe153069-6018-4026-9d8a-b4685d64ed38\", \"9c5c5819-4547-48ff-8f13-f22d11438a3a\": \"9c5c5819-4547-48ff-8f13-f22d11438a3a\", \"c6fb2fef-6474-47b8-9092-b026ab781d2b\": \"c6fb2fef-6474-47b8-9092-b026ab781d2b\", \"ecf9695e-b641-4b46-a467-0bd43708cbe0\": \"ecf9695e-b641-4b46-a467-0bd43708cbe0\", \"d724a087-a3d0-4009-bd38-b806365de8c7\": \"d724a087-a3d0-4009-bd38-b806365de8c7\", \"63e59f20-3737-45d8-9a92-d32c74bb7166\": \"63e59f20-3737-45d8-9a92-d32c74bb7166\", \"066b93a5-3342-4cc4-8571-98cadb2a3164\": \"066b93a5-3342-4cc4-8571-98cadb2a3164\", \"58ad19b7-324c-4be6-a9b8-3fb64690c7bb\": \"58ad19b7-324c-4be6-a9b8-3fb64690c7bb\", \"9922a1b5-0c0b-4c9f-bfe5-b353886f674a\": \"9922a1b5-0c0b-4c9f-bfe5-b353886f674a\", \"1a2a3619-bfd3-4b5d-8688-08beca93b771\": \"1a2a3619-bfd3-4b5d-8688-08beca93b771\", \"fd961f40-ff40-4c4d-9fc2-ae381ad4b060\": \"fd961f40-ff40-4c4d-9fc2-ae381ad4b060\", \"e99348f4-24f8-4709-b5b5-311c923a356a\": \"e99348f4-24f8-4709-b5b5-311c923a356a\", \"fa140f79-d1f4-444a-b07c-c3c2b00bcb6a\": \"fa140f79-d1f4-444a-b07c-c3c2b00bcb6a\", \"9071c131-fcde-4881-9954-dfdd8b1909a3\": \"9071c131-fcde-4881-9954-dfdd8b1909a3\", \"4dec264a-3556-4369-a51f-fa460f2e3e67\": \"4dec264a-3556-4369-a51f-fa460f2e3e67\", \"88b2e2ea-51e1-407b-a3db-6120ddb7a953\": \"88b2e2ea-51e1-407b-a3db-6120ddb7a953\", \"612e4522-6b23-4b77-a1a6-cc15d93cbfad\": \"612e4522-6b23-4b77-a1a6-cc15d93cbfad\", \"b1a24e8a-e7a6-490c-92fb-bf690a24a9cc\": \"b1a24e8a-e7a6-490c-92fb-bf690a24a9cc\", \"b4e16def-e018-4af7-9216-35a9f688b90e\": \"b4e16def-e018-4af7-9216-35a9f688b90e\", \"ba7f5982-2eee-425b-813b-c17de4d0bf65\": \"ba7f5982-2eee-425b-813b-c17de4d0bf65\", \"8159166f-ff8a-4af8-9c75-6af7349664af\": \"8159166f-ff8a-4af8-9c75-6af7349664af\", \"507c5eea-78f1-422c-a6d3-a187e4291c1f\": \"507c5eea-78f1-422c-a6d3-a187e4291c1f\", \"f78e2e16-decf-4534-bbd2-c0aab0f37021\": \"f78e2e16-decf-4534-bbd2-c0aab0f37021\", \"9fdd0b73-5596-42cb-9361-ac375d6f0415\": \"9fdd0b73-5596-42cb-9361-ac375d6f0415\", \"b62c885a-a745-427f-b3a1-5fb1988990bc\": \"b62c885a-a745-427f-b3a1-5fb1988990bc\", \"c313afad-778d-4960-b5ae-858e82effd8d\": \"c313afad-778d-4960-b5ae-858e82effd8d\", \"a6693f1e-bd6a-46b9-adc4-056ea375dab0\": \"a6693f1e-bd6a-46b9-adc4-056ea375dab0\", \"6b4320f6-7f58-47bd-82b4-42567bcb1bc8\": \"6b4320f6-7f58-47bd-82b4-42567bcb1bc8\", \"0a2e62df-7f45-47ee-9a98-86d449d32648\": \"0a2e62df-7f45-47ee-9a98-86d449d32648\", \"676f69bd-4650-4c90-a9fd-253354c2fd06\": \"676f69bd-4650-4c90-a9fd-253354c2fd06\", \"ea0f952a-51a3-4033-8131-4a6f189c4cba\": \"ea0f952a-51a3-4033-8131-4a6f189c4cba\", \"c9728fd6-54ea-4466-b7e4-71bf94f2db36\": \"c9728fd6-54ea-4466-b7e4-71bf94f2db36\", \"33a22272-74c1-4053-899b-f52ee8721489\": \"33a22272-74c1-4053-899b-f52ee8721489\", \"4e54f3b3-3cec-4266-9bfd-b7ba95ce2939\": \"4e54f3b3-3cec-4266-9bfd-b7ba95ce2939\", \"b11f795c-9011-4923-9346-e9949df69467\": \"b11f795c-9011-4923-9346-e9949df69467\", \"55eec99b-78ea-4843-af4a-46d61411510b\": \"55eec99b-78ea-4843-af4a-46d61411510b\", \"1b591c75-28e9-440c-8e65-5eaf1b979fc5\": \"1b591c75-28e9-440c-8e65-5eaf1b979fc5\", \"fd91c70a-adcb-4a30-9a73-73a17707624d\": \"fd91c70a-adcb-4a30-9a73-73a17707624d\", \"cc12d5e5-c058-4c6e-97f4-232c2f91af7a\": \"cc12d5e5-c058-4c6e-97f4-232c2f91af7a\", \"5713b650-3c69-4369-b4a1-25e79a3f4b4b\": \"5713b650-3c69-4369-b4a1-25e79a3f4b4b\", \"6216ecc3-4580-4861-905a-51f0c233a993\": \"6216ecc3-4580-4861-905a-51f0c233a993\", \"202ed815-c9f3-4180-976f-2b3d0fc99b3a\": \"202ed815-c9f3-4180-976f-2b3d0fc99b3a\", \"fc45659e-9e72-42e5-b1f5-e964f6474119\": \"fc45659e-9e72-42e5-b1f5-e964f6474119\", \"57d310ce-6e84-4805-b3ee-4137f01377e4\": \"57d310ce-6e84-4805-b3ee-4137f01377e4\", \"da804d94-7c21-4a4b-91f5-00ba625e411f\": \"da804d94-7c21-4a4b-91f5-00ba625e411f\", \"ae3e487e-3473-4c6c-a5bc-d6cec3cb8565\": \"ae3e487e-3473-4c6c-a5bc-d6cec3cb8565\", \"086f66d3-b416-41bd-af49-179420725658\": \"086f66d3-b416-41bd-af49-179420725658\", \"45da310f-48f1-4072-930a-ed4abccd4067\": \"45da310f-48f1-4072-930a-ed4abccd4067\", \"ebae6ed2-50e2-4e10-8c43-8d892eece566\": \"ebae6ed2-50e2-4e10-8c43-8d892eece566\", \"23c1c50a-828e-403c-ae44-4fcba39d6197\": \"23c1c50a-828e-403c-ae44-4fcba39d6197\", \"8d13191d-7fa0-401f-909f-e761626681de\": \"8d13191d-7fa0-401f-909f-e761626681de\", \"5f40ffb2-adf0-45f0-8837-0edd1db895a5\": \"5f40ffb2-adf0-45f0-8837-0edd1db895a5\", \"1d25b9e7-defd-4782-b095-1562832e7d21\": \"1d25b9e7-defd-4782-b095-1562832e7d21\", \"9013bbb3-cb21-4429-99cc-e8c7426f10fb\": \"9013bbb3-cb21-4429-99cc-e8c7426f10fb\", \"978359a5-1e65-4640-8a9f-343a2a59698d\": \"978359a5-1e65-4640-8a9f-343a2a59698d\", \"30c71308-f900-4bae-8ec6-d86d4dd13419\": \"30c71308-f900-4bae-8ec6-d86d4dd13419\", \"9819cc86-9766-441a-aa8c-40e186e24fea\": \"9819cc86-9766-441a-aa8c-40e186e24fea\", \"5c6de902-b23a-4c7e-855f-cd5eb652add2\": \"5c6de902-b23a-4c7e-855f-cd5eb652add2\", \"bb921f3b-6a8e-48ca-b4b5-e599a6d6c406\": \"bb921f3b-6a8e-48ca-b4b5-e599a6d6c406\", \"d4c91e89-461c-4294-ace0-23d5ce6747c0\": \"d4c91e89-461c-4294-ace0-23d5ce6747c0\", \"8f9fd36f-b761-4f40-a66c-6c4f75d4207b\": \"8f9fd36f-b761-4f40-a66c-6c4f75d4207b\", \"15c3b984-ff09-4f68-9d55-fe705c4f4ae4\": \"15c3b984-ff09-4f68-9d55-fe705c4f4ae4\", \"0526c5ab-8527-42b1-b1d8-1cfe710874f4\": \"0526c5ab-8527-42b1-b1d8-1cfe710874f4\", \"d4aa2f5a-33a8-453a-a6ad-c4bacf576fab\": \"d4aa2f5a-33a8-453a-a6ad-c4bacf576fab\", \"0ccd7498-9105-4204-bafd-6edc9cf75849\": \"0ccd7498-9105-4204-bafd-6edc9cf75849\", \"bd6b344e-d289-4006-a983-d9ffff88650f\": \"bd6b344e-d289-4006-a983-d9ffff88650f\", \"1fb579b0-32b1-4482-93af-f8335ef3d5e9\": \"1fb579b0-32b1-4482-93af-f8335ef3d5e9\", \"ab58e4bc-757c-4b83-8279-13bf36e223f4\": \"ab58e4bc-757c-4b83-8279-13bf36e223f4\", \"d6db2978-cf10-4fd6-949e-fc950bbf2598\": \"d6db2978-cf10-4fd6-949e-fc950bbf2598\", \"c2191d8c-3f02-4542-a078-164beec931a8\": \"c2191d8c-3f02-4542-a078-164beec931a8\", \"ca4bf718-63a5-4fb5-b024-7c448c880f95\": \"ca4bf718-63a5-4fb5-b024-7c448c880f95\", \"faf19732-15b6-4344-8678-a4b2c6a85dd1\": \"faf19732-15b6-4344-8678-a4b2c6a85dd1\", \"dddf8bf2-5179-4791-a310-7710be80004b\": \"dddf8bf2-5179-4791-a310-7710be80004b\", \"113bc7b3-5f99-45ca-b093-0b867520f051\": \"113bc7b3-5f99-45ca-b093-0b867520f051\", \"c6d6677b-f019-4c31-a8d0-503541971456\": \"c6d6677b-f019-4c31-a8d0-503541971456\", \"6b031f5f-0e40-4434-8ce5-d1e0ba08f7a0\": \"6b031f5f-0e40-4434-8ce5-d1e0ba08f7a0\", \"66cd63ad-1f7e-415f-99fb-100448af6aa2\": \"66cd63ad-1f7e-415f-99fb-100448af6aa2\", \"78390fc6-05b5-471f-9ed8-a2f24c6d53a8\": \"78390fc6-05b5-471f-9ed8-a2f24c6d53a8\", \"10d7b5e0-3e25-416f-bc00-2a01933c6b9b\": \"10d7b5e0-3e25-416f-bc00-2a01933c6b9b\", \"521b4f69-fc53-4ee1-aa3b-9a64d4689aab\": \"521b4f69-fc53-4ee1-aa3b-9a64d4689aab\", \"a63ebc03-cc47-4b22-bcbe-b26c328d8ad4\": \"a63ebc03-cc47-4b22-bcbe-b26c328d8ad4\", \"05cd4e21-ea82-46c9-9edb-0ce5101680e8\": \"05cd4e21-ea82-46c9-9edb-0ce5101680e8\", \"9ab01bf2-af0f-4b17-9232-46eca13fd2cc\": \"9ab01bf2-af0f-4b17-9232-46eca13fd2cc\", \"b8885c9f-be74-4eac-b33e-6388fca0496f\": \"b8885c9f-be74-4eac-b33e-6388fca0496f\", \"f6111520-529e-4fde-8438-3bdb22626f29\": \"f6111520-529e-4fde-8438-3bdb22626f29\", \"305361e0-447c-4b9a-bd75-cb60ce8e508a\": \"305361e0-447c-4b9a-bd75-cb60ce8e508a\", \"6d55016e-f6c2-4b28-9228-eae4e5982a3d\": \"6d55016e-f6c2-4b28-9228-eae4e5982a3d\", \"da53c1af-ff4e-4e59-a3b4-8ba2e6b62195\": \"da53c1af-ff4e-4e59-a3b4-8ba2e6b62195\", \"7cc0cbb7-3405-4440-bd3a-fe94b1d814f1\": \"7cc0cbb7-3405-4440-bd3a-fe94b1d814f1\", \"aa03814b-1151-47bf-b2c3-d5f728e0ed36\": \"aa03814b-1151-47bf-b2c3-d5f728e0ed36\", \"8fe63cef-848b-4b24-a35e-d09771946411\": \"8fe63cef-848b-4b24-a35e-d09771946411\", \"cfc29151-e033-4efa-8330-099a55348c4e\": \"cfc29151-e033-4efa-8330-099a55348c4e\", \"967d28de-3d2f-4905-8de3-660bf3740fca\": \"967d28de-3d2f-4905-8de3-660bf3740fca\", \"49bb4996-036c-48d4-9e30-5295bc7ac314\": \"49bb4996-036c-48d4-9e30-5295bc7ac314\", \"8fe6d34d-9c21-4656-86d3-732a702b4ec2\": \"8fe6d34d-9c21-4656-86d3-732a702b4ec2\", \"e09fef37-7df1-48a1-bfae-7d3cf85d77e9\": \"e09fef37-7df1-48a1-bfae-7d3cf85d77e9\", \"1efa753a-6d54-4d40-8bdf-0f52e6fea8b5\": \"1efa753a-6d54-4d40-8bdf-0f52e6fea8b5\", \"9d9f1478-1daf-40ec-8cd4-87f4fa12250a\": \"9d9f1478-1daf-40ec-8cd4-87f4fa12250a\", \"d77ea73b-146d-4d3a-b6b5-3f7d5bdca293\": \"d77ea73b-146d-4d3a-b6b5-3f7d5bdca293\", \"dfb85eda-0acd-4182-a136-c8692ca0a786\": \"dfb85eda-0acd-4182-a136-c8692ca0a786\", \"afd5e88c-7489-4f00-a9ad-44ecd97dbfb2\": \"afd5e88c-7489-4f00-a9ad-44ecd97dbfb2\", \"cb588028-5fd3-4f58-8384-ff9ebde0ae11\": \"cb588028-5fd3-4f58-8384-ff9ebde0ae11\", \"195ea30a-2725-4638-8edb-425d24fc565d\": \"195ea30a-2725-4638-8edb-425d24fc565d\", \"63e42cac-6151-43e7-85db-f41c374bb94b\": \"63e42cac-6151-43e7-85db-f41c374bb94b\", \"aac01576-0a03-407b-91cd-e222aecfd721\": \"aac01576-0a03-407b-91cd-e222aecfd721\", \"ab56bea2-c39a-444a-9434-dff308adbbea\": \"ab56bea2-c39a-444a-9434-dff308adbbea\", \"637776b7-4a00-4514-ae55-e2dfe5c640f6\": \"637776b7-4a00-4514-ae55-e2dfe5c640f6\", \"bbdf5750-fabd-4e11-b784-19a0b62b520b\": \"bbdf5750-fabd-4e11-b784-19a0b62b520b\", \"b10f58ab-250a-4f49-8497-16be51d652ee\": \"b10f58ab-250a-4f49-8497-16be51d652ee\", \"a0e4e6e7-b68e-4e73-b5e0-8d5d38b1bf70\": \"a0e4e6e7-b68e-4e73-b5e0-8d5d38b1bf70\", \"50d36569-0c7c-4cae-a1c0-6bb1c8990fc0\": \"50d36569-0c7c-4cae-a1c0-6bb1c8990fc0\", \"d8b051cc-f398-40c7-9326-433d79f1051d\": \"d8b051cc-f398-40c7-9326-433d79f1051d\", \"6859747b-0a8a-48c4-980f-6600405219ab\": \"6859747b-0a8a-48c4-980f-6600405219ab\", \"b682c9ff-86e2-42e3-a4e9-c188dd7cfb8f\": \"b682c9ff-86e2-42e3-a4e9-c188dd7cfb8f\", \"c4c3dd73-196b-4290-82dd-4046b9bb0ad8\": \"c4c3dd73-196b-4290-82dd-4046b9bb0ad8\", \"80817e53-55d3-43d3-83d1-a719a414926a\": \"80817e53-55d3-43d3-83d1-a719a414926a\", \"8e6e18c6-4421-4f3d-9290-ba27dfcaf95d\": \"8e6e18c6-4421-4f3d-9290-ba27dfcaf95d\", \"d71c30f3-4e73-4117-8927-b336191f9d1c\": \"d71c30f3-4e73-4117-8927-b336191f9d1c\", \"694102af-3409-4464-bb8d-19b1f381f253\": \"694102af-3409-4464-bb8d-19b1f381f253\", \"7a70e0d5-0a6d-4ae6-a4c2-e0290bc97d7d\": \"7a70e0d5-0a6d-4ae6-a4c2-e0290bc97d7d\", \"7a93bcb4-c2e9-4717-9c8b-2f99c1b5c97f\": \"7a93bcb4-c2e9-4717-9c8b-2f99c1b5c97f\", \"0faa8361-dc17-4fba-a3c1-eae0d76cc290\": \"0faa8361-dc17-4fba-a3c1-eae0d76cc290\", \"97b47867-e66b-4e98-bc15-fbde3135ba50\": \"97b47867-e66b-4e98-bc15-fbde3135ba50\", \"df6864be-1e32-4467-aeb8-81f7f38176e1\": \"df6864be-1e32-4467-aeb8-81f7f38176e1\", \"332fe951-d55a-44b7-b5a2-bfbeb1c57151\": \"332fe951-d55a-44b7-b5a2-bfbeb1c57151\", \"c0212a98-a9a5-4e81-abc4-7bcef93be942\": \"c0212a98-a9a5-4e81-abc4-7bcef93be942\", \"b42544bb-3912-482e-8047-ad5386607134\": \"b42544bb-3912-482e-8047-ad5386607134\", \"6feacb6d-c009-40f0-9392-dc2d6bd0cb7b\": \"6feacb6d-c009-40f0-9392-dc2d6bd0cb7b\", \"981cfbe8-bed1-4d15-97e9-dddfc96c1123\": \"981cfbe8-bed1-4d15-97e9-dddfc96c1123\", \"51b4e8a9-f9dd-4153-9ca8-36e2bb679353\": \"51b4e8a9-f9dd-4153-9ca8-36e2bb679353\", \"5b10e014-d531-451f-8425-864e25fa5558\": \"5b10e014-d531-451f-8425-864e25fa5558\", \"e44bddbb-d644-4ac6-8a1c-726ebf9627f4\": \"e44bddbb-d644-4ac6-8a1c-726ebf9627f4\", \"03f8aa5e-45cf-4f50-bd7e-d4f06ebefc23\": \"03f8aa5e-45cf-4f50-bd7e-d4f06ebefc23\", \"b7723603-864a-4164-9084-336d25f2890d\": \"b7723603-864a-4164-9084-336d25f2890d\", \"b2342ae1-ec3e-4d96-aaa4-1dfc15e8279b\": \"b2342ae1-ec3e-4d96-aaa4-1dfc15e8279b\", \"1c7a3a4e-db10-480e-9851-4b91640abc16\": \"1c7a3a4e-db10-480e-9851-4b91640abc16\", \"ef773976-8799-4129-979e-4d74984bd69d\": \"ef773976-8799-4129-979e-4d74984bd69d\", \"f85638bc-678e-4101-b240-962c6c1c8137\": \"f85638bc-678e-4101-b240-962c6c1c8137\", \"aba94add-6b16-4f14-9b0a-e5ec1ea041a0\": \"aba94add-6b16-4f14-9b0a-e5ec1ea041a0\", \"07e6b100-c219-485e-b418-d1d938ca0102\": \"07e6b100-c219-485e-b418-d1d938ca0102\", \"fced2909-7e86-41c3-924f-d06b644b76a7\": \"fced2909-7e86-41c3-924f-d06b644b76a7\", \"284789e6-68c0-4bf2-982c-288eaa266705\": \"284789e6-68c0-4bf2-982c-288eaa266705\", \"64b45aed-c073-4f5a-b33f-939104b0f21f\": \"64b45aed-c073-4f5a-b33f-939104b0f21f\", \"244967f0-6448-4a4f-9e2b-f96e26f682dd\": \"244967f0-6448-4a4f-9e2b-f96e26f682dd\", \"62e784ee-a4cf-474d-aabf-0044130aa013\": \"62e784ee-a4cf-474d-aabf-0044130aa013\", \"3f460913-76d7-44b2-820e-d4b1160f4b20\": \"3f460913-76d7-44b2-820e-d4b1160f4b20\", \"cb16543a-5ec6-4947-bb55-b9784e6e7c7c\": \"cb16543a-5ec6-4947-bb55-b9784e6e7c7c\", \"5235a666-1e07-4abc-8742-8e1bcf0e790d\": \"5235a666-1e07-4abc-8742-8e1bcf0e790d\", \"496a456f-2ec3-48cf-940f-602619bd2c6c\": \"496a456f-2ec3-48cf-940f-602619bd2c6c\", \"cb50208d-24e7-4954-972f-900604a9ab61\": \"cb50208d-24e7-4954-972f-900604a9ab61\", \"aaec41e4-2cd2-44d8-8c78-5ca1268a0780\": \"aaec41e4-2cd2-44d8-8c78-5ca1268a0780\", \"bae30186-8436-4dd3-ab19-b6c86c3d67c1\": \"bae30186-8436-4dd3-ab19-b6c86c3d67c1\", \"1fc0e546-5e5b-43fd-8efe-1aa0db4fad69\": \"1fc0e546-5e5b-43fd-8efe-1aa0db4fad69\"}, \"doc_id_dict\": {}, \"embeddings_dict\": {}}"}}}

data/indices/auto_merging/d5c92a9b2f/meta.json

@@ -0,0 +1,9 @@
+{
+  "doc_source": "Master_Thesis.pdf",
+  "embed_model": "BAAI/bge-small-en-v1.5",
+  "chunk_sizes": [
+    2048,
+    512,
+    128
+  ]
+}

data/indices/basic/graph_store.json

@@ -0,0 +1 @@
+{"graph_dict": {}}

data/indices/basic/image__vector_store.json

@@ -0,0 +1 @@
+{"embedding_dict": {}, "text_id_to_ref_doc_id": {}, "metadata_dict": {}}

data/indices/basic/index_store.json

@@ -0,0 +1 @@
+{"index_store/data": {"0765479e-c27c-4c88-b41f-dc98ed86f3a5": {"__type__": "vector_store", "__data__": "{\"index_id\": \"0765479e-c27c-4c88-b41f-dc98ed86f3a5\", \"summary\": null, \"nodes_dict\": {\"a31b64b9-d4c7-4ec6-8ca0-10d085d48205\": \"a31b64b9-d4c7-4ec6-8ca0-10d085d48205\", \"b17212a2-233f-492c-b53b-6a03c84d9f4f\": \"b17212a2-233f-492c-b53b-6a03c84d9f4f\", \"21f04a43-0875-4b86-94e4-93b11843969f\": \"21f04a43-0875-4b86-94e4-93b11843969f\", \"35cb1ff3-4a4f-4616-a7c1-af26f0b9d1e9\": \"35cb1ff3-4a4f-4616-a7c1-af26f0b9d1e9\", \"db73e86c-4c24-4992-b2c4-96c3cb23e27e\": \"db73e86c-4c24-4992-b2c4-96c3cb23e27e\", \"66023780-9b49-46a9-9204-8a4786f537e9\": \"66023780-9b49-46a9-9204-8a4786f537e9\", \"3ed814ba-a0a4-4328-802d-91780bc5964e\": \"3ed814ba-a0a4-4328-802d-91780bc5964e\", \"2ce891bb-b95c-457c-859e-7a8980eb98c2\": \"2ce891bb-b95c-457c-859e-7a8980eb98c2\", \"04a0e9e2-4399-4f8e-9bb7-4a92ca7f3dfa\": \"04a0e9e2-4399-4f8e-9bb7-4a92ca7f3dfa\", \"39a9562c-9040-4276-b6a3-0686a44bcf50\": \"39a9562c-9040-4276-b6a3-0686a44bcf50\", \"fe41da69-d1ee-4ce6-8888-34e9c9a759b5\": \"fe41da69-d1ee-4ce6-8888-34e9c9a759b5\", \"5e20f7b6-ac06-4a68-913b-419a2b585f5d\": \"5e20f7b6-ac06-4a68-913b-419a2b585f5d\", \"c0f8c088-1ff8-4839-a325-2522f014510a\": \"c0f8c088-1ff8-4839-a325-2522f014510a\", \"e29d9346-c846-4203-8c35-b8bb1e8fa481\": \"e29d9346-c846-4203-8c35-b8bb1e8fa481\", \"004767a1-85b2-466e-b117-b264a4667205\": \"004767a1-85b2-466e-b117-b264a4667205\", \"bd3c9a8c-f95b-478e-a4a2-01ed7f774d64\": \"bd3c9a8c-f95b-478e-a4a2-01ed7f774d64\", \"5084cc68-c3d7-4136-83c1-b8c48955719a\": \"5084cc68-c3d7-4136-83c1-b8c48955719a\", \"a1381f9d-d625-43ab-8869-28f2ec055ddd\": \"a1381f9d-d625-43ab-8869-28f2ec055ddd\", \"7e4d7d91-e79e-49e5-8c71-8a48fc38cb46\": \"7e4d7d91-e79e-49e5-8c71-8a48fc38cb46\", \"379dd8d5-bf98-4615-93a9-cba410df45ec\": \"379dd8d5-bf98-4615-93a9-cba410df45ec\"}, \"doc_id_dict\": {}, \"embeddings_dict\": {}}"}}}

data/indices/sentence_window/61a981e27b/graph_store.json

@@ -0,0 +1 @@
+{"graph_dict": {}}

data/indices/sentence_window/61a981e27b/image__vector_store.json

@@ -0,0 +1 @@
+{"embedding_dict": {}, "text_id_to_ref_doc_id": {}, "metadata_dict": {}}

data/indices/sentence_window/61a981e27b/index_store.json

@@ -0,0 +1 @@
+{"index_store/data": {"dcb26062-97ca-4980-98ba-3ac367ae9e38": {"__type__": "vector_store", "__data__": "{\"index_id\": \"dcb26062-97ca-4980-98ba-3ac367ae9e38\", \"summary\": null, \"nodes_dict\": {\"9ab5067f-1cba-4f14-b483-f13e4943eb6f\": \"9ab5067f-1cba-4f14-b483-f13e4943eb6f\", \"4b7406f1-4038-428d-a41b-ed15736a3ad2\": \"4b7406f1-4038-428d-a41b-ed15736a3ad2\", \"19f2c43a-c9ef-4afc-9607-38a5a898b98b\": \"19f2c43a-c9ef-4afc-9607-38a5a898b98b\", \"0fd054f0-bd92-4e92-8037-4fcc7393ef0c\": \"0fd054f0-bd92-4e92-8037-4fcc7393ef0c\", \"d898ce41-24a0-4315-9a8e-056fc948525e\": \"d898ce41-24a0-4315-9a8e-056fc948525e\", \"cd43f204-6cd3-414d-82b8-bcbde2cceace\": \"cd43f204-6cd3-414d-82b8-bcbde2cceace\", \"dcc52b5f-0e96-4574-a39a-ab928669c990\": \"dcc52b5f-0e96-4574-a39a-ab928669c990\", \"5df6f5f8-cc1d-408c-b9a1-85e01ec94403\": \"5df6f5f8-cc1d-408c-b9a1-85e01ec94403\", \"2521c213-0049-4cc6-875f-a868a9ca718e\": \"2521c213-0049-4cc6-875f-a868a9ca718e\", \"12ee176d-2954-4255-ab0a-65a6b867ea3a\": \"12ee176d-2954-4255-ab0a-65a6b867ea3a\", \"3e0d57bd-2b67-4083-b6b4-566dd1f91eab\": \"3e0d57bd-2b67-4083-b6b4-566dd1f91eab\", \"aa04a66d-6118-40d8-8290-3cf7d233ebad\": \"aa04a66d-6118-40d8-8290-3cf7d233ebad\", \"0a5e52be-600c-465e-aad7-10b04c9a7c8a\": \"0a5e52be-600c-465e-aad7-10b04c9a7c8a\", \"489b851b-8e96-4f53-a3bb-32fadb4ccbf0\": \"489b851b-8e96-4f53-a3bb-32fadb4ccbf0\", \"75c6b57b-009a-4b1d-9099-f188ecd51a0a\": \"75c6b57b-009a-4b1d-9099-f188ecd51a0a\", \"6ac79648-9763-4226-ad60-ac9f6ce20778\": \"6ac79648-9763-4226-ad60-ac9f6ce20778\", \"45842347-a5ac-43ac-b405-f77ace8a56a7\": \"45842347-a5ac-43ac-b405-f77ace8a56a7\", \"26cefadf-7040-43ea-9003-a1a4ad189372\": \"26cefadf-7040-43ea-9003-a1a4ad189372\", \"b6cbbc8a-2737-4272-a9d4-2f608931f1df\": \"b6cbbc8a-2737-4272-a9d4-2f608931f1df\", \"b5e2bdd2-6e23-4ccc-9bbd-4f08b831e046\": \"b5e2bdd2-6e23-4ccc-9bbd-4f08b831e046\"}, \"doc_id_dict\": {}, \"embeddings_dict\": {}}"}}}

data/indices/sentence_window/61a981e27b/meta.json

@@ -0,0 +1,5 @@
+{
+  "doc_source": "Master_Thesis.pdf",
+  "embed_model": "BAAI/bge-small-en-v1.5",
+  "sentence_window_size": 3
+}

pyproject.toml

@@ -0,0 +1,51 @@
+[project]
+name = "mythesis-chatbot"
+version = "0.1.0"
+description = "RAG chatbot trained on my master thesis."
+authors = [
+    {name = "Léonard Pasi",email = "leonardpasi@gmail.com"}
+]
+readme = "README.md"
+requires-python = "^3.10"
+dependencies = [
+    "trulens (>=1.4.7,<2.0.0)",
+    "llama-index (>=0.12.28,<0.13.0)",
+    "pandas (>=2.2.3,<3.0.0)",
+    "llama-index-embeddings-huggingface (>=0.5.2,<0.6.0)",
+    "trulens-providers-openai (>=1.4.8,<2.0.0)",
+    "trulens-apps-llamaindex (>=1.4.8,<2.0.0)",
+    "gradio (==5.24.0)",
+]
+
+[tool.poetry]
+packages = [{include = "mythesis_chatbot", from = "src"}]
+
+[tool.poetry.group.dev.dependencies]
+pre-commit = "^4.2.0"
+flake8 = "^7.2.0"
+black = "^25.1.0"
+isort = "^6.0.1"
+
+[tool.black]
+line-length = 88
+exclude = '''
+/(
+    \.git
+    | \.mypy_cache
+    | \.tox
+    | \.venv
+    | _build
+    | buck-out
+    | build
+    | dist
+)/
+'''
+
+[tool.isort]
+profile="black"
+known_first_party=["mythesis_chatbot"]
+known_third_party=["llamaindex", "trulens"]
+
+[build-system]
+requires = ["poetry-core>=2.0.0,<3.0.0"]
+build-backend = "poetry.core.masonry.api"

requirements.txt

@@ -0,0 +1,266 @@
+aiohappyeyeballs==2.6.1 ; python_version >= "3.10" and python_version < "4.0"
+aiohttp==3.11.16 ; python_version >= "3.10" and python_version < "4.0"
+aiosignal==1.3.2 ; python_version >= "3.10" and python_version < "4.0"
+alembic==1.15.2 ; python_version >= "3.10" and python_version < "4.0"
+altair==5.5.0 ; python_version >= "3.10" and python_version < "4.0"
+annotated-types==0.7.0 ; python_version >= "3.10" and python_version < "4.0"
+anyio==4.9.0 ; python_version >= "3.10" and python_version < "4.0"
+appnope==0.1.4 ; python_version >= "3.10" and python_version < "4.0" and platform_system == "Darwin"
+argon2-cffi-bindings==21.2.0 ; python_version >= "3.10" and python_version < "4.0"
+argon2-cffi==23.1.0 ; python_version >= "3.10" and python_version < "4.0"
+arrow==1.3.0 ; python_version >= "3.10" and python_version < "4.0"
+asttokens==3.0.0 ; python_version >= "3.10" and python_version < "4.0"
+async-lru==2.0.5 ; python_version >= "3.10" and python_version < "4.0"
+async-timeout==4.0.3 ; python_version == "3.10"
+attrs==25.3.0 ; python_version >= "3.10" and python_version < "4.0"
+babel==2.17.0 ; python_version >= "3.10" and python_version < "4.0"
+banks==2.1.1 ; python_version >= "3.10" and python_version < "4.0"
+beautifulsoup4==4.13.3 ; python_version >= "3.10" and python_version < "4.0"
+bleach==6.2.0 ; python_version >= "3.10" and python_version < "4.0"
+blinker==1.9.0 ; python_version >= "3.10" and python_version < "4.0"
+build==1.2.2.post1 ; python_version >= "3.10" and python_version < "4.0"
+cachecontrol==0.14.2 ; python_version >= "3.10" and python_version < "4.0"
+cachetools==5.5.2 ; python_version >= "3.10" and python_version < "4.0"
+certifi==2025.1.31 ; python_version >= "3.10" and python_version < "4.0"
+cffi==1.17.1 ; python_version >= "3.10" and python_version < "4.0"
+charset-normalizer==3.4.1 ; python_version >= "3.10" and python_version < "4.0"
+cleo==2.1.0 ; python_version >= "3.10" and python_version < "4.0"
+click==8.1.8 ; python_version >= "3.10" and python_version < "4.0"
+colorama==0.4.6 ; python_version >= "3.10" and python_version < "4.0"
+comm==0.2.2 ; python_version >= "3.10" and python_version < "4.0"
+crashtest==0.4.1 ; python_version >= "3.10" and python_version < "4.0"
+cryptography==44.0.2 ; python_version >= "3.10" and python_version < "4.0" and sys_platform == "linux"
+dataclasses-json==0.6.7 ; python_version >= "3.10" and python_version < "4.0"
+debugpy==1.8.13 ; python_version >= "3.10" and python_version < "4.0"
+decorator==5.2.1 ; python_version >= "3.10" and python_version < "4.0"
+defusedxml==0.7.1 ; python_version >= "3.10" and python_version < "4.0"
+deprecated==1.2.18 ; python_version >= "3.10" and python_version < "4.0"
+dill==0.3.9 ; python_version >= "3.10" and python_version < "4.0"
+dirtyjson==1.0.8 ; python_version >= "3.10" and python_version < "4.0"
+distlib==0.3.9 ; python_version >= "3.10" and python_version < "4.0"
+distro==1.9.0 ; python_version >= "3.10" and python_version < "4.0"
+dulwich==0.21.7 ; python_version >= "3.10" and python_version < "4.0"
+exceptiongroup==1.2.2 ; python_version == "3.10"
+executing==2.2.0 ; python_version >= "3.10" and python_version < "4.0"
+fastjsonschema==2.21.1 ; python_version >= "3.10" and python_version < "4.0"
+filelock==3.18.0 ; python_version >= "3.10" and python_version < "4.0"
+filetype==1.2.0 ; python_version >= "3.10" and python_version < "4.0"
+fqdn==1.5.1 ; python_version >= "3.10" and python_version < "4.0"
+frozenlist==1.5.0 ; python_version >= "3.10" and python_version < "4.0"
+fsspec==2025.3.2 ; python_version >= "3.10" and python_version < "4.0"
+gitdb==4.0.12 ; python_version >= "3.10" and python_version < "4.0"
+gitpython==3.1.44 ; python_version >= "3.10" and python_version < "4.0"
+greenlet==3.1.1 ; python_version >= "3.10" and python_version <= "3.13" and (platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "AMD64" or platform_machine == "win32" or platform_machine == "WIN32")
+griffe==1.7.2 ; python_version >= "3.10" and python_version < "4.0"
+h11==0.14.0 ; python_version >= "3.10" and python_version < "4.0"
+httpcore==1.0.7 ; python_version >= "3.10" and python_version < "4.0"
+httpx-sse==0.4.0 ; python_version >= "3.10" and python_version < "4.0"
+httpx==0.28.1 ; python_version >= "3.10" and python_version < "4.0"
+huggingface-hub==0.30.1 ; python_version >= "3.10" and python_version < "4.0"
+idna==3.10 ; python_version >= "3.10" and python_version < "4.0"
+importlib-metadata==8.6.1 ; python_version >= "3.10" and python_version < "4.0"
+importlib-resources==6.5.2 ; python_version >= "3.10" and python_version < "4.0"
+installer==0.7.0 ; python_version >= "3.10" and python_version < "4.0"
+ipykernel==6.29.5 ; python_version >= "3.10" and python_version < "4.0"
+ipython-pygments-lexers==1.1.1 ; python_version >= "3.11" and python_version < "4.0"
+ipython==8.34.0 ; python_version == "3.10"
+ipython==9.0.2 ; python_version >= "3.11" and python_version < "4.0"
+ipywidgets==8.1.5 ; python_version >= "3.10" and python_version < "4.0"
+isoduration==20.11.0 ; python_version >= "3.10" and python_version < "4.0"
+jaraco-classes==3.4.0 ; python_version >= "3.10" and python_version < "4.0"
+jedi==0.19.2 ; python_version >= "3.10" and python_version < "4.0"
+jeepney==0.9.0 ; python_version >= "3.10" and python_version < "4.0" and sys_platform == "linux"
+jinja2==3.1.6 ; python_version >= "3.10" and python_version < "4.0"
+jiter==0.9.0 ; python_version >= "3.10" and python_version < "4.0"
+joblib==1.4.2 ; python_version >= "3.10" and python_version < "4.0"
+json5==0.11.0 ; python_version >= "3.10" and python_version < "4.0"
+jsonpatch==1.33 ; python_version >= "3.10" and python_version < "4.0"
+jsonpointer==3.0.0 ; python_version >= "3.10" and python_version < "4.0"
+jsonschema-specifications==2024.10.1 ; python_version >= "3.10" and python_version < "4.0"
+jsonschema==4.23.0 ; python_version >= "3.10" and python_version < "4.0"
+jupyter-client==8.6.3 ; python_version >= "3.10" and python_version < "4.0"
+jupyter-console==6.6.3 ; python_version >= "3.10" and python_version < "4.0"
+jupyter-core==5.7.2 ; python_version >= "3.10" and python_version < "4.0"
+jupyter-events==0.12.0 ; python_version >= "3.10" and python_version < "4.0"
+jupyter-lsp==2.2.5 ; python_version >= "3.10" and python_version < "4.0"
+jupyter-server-terminals==0.5.3 ; python_version >= "3.10" and python_version < "4.0"
+jupyter-server==2.15.0 ; python_version >= "3.10" and python_version < "4.0"
+jupyter==1.1.1 ; python_version >= "3.10" and python_version < "4.0"
+jupyterlab-pygments==0.3.0 ; python_version >= "3.10" and python_version < "4.0"
+jupyterlab-server==2.27.3 ; python_version >= "3.10" and python_version < "4.0"
+jupyterlab-widgets==3.0.13 ; python_version >= "3.10" and python_version < "4.0"
+jupyterlab==4.3.6 ; python_version >= "3.10" and python_version < "4.0"
+keyring==24.3.1 ; python_version >= "3.10" and python_version < "4.0"
+langchain-community==0.3.21 ; python_version >= "3.10" and python_version < "4.0"
+langchain-core==0.3.51 ; python_version >= "3.10" and python_version < "4.0"
+langchain-text-splitters==0.3.8 ; python_version >= "3.10" and python_version < "4.0"
+langchain==0.3.23 ; python_version >= "3.10" and python_version < "4.0"
+langsmith==0.3.24 ; python_version >= "3.10" and python_version < "4.0"
+llama-cloud-services==0.6.9 ; python_version >= "3.10" and python_version < "4.0"
+llama-cloud==0.1.17 ; python_version >= "3.10" and python_version < "4.0"
+llama-index-agent-openai==0.4.6 ; python_version >= "3.10" and python_version < "4.0"
+llama-index-cli==0.4.1 ; python_version >= "3.10" and python_version < "4.0"
+llama-index-core==0.12.28 ; python_version >= "3.10" and python_version < "4.0"
+llama-index-embeddings-huggingface==0.5.2 ; python_version >= "3.10" and python_version < "4.0"
+llama-index-embeddings-openai==0.3.1 ; python_version >= "3.10" and python_version < "4.0"
+llama-index-indices-managed-llama-cloud==0.6.10 ; python_version >= "3.10" and python_version < "4.0"
+llama-index-llms-openai==0.3.29 ; python_version >= "3.10" and python_version < "4.0"
+llama-index-multi-modal-llms-openai==0.4.3 ; python_version >= "3.10" and python_version < "4.0"
+llama-index-program-openai==0.3.1 ; python_version >= "3.10" and python_version < "4.0"
+llama-index-question-gen-openai==0.3.0 ; python_version >= "3.10" and python_version < "4.0"
+llama-index-readers-file==0.4.7 ; python_version >= "3.10" and python_version < "4.0"
+llama-index-readers-llama-parse==0.4.0 ; python_version >= "3.10" and python_version < "4.0"
+llama-index==0.12.28 ; python_version >= "3.10" and python_version < "4.0"
+llama-parse==0.6.4.post1 ; python_version >= "3.10" and python_version < "4.0"
+mako==1.3.9 ; python_version >= "3.10" and python_version < "4.0"
+markdown-it-py==3.0.0 ; python_version >= "3.10" and python_version < "4.0"
+markupsafe==3.0.2 ; python_version >= "3.10" and python_version < "4.0"
+marshmallow==3.26.1 ; python_version >= "3.10" and python_version < "4.0"
+matplotlib-inline==0.1.7 ; python_version >= "3.10" and python_version < "4.0"
+mdurl==0.1.2 ; python_version >= "3.10" and python_version < "4.0"
+mistune==3.1.3 ; python_version >= "3.10" and python_version < "4.0"
+more-itertools==10.6.0 ; python_version >= "3.10" and python_version < "4.0"
+mpmath==1.3.0 ; python_version >= "3.10" and python_version < "4.0"
+msgpack==1.1.0 ; python_version >= "3.10" and python_version < "4.0"
+multidict==6.3.1 ; python_version >= "3.10" and python_version < "4.0"
+munch==2.5.0 ; python_version >= "3.10" and python_version < "4.0"
+mypy-extensions==1.0.0 ; python_version >= "3.10" and python_version < "4.0"
+narwhals==1.33.0 ; python_version >= "3.10" and python_version < "4.0"
+nbclient==0.10.2 ; python_version >= "3.10" and python_version < "4.0"
+nbconvert==7.16.6 ; python_version >= "3.10" and python_version < "4.0"
+nbformat==5.10.4 ; python_version >= "3.10" and python_version < "4.0"
+nest-asyncio==1.6.0 ; python_version >= "3.10" and python_version < "4.0"
+networkx==3.4.2 ; python_version >= "3.10" and python_version < "4.0"
+nltk==3.9.1 ; python_version >= "3.10" and python_version < "4.0"
+notebook-shim==0.2.4 ; python_version >= "3.10" and python_version < "4.0"
+notebook==7.3.3 ; python_version >= "3.10" and python_version < "4.0"
+numpy==2.2.4 ; python_version >= "3.10" and python_version < "4.0"
+nvidia-cublas-cu12==12.4.5.8 ; python_version >= "3.10" and python_version < "4.0" and platform_machine == "x86_64" and platform_system == "Linux"
+nvidia-cuda-cupti-cu12==12.4.127 ; python_version >= "3.10" and python_version < "4.0" and platform_machine == "x86_64" and platform_system == "Linux"
+nvidia-cuda-nvrtc-cu12==12.4.127 ; python_version >= "3.10" and python_version < "4.0" and platform_machine == "x86_64" and platform_system == "Linux"
+nvidia-cuda-runtime-cu12==12.4.127 ; python_version >= "3.10" and python_version < "4.0" and platform_machine == "x86_64" and platform_system == "Linux"
+nvidia-cudnn-cu12==9.1.0.70 ; python_version >= "3.10" and python_version < "4.0" and platform_machine == "x86_64" and platform_system == "Linux"
+nvidia-cufft-cu12==11.2.1.3 ; python_version >= "3.10" and python_version < "4.0" and platform_machine == "x86_64" and platform_system == "Linux"
+nvidia-curand-cu12==10.3.5.147 ; python_version >= "3.10" and python_version < "4.0" and platform_machine == "x86_64" and platform_system == "Linux"
+nvidia-cusolver-cu12==11.6.1.9 ; python_version >= "3.10" and python_version < "4.0" and platform_machine == "x86_64" and platform_system == "Linux"
+nvidia-cusparse-cu12==12.3.1.170 ; python_version >= "3.10" and python_version < "4.0" and platform_machine == "x86_64" and platform_system == "Linux"
+nvidia-cusparselt-cu12==0.6.2 ; python_version >= "3.10" and python_version < "4.0" and platform_machine == "x86_64" and platform_system == "Linux"
+nvidia-nccl-cu12==2.21.5 ; python_version >= "3.10" and python_version < "4.0" and platform_machine == "x86_64" and platform_system == "Linux"
+nvidia-nvjitlink-cu12==12.4.127 ; python_version >= "3.10" and python_version < "4.0" and platform_machine == "x86_64" and platform_system == "Linux"
+nvidia-nvtx-cu12==12.4.127 ; python_version >= "3.10" and python_version < "4.0" and platform_machine == "x86_64" and platform_system == "Linux"
+openai==1.70.0 ; python_version >= "3.10" and python_version < "4.0"
+opentelemetry-api==1.31.1 ; python_version >= "3.10" and python_version < "4.0"
+opentelemetry-proto==1.31.1 ; python_version >= "3.10" and python_version < "4.0"
+opentelemetry-sdk==1.31.1 ; python_version >= "3.10" and python_version < "4.0"
+opentelemetry-semantic-conventions==0.52b1 ; python_version >= "3.10" and python_version < "4.0"
+orjson==3.10.16 ; python_version >= "3.10" and python_version < "4.0" and platform_python_implementation != "PyPy"
+overrides==7.7.0 ; python_version >= "3.10" and python_version < "4.0"
+packaging==24.2 ; python_version >= "3.10" and python_version < "4.0"
+pandas==2.2.3 ; python_version >= "3.10" and python_version < "4.0"
+pandocfilters==1.5.1 ; python_version >= "3.10" and python_version < "4.0"
+parso==0.8.4 ; python_version >= "3.10" and python_version < "4.0"
+pexpect==4.9.0 ; python_version >= "3.10" and python_version < "4.0"
+pillow==11.1.0 ; python_version >= "3.10" and python_version < "4.0"
+pkginfo==1.12.1.2 ; python_version >= "3.10" and python_version < "4.0"
+platformdirs==4.3.7 ; python_version >= "3.10" and python_version < "4.0"
+plotly==5.24.1 ; python_version >= "3.10" and python_version < "4.0"
+poetry-core==1.9.1 ; python_version >= "3.10" and python_version < "4.0"
+poetry-plugin-export==1.8.0 ; python_version >= "3.10" and python_version < "4.0"
+poetry==1.8.5 ; python_version >= "3.10" and python_version < "4.0"
+prometheus-client==0.21.1 ; python_version >= "3.10" and python_version < "4.0"
+prompt-toolkit==3.0.50 ; python_version >= "3.10" and python_version < "4.0"
+propcache==0.3.1 ; python_version >= "3.10" and python_version < "4.0"
+protobuf==5.29.4 ; python_version >= "3.10" and python_version < "4.0"
+psutil==5.9.8 ; python_version >= "3.10" and python_version < "4.0"
+ptyprocess==0.7.0 ; python_version >= "3.10" and python_version < "4.0"
+pure-eval==0.2.3 ; python_version >= "3.10" and python_version < "4.0"
+pyarrow==19.0.1 ; python_version >= "3.10" and python_version < "4.0"
+pycparser==2.22 ; python_version >= "3.10" and python_version < "4.0"
+pydantic-core==2.33.0 ; python_version >= "3.10" and python_version < "4.0"
+pydantic-settings==2.8.1 ; python_version >= "3.10" and python_version < "4.0"
+pydantic==2.11.1 ; python_version >= "3.10" and python_version < "4.0"
+pydeck==0.9.1 ; python_version >= "3.10" and python_version < "4.0"
+pygments==2.19.1 ; python_version >= "3.10" and python_version < "4.0"
+pypdf==5.4.0 ; python_version >= "3.10" and python_version < "4.0"
+pyproject-hooks==1.2.0 ; python_version >= "3.10" and python_version < "4.0"
+python-dateutil==2.9.0.post0 ; python_version >= "3.10" and python_version < "4.0"
+python-decouple==3.8 ; python_version >= "3.10" and python_version < "4.0"
+python-dotenv==1.1.0 ; python_version >= "3.10" and python_version < "4.0"
+python-json-logger==3.3.0 ; python_version >= "3.10" and python_version < "4.0"
+pytz==2025.2 ; python_version >= "3.10" and python_version < "4.0"
+pywin32-ctypes==0.2.3 ; python_version >= "3.10" and python_version < "4.0" and sys_platform == "win32"
+pywin32==310 ; python_version >= "3.10" and python_version < "4.0" and platform_python_implementation != "PyPy" and sys_platform == "win32"
+pywinpty==2.0.15 ; python_version >= "3.10" and python_version < "4.0" and os_name == "nt"
+pyyaml==6.0.2 ; python_version >= "3.10" and python_version < "4.0"
+pyzmq==26.3.0 ; python_version >= "3.10" and python_version < "4.0"
+rapidfuzz==3.12.2 ; python_version >= "3.10" and python_version < "4.0"
+referencing==0.36.2 ; python_version >= "3.10" and python_version < "4.0"
+regex==2024.11.6 ; python_version >= "3.10" and python_version < "4.0"
+requests-toolbelt==1.0.0 ; python_version >= "3.10" and python_version < "4.0"
+requests==2.32.3 ; python_version >= "3.10" and python_version < "4.0"
+rfc3339-validator==0.1.4 ; python_version >= "3.10" and python_version < "4.0"
+rfc3986-validator==0.1.1 ; python_version >= "3.10" and python_version < "4.0"
+rich==13.9.4 ; python_version >= "3.10" and python_version < "4.0"
+rpds-py==0.24.0 ; python_version >= "3.10" and python_version < "4.0"
+safetensors==0.5.3 ; python_version >= "3.10" and python_version < "4.0"
+scikit-learn==1.6.1 ; python_version >= "3.10" and python_version < "4.0"
+scipy==1.15.2 ; python_version >= "3.10" and python_version < "4.0"
+secretstorage==3.3.3 ; python_version >= "3.10" and python_version < "4.0" and sys_platform == "linux"
+send2trash==1.8.3 ; python_version >= "3.10" and python_version < "4.0"
+sentence-transformers==4.0.2 ; python_version >= "3.10" and python_version < "4.0"
+setuptools==78.1.0 ; python_version >= "3.10" and python_version < "4.0"
+shellingham==1.5.4 ; python_version >= "3.10" and python_version < "4.0"
+six==1.17.0 ; python_version >= "3.10" and python_version < "4.0"
+smmap==5.0.2 ; python_version >= "3.10" and python_version < "4.0"
+sniffio==1.3.1 ; python_version >= "3.10" and python_version < "4.0"
+soupsieve==2.6 ; python_version >= "3.10" and python_version < "4.0"
+sqlalchemy==2.0.40 ; python_version >= "3.10" and python_version < "4.0"
+stack-data==0.6.3 ; python_version >= "3.10" and python_version < "4.0"
+streamlit-aggrid==1.1.2 ; python_version >= "3.10" and python_version < "4.0"
+streamlit==1.44.1 ; python_version >= "3.10" and python_version < "4.0"
+striprtf==0.0.26 ; python_version >= "3.10" and python_version < "4.0"
+sympy==1.13.1 ; python_version >= "3.10" and python_version < "4.0"
+tenacity==9.1.2 ; python_version >= "3.10" and python_version < "4.0"
+terminado==0.18.1 ; python_version >= "3.10" and python_version < "4.0"
+threadpoolctl==3.6.0 ; python_version >= "3.10" and python_version < "4.0"
+tiktoken==0.9.0 ; python_version >= "3.10" and python_version < "4.0"
+tinycss2==1.4.0 ; python_version >= "3.10" and python_version < "4.0"
+tokenizers==0.21.1 ; python_version >= "3.10" and python_version < "4.0"
+toml==0.10.2 ; python_version >= "3.10" and python_version < "4.0"
+tomli==2.2.1 ; python_version == "3.10"
+tomlkit==0.13.2 ; python_version >= "3.10" and python_version < "4.0"
+torch==2.6.0 ; python_version >= "3.10" and python_version < "4.0"
+tornado==6.4.2 ; python_version >= "3.10" and python_version < "4.0"
+tqdm==4.67.1 ; python_version >= "3.10" and python_version < "4.0"
+traitlets==5.14.3 ; python_version >= "3.10" and python_version < "4.0"
+transformers==4.50.3 ; python_version >= "3.10" and python_version < "4.0"
+triton==3.2.0 ; python_version >= "3.10" and python_version < "4.0" and platform_machine == "x86_64" and platform_system == "Linux"
+trove-classifiers==2025.3.19.19 ; python_version >= "3.10" and python_version < "4.0"
+trulens-apps-langchain==1.4.8 ; python_version >= "3.10" and python_version < "4.0"
+trulens-apps-llamaindex==1.4.8 ; python_version >= "3.10" and python_version < "4.0"
+trulens-core==1.4.7 ; python_version >= "3.10" and python_version < "4.0"
+trulens-dashboard==1.4.7 ; python_version >= "3.10" and python_version < "4.0"
+trulens-eval==1.4.7 ; python_version >= "3.10" and python_version < "4.0"
+trulens-feedback==1.4.7 ; python_version >= "3.10" and python_version < "4.0"
+trulens-otel-semconv==1.4.7 ; python_version >= "3.10" and python_version < "4.0"
+trulens-providers-openai==1.4.8 ; python_version >= "3.10" and python_version < "4.0"
+trulens==1.4.7 ; python_version >= "3.10" and python_version < "4.0"
+types-python-dateutil==2.9.0.20241206 ; python_version >= "3.10" and python_version < "4.0"
+typing-extensions==4.13.0 ; python_version >= "3.10" and python_version < "4.0"
+typing-inspect==0.9.0 ; python_version >= "3.10" and python_version < "4.0"
+typing-inspection==0.4.0 ; python_version >= "3.10" and python_version < "4.0"
+tzdata==2025.2 ; python_version >= "3.10" and python_version < "4.0"
+uri-template==1.3.0 ; python_version >= "3.10" and python_version < "4.0"
+urllib3==2.3.0 ; python_version >= "3.10" and python_version < "4.0"
+virtualenv==20.30.0 ; python_version >= "3.10" and python_version < "4.0"
+watchdog==6.0.0 ; python_version >= "3.10" and python_version < "4.0" and platform_system != "Darwin"
+wcwidth==0.2.13 ; python_version >= "3.10" and python_version < "4.0"
+webcolors==24.11.1 ; python_version >= "3.10" and python_version < "4.0"
+webencodings==0.5.1 ; python_version >= "3.10" and python_version < "4.0"
+websocket-client==1.8.0 ; python_version >= "3.10" and python_version < "4.0"
+widgetsnbextension==4.0.13 ; python_version >= "3.10" and python_version < "4.0"
+wrapt==1.17.2 ; python_version >= "3.10" and python_version < "4.0"
+xattr==1.1.4 ; python_version >= "3.10" and python_version < "4.0" and sys_platform == "darwin"
+yarl==1.18.3 ; python_version >= "3.10" and python_version < "4.0"
+zipp==3.21.0 ; python_version >= "3.10" and python_version < "4.0"
+zstandard==0.23.0 ; python_version >= "3.10" and python_version < "4.0"

src/mythesis_chatbot/rag_setup.py

@@ -0,0 +1,264 @@
+import json
+import os
+from typing import Literal
+
+import openai
+from llama_index.core import (
+    Document,
+    Settings,
+    SimpleDirectoryReader,
+    StorageContext,
+    VectorStoreIndex,
+    load_index_from_storage,
+)
+from llama_index.core.node_parser import (
+    HierarchicalNodeParser,
+    SentenceWindowNodeParser,
+    get_leaf_nodes,
+)
+from llama_index.core.postprocessor import (
+    MetadataReplacementPostProcessor,
+    SentenceTransformerRerank,
+)
+from llama_index.core.query_engine import RetrieverQueryEngine
+from llama_index.core.retrievers import AutoMergingRetriever
+from llama_index.embeddings.huggingface import HuggingFaceEmbedding
+from llama_index.llms.openai import OpenAI
+
+from mythesis_chatbot.utils import get_config_hash, get_openai_api_key
+
+SupportedRags = Literal["basic", "sentence window retrieval", "auto-merging retrieval"]
+SupportedOpenAIllms = Literal["gpt-4o-mini", "gpt-3.5-turbo"]
+SupportedEmbedModels = Literal["BAAI/bge-small-en-v1.5"]
+SupportedRerankModels = Literal["cross-encoder/ms-marco-MiniLM-L-2-v2"]
+
+
+def load_data(input_file: str) -> Document:
+
+    reader = SimpleDirectoryReader(input_files=[input_file])
+    documents = reader.load_data()  # List of Document objects (one object per page)
+    # Merge into single document
+    document = Document(text="\n\n".join([doc.text for doc in documents]))
+
+    return document
+
+
+def build_sentence_window_index(
+    input_file: str,
+    save_dir: str,
+    index_config: dict[str, str | int],
+):
+    config_hash = get_config_hash(index_config)
+    save_dir = os.path.join(save_dir, "sentence_window", config_hash)
+
+    Settings.embed_model = HuggingFaceEmbedding(model_name=index_config["embed_model"])
+
+    if not os.path.exists(save_dir):
+
+        document = load_data(input_file)
+
+        # Create the sentence window node parser w/ default settings.
+        # A node is a chunck of text. Each node returned by the sentence window node
+        # parser also contains its context as metadata (closest chuncks of texts)
+        node_parser = SentenceWindowNodeParser.from_defaults(
+            window_size=index_config["sentence_window_size"],
+            window_metadata_key="window",
+            original_text_metadata_key="original_text",
+        )
+
+        Settings.node_parser = node_parser
+
+        sentence_index = VectorStoreIndex.from_documents([document])
+        sentence_index.storage_context.persist(persist_dir=save_dir)
+        with open(os.path.join(save_dir, "meta.json"), "w") as f:
+            json.dump(index_config, f, indent=2)
+
+    else:
+        sentence_index = load_index_from_storage(
+            StorageContext.from_defaults(persist_dir=save_dir)
+        )
+
+    return sentence_index
+
+
+def build_automerging_index(
+    input_file: str,
+    save_dir: str,
+    index_config: dict[str, str | list[int]],
+):
+
+    config_hash = get_config_hash(index_config)
+    save_dir = os.path.join(save_dir, "auto_merging", config_hash)
+
+    Settings.embed_model = HuggingFaceEmbedding(model_name=index_config["embed_model"])
+
+    if not os.path.exists(save_dir):
+
+        document = load_data(input_file)
+        node_parser = HierarchicalNodeParser.from_defaults(
+            chunk_sizes=index_config["chunk_sizes"]
+        )
+        nodes = node_parser.get_nodes_from_documents([document])
+        leaf_nodes = get_leaf_nodes(nodes)
+
+        Settings.node_parser = node_parser
+
+        storage_context = StorageContext.from_defaults()
+        storage_context.docstore.add_documents(nodes)
+
+        automerging_index = VectorStoreIndex(
+            leaf_nodes,
+            storage_context=storage_context,
+        )
+        automerging_index.storage_context.persist(persist_dir=save_dir)
+        with open(os.path.join(save_dir, "meta.json"), "w") as f:
+            json.dump(index_config, f, indent=2)
+
+    else:
+        automerging_index = load_index_from_storage(
+            StorageContext.from_defaults(persist_dir=save_dir),
+        )
+    return automerging_index
+
+
+def get_sentence_window_query_engine(
+    sentence_index,
+    similarity_top_k: int = 6,
+    rerank_top_n: int = 2,
+    rerank_model: str = "cross-encoder/ms-marco-MiniLM-L-2-v2",
+):
+    # Used to replace the node content with a field from the node metadata.
+    postproc = MetadataReplacementPostProcessor(target_metadata_key="window")
+
+    # Rerank can speed up an LLM query without sacrificing accuracy. It does so by
+    # pruning away irrelevant nodes from the context.
+    rerank = SentenceTransformerRerank(top_n=rerank_top_n, model=rerank_model)
+
+    sentence_window_engine = sentence_index.as_query_engine(
+        similarity_top_k=similarity_top_k, node_postprocessors=[postproc, rerank]
+    )
+    return sentence_window_engine
+
+
+def get_automerging_query_engine(
+    automerging_index,
+    similarity_top_k: int = 12,
+    rerank_top_n: int = 6,
+    rerank_model: str = "cross-encoder/ms-marco-MiniLM-L-2-v2",
+):
+    base_retriever = automerging_index.as_retriever(similarity_top_k=similarity_top_k)
+    retriever = AutoMergingRetriever(
+        base_retriever, automerging_index.storage_context, verbose=True
+    )
+    rerank = SentenceTransformerRerank(top_n=rerank_top_n, model=rerank_model)
+    auto_merging_engine = RetrieverQueryEngine.from_args(
+        retriever, node_postprocessors=[rerank]
+    )
+    return auto_merging_engine
+
+
+def sentence_window_retrieval_setup(
+    input_file: str,
+    save_dir: str,
+    llm_openai_model: SupportedOpenAIllms = "gpt-4o-mini",
+    temperature: float = 0.1,
+    embed_model: SupportedEmbedModels = "BAAI/bge-small-en-v1.5",
+    sentence_window_size: int = 3,
+    similarity_top_k: int = 6,
+    rerank_model: SupportedRerankModels = "cross-encoder/ms-marco-MiniLM-L-2-v2",
+    rerank_top_n: int = 2,
+):
+
+    openai.api_key = get_openai_api_key()
+
+    # This allows to uniquely identify the index
+    config = {
+        "doc_source": os.path.basename(input_file),
+        "embed_model": embed_model,
+        "sentence_window_size": sentence_window_size,
+    }
+
+    # 1. Build index
+    index = build_sentence_window_index(input_file, save_dir, config)
+
+    Settings.llm = OpenAI(model=llm_openai_model, temperature=temperature)
+
+    # 2. Get engine
+    sentence_window_engine = get_sentence_window_query_engine(
+        index,
+        similarity_top_k=similarity_top_k,
+        rerank_model=rerank_model,
+        rerank_top_n=rerank_top_n,
+    )
+
+    return sentence_window_engine
+
+
+def automerging_retrieval_setup(
+    input_file: str,
+    save_dir: str,
+    llm_openai_model: SupportedOpenAIllms = "gpt-4o-mini",
+    temperature: float = 0.1,
+    embed_model: SupportedEmbedModels = "BAAI/bge-small-en-v1.5",
+    chunk_sizes=[2048, 512, 128],
+    similarity_top_k: int = 6,
+    rerank_model: SupportedRerankModels = "cross-encoder/ms-marco-MiniLM-L-2-v2",
+    rerank_top_n: int = 2,
+):
+    openai.api_key = get_openai_api_key()
+
+    # This allows to uniquely identify the index
+    config = {
+        "doc_source": os.path.basename(input_file),
+        "embed_model": embed_model,
+        "chunk_sizes": chunk_sizes,
+    }
+
+    # 1. Build index
+    index = build_automerging_index(input_file, save_dir, config)
+
+    Settings.llm = OpenAI(model=llm_openai_model, temperature=temperature)
+
+    # 2. Get engine
+    automerging_engine = get_sentence_window_query_engine(
+        index,
+        similarity_top_k=similarity_top_k,
+        rerank_model=rerank_model,
+        rerank_top_n=rerank_top_n,
+    )
+
+    return automerging_engine
+
+
+def basic_rag_setup(
+    input_file: str,
+    save_dir: str,
+    llm_openai_model: SupportedOpenAIllms = "gpt-4o-mini",
+    temperature: float = 0.1,
+    embed_model: SupportedEmbedModels = "BAAI/bge-small-en-v1.5",
+    similarity_top_k: int = 6,
+    rerank_model: SupportedRerankModels = "cross-encoder/ms-marco-MiniLM-L-2-v2",
+    rerank_top_n: int = 2,
+):
+    openai.api_key = get_openai_api_key()
+
+    Settings.embed_model = HuggingFaceEmbedding(model_name=embed_model)
+
+    save_dir = os.path.join(save_dir, "basic")
+    if not os.path.exists(save_dir):
+        document = load_data(input_file)
+        index = VectorStoreIndex.from_documents([document])
+        index.storage_context.persist(persist_dir=save_dir)
+    else:
+        index = load_index_from_storage(
+            StorageContext.from_defaults(persist_dir=save_dir)
+        )
+
+    rerank = SentenceTransformerRerank(top_n=rerank_top_n, model=rerank_model)
+
+    engine = index.as_query_engine(
+        llm=OpenAI(model=llm_openai_model, temperature=temperature),
+        similarity_top_k=similarity_top_k,
+        node_postprocessors=[rerank],
+    )
+    return engine

src/mythesis_chatbot/utils.py

@@ -0,0 +1,23 @@
+import hashlib
+import json
+import os
+
+
+def get_config_hash(config: dict) -> str:
+    # Use JSON to serialize and sort keys for deterministic output
+    config_str = json.dumps(config, sort_keys=True)
+
+    return hashlib.sha256(config_str.encode()).hexdigest()[:10]  # short hash
+
+
+def get_openai_api_key():
+    """
+    Get the OpenAI API key from an environment variable.
+    """
+    api_key = os.getenv("OPENAI_API_KEY")
+    if api_key:
+        return api_key
+
+    raise ValueError(
+        "OpenAI API key not found. Please follow the instruction in the readme file."
+    )