File size: 2,839 Bytes
2af0eb7 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 |
# This file contains shared functions used by multiple files
import os
import sys
import pymongo
from dotenv import load_dotenv
from langchain_community.embeddings import OllamaEmbeddings
from langchain_openai.chat_models import ChatOpenAI
from qdrant_client import QdrantClient
from langchain_community.llms import Ollama
# Unused since usage limit reached since years ago...rip
def getOpenAiModel():
MODEL = "gpt-3.5-turbo"
try:
load_dotenv(override=True)
except Exception:
load_dotenv(sys.path[1] + "/.env", override=True)
OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
return ChatOpenAI(openai_api_key=OPENAI_API_KEY, model=MODEL)
# Create a mongoDB connection
def getMongoClient():
try:
load_dotenv(override=True)
except Exception:
load_dotenv(sys.path[1] + "/.env", override=True)
DATABASE_HOST = os.getenv("DATABASE_HOST")
return pymongo.MongoClient(DATABASE_HOST)
# Create a qdrant connection
def getQdrantClient():
try:
load_dotenv(override=True)
except Exception:
load_dotenv(sys.path[1] + "/.env", override=True)
USE_QDRANT_CLOUD = os.getenv("USE_QDRANT_CLOUD")
QDRANT_CLOUD_URL = os.getenv("QDRANT_CLOUD_URL")
QDRANT_APIKEY = os.getenv("QDRANT_APIKEY")
if USE_QDRANT_CLOUD=="True":
return QdrantClient(url=QDRANT_CLOUD_URL, api_key=QDRANT_APIKEY)
else:
return QdrantClient(url=QDRANT_CLOUD_URL)
# Setup the text embedder
def getEmbeddingsModel(MODEL="llama3.2"):
try:
load_dotenv(override=True)
except Exception:
load_dotenv(sys.path[1] + "/.env", override=True)
USE_DOCKER = os.getenv("USE_DOCKER")
if USE_DOCKER == "True":
return OllamaEmbeddings(model=MODEL, base_url="http://host.docker.internal:11434")
else:
return OllamaEmbeddings(model=MODEL)
# Setup the model
def getModel(MODEL="llama3.2"):
try:
load_dotenv(override=True)
except Exception:
load_dotenv(sys.path[1] + "/.env", override=True)
USE_DOCKER = os.getenv("USE_DOCKER")
if USE_DOCKER == "True":
return Ollama(model=MODEL, base_url="http://host.docker.internal:11434")
else:
return Ollama(model=MODEL)
# Setup clearML
def setupClearML():
try:
load_dotenv(override=True)
except Exception:
load_dotenv(sys.path[1] + "/.env", override=True)
CLEARML_WEB_HOST = os.getenv("CLEARML_WEB_HOST")
CLEARML_API_HOST = os.getenv("CLEARML_API_HOST")
CLEARML_FILES_HOST = os.getenv("CLEARML_FILES_HOST")
CLEARML_API_ACCESS_KEY = os.getenv("CLEARML_API_ACCESS_KEY")
CLEARML_API_SECRET_KEY = os.getenv("CLEARML_API_SECRETKEY")
return (
CLEARML_WEB_HOST,
CLEARML_API_HOST,
CLEARML_FILES_HOST,
CLEARML_API_ACCESS_KEY,
CLEARML_API_SECRET_KEY,
)
|