File size: 3,685 Bytes
1476939 dc94b28 1476939 dc94b28 904d53f 7e51da8 1476939 dc94b28 904d53f 7e51da8 1476939 2c0c391 dc94b28 904d53f 7e51da8 2c0c391 7d69384 dc94b28 657847e eb1e30c 1476939 7d69384 dc94b28 657847e eb1e30c 1476939 dc94b28 657847e eb1e30c 1476939 dc94b28 657847e eb1e30c 1476939 dc94b28 657847e eb1e30c 1476939 657847e eb1e30c 1476939 1b3d55c eb1e30c 1b3d55c eb1e30c 1b3d55c fcdfb63 1476939 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 |
__all__ = ['AgentCostManager']
class AgentCostManager():
dependencies = []
inputSchema = {
"name": "AgentCostManager",
"description": "Retrieves the cost of creating and invoking an agent. Also includes the strengths of each model. Please make sure to use this before creating an agent.",
"parameters": {
"type": "object",
"properties": {},
"required": [],
}
}
costs = {
"llama3.2": {
"description": "Avg Accuracy: 49.75%, Latency 0.9s, 63.4% on multi-task understanding, 40.8% on rewriting, 78.6% on reasoning.",
"create_resource_cost": 10,
"invoke_resource_cost": 40,
},
"mistral": {
"description": "Avg Accuracy: 51.3%, Latency 9.7s, 51% on LegalBench, 60.1% on multi-task understanding, 69.9% on TriviaQA, 67.9% on reasoning",
"create_resource_cost": 20,
"invoke_resource_cost": 100,
},
"deepseek-r1": {
"description": "Avg Accuracy: 77.3%, Latency: 120s, 69.9% on LegalBench, 71.1% on multi-task understanding, 92.2% on Math",
"create_resource_cost": 20,
"invoke_resource_cost": 150,
},
"gemini-2.5-flash-preview-05-20": {
"description": "Avg Accuracy: 75.8%, 82.8% on LegalBench, 81.6% on multi-task understanding, 91.6% on Math",
"create_expense_cost": 0,
"invoke_expense_cost": 0.15,
"output_expense_cost": 0.60,
},
"gemini-2.5-pro-exp-03-25": {
"description": "Avg Accuracy: 64.3%, 83.6% on LegalBench, 84.1% on multi-task understanding, 95.2% on Math, 63.8% on Coding",
"create_expense_cost": 0,
"invoke_expense_cost": 1.25,
"output_expense_cost": 10.00,
},
"gemini-2.0-flash": {
"description": "Avg Accuracy: 64.3%, 79.9% on LegalBench, 77.4% on multi-task understanding, 90.9% on Math, 34.5% on Coding",
"create_expense_cost": 0,
"invoke_expense_cost": 0.10,
"output_expense_cost": 0.40,
},
"gemini-2.0-flash-lite": {
"description": "Avg Accuracy: 64.1%, 71.6% on multi-task understanding, 86.8% on Math, 28.9% on Coding",
"create_expense_cost": 0,
"invoke_expense_cost": 0.075,
"output_expense_cost": 0.30,
},
"gemini-1.5-flash": {
"description": "62.0% on LegalBench, 61.0% on MMLU, 59.0% on MATH",
"create_expense_cost": 0,
"invoke_expense_cost": 0.075,
"output_expense_cost": 0.30,
},
"gemini-1.5-flash-8b": {
"description": "High volume and lower intelligence tasks",
"create_expense_cost": 0,
"invoke_expense_cost": 0.0375,
"output_expense_cost": 0.15,
},
"groq-qwen-qwq-32b": {
"description": "79.5% on AIME24, is comparable to o1-mini and DeepSeek-R1 on all reasonig tasks",
"create_expense_cost": 0,
"invoke_expense_cost": 0.29,
"output_expense_cost": 0.39,
},
"lambda-hermes3-8b": {
"description": "High volume and lower intelligence tasks, 60.0% on MMLU, 58.0% on MATH",
"create_expense_cost": 0,
"invoke_expense_cost": 0.025,
"output_expense_cost": 0.04,
},
}
def get_costs(self):
return self.costs
def run(self, **kwargs):
return {
"status": "success",
"message": "Cost of creating and invoking an agent",
"output": self.costs,
}
|