[ { "model":"amazon\/nova-micro-v1", "bcp_47":"ar", "task":"classification", "metric":"accuracy", "score":0.8 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"ar", "task":"mmlu", "metric":"accuracy", "score":0.6 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"ar", "task":"translation_from", "metric":"bleu", "score":0.2294045445 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"ar", "task":"translation_from", "metric":"chrf", "score":0.4952691669 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"ar", "task":"translation_to", "metric":"bleu", "score":0.3288016428 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"ar", "task":"translation_to", "metric":"chrf", "score":0.5463705477 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"bn", "task":"classification", "metric":"accuracy", "score":0.8 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"bn", "task":"mmlu", "metric":"accuracy", "score":0.3 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"bn", "task":"translation_from", "metric":"bleu", "score":0.2027365964 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"bn", "task":"translation_from", "metric":"chrf", "score":0.4429221375 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"bn", "task":"translation_to", "metric":"bleu", "score":0.2592014549 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"bn", "task":"translation_to", "metric":"chrf", "score":0.4502390132 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"de", "task":"classification", "metric":"accuracy", "score":0.7 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"de", "task":"mmlu", "metric":"accuracy", "score":0.5 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"de", "task":"translation_from", "metric":"bleu", "score":0.2486422994 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"de", "task":"translation_from", "metric":"chrf", "score":0.5046441311 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"de", "task":"translation_to", "metric":"bleu", "score":0.390442231 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"de", "task":"translation_to", "metric":"chrf", "score":0.6258445826 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"en", "task":"classification", "metric":"accuracy", "score":0.8 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"en", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"en", "task":"translation_from", "metric":"bleu", "score":0.4829593782 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"en", "task":"translation_from", "metric":"chrf", "score":0.6670785718 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"en", "task":"translation_to", "metric":"bleu", "score":0.5654501085 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"en", "task":"translation_to", "metric":"chrf", "score":0.7810071072 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"es", "task":"classification", "metric":"accuracy", "score":0.8 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"es", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"es", "task":"translation_from", "metric":"bleu", "score":0.2334919219 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"es", "task":"translation_from", "metric":"chrf", "score":0.488808171 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"es", "task":"translation_to", "metric":"bleu", "score":0.3534859166 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"es", "task":"translation_to", "metric":"chrf", "score":0.5837022928 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"fr", "task":"classification", "metric":"accuracy", "score":0.8 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"fr", "task":"mmlu", "metric":"accuracy", "score":0.6 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"fr", "task":"translation_from", "metric":"bleu", "score":0.2616894756 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"fr", "task":"translation_from", "metric":"chrf", "score":0.5082668021 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"fr", "task":"translation_to", "metric":"bleu", "score":0.4039820936 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"fr", "task":"translation_to", "metric":"chrf", "score":0.5963334297 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"hi", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"hi", "task":"mmlu", "metric":"accuracy", "score":0.5 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"hi", "task":"translation_from", "metric":"bleu", "score":0.3412962275 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"hi", "task":"translation_from", "metric":"chrf", "score":0.5789604921 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"hi", "task":"translation_to", "metric":"bleu", "score":0.3866780476 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"hi", "task":"translation_to", "metric":"chrf", "score":0.60404499 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"id", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"id", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"id", "task":"translation_from", "metric":"bleu", "score":0.1741677254 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"id", "task":"translation_from", "metric":"chrf", "score":0.4469342589 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"id", "task":"translation_to", "metric":"bleu", "score":0.1716570673 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"id", "task":"translation_to", "metric":"chrf", "score":0.5608938423 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"ja", "task":"classification", "metric":"accuracy", "score":0.7 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"ja", "task":"mmlu", "metric":"accuracy", "score":0.5 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"ja", "task":"translation_from", "metric":"bleu", "score":0.2197709837 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"ja", "task":"translation_from", "metric":"chrf", "score":0.4784848367 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"ja", "task":"translation_to", "metric":"bleu", "score":0.3426239655 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"ja", "task":"translation_to", "metric":"chrf", "score":0.4822747548 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"mr", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"mr", "task":"translation_from", "metric":"bleu", "score":0.1387332093 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"mr", "task":"translation_from", "metric":"chrf", "score":0.4231622054 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"mr", "task":"translation_to", "metric":"bleu", "score":0.1326285083 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"mr", "task":"translation_to", "metric":"chrf", "score":0.3320956129 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"pa", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"pa", "task":"translation_from", "metric":"bleu", "score":0.3548435009 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"pa", "task":"translation_from", "metric":"chrf", "score":0.6046727327 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"pa", "task":"translation_to", "metric":"bleu", "score":0.3341282505 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"pa", "task":"translation_to", "metric":"chrf", "score":0.4931240563 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"pt", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"pt", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"pt", "task":"translation_from", "metric":"bleu", "score":0.2534618489 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"pt", "task":"translation_from", "metric":"chrf", "score":0.4757478619 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"pt", "task":"translation_to", "metric":"bleu", "score":0.4150555406 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"pt", "task":"translation_to", "metric":"chrf", "score":0.632650236 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"ru", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"ru", "task":"mmlu", "metric":"accuracy", "score":0.6 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"ru", "task":"translation_from", "metric":"bleu", "score":0.173300895 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"ru", "task":"translation_from", "metric":"chrf", "score":0.4267275321 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"ru", "task":"translation_to", "metric":"bleu", "score":0.2934481188 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"ru", "task":"translation_to", "metric":"chrf", "score":0.5432096638 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"sw", "task":"classification", "metric":"accuracy", "score":0.8 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"sw", "task":"mmlu", "metric":"accuracy", "score":0.5 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"sw", "task":"translation_from", "metric":"bleu", "score":0.203432711 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"sw", "task":"translation_from", "metric":"chrf", "score":0.4661592161 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"sw", "task":"translation_to", "metric":"bleu", "score":0.260138004 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"sw", "task":"translation_to", "metric":"chrf", "score":0.5610336232 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"te", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"te", "task":"mmlu", "metric":"accuracy", "score":0.4 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"te", "task":"translation_from", "metric":"bleu", "score":0.3031235597 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"te", "task":"translation_from", "metric":"chrf", "score":0.537458124 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"te", "task":"translation_to", "metric":"bleu", "score":0.3112392832 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"te", "task":"translation_to", "metric":"chrf", "score":0.5195600347 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"ur", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"ur", "task":"translation_from", "metric":"bleu", "score":0.2338522454 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"ur", "task":"translation_from", "metric":"chrf", "score":0.4761317098 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"ur", "task":"translation_to", "metric":"bleu", "score":0.1016530484 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"ur", "task":"translation_to", "metric":"chrf", "score":0.3475324071 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"zh", "task":"classification", "metric":"accuracy", "score":0.7 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"zh", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"zh", "task":"translation_from", "metric":"bleu", "score":0.1734447333 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"zh", "task":"translation_from", "metric":"chrf", "score":0.4737881775 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"zh", "task":"translation_to", "metric":"bleu", "score":0.2855931625 }, { "model":"amazon\/nova-micro-v1", "bcp_47":"zh", "task":"translation_to", "metric":"chrf", "score":0.3243323503 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"ar", "task":"classification", "metric":"accuracy", "score":0.8 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"ar", "task":"mmlu", "metric":"accuracy", "score":1.0 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"ar", "task":"translation_from", "metric":"bleu", "score":0.1806181405 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"ar", "task":"translation_from", "metric":"chrf", "score":0.4614154717 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"ar", "task":"translation_to", "metric":"bleu", "score":0.3184146682 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"ar", "task":"translation_to", "metric":"chrf", "score":0.4885651648 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"bn", "task":"classification", "metric":"accuracy", "score":0.8 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"bn", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"bn", "task":"translation_from", "metric":"bleu", "score":0.1707650511 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"bn", "task":"translation_from", "metric":"chrf", "score":0.4191623205 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"bn", "task":"translation_to", "metric":"bleu", "score":0.1747239771 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"bn", "task":"translation_to", "metric":"chrf", "score":0.4138755715 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"de", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"de", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"de", "task":"translation_from", "metric":"bleu", "score":0.214232349 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"de", "task":"translation_from", "metric":"chrf", "score":0.4760523398 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"de", "task":"translation_to", "metric":"bleu", "score":0.3694909412 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"de", "task":"translation_to", "metric":"chrf", "score":0.6379394313 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"en", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"en", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"en", "task":"translation_from", "metric":"bleu", "score":0.3898301226 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"en", "task":"translation_from", "metric":"chrf", "score":0.6012929861 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"en", "task":"translation_to", "metric":"bleu", "score":0.488423591 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"en", "task":"translation_to", "metric":"chrf", "score":0.7529549619 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"es", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"es", "task":"mmlu", "metric":"accuracy", "score":0.9 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"es", "task":"translation_from", "metric":"bleu", "score":0.2337674235 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"es", "task":"translation_from", "metric":"chrf", "score":0.4951544114 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"es", "task":"translation_to", "metric":"bleu", "score":0.341066665 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"es", "task":"translation_to", "metric":"chrf", "score":0.6161709719 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"fr", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"fr", "task":"mmlu", "metric":"accuracy", "score":0.6 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"fr", "task":"translation_from", "metric":"bleu", "score":0.199697024 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"fr", "task":"translation_from", "metric":"chrf", "score":0.4970728001 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"fr", "task":"translation_to", "metric":"bleu", "score":0.548967737 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"fr", "task":"translation_to", "metric":"chrf", "score":0.7095016995 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"hi", "task":"classification", "metric":"accuracy", "score":0.8 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"hi", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"hi", "task":"translation_from", "metric":"bleu", "score":0.367723046 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"hi", "task":"translation_from", "metric":"chrf", "score":0.5881358889 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"hi", "task":"translation_to", "metric":"bleu", "score":0.3278391831 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"hi", "task":"translation_to", "metric":"chrf", "score":0.5487381665 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"id", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"id", "task":"mmlu", "metric":"accuracy", "score":0.9 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"id", "task":"translation_from", "metric":"bleu", "score":0.2012670144 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"id", "task":"translation_from", "metric":"chrf", "score":0.4903423287 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"id", "task":"translation_to", "metric":"bleu", "score":0.2590870164 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"id", "task":"translation_to", "metric":"chrf", "score":0.5931106206 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"ja", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"ja", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"ja", "task":"translation_from", "metric":"bleu", "score":0.1789581673 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"ja", "task":"translation_from", "metric":"chrf", "score":0.4444827041 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"ja", "task":"translation_to", "metric":"bleu", "score":0.175930196 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"ja", "task":"translation_to", "metric":"chrf", "score":0.329462033 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"mr", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"mr", "task":"translation_from", "metric":"bleu", "score":0.1834187267 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"mr", "task":"translation_from", "metric":"chrf", "score":0.470878386 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"mr", "task":"translation_to", "metric":"bleu", "score":0.1744972384 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"mr", "task":"translation_to", "metric":"chrf", "score":0.4213302564 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"pa", "task":"classification", "metric":"accuracy", "score":0.8 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"pa", "task":"translation_from", "metric":"bleu", "score":0.3488186237 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"pa", "task":"translation_from", "metric":"chrf", "score":0.5904301709 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"pa", "task":"translation_to", "metric":"bleu", "score":0.3621810085 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"pa", "task":"translation_to", "metric":"chrf", "score":0.51265991 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"pt", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"pt", "task":"mmlu", "metric":"accuracy", "score":1.0 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"pt", "task":"translation_from", "metric":"bleu", "score":0.2014349032 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"pt", "task":"translation_from", "metric":"chrf", "score":0.458360538 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"pt", "task":"translation_to", "metric":"bleu", "score":0.4537177243 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"pt", "task":"translation_to", "metric":"chrf", "score":0.6780859486 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"ru", "task":"classification", "metric":"accuracy", "score":0.8 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"ru", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"ru", "task":"translation_from", "metric":"bleu", "score":0.2192993586 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"ru", "task":"translation_from", "metric":"chrf", "score":0.5037493911 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"ru", "task":"translation_to", "metric":"bleu", "score":0.3365435366 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"ru", "task":"translation_to", "metric":"chrf", "score":0.5803139095 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"sw", "task":"classification", "metric":"accuracy", "score":0.8 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"sw", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"sw", "task":"translation_from", "metric":"bleu", "score":0.1629528612 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"sw", "task":"translation_from", "metric":"chrf", "score":0.3934230515 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"sw", "task":"translation_to", "metric":"bleu", "score":0.2569355275 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"sw", "task":"translation_to", "metric":"chrf", "score":0.5756052088 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"te", "task":"classification", "metric":"accuracy", "score":0.8 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"te", "task":"mmlu", "metric":"accuracy", "score":0.3 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"te", "task":"translation_from", "metric":"bleu", "score":0.3030045526 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"te", "task":"translation_from", "metric":"chrf", "score":0.5524023581 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"te", "task":"translation_to", "metric":"bleu", "score":0.197839972 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"te", "task":"translation_to", "metric":"chrf", "score":0.4780950203 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"ur", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"ur", "task":"translation_from", "metric":"bleu", "score":0.1997540464 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"ur", "task":"translation_from", "metric":"chrf", "score":0.4750614684 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"ur", "task":"translation_to", "metric":"bleu", "score":0.1234051854 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"ur", "task":"translation_to", "metric":"chrf", "score":0.2799574404 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"zh", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"zh", "task":"mmlu", "metric":"accuracy", "score":1.0 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"zh", "task":"translation_from", "metric":"bleu", "score":0.1922766267 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"zh", "task":"translation_from", "metric":"chrf", "score":0.4644593408 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"zh", "task":"translation_to", "metric":"bleu", "score":0.2227195824 }, { "model":"anthropic\/claude-3-haiku", "bcp_47":"zh", "task":"translation_to", "metric":"chrf", "score":0.2687577774 }, { "model":"cohere\/command-r", "bcp_47":"ar", "task":"classification", "metric":"accuracy", "score":0.8 }, { "model":"cohere\/command-r", "bcp_47":"ar", "task":"mmlu", "metric":"accuracy", "score":0.6 }, { "model":"cohere\/command-r", "bcp_47":"ar", "task":"translation_from", "metric":"bleu", "score":0.2126059737 }, { "model":"cohere\/command-r", "bcp_47":"ar", "task":"translation_from", "metric":"chrf", "score":0.4462765689 }, { "model":"cohere\/command-r", "bcp_47":"ar", "task":"translation_to", "metric":"bleu", "score":0.1836371582 }, { "model":"cohere\/command-r", "bcp_47":"ar", "task":"translation_to", "metric":"chrf", "score":0.3778102006 }, { "model":"cohere\/command-r", "bcp_47":"bn", "task":"classification", "metric":"accuracy", "score":0.8 }, { "model":"cohere\/command-r", "bcp_47":"bn", "task":"mmlu", "metric":"accuracy", "score":0.4 }, { "model":"cohere\/command-r", "bcp_47":"bn", "task":"translation_from", "metric":"bleu", "score":0.0953670352 }, { "model":"cohere\/command-r", "bcp_47":"bn", "task":"translation_from", "metric":"chrf", "score":0.3310320034 }, { "model":"cohere\/command-r", "bcp_47":"bn", "task":"translation_to", "metric":"bleu", "score":0.024360011 }, { "model":"cohere\/command-r", "bcp_47":"bn", "task":"translation_to", "metric":"chrf", "score":0.2108567474 }, { "model":"cohere\/command-r", "bcp_47":"de", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"cohere\/command-r", "bcp_47":"de", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"cohere\/command-r", "bcp_47":"de", "task":"translation_from", "metric":"bleu", "score":0.2306746115 }, { "model":"cohere\/command-r", "bcp_47":"de", "task":"translation_from", "metric":"chrf", "score":0.4506487927 }, { "model":"cohere\/command-r", "bcp_47":"de", "task":"translation_to", "metric":"bleu", "score":0.2707214129 }, { "model":"cohere\/command-r", "bcp_47":"de", "task":"translation_to", "metric":"chrf", "score":0.4862384022 }, { "model":"cohere\/command-r", "bcp_47":"en", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"cohere\/command-r", "bcp_47":"en", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"cohere\/command-r", "bcp_47":"en", "task":"translation_from", "metric":"bleu", "score":0.2371132106 }, { "model":"cohere\/command-r", "bcp_47":"en", "task":"translation_from", "metric":"chrf", "score":0.4776904622 }, { "model":"cohere\/command-r", "bcp_47":"en", "task":"translation_to", "metric":"bleu", "score":0.2328322481 }, { "model":"cohere\/command-r", "bcp_47":"en", "task":"translation_to", "metric":"chrf", "score":0.5236467405 }, { "model":"cohere\/command-r", "bcp_47":"es", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"cohere\/command-r", "bcp_47":"es", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"cohere\/command-r", "bcp_47":"es", "task":"translation_from", "metric":"bleu", "score":0.2281708898 }, { "model":"cohere\/command-r", "bcp_47":"es", "task":"translation_from", "metric":"chrf", "score":0.4658048059 }, { "model":"cohere\/command-r", "bcp_47":"es", "task":"translation_to", "metric":"bleu", "score":0.302206669 }, { "model":"cohere\/command-r", "bcp_47":"es", "task":"translation_to", "metric":"chrf", "score":0.5004953235 }, { "model":"cohere\/command-r", "bcp_47":"fr", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"cohere\/command-r", "bcp_47":"fr", "task":"mmlu", "metric":"accuracy", "score":0.5 }, { "model":"cohere\/command-r", "bcp_47":"fr", "task":"translation_from", "metric":"bleu", "score":0.2485056339 }, { "model":"cohere\/command-r", "bcp_47":"fr", "task":"translation_from", "metric":"chrf", "score":0.4746922491 }, { "model":"cohere\/command-r", "bcp_47":"fr", "task":"translation_to", "metric":"bleu", "score":0.3589308099 }, { "model":"cohere\/command-r", "bcp_47":"fr", "task":"translation_to", "metric":"chrf", "score":0.5387167593 }, { "model":"cohere\/command-r", "bcp_47":"hi", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"cohere\/command-r", "bcp_47":"hi", "task":"mmlu", "metric":"accuracy", "score":0.6 }, { "model":"cohere\/command-r", "bcp_47":"hi", "task":"translation_from", "metric":"bleu", "score":0.176310372 }, { "model":"cohere\/command-r", "bcp_47":"hi", "task":"translation_from", "metric":"chrf", "score":0.3718450169 }, { "model":"cohere\/command-r", "bcp_47":"hi", "task":"translation_to", "metric":"bleu", "score":0.1638320264 }, { "model":"cohere\/command-r", "bcp_47":"hi", "task":"translation_to", "metric":"chrf", "score":0.4001947091 }, { "model":"cohere\/command-r", "bcp_47":"id", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"cohere\/command-r", "bcp_47":"id", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"cohere\/command-r", "bcp_47":"id", "task":"translation_from", "metric":"bleu", "score":0.2256160181 }, { "model":"cohere\/command-r", "bcp_47":"id", "task":"translation_from", "metric":"chrf", "score":0.4634218276 }, { "model":"cohere\/command-r", "bcp_47":"id", "task":"translation_to", "metric":"bleu", "score":0.2436068148 }, { "model":"cohere\/command-r", "bcp_47":"id", "task":"translation_to", "metric":"chrf", "score":0.5064853457 }, { "model":"cohere\/command-r", "bcp_47":"ja", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"cohere\/command-r", "bcp_47":"ja", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"cohere\/command-r", "bcp_47":"ja", "task":"translation_from", "metric":"bleu", "score":0.2136903951 }, { "model":"cohere\/command-r", "bcp_47":"ja", "task":"translation_from", "metric":"chrf", "score":0.4243463352 }, { "model":"cohere\/command-r", "bcp_47":"ja", "task":"translation_to", "metric":"bleu", "score":0.0970742193 }, { "model":"cohere\/command-r", "bcp_47":"ja", "task":"translation_to", "metric":"chrf", "score":0.2293867344 }, { "model":"cohere\/command-r", "bcp_47":"mr", "task":"classification", "metric":"accuracy", "score":0.8 }, { "model":"cohere\/command-r", "bcp_47":"mr", "task":"translation_from", "metric":"bleu", "score":0.0799150208 }, { "model":"cohere\/command-r", "bcp_47":"mr", "task":"translation_from", "metric":"chrf", "score":0.2893419537 }, { "model":"cohere\/command-r", "bcp_47":"mr", "task":"translation_to", "metric":"bleu", "score":0.0581409044 }, { "model":"cohere\/command-r", "bcp_47":"mr", "task":"translation_to", "metric":"chrf", "score":0.2268165833 }, { "model":"cohere\/command-r", "bcp_47":"pa", "task":"classification", "metric":"accuracy", "score":0.1 }, { "model":"cohere\/command-r", "bcp_47":"pa", "task":"translation_from", "metric":"bleu", "score":0.1097931752 }, { "model":"cohere\/command-r", "bcp_47":"pa", "task":"translation_from", "metric":"chrf", "score":0.2693246745 }, { "model":"cohere\/command-r", "bcp_47":"pa", "task":"translation_to", "metric":"bleu", "score":0.056288688 }, { "model":"cohere\/command-r", "bcp_47":"pa", "task":"translation_to", "metric":"chrf", "score":0.1832547974 }, { "model":"cohere\/command-r", "bcp_47":"pt", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"cohere\/command-r", "bcp_47":"pt", "task":"mmlu", "metric":"accuracy", "score":0.9 }, { "model":"cohere\/command-r", "bcp_47":"pt", "task":"translation_from", "metric":"bleu", "score":0.2273037816 }, { "model":"cohere\/command-r", "bcp_47":"pt", "task":"translation_from", "metric":"chrf", "score":0.4505835688 }, { "model":"cohere\/command-r", "bcp_47":"pt", "task":"translation_to", "metric":"bleu", "score":0.3286476013 }, { "model":"cohere\/command-r", "bcp_47":"pt", "task":"translation_to", "metric":"chrf", "score":0.5091531561 }, { "model":"cohere\/command-r", "bcp_47":"ru", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"cohere\/command-r", "bcp_47":"ru", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"cohere\/command-r", "bcp_47":"ru", "task":"translation_from", "metric":"bleu", "score":0.1855609032 }, { "model":"cohere\/command-r", "bcp_47":"ru", "task":"translation_from", "metric":"chrf", "score":0.4153884364 }, { "model":"cohere\/command-r", "bcp_47":"ru", "task":"translation_to", "metric":"bleu", "score":0.277880372 }, { "model":"cohere\/command-r", "bcp_47":"ru", "task":"translation_to", "metric":"chrf", "score":0.4652042594 }, { "model":"cohere\/command-r", "bcp_47":"sw", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"cohere\/command-r", "bcp_47":"sw", "task":"mmlu", "metric":"accuracy", "score":0.5 }, { "model":"cohere\/command-r", "bcp_47":"sw", "task":"translation_from", "metric":"bleu", "score":0.0369694992 }, { "model":"cohere\/command-r", "bcp_47":"sw", "task":"translation_from", "metric":"chrf", "score":0.240137357 }, { "model":"cohere\/command-r", "bcp_47":"sw", "task":"translation_to", "metric":"bleu", "score":0.0127960718 }, { "model":"cohere\/command-r", "bcp_47":"sw", "task":"translation_to", "metric":"chrf", "score":0.2182424065 }, { "model":"cohere\/command-r", "bcp_47":"te", "task":"classification", "metric":"accuracy", "score":0.8 }, { "model":"cohere\/command-r", "bcp_47":"te", "task":"mmlu", "metric":"accuracy", "score":0.4 }, { "model":"cohere\/command-r", "bcp_47":"te", "task":"translation_from", "metric":"bleu", "score":0.0940291711 }, { "model":"cohere\/command-r", "bcp_47":"te", "task":"translation_from", "metric":"chrf", "score":0.2872804994 }, { "model":"cohere\/command-r", "bcp_47":"te", "task":"translation_to", "metric":"bleu", "score":0.0398689493 }, { "model":"cohere\/command-r", "bcp_47":"te", "task":"translation_to", "metric":"chrf", "score":0.2075817756 }, { "model":"cohere\/command-r", "bcp_47":"ur", "task":"classification", "metric":"accuracy", "score":0.7 }, { "model":"cohere\/command-r", "bcp_47":"ur", "task":"translation_from", "metric":"bleu", "score":0.117904807 }, { "model":"cohere\/command-r", "bcp_47":"ur", "task":"translation_from", "metric":"chrf", "score":0.3035503387 }, { "model":"cohere\/command-r", "bcp_47":"ur", "task":"translation_to", "metric":"bleu", "score":0.0043914583 }, { "model":"cohere\/command-r", "bcp_47":"ur", "task":"translation_to", "metric":"chrf", "score":0.1973427325 }, { "model":"cohere\/command-r", "bcp_47":"zh", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"cohere\/command-r", "bcp_47":"zh", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"cohere\/command-r", "bcp_47":"zh", "task":"translation_from", "metric":"bleu", "score":0.1818080543 }, { "model":"cohere\/command-r", "bcp_47":"zh", "task":"translation_from", "metric":"chrf", "score":0.4402625664 }, { "model":"cohere\/command-r", "bcp_47":"zh", "task":"translation_to", "metric":"bleu", "score":0.1788431133 }, { "model":"cohere\/command-r", "bcp_47":"zh", "task":"translation_to", "metric":"chrf", "score":0.2242902764 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"ar", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"ar", "task":"mmlu", "metric":"accuracy", "score":0.9 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"ar", "task":"translation_from", "metric":"bleu", "score":0.2660357996 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"ar", "task":"translation_from", "metric":"chrf", "score":0.5489237487 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"ar", "task":"translation_to", "metric":"bleu", "score":0.3431691167 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"ar", "task":"translation_to", "metric":"chrf", "score":0.5659025601 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"bn", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"bn", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"bn", "task":"translation_from", "metric":"bleu", "score":0.2999810338 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"bn", "task":"translation_from", "metric":"chrf", "score":0.5315905896 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"bn", "task":"translation_to", "metric":"bleu", "score":0.3568113924 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"bn", "task":"translation_to", "metric":"chrf", "score":0.5379558638 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"de", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"de", "task":"mmlu", "metric":"accuracy", "score":1.0 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"de", "task":"translation_from", "metric":"bleu", "score":0.3410327538 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"de", "task":"translation_from", "metric":"chrf", "score":0.5702449417 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"de", "task":"translation_to", "metric":"bleu", "score":0.4621824412 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"de", "task":"translation_to", "metric":"chrf", "score":0.6644677733 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"en", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"en", "task":"mmlu", "metric":"accuracy", "score":1.0 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"en", "task":"translation_from", "metric":"bleu", "score":0.5530727537 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"en", "task":"translation_from", "metric":"chrf", "score":0.7075859327 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"en", "task":"translation_to", "metric":"bleu", "score":0.6422296046 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"en", "task":"translation_to", "metric":"chrf", "score":0.81811999 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"es", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"es", "task":"mmlu", "metric":"accuracy", "score":0.9 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"es", "task":"translation_from", "metric":"bleu", "score":0.3558328881 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"es", "task":"translation_from", "metric":"chrf", "score":0.5816613686 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"es", "task":"translation_to", "metric":"bleu", "score":0.3018683161 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"es", "task":"translation_to", "metric":"chrf", "score":0.5862734644 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"fr", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"fr", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"fr", "task":"translation_from", "metric":"bleu", "score":0.3482763208 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"fr", "task":"translation_from", "metric":"chrf", "score":0.5982327963 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"fr", "task":"translation_to", "metric":"bleu", "score":0.452269216 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"fr", "task":"translation_to", "metric":"chrf", "score":0.6590312745 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"hi", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"hi", "task":"mmlu", "metric":"accuracy", "score":1.0 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"hi", "task":"translation_from", "metric":"bleu", "score":0.3747394322 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"hi", "task":"translation_from", "metric":"chrf", "score":0.6079633657 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"hi", "task":"translation_to", "metric":"bleu", "score":0.4420465241 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"hi", "task":"translation_to", "metric":"chrf", "score":0.6372857982 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"id", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"id", "task":"mmlu", "metric":"accuracy", "score":1.0 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"id", "task":"translation_from", "metric":"bleu", "score":0.3120299438 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"id", "task":"translation_from", "metric":"chrf", "score":0.5421647326 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"id", "task":"translation_to", "metric":"bleu", "score":0.3139800299 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"id", "task":"translation_to", "metric":"chrf", "score":0.6305869448 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"ja", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"ja", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"ja", "task":"translation_from", "metric":"bleu", "score":0.2121435327 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"ja", "task":"translation_from", "metric":"chrf", "score":0.5165098934 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"ja", "task":"translation_to", "metric":"bleu", "score":0.3965805608 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"ja", "task":"translation_to", "metric":"chrf", "score":0.5197322727 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"mr", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"mr", "task":"translation_from", "metric":"bleu", "score":0.2536168105 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"mr", "task":"translation_from", "metric":"chrf", "score":0.5193737639 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"mr", "task":"translation_to", "metric":"bleu", "score":0.2743904396 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"mr", "task":"translation_to", "metric":"chrf", "score":0.4918283752 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"pa", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"pa", "task":"translation_from", "metric":"bleu", "score":0.4307107958 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"pa", "task":"translation_from", "metric":"chrf", "score":0.6547934264 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"pa", "task":"translation_to", "metric":"bleu", "score":0.3789232965 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"pa", "task":"translation_to", "metric":"chrf", "score":0.5490387026 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"pt", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"pt", "task":"mmlu", "metric":"accuracy", "score":0.9 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"pt", "task":"translation_from", "metric":"bleu", "score":0.3491027097 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"pt", "task":"translation_from", "metric":"chrf", "score":0.5771246685 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"pt", "task":"translation_to", "metric":"bleu", "score":0.3737440632 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"pt", "task":"translation_to", "metric":"chrf", "score":0.6547073078 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"ru", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"ru", "task":"mmlu", "metric":"accuracy", "score":0.9 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"ru", "task":"translation_from", "metric":"bleu", "score":0.2584319121 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"ru", "task":"translation_from", "metric":"chrf", "score":0.5177022275 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"ru", "task":"translation_to", "metric":"bleu", "score":0.371218137 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"ru", "task":"translation_to", "metric":"chrf", "score":0.5948906549 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"sw", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"sw", "task":"mmlu", "metric":"accuracy", "score":0.9 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"sw", "task":"translation_from", "metric":"bleu", "score":0.2583542451 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"sw", "task":"translation_from", "metric":"chrf", "score":0.5157482031 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"sw", "task":"translation_to", "metric":"bleu", "score":0.3458513734 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"sw", "task":"translation_to", "metric":"chrf", "score":0.634706105 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"te", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"te", "task":"mmlu", "metric":"accuracy", "score":0.6 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"te", "task":"translation_from", "metric":"bleu", "score":0.3887890595 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"te", "task":"translation_from", "metric":"chrf", "score":0.6100201392 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"te", "task":"translation_to", "metric":"bleu", "score":0.3146346509 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"te", "task":"translation_to", "metric":"chrf", "score":0.5476489231 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"ur", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"ur", "task":"translation_from", "metric":"bleu", "score":0.2324983634 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"ur", "task":"translation_from", "metric":"chrf", "score":0.497221173 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"ur", "task":"translation_to", "metric":"bleu", "score":0.2976387517 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"ur", "task":"translation_to", "metric":"chrf", "score":0.4761547661 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"zh", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"zh", "task":"mmlu", "metric":"accuracy", "score":1.0 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"zh", "task":"translation_from", "metric":"bleu", "score":0.2377253988 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"zh", "task":"translation_from", "metric":"chrf", "score":0.5405656214 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"zh", "task":"translation_to", "metric":"bleu", "score":0.2660263348 }, { "model":"deepseek\/deepseek-chat", "bcp_47":"zh", "task":"translation_to", "metric":"chrf", "score":0.3459946232 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"ar", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"ar", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"ar", "task":"translation_from", "metric":"bleu", "score":0.2993048546 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"ar", "task":"translation_from", "metric":"chrf", "score":0.5651988199 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"ar", "task":"translation_to", "metric":"bleu", "score":0.3853695566 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"ar", "task":"translation_to", "metric":"chrf", "score":0.602672086 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"bn", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"bn", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"bn", "task":"translation_from", "metric":"bleu", "score":0.2798537803 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"bn", "task":"translation_from", "metric":"chrf", "score":0.5394779979 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"bn", "task":"translation_to", "metric":"bleu", "score":0.3719179468 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"bn", "task":"translation_to", "metric":"chrf", "score":0.539682577 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"de", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"de", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"de", "task":"translation_from", "metric":"bleu", "score":0.3207762021 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"de", "task":"translation_from", "metric":"chrf", "score":0.5555389401 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"de", "task":"translation_to", "metric":"bleu", "score":0.4227752207 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"de", "task":"translation_to", "metric":"chrf", "score":0.6601015066 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"en", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"en", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"en", "task":"translation_from", "metric":"bleu", "score":0.569880154 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"en", "task":"translation_from", "metric":"chrf", "score":0.7141874172 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"en", "task":"translation_to", "metric":"bleu", "score":0.6432470265 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"en", "task":"translation_to", "metric":"chrf", "score":0.8278285651 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"es", "task":"classification", "metric":"accuracy", "score":0.8 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"es", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"es", "task":"translation_from", "metric":"bleu", "score":0.3223436235 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"es", "task":"translation_from", "metric":"chrf", "score":0.5379000389 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"es", "task":"translation_to", "metric":"bleu", "score":0.3061941236 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"es", "task":"translation_to", "metric":"chrf", "score":0.5918382188 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"fr", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"fr", "task":"mmlu", "metric":"accuracy", "score":0.5 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"fr", "task":"translation_from", "metric":"bleu", "score":0.3297556296 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"fr", "task":"translation_from", "metric":"chrf", "score":0.5925809306 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"fr", "task":"translation_to", "metric":"bleu", "score":0.4619761505 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"fr", "task":"translation_to", "metric":"chrf", "score":0.6734036273 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"hi", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"hi", "task":"mmlu", "metric":"accuracy", "score":1.0 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"hi", "task":"translation_from", "metric":"bleu", "score":0.3670345278 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"hi", "task":"translation_from", "metric":"chrf", "score":0.6162795112 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"hi", "task":"translation_to", "metric":"bleu", "score":0.3937102354 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"hi", "task":"translation_to", "metric":"chrf", "score":0.6229439454 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"id", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"id", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"id", "task":"translation_from", "metric":"bleu", "score":0.3020610187 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"id", "task":"translation_from", "metric":"chrf", "score":0.5462026627 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"id", "task":"translation_to", "metric":"bleu", "score":0.299038365 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"id", "task":"translation_to", "metric":"chrf", "score":0.625735911 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"ja", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"ja", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"ja", "task":"translation_from", "metric":"bleu", "score":0.1875636541 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"ja", "task":"translation_from", "metric":"chrf", "score":0.5136106256 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"ja", "task":"translation_to", "metric":"bleu", "score":0.3333377273 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"ja", "task":"translation_to", "metric":"chrf", "score":0.4709407515 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"mr", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"mr", "task":"translation_from", "metric":"bleu", "score":0.2487969868 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"mr", "task":"translation_from", "metric":"chrf", "score":0.518708582 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"mr", "task":"translation_to", "metric":"bleu", "score":0.2238578938 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"mr", "task":"translation_to", "metric":"chrf", "score":0.4748109447 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"pa", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"pa", "task":"translation_from", "metric":"bleu", "score":0.4358296524 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"pa", "task":"translation_from", "metric":"chrf", "score":0.6732700357 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"pa", "task":"translation_to", "metric":"bleu", "score":0.3548387061 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"pa", "task":"translation_to", "metric":"chrf", "score":0.5492435889 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"pt", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"pt", "task":"mmlu", "metric":"accuracy", "score":0.9 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"pt", "task":"translation_from", "metric":"bleu", "score":0.3091555064 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"pt", "task":"translation_from", "metric":"chrf", "score":0.5528775735 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"pt", "task":"translation_to", "metric":"bleu", "score":0.3860807525 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"pt", "task":"translation_to", "metric":"chrf", "score":0.6710753294 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"ru", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"ru", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"ru", "task":"translation_from", "metric":"bleu", "score":0.2464304597 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"ru", "task":"translation_from", "metric":"chrf", "score":0.5343201712 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"ru", "task":"translation_to", "metric":"bleu", "score":0.3592456339 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"ru", "task":"translation_to", "metric":"chrf", "score":0.5816925415 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"sw", "task":"classification", "metric":"accuracy", "score":0.8 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"sw", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"sw", "task":"translation_from", "metric":"bleu", "score":0.2438930348 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"sw", "task":"translation_from", "metric":"chrf", "score":0.5213646779 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"sw", "task":"translation_to", "metric":"bleu", "score":0.2864351463 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"sw", "task":"translation_to", "metric":"chrf", "score":0.6249321785 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"te", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"te", "task":"mmlu", "metric":"accuracy", "score":0.4 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"te", "task":"translation_from", "metric":"bleu", "score":0.3465514125 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"te", "task":"translation_from", "metric":"chrf", "score":0.588783736 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"te", "task":"translation_to", "metric":"bleu", "score":0.2948086539 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"te", "task":"translation_to", "metric":"chrf", "score":0.5417485172 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"ur", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"ur", "task":"translation_from", "metric":"bleu", "score":0.2290327476 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"ur", "task":"translation_from", "metric":"chrf", "score":0.5229250115 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"ur", "task":"translation_to", "metric":"bleu", "score":0.3122409611 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"ur", "task":"translation_to", "metric":"chrf", "score":0.4921734247 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"zh", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"zh", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"zh", "task":"translation_from", "metric":"bleu", "score":0.2350235637 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"zh", "task":"translation_from", "metric":"chrf", "score":0.528086246 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"zh", "task":"translation_to", "metric":"bleu", "score":0.2524231151 }, { "model":"deepseek\/deepseek-chat-v3-0324", "bcp_47":"zh", "task":"translation_to", "metric":"chrf", "score":0.3456705882 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"ar", "task":"classification", "metric":"accuracy", "score":0.1 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"ar", "task":"mmlu", "metric":"accuracy", "score":0.0 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"ar", "task":"translation_from", "metric":"bleu", "score":0.2238237549 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"ar", "task":"translation_from", "metric":"chrf", "score":0.4534366926 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"ar", "task":"translation_to", "metric":"bleu", "score":0.2689055687 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"ar", "task":"translation_to", "metric":"chrf", "score":0.4149416248 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"bn", "task":"classification", "metric":"accuracy", "score":0.2 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"bn", "task":"mmlu", "metric":"accuracy", "score":0.0 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"bn", "task":"translation_from", "metric":"bleu", "score":0.2443635406 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"bn", "task":"translation_from", "metric":"chrf", "score":0.4676159664 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"bn", "task":"translation_to", "metric":"bleu", "score":0.2223182846 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"bn", "task":"translation_to", "metric":"chrf", "score":0.3685961254 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"de", "task":"classification", "metric":"accuracy", "score":0.0 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"de", "task":"mmlu", "metric":"accuracy", "score":0.0 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"de", "task":"translation_from", "metric":"bleu", "score":0.2633055293 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"de", "task":"translation_from", "metric":"chrf", "score":0.451378667 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"de", "task":"translation_to", "metric":"bleu", "score":0.2576565152 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"de", "task":"translation_to", "metric":"chrf", "score":0.3874723625 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"en", "task":"classification", "metric":"accuracy", "score":0.1 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"en", "task":"mmlu", "metric":"accuracy", "score":0.0 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"en", "task":"translation_from", "metric":"bleu", "score":0.3516668412 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"en", "task":"translation_from", "metric":"chrf", "score":0.4533116867 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"en", "task":"translation_to", "metric":"bleu", "score":0.5868227988 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"en", "task":"translation_to", "metric":"chrf", "score":0.7574063883 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"es", "task":"classification", "metric":"accuracy", "score":0.0 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"es", "task":"mmlu", "metric":"accuracy", "score":0.0 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"es", "task":"translation_from", "metric":"bleu", "score":0.2163623393 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"es", "task":"translation_from", "metric":"chrf", "score":0.4705179867 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"es", "task":"translation_to", "metric":"bleu", "score":0.2458237388 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"es", "task":"translation_to", "metric":"chrf", "score":0.426538099 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"fr", "task":"classification", "metric":"accuracy", "score":0.1 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"fr", "task":"mmlu", "metric":"accuracy", "score":0.0 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"fr", "task":"translation_from", "metric":"bleu", "score":0.3182982487 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"fr", "task":"translation_from", "metric":"chrf", "score":0.5389072956 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"fr", "task":"translation_to", "metric":"bleu", "score":0.47314841 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"fr", "task":"translation_to", "metric":"chrf", "score":0.6884710951 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"hi", "task":"classification", "metric":"accuracy", "score":0.2 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"hi", "task":"mmlu", "metric":"accuracy", "score":0.1 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"hi", "task":"translation_from", "metric":"bleu", "score":0.2751556219 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"hi", "task":"translation_from", "metric":"chrf", "score":0.4216960004 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"hi", "task":"translation_to", "metric":"bleu", "score":0.2827365983 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"hi", "task":"translation_to", "metric":"chrf", "score":0.5128198247 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"id", "task":"classification", "metric":"accuracy", "score":0.2 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"id", "task":"mmlu", "metric":"accuracy", "score":0.0 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"id", "task":"translation_from", "metric":"bleu", "score":0.223787985 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"id", "task":"translation_from", "metric":"chrf", "score":0.485186041 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"id", "task":"translation_to", "metric":"bleu", "score":0.3571004344 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"id", "task":"translation_to", "metric":"chrf", "score":0.6398491182 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"ja", "task":"classification", "metric":"accuracy", "score":0.2 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"ja", "task":"mmlu", "metric":"accuracy", "score":0.0 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"ja", "task":"translation_from", "metric":"bleu", "score":0.1711796281 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"ja", "task":"translation_from", "metric":"chrf", "score":0.4108641598 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"ja", "task":"translation_to", "metric":"bleu", "score":0.2194500975 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"ja", "task":"translation_to", "metric":"chrf", "score":0.3589526769 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"mr", "task":"classification", "metric":"accuracy", "score":0.1 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"mr", "task":"translation_from", "metric":"bleu", "score":0.1828389227 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"mr", "task":"translation_from", "metric":"chrf", "score":0.3179756072 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"mr", "task":"translation_to", "metric":"bleu", "score":0.2129586558 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"mr", "task":"translation_to", "metric":"chrf", "score":0.4314516197 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"pa", "task":"classification", "metric":"accuracy", "score":0.1 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"pa", "task":"translation_from", "metric":"bleu", "score":0.252814761 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"pa", "task":"translation_from", "metric":"chrf", "score":0.4131775231 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"pa", "task":"translation_to", "metric":"bleu", "score":0.2671823746 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"pa", "task":"translation_to", "metric":"chrf", "score":0.3966391033 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"pt", "task":"classification", "metric":"accuracy", "score":0.1 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"pt", "task":"mmlu", "metric":"accuracy", "score":0.0 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"pt", "task":"translation_from", "metric":"bleu", "score":0.2348823133 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"pt", "task":"translation_from", "metric":"chrf", "score":0.4500923911 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"pt", "task":"translation_to", "metric":"bleu", "score":0.4182790857 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"pt", "task":"translation_to", "metric":"chrf", "score":0.649967582 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"ru", "task":"classification", "metric":"accuracy", "score":0.1 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"ru", "task":"mmlu", "metric":"accuracy", "score":0.0 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"ru", "task":"translation_from", "metric":"bleu", "score":0.1839314203 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"ru", "task":"translation_from", "metric":"chrf", "score":0.4234191674 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"ru", "task":"translation_to", "metric":"bleu", "score":0.3489148579 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"ru", "task":"translation_to", "metric":"chrf", "score":0.5567945257 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"sw", "task":"classification", "metric":"accuracy", "score":0.1 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"sw", "task":"mmlu", "metric":"accuracy", "score":0.0 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"sw", "task":"translation_from", "metric":"bleu", "score":0.2041105012 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"sw", "task":"translation_from", "metric":"chrf", "score":0.3657796945 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"sw", "task":"translation_to", "metric":"bleu", "score":0.3130978532 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"sw", "task":"translation_to", "metric":"chrf", "score":0.5950962977 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"te", "task":"classification", "metric":"accuracy", "score":0.1 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"te", "task":"mmlu", "metric":"accuracy", "score":0.0 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"te", "task":"translation_from", "metric":"bleu", "score":0.1200373123 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"te", "task":"translation_from", "metric":"chrf", "score":0.3196364935 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"te", "task":"translation_to", "metric":"bleu", "score":0.1814754432 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"te", "task":"translation_to", "metric":"chrf", "score":0.3157059838 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"ur", "task":"classification", "metric":"accuracy", "score":0.0 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"ur", "task":"translation_from", "metric":"bleu", "score":0.181272453 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"ur", "task":"translation_from", "metric":"chrf", "score":0.386776605 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"ur", "task":"translation_to", "metric":"bleu", "score":0.2603415771 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"ur", "task":"translation_to", "metric":"chrf", "score":0.4241396601 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"zh", "task":"classification", "metric":"accuracy", "score":0.1 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"zh", "task":"mmlu", "metric":"accuracy", "score":0.0 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"zh", "task":"translation_from", "metric":"bleu", "score":0.2137844239 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"zh", "task":"translation_from", "metric":"chrf", "score":0.5028557922 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"zh", "task":"translation_to", "metric":"bleu", "score":0.3129908127 }, { "model":"deepseek\/deepseek-r1", "bcp_47":"zh", "task":"translation_to", "metric":"chrf", "score":0.3710290799 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"ar", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"ar", "task":"mmlu", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"ar", "task":"translation_from", "metric":"bleu", "score":0.3412745465 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"ar", "task":"translation_from", "metric":"chrf", "score":0.5826925715 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"ar", "task":"translation_to", "metric":"bleu", "score":0.390058702 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"ar", "task":"translation_to", "metric":"chrf", "score":0.597901367 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"bn", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"bn", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"bn", "task":"translation_from", "metric":"bleu", "score":0.2988263355 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"bn", "task":"translation_from", "metric":"chrf", "score":0.5680625724 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"bn", "task":"translation_to", "metric":"bleu", "score":0.3895386095 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"bn", "task":"translation_to", "metric":"chrf", "score":0.5693830083 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"de", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"de", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"de", "task":"translation_from", "metric":"bleu", "score":0.4245142301 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"de", "task":"translation_from", "metric":"chrf", "score":0.6390046108 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"de", "task":"translation_to", "metric":"bleu", "score":0.4937713215 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"de", "task":"translation_to", "metric":"chrf", "score":0.6856946146 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"en", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"en", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"en", "task":"translation_from", "metric":"bleu", "score":0.5700014681 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"en", "task":"translation_from", "metric":"chrf", "score":0.7211557733 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"en", "task":"translation_to", "metric":"bleu", "score":0.6711231356 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"en", "task":"translation_to", "metric":"chrf", "score":0.8279616884 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"es", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"es", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"es", "task":"translation_from", "metric":"bleu", "score":0.3344072959 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"es", "task":"translation_from", "metric":"chrf", "score":0.5751886204 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"es", "task":"translation_to", "metric":"bleu", "score":0.3905772718 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"es", "task":"translation_to", "metric":"chrf", "score":0.6166676981 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"fr", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"fr", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"fr", "task":"translation_from", "metric":"bleu", "score":0.3411643102 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"fr", "task":"translation_from", "metric":"chrf", "score":0.5823324237 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"fr", "task":"translation_to", "metric":"bleu", "score":0.549646027 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"fr", "task":"translation_to", "metric":"chrf", "score":0.7334440232 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"hi", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"hi", "task":"mmlu", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"hi", "task":"translation_from", "metric":"bleu", "score":0.4434404142 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"hi", "task":"translation_from", "metric":"chrf", "score":0.6607832834 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"hi", "task":"translation_to", "metric":"bleu", "score":0.4308263411 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"hi", "task":"translation_to", "metric":"chrf", "score":0.6353494719 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"id", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"id", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"id", "task":"translation_from", "metric":"bleu", "score":0.3626330278 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"id", "task":"translation_from", "metric":"chrf", "score":0.5869542517 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"id", "task":"translation_to", "metric":"bleu", "score":0.4368014195 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"id", "task":"translation_to", "metric":"chrf", "score":0.6964617832 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"ja", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"ja", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"ja", "task":"translation_from", "metric":"bleu", "score":0.3331002174 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"ja", "task":"translation_from", "metric":"chrf", "score":0.579659503 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"ja", "task":"translation_to", "metric":"bleu", "score":0.336969824 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"ja", "task":"translation_to", "metric":"chrf", "score":0.4913494739 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"mr", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"mr", "task":"translation_from", "metric":"bleu", "score":0.3338185652 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"mr", "task":"translation_from", "metric":"chrf", "score":0.5787491818 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"mr", "task":"translation_to", "metric":"bleu", "score":0.2806196555 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"mr", "task":"translation_to", "metric":"chrf", "score":0.4516145469 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"pa", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"pa", "task":"translation_from", "metric":"bleu", "score":0.4372599799 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"pa", "task":"translation_from", "metric":"chrf", "score":0.658993109 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"pa", "task":"translation_to", "metric":"bleu", "score":0.4174871385 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"pa", "task":"translation_to", "metric":"chrf", "score":0.5931595705 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"pt", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"pt", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"pt", "task":"translation_from", "metric":"bleu", "score":0.3198419121 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"pt", "task":"translation_from", "metric":"chrf", "score":0.578067895 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"pt", "task":"translation_to", "metric":"bleu", "score":0.5056809967 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"pt", "task":"translation_to", "metric":"chrf", "score":0.6966172136 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"ru", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"ru", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"ru", "task":"translation_from", "metric":"bleu", "score":0.2839962776 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"ru", "task":"translation_from", "metric":"chrf", "score":0.5461994333 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"ru", "task":"translation_to", "metric":"bleu", "score":0.4589390255 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"ru", "task":"translation_to", "metric":"chrf", "score":0.6469214364 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"sw", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"sw", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"sw", "task":"translation_from", "metric":"bleu", "score":0.4043048116 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"sw", "task":"translation_from", "metric":"chrf", "score":0.5944297519 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"sw", "task":"translation_to", "metric":"bleu", "score":0.499303533 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"sw", "task":"translation_to", "metric":"chrf", "score":0.7212747243 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"te", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"te", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"te", "task":"translation_from", "metric":"bleu", "score":0.4243816819 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"te", "task":"translation_from", "metric":"chrf", "score":0.6336752528 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"te", "task":"translation_to", "metric":"bleu", "score":0.3644641049 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"te", "task":"translation_to", "metric":"chrf", "score":0.5789291672 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"ur", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"ur", "task":"translation_from", "metric":"bleu", "score":0.31244285 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"ur", "task":"translation_from", "metric":"chrf", "score":0.5630479496 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"ur", "task":"translation_to", "metric":"bleu", "score":0.2911146353 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"ur", "task":"translation_to", "metric":"chrf", "score":0.4826415387 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"zh", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"zh", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"zh", "task":"translation_from", "metric":"bleu", "score":0.3075560511 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"zh", "task":"translation_from", "metric":"chrf", "score":0.5721434339 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"zh", "task":"translation_to", "metric":"bleu", "score":0.3628421643 }, { "model":"google\/gemini-2.0-flash-001", "bcp_47":"zh", "task":"translation_to", "metric":"chrf", "score":0.3921364269 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"ar", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"ar", "task":"mmlu", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"ar", "task":"translation_from", "metric":"bleu", "score":0.2723646085 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"ar", "task":"translation_from", "metric":"chrf", "score":0.523730035 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"ar", "task":"translation_to", "metric":"bleu", "score":0.3878873306 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"ar", "task":"translation_to", "metric":"chrf", "score":0.592059186 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"bn", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"bn", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"bn", "task":"translation_from", "metric":"bleu", "score":0.2672489266 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"bn", "task":"translation_from", "metric":"chrf", "score":0.4952818431 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"bn", "task":"translation_to", "metric":"bleu", "score":0.3639249804 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"bn", "task":"translation_to", "metric":"chrf", "score":0.5538942616 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"de", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"de", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"de", "task":"translation_from", "metric":"bleu", "score":0.3382313883 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"de", "task":"translation_from", "metric":"chrf", "score":0.5705047774 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"de", "task":"translation_to", "metric":"bleu", "score":0.5354923841 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"de", "task":"translation_to", "metric":"chrf", "score":0.7125417889 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"en", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"en", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"en", "task":"translation_from", "metric":"bleu", "score":0.4617498931 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"en", "task":"translation_from", "metric":"chrf", "score":0.6321211549 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"en", "task":"translation_to", "metric":"bleu", "score":0.5951204739 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"en", "task":"translation_to", "metric":"chrf", "score":0.7613397345 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"es", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"es", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"es", "task":"translation_from", "metric":"bleu", "score":0.3547349164 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"es", "task":"translation_from", "metric":"chrf", "score":0.5728186386 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"es", "task":"translation_to", "metric":"bleu", "score":0.3576086442 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"es", "task":"translation_to", "metric":"chrf", "score":0.6061019948 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"fr", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"fr", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"fr", "task":"translation_from", "metric":"bleu", "score":0.2978444639 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"fr", "task":"translation_from", "metric":"chrf", "score":0.5625976718 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"fr", "task":"translation_to", "metric":"bleu", "score":0.5244850062 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"fr", "task":"translation_to", "metric":"chrf", "score":0.7081676298 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"hi", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"hi", "task":"mmlu", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"hi", "task":"translation_from", "metric":"bleu", "score":0.4286739426 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"hi", "task":"translation_from", "metric":"chrf", "score":0.6359453866 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"hi", "task":"translation_to", "metric":"bleu", "score":0.4344801023 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"hi", "task":"translation_to", "metric":"chrf", "score":0.6474630132 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"id", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"id", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"id", "task":"translation_from", "metric":"bleu", "score":0.3126497326 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"id", "task":"translation_from", "metric":"chrf", "score":0.5733116043 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"id", "task":"translation_to", "metric":"bleu", "score":0.3882402101 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"id", "task":"translation_to", "metric":"chrf", "score":0.6624914478 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"ja", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"ja", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"ja", "task":"translation_from", "metric":"bleu", "score":0.289402493 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"ja", "task":"translation_from", "metric":"chrf", "score":0.5452343766 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"ja", "task":"translation_to", "metric":"bleu", "score":0.1879084772 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"ja", "task":"translation_to", "metric":"chrf", "score":0.3787664659 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"mr", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"mr", "task":"translation_from", "metric":"bleu", "score":0.2906692433 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"mr", "task":"translation_from", "metric":"chrf", "score":0.5058194876 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"mr", "task":"translation_to", "metric":"bleu", "score":0.2450137243 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"mr", "task":"translation_to", "metric":"chrf", "score":0.4736891146 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"pa", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"pa", "task":"translation_from", "metric":"bleu", "score":0.3792889323 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"pa", "task":"translation_from", "metric":"chrf", "score":0.6001893022 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"pa", "task":"translation_to", "metric":"bleu", "score":0.4267930078 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"pa", "task":"translation_to", "metric":"chrf", "score":0.583867208 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"pt", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"pt", "task":"mmlu", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"pt", "task":"translation_from", "metric":"bleu", "score":0.3275326122 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"pt", "task":"translation_from", "metric":"chrf", "score":0.5672682921 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"pt", "task":"translation_to", "metric":"bleu", "score":0.4390630519 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"pt", "task":"translation_to", "metric":"chrf", "score":0.6588223005 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"ru", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"ru", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"ru", "task":"translation_from", "metric":"bleu", "score":0.266439807 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"ru", "task":"translation_from", "metric":"chrf", "score":0.5101737799 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"ru", "task":"translation_to", "metric":"bleu", "score":0.4085478092 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"ru", "task":"translation_to", "metric":"chrf", "score":0.5658655698 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"sw", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"sw", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"sw", "task":"translation_from", "metric":"bleu", "score":0.2763965367 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"sw", "task":"translation_from", "metric":"chrf", "score":0.520411275 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"sw", "task":"translation_to", "metric":"bleu", "score":0.4350912598 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"sw", "task":"translation_to", "metric":"chrf", "score":0.6853284539 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"te", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"te", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"te", "task":"translation_from", "metric":"bleu", "score":0.3588325355 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"te", "task":"translation_from", "metric":"chrf", "score":0.5889091872 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"te", "task":"translation_to", "metric":"bleu", "score":0.3231474889 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"te", "task":"translation_to", "metric":"chrf", "score":0.5470305035 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"ur", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"ur", "task":"translation_from", "metric":"bleu", "score":0.2859928961 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"ur", "task":"translation_from", "metric":"chrf", "score":0.5295533045 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"ur", "task":"translation_to", "metric":"bleu", "score":0.3010091175 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"ur", "task":"translation_to", "metric":"chrf", "score":0.493353195 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"zh", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"zh", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"zh", "task":"translation_from", "metric":"bleu", "score":0.2557280993 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"zh", "task":"translation_from", "metric":"chrf", "score":0.5212143675 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"zh", "task":"translation_to", "metric":"bleu", "score":0.2610248692 }, { "model":"google\/gemini-2.0-flash-lite-001", "bcp_47":"zh", "task":"translation_to", "metric":"chrf", "score":0.314329989 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"ar", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"ar", "task":"mmlu", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"ar", "task":"translation_from", "metric":"bleu", "score":0.3462011916 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"ar", "task":"translation_from", "metric":"chrf", "score":0.5813984303 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"ar", "task":"translation_to", "metric":"bleu", "score":0.4319399394 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"ar", "task":"translation_to", "metric":"chrf", "score":0.6269389132 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"bn", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"bn", "task":"mmlu", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"bn", "task":"translation_from", "metric":"bleu", "score":0.3332105938 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"bn", "task":"translation_from", "metric":"chrf", "score":0.5788748413 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"bn", "task":"translation_to", "metric":"bleu", "score":0.3925309816 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"bn", "task":"translation_to", "metric":"chrf", "score":0.5671456285 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"de", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"de", "task":"mmlu", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"de", "task":"translation_from", "metric":"bleu", "score":0.4080538745 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"de", "task":"translation_from", "metric":"chrf", "score":0.6278026445 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"de", "task":"translation_to", "metric":"bleu", "score":0.4940498755 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"de", "task":"translation_to", "metric":"chrf", "score":0.6915113583 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"en", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"en", "task":"mmlu", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"en", "task":"translation_from", "metric":"bleu", "score":0.5780592594 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"en", "task":"translation_from", "metric":"chrf", "score":0.727062887 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"en", "task":"translation_to", "metric":"bleu", "score":0.725042796 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"en", "task":"translation_to", "metric":"chrf", "score":0.868453404 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"es", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"es", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"es", "task":"translation_from", "metric":"bleu", "score":0.3228577771 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"es", "task":"translation_from", "metric":"chrf", "score":0.5691176688 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"es", "task":"translation_to", "metric":"bleu", "score":0.4211069488 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"es", "task":"translation_to", "metric":"chrf", "score":0.6329026504 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"fr", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"fr", "task":"mmlu", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"fr", "task":"translation_from", "metric":"bleu", "score":0.3683003934 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"fr", "task":"translation_from", "metric":"chrf", "score":0.5961800274 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"fr", "task":"translation_to", "metric":"bleu", "score":0.5899042948 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"fr", "task":"translation_to", "metric":"chrf", "score":0.7506595836 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"hi", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"hi", "task":"mmlu", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"hi", "task":"translation_from", "metric":"bleu", "score":0.4661333335 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"hi", "task":"translation_from", "metric":"chrf", "score":0.6599650679 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"hi", "task":"translation_to", "metric":"bleu", "score":0.4319599559 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"hi", "task":"translation_to", "metric":"chrf", "score":0.6493733048 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"id", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"id", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"id", "task":"translation_from", "metric":"bleu", "score":0.3507436052 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"id", "task":"translation_from", "metric":"chrf", "score":0.5856872774 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"id", "task":"translation_to", "metric":"bleu", "score":0.387557065 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"id", "task":"translation_to", "metric":"chrf", "score":0.6730969345 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"ja", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"ja", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"ja", "task":"translation_from", "metric":"bleu", "score":0.3590987818 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"ja", "task":"translation_from", "metric":"chrf", "score":0.5958999814 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"ja", "task":"translation_to", "metric":"bleu", "score":0.4151803591 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"ja", "task":"translation_to", "metric":"chrf", "score":0.5319028037 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"mr", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"mr", "task":"translation_from", "metric":"bleu", "score":0.2888720948 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"mr", "task":"translation_from", "metric":"chrf", "score":0.5411662197 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"mr", "task":"translation_to", "metric":"bleu", "score":0.2820351359 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"mr", "task":"translation_to", "metric":"chrf", "score":0.5017455952 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"pa", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"pa", "task":"translation_from", "metric":"bleu", "score":0.4833653569 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"pa", "task":"translation_from", "metric":"chrf", "score":0.6877252787 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"pa", "task":"translation_to", "metric":"bleu", "score":0.4959043786 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"pa", "task":"translation_to", "metric":"chrf", "score":0.631939007 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"pt", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"pt", "task":"mmlu", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"pt", "task":"translation_from", "metric":"bleu", "score":0.4185968559 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"pt", "task":"translation_from", "metric":"chrf", "score":0.6194298291 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"pt", "task":"translation_to", "metric":"bleu", "score":0.5066913819 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"pt", "task":"translation_to", "metric":"chrf", "score":0.7045563325 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"ru", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"ru", "task":"mmlu", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"ru", "task":"translation_from", "metric":"bleu", "score":0.2835606416 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"ru", "task":"translation_from", "metric":"chrf", "score":0.5525332546 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"ru", "task":"translation_to", "metric":"bleu", "score":0.5294387768 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"ru", "task":"translation_to", "metric":"chrf", "score":0.6814959444 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"sw", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"sw", "task":"mmlu", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"sw", "task":"translation_from", "metric":"bleu", "score":0.4452506057 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"sw", "task":"translation_from", "metric":"chrf", "score":0.6299322033 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"sw", "task":"translation_to", "metric":"bleu", "score":0.4773237879 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"sw", "task":"translation_to", "metric":"chrf", "score":0.6983325706 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"te", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"te", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"te", "task":"translation_from", "metric":"bleu", "score":0.4838983303 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"te", "task":"translation_from", "metric":"chrf", "score":0.6878651593 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"te", "task":"translation_to", "metric":"bleu", "score":0.4225689448 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"te", "task":"translation_to", "metric":"chrf", "score":0.6171822766 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"ur", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"ur", "task":"translation_from", "metric":"bleu", "score":0.3194163699 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"ur", "task":"translation_from", "metric":"chrf", "score":0.5758075077 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"ur", "task":"translation_to", "metric":"bleu", "score":0.2609063103 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"ur", "task":"translation_to", "metric":"chrf", "score":0.4351160146 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"zh", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"zh", "task":"mmlu", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"zh", "task":"translation_from", "metric":"bleu", "score":0.2632355471 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"zh", "task":"translation_from", "metric":"chrf", "score":0.5527183044 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"zh", "task":"translation_to", "metric":"bleu", "score":0.3128970722 }, { "model":"google\/gemini-2.5-flash-preview", "bcp_47":"zh", "task":"translation_to", "metric":"chrf", "score":0.3699956681 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"ar", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"ar", "task":"mmlu", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"ar", "task":"translation_from", "metric":"bleu", "score":0.3509446122 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"ar", "task":"translation_from", "metric":"chrf", "score":0.5872861306 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"ar", "task":"translation_to", "metric":"bleu", "score":0.4369226272 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"ar", "task":"translation_to", "metric":"chrf", "score":0.6357392299 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"bn", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"bn", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"bn", "task":"translation_from", "metric":"bleu", "score":0.2928560114 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"bn", "task":"translation_from", "metric":"chrf", "score":0.5373877759 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"bn", "task":"translation_to", "metric":"bleu", "score":0.371713483 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"bn", "task":"translation_to", "metric":"chrf", "score":0.5467648432 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"de", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"de", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"de", "task":"translation_from", "metric":"bleu", "score":0.3827187056 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"de", "task":"translation_from", "metric":"chrf", "score":0.6092943987 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"de", "task":"translation_to", "metric":"bleu", "score":0.5181328066 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"de", "task":"translation_to", "metric":"chrf", "score":0.6981190119 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"en", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"en", "task":"mmlu", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"en", "task":"translation_from", "metric":"bleu", "score":0.5942868447 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"en", "task":"translation_from", "metric":"chrf", "score":0.7298379077 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"en", "task":"translation_to", "metric":"bleu", "score":0.7309888475 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"en", "task":"translation_to", "metric":"chrf", "score":0.8534239972 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"es", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"es", "task":"mmlu", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"es", "task":"translation_from", "metric":"bleu", "score":0.3497891701 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"es", "task":"translation_from", "metric":"chrf", "score":0.5889817614 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"es", "task":"translation_to", "metric":"bleu", "score":0.395885417 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"es", "task":"translation_to", "metric":"chrf", "score":0.6273835234 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"fr", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"fr", "task":"mmlu", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"fr", "task":"translation_from", "metric":"bleu", "score":0.3737227872 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"fr", "task":"translation_from", "metric":"chrf", "score":0.6039484183 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"fr", "task":"translation_to", "metric":"bleu", "score":0.5555743798 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"fr", "task":"translation_to", "metric":"chrf", "score":0.7229519037 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"hi", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"hi", "task":"mmlu", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"hi", "task":"translation_from", "metric":"bleu", "score":0.4387215588 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"hi", "task":"translation_from", "metric":"chrf", "score":0.6443095169 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"hi", "task":"translation_to", "metric":"bleu", "score":0.4255607348 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"hi", "task":"translation_to", "metric":"chrf", "score":0.6457270128 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"id", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"id", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"id", "task":"translation_from", "metric":"bleu", "score":0.3789722313 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"id", "task":"translation_from", "metric":"chrf", "score":0.5973074889 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"id", "task":"translation_to", "metric":"bleu", "score":0.406320567 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"id", "task":"translation_to", "metric":"chrf", "score":0.6828299565 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"ja", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"ja", "task":"mmlu", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"ja", "task":"translation_from", "metric":"bleu", "score":0.3423388267 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"ja", "task":"translation_from", "metric":"chrf", "score":0.5884893314 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"ja", "task":"translation_to", "metric":"bleu", "score":0.3740472584 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"ja", "task":"translation_to", "metric":"chrf", "score":0.4832664839 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"mr", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"mr", "task":"translation_from", "metric":"bleu", "score":0.3055522766 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"mr", "task":"translation_from", "metric":"chrf", "score":0.5435968357 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"mr", "task":"translation_to", "metric":"bleu", "score":0.257460039 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"mr", "task":"translation_to", "metric":"chrf", "score":0.4845716024 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"pa", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"pa", "task":"translation_from", "metric":"bleu", "score":0.4182659606 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"pa", "task":"translation_from", "metric":"chrf", "score":0.6696655487 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"pa", "task":"translation_to", "metric":"bleu", "score":0.4665320226 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"pa", "task":"translation_to", "metric":"chrf", "score":0.6280330147 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"pt", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"pt", "task":"mmlu", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"pt", "task":"translation_from", "metric":"bleu", "score":0.3485890022 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"pt", "task":"translation_from", "metric":"chrf", "score":0.574915747 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"pt", "task":"translation_to", "metric":"bleu", "score":0.4760104707 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"pt", "task":"translation_to", "metric":"chrf", "score":0.6891420805 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"ru", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"ru", "task":"mmlu", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"ru", "task":"translation_from", "metric":"bleu", "score":0.2893557086 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"ru", "task":"translation_from", "metric":"chrf", "score":0.5514512546 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"ru", "task":"translation_to", "metric":"bleu", "score":0.4910352633 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"ru", "task":"translation_to", "metric":"chrf", "score":0.6636379715 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"sw", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"sw", "task":"mmlu", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"sw", "task":"translation_from", "metric":"bleu", "score":0.397583037 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"sw", "task":"translation_from", "metric":"chrf", "score":0.6014699575 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"sw", "task":"translation_to", "metric":"bleu", "score":0.4540021563 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"sw", "task":"translation_to", "metric":"chrf", "score":0.7002830052 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"te", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"te", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"te", "task":"translation_from", "metric":"bleu", "score":0.4025505277 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"te", "task":"translation_from", "metric":"chrf", "score":0.6339036498 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"te", "task":"translation_to", "metric":"bleu", "score":0.4166086163 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"te", "task":"translation_to", "metric":"chrf", "score":0.6171953291 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"ur", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"ur", "task":"translation_from", "metric":"bleu", "score":0.3212777323 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"ur", "task":"translation_from", "metric":"chrf", "score":0.5735084601 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"ur", "task":"translation_to", "metric":"bleu", "score":0.3027958989 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"ur", "task":"translation_to", "metric":"chrf", "score":0.4912965943 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"zh", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"zh", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"zh", "task":"translation_from", "metric":"bleu", "score":0.2989639846 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"zh", "task":"translation_from", "metric":"chrf", "score":0.5646599732 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"zh", "task":"translation_to", "metric":"bleu", "score":0.2915310914 }, { "model":"google\/gemini-2.5-flash-preview-05-20", "bcp_47":"zh", "task":"translation_to", "metric":"chrf", "score":0.3358004159 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"ar", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"ar", "task":"mmlu", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"ar", "task":"translation_from", "metric":"bleu", "score":0.2747843596 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"ar", "task":"translation_from", "metric":"chrf", "score":0.5519960681 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"ar", "task":"translation_to", "metric":"bleu", "score":0.2863967069 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"ar", "task":"translation_to", "metric":"chrf", "score":0.5318173199 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"bn", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"bn", "task":"mmlu", "metric":"accuracy", "score":0.6 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"bn", "task":"translation_from", "metric":"bleu", "score":0.271237739 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"bn", "task":"translation_from", "metric":"chrf", "score":0.5173954387 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"bn", "task":"translation_to", "metric":"bleu", "score":0.3067537945 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"bn", "task":"translation_to", "metric":"chrf", "score":0.5194482945 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"de", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"de", "task":"mmlu", "metric":"accuracy", "score":0.6 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"de", "task":"translation_from", "metric":"bleu", "score":0.3189602129 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"de", "task":"translation_from", "metric":"chrf", "score":0.5548503533 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"de", "task":"translation_to", "metric":"bleu", "score":0.5229096392 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"de", "task":"translation_to", "metric":"chrf", "score":0.7023434262 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"en", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"en", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"en", "task":"translation_from", "metric":"bleu", "score":0.3889146477 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"en", "task":"translation_from", "metric":"chrf", "score":0.628092835 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"en", "task":"translation_to", "metric":"bleu", "score":0.4660772497 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"en", "task":"translation_to", "metric":"chrf", "score":0.7280386297 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"es", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"es", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"es", "task":"translation_from", "metric":"bleu", "score":0.3593767686 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"es", "task":"translation_from", "metric":"chrf", "score":0.5668073679 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"es", "task":"translation_to", "metric":"bleu", "score":0.3662275621 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"es", "task":"translation_to", "metric":"chrf", "score":0.6102640711 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"fr", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"fr", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"fr", "task":"translation_from", "metric":"bleu", "score":0.3180384008 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"fr", "task":"translation_from", "metric":"chrf", "score":0.5571267732 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"fr", "task":"translation_to", "metric":"bleu", "score":0.5778354146 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"fr", "task":"translation_to", "metric":"chrf", "score":0.7490356238 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"hi", "task":"classification", "metric":"accuracy", "score":0.8 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"hi", "task":"mmlu", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"hi", "task":"translation_from", "metric":"bleu", "score":0.3722751955 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"hi", "task":"translation_from", "metric":"chrf", "score":0.6207213131 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"hi", "task":"translation_to", "metric":"bleu", "score":0.3378499277 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"hi", "task":"translation_to", "metric":"chrf", "score":0.554090013 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"id", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"id", "task":"mmlu", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"id", "task":"translation_from", "metric":"bleu", "score":0.3583744222 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"id", "task":"translation_from", "metric":"chrf", "score":0.5732194975 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"id", "task":"translation_to", "metric":"bleu", "score":0.3364664006 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"id", "task":"translation_to", "metric":"chrf", "score":0.6438910651 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"ja", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"ja", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"ja", "task":"translation_from", "metric":"bleu", "score":0.2581140706 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"ja", "task":"translation_from", "metric":"chrf", "score":0.5395853617 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"ja", "task":"translation_to", "metric":"bleu", "score":0.3298839393 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"ja", "task":"translation_to", "metric":"chrf", "score":0.4471547552 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"mr", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"mr", "task":"translation_from", "metric":"bleu", "score":0.2370074805 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"mr", "task":"translation_from", "metric":"chrf", "score":0.4804215458 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"mr", "task":"translation_to", "metric":"bleu", "score":0.2399769139 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"mr", "task":"translation_to", "metric":"chrf", "score":0.4726429935 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"pa", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"pa", "task":"translation_from", "metric":"bleu", "score":0.4180718844 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"pa", "task":"translation_from", "metric":"chrf", "score":0.6426219278 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"pa", "task":"translation_to", "metric":"bleu", "score":0.3938693136 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"pa", "task":"translation_to", "metric":"chrf", "score":0.5573992167 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"pt", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"pt", "task":"mmlu", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"pt", "task":"translation_from", "metric":"bleu", "score":0.3537745123 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"pt", "task":"translation_from", "metric":"chrf", "score":0.5904429929 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"pt", "task":"translation_to", "metric":"bleu", "score":0.4588664196 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"pt", "task":"translation_to", "metric":"chrf", "score":0.6844540285 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"ru", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"ru", "task":"mmlu", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"ru", "task":"translation_from", "metric":"bleu", "score":0.2449777422 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"ru", "task":"translation_from", "metric":"chrf", "score":0.5268764903 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"ru", "task":"translation_to", "metric":"bleu", "score":0.3789708434 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"ru", "task":"translation_to", "metric":"chrf", "score":0.5790333031 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"sw", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"sw", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"sw", "task":"translation_from", "metric":"bleu", "score":0.3005035588 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"sw", "task":"translation_from", "metric":"chrf", "score":0.5210660172 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"sw", "task":"translation_to", "metric":"bleu", "score":0.3401968092 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"sw", "task":"translation_to", "metric":"chrf", "score":0.6217197146 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"te", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"te", "task":"mmlu", "metric":"accuracy", "score":0.6 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"te", "task":"translation_from", "metric":"bleu", "score":0.3330093484 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"te", "task":"translation_from", "metric":"chrf", "score":0.5694168709 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"te", "task":"translation_to", "metric":"bleu", "score":0.2667299931 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"te", "task":"translation_to", "metric":"chrf", "score":0.5208033902 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"ur", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"ur", "task":"translation_from", "metric":"bleu", "score":0.322448107 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"ur", "task":"translation_from", "metric":"chrf", "score":0.5887654616 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"ur", "task":"translation_to", "metric":"bleu", "score":0.2148139783 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"ur", "task":"translation_to", "metric":"chrf", "score":0.4226865444 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"zh", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"zh", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"zh", "task":"translation_from", "metric":"bleu", "score":0.2201641871 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"zh", "task":"translation_from", "metric":"chrf", "score":0.5051068628 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"zh", "task":"translation_to", "metric":"bleu", "score":0.2508351517 }, { "model":"google\/gemini-flash-1.5", "bcp_47":"zh", "task":"translation_to", "metric":"chrf", "score":0.3110461024 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"ar", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"ar", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"ar", "task":"translation_from", "metric":"bleu", "score":0.260683336 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"ar", "task":"translation_from", "metric":"chrf", "score":0.5383651277 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"ar", "task":"translation_to", "metric":"bleu", "score":0.2962406565 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"ar", "task":"translation_to", "metric":"chrf", "score":0.5190026627 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"bn", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"bn", "task":"mmlu", "metric":"accuracy", "score":0.6 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"bn", "task":"translation_from", "metric":"bleu", "score":0.2183929994 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"bn", "task":"translation_from", "metric":"chrf", "score":0.4877941086 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"bn", "task":"translation_to", "metric":"bleu", "score":0.2700916391 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"bn", "task":"translation_to", "metric":"chrf", "score":0.5150577414 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"de", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"de", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"de", "task":"translation_from", "metric":"bleu", "score":0.3440655166 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"de", "task":"translation_from", "metric":"chrf", "score":0.563902418 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"de", "task":"translation_to", "metric":"bleu", "score":0.4305522274 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"de", "task":"translation_to", "metric":"chrf", "score":0.6477508732 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"en", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"en", "task":"mmlu", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"en", "task":"translation_from", "metric":"bleu", "score":0.4113463435 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"en", "task":"translation_from", "metric":"chrf", "score":0.6260248317 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"en", "task":"translation_to", "metric":"bleu", "score":0.5310035709 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"en", "task":"translation_to", "metric":"chrf", "score":0.7595845064 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"es", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"es", "task":"mmlu", "metric":"accuracy", "score":0.6 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"es", "task":"translation_from", "metric":"bleu", "score":0.3031284355 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"es", "task":"translation_from", "metric":"chrf", "score":0.5241309352 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"es", "task":"translation_to", "metric":"bleu", "score":0.3289699508 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"es", "task":"translation_to", "metric":"chrf", "score":0.5811203167 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"fr", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"fr", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"fr", "task":"translation_from", "metric":"bleu", "score":0.3349110908 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"fr", "task":"translation_from", "metric":"chrf", "score":0.564806297 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"fr", "task":"translation_to", "metric":"bleu", "score":0.4574014191 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"fr", "task":"translation_to", "metric":"chrf", "score":0.6626552528 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"hi", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"hi", "task":"mmlu", "metric":"accuracy", "score":0.6 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"hi", "task":"translation_from", "metric":"bleu", "score":0.3184042229 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"hi", "task":"translation_from", "metric":"chrf", "score":0.5712698408 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"hi", "task":"translation_to", "metric":"bleu", "score":0.3659029431 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"hi", "task":"translation_to", "metric":"chrf", "score":0.5796195236 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"id", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"id", "task":"mmlu", "metric":"accuracy", "score":0.5 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"id", "task":"translation_from", "metric":"bleu", "score":0.2301748885 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"id", "task":"translation_from", "metric":"chrf", "score":0.4682741896 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"id", "task":"translation_to", "metric":"bleu", "score":0.3346592082 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"id", "task":"translation_to", "metric":"chrf", "score":0.6514874668 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"ja", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"ja", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"ja", "task":"translation_from", "metric":"bleu", "score":0.2664966821 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"ja", "task":"translation_from", "metric":"chrf", "score":0.5386982677 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"ja", "task":"translation_to", "metric":"bleu", "score":0.2651365589 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"ja", "task":"translation_to", "metric":"chrf", "score":0.409095006 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"mr", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"mr", "task":"translation_from", "metric":"bleu", "score":0.2256246926 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"mr", "task":"translation_from", "metric":"chrf", "score":0.4703189943 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"mr", "task":"translation_to", "metric":"bleu", "score":0.2254492518 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"mr", "task":"translation_to", "metric":"chrf", "score":0.447826525 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"pa", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"pa", "task":"translation_from", "metric":"bleu", "score":0.3706063992 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"pa", "task":"translation_from", "metric":"chrf", "score":0.6167676482 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"pa", "task":"translation_to", "metric":"bleu", "score":0.4234596823 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"pa", "task":"translation_to", "metric":"chrf", "score":0.5629443923 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"pt", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"pt", "task":"mmlu", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"pt", "task":"translation_from", "metric":"bleu", "score":0.2971403532 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"pt", "task":"translation_from", "metric":"chrf", "score":0.5321068893 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"pt", "task":"translation_to", "metric":"bleu", "score":0.4364286549 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"pt", "task":"translation_to", "metric":"chrf", "score":0.6509885745 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"ru", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"ru", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"ru", "task":"translation_from", "metric":"bleu", "score":0.2300270544 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"ru", "task":"translation_from", "metric":"chrf", "score":0.4839384065 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"ru", "task":"translation_to", "metric":"bleu", "score":0.4236492288 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"ru", "task":"translation_to", "metric":"chrf", "score":0.6116207052 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"sw", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"sw", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"sw", "task":"translation_from", "metric":"bleu", "score":0.2501435914 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"sw", "task":"translation_from", "metric":"chrf", "score":0.5088299265 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"sw", "task":"translation_to", "metric":"bleu", "score":0.262372343 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"sw", "task":"translation_to", "metric":"chrf", "score":0.5806899403 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"te", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"te", "task":"mmlu", "metric":"accuracy", "score":0.6 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"te", "task":"translation_from", "metric":"bleu", "score":0.3568851036 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"te", "task":"translation_from", "metric":"chrf", "score":0.5825326367 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"te", "task":"translation_to", "metric":"bleu", "score":0.3112091725 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"te", "task":"translation_to", "metric":"chrf", "score":0.5431414206 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"ur", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"ur", "task":"translation_from", "metric":"bleu", "score":0.2402951661 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"ur", "task":"translation_from", "metric":"chrf", "score":0.5033005385 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"ur", "task":"translation_to", "metric":"bleu", "score":0.2302239803 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"ur", "task":"translation_to", "metric":"chrf", "score":0.4066956434 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"zh", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"zh", "task":"mmlu", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"zh", "task":"translation_from", "metric":"bleu", "score":0.249810194 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"zh", "task":"translation_from", "metric":"chrf", "score":0.5151255506 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"zh", "task":"translation_to", "metric":"bleu", "score":0.1854861198 }, { "model":"google\/gemini-flash-1.5-8b", "bcp_47":"zh", "task":"translation_to", "metric":"chrf", "score":0.2463326959 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"ar", "task":"classification", "metric":"accuracy", "score":0.7 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"ar", "task":"mmlu", "metric":"accuracy", "score":0.0 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"ar", "task":"translation_from", "metric":"bleu", "score":0.2170512634 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"ar", "task":"translation_from", "metric":"chrf", "score":0.4780493622 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"ar", "task":"translation_to", "metric":"bleu", "score":0.2844387275 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"ar", "task":"translation_to", "metric":"chrf", "score":0.4979841291 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"bn", "task":"classification", "metric":"accuracy", "score":0.8 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"bn", "task":"mmlu", "metric":"accuracy", "score":0.0 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"bn", "task":"translation_from", "metric":"bleu", "score":0.1842659234 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"bn", "task":"translation_from", "metric":"chrf", "score":0.4331372452 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"bn", "task":"translation_to", "metric":"bleu", "score":0.2644024965 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"bn", "task":"translation_to", "metric":"chrf", "score":0.462996411 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"de", "task":"classification", "metric":"accuracy", "score":0.8 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"de", "task":"mmlu", "metric":"accuracy", "score":0.0 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"de", "task":"translation_from", "metric":"bleu", "score":0.2470808909 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"de", "task":"translation_from", "metric":"chrf", "score":0.4959472303 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"de", "task":"translation_to", "metric":"bleu", "score":0.3105509871 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"de", "task":"translation_to", "metric":"chrf", "score":0.5118330277 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"en", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"en", "task":"mmlu", "metric":"accuracy", "score":0.0 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"en", "task":"translation_from", "metric":"bleu", "score":0.4810009804 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"en", "task":"translation_from", "metric":"chrf", "score":0.6244199222 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"en", "task":"translation_to", "metric":"bleu", "score":0.6112171119 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"en", "task":"translation_to", "metric":"chrf", "score":0.8046069838 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"es", "task":"classification", "metric":"accuracy", "score":0.8 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"es", "task":"mmlu", "metric":"accuracy", "score":0.1 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"es", "task":"translation_from", "metric":"bleu", "score":0.1990314823 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"es", "task":"translation_from", "metric":"chrf", "score":0.4557184175 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"es", "task":"translation_to", "metric":"bleu", "score":0.3019092795 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"es", "task":"translation_to", "metric":"chrf", "score":0.5373780713 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"fr", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"fr", "task":"mmlu", "metric":"accuracy", "score":0.0 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"fr", "task":"translation_from", "metric":"bleu", "score":0.1741359314 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"fr", "task":"translation_from", "metric":"chrf", "score":0.4175910256 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"fr", "task":"translation_to", "metric":"bleu", "score":0.4029915732 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"fr", "task":"translation_to", "metric":"chrf", "score":0.618701934 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"hi", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"hi", "task":"mmlu", "metric":"accuracy", "score":0.1 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"hi", "task":"translation_from", "metric":"bleu", "score":0.3016091966 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"hi", "task":"translation_from", "metric":"chrf", "score":0.5281403748 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"hi", "task":"translation_to", "metric":"bleu", "score":0.2736270122 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"hi", "task":"translation_to", "metric":"chrf", "score":0.5159089396 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"id", "task":"classification", "metric":"accuracy", "score":0.8 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"id", "task":"mmlu", "metric":"accuracy", "score":0.0 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"id", "task":"translation_from", "metric":"bleu", "score":0.1930481943 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"id", "task":"translation_from", "metric":"chrf", "score":0.420733839 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"id", "task":"translation_to", "metric":"bleu", "score":0.344885086 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"id", "task":"translation_to", "metric":"chrf", "score":0.6497564532 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"ja", "task":"classification", "metric":"accuracy", "score":0.8 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"ja", "task":"mmlu", "metric":"accuracy", "score":0.0 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"ja", "task":"translation_from", "metric":"bleu", "score":0.2254647544 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"ja", "task":"translation_from", "metric":"chrf", "score":0.4810564434 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"ja", "task":"translation_to", "metric":"bleu", "score":0.1874016498 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"ja", "task":"translation_to", "metric":"chrf", "score":0.2916976073 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"mr", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"mr", "task":"translation_from", "metric":"bleu", "score":0.1616328428 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"mr", "task":"translation_from", "metric":"chrf", "score":0.414450627 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"mr", "task":"translation_to", "metric":"bleu", "score":0.1123956753 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"mr", "task":"translation_to", "metric":"chrf", "score":0.3542849129 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"pa", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"pa", "task":"translation_from", "metric":"bleu", "score":0.34113337 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"pa", "task":"translation_from", "metric":"chrf", "score":0.5187033084 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"pa", "task":"translation_to", "metric":"bleu", "score":0.3611338165 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"pa", "task":"translation_to", "metric":"chrf", "score":0.5059192583 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"pt", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"pt", "task":"mmlu", "metric":"accuracy", "score":0.1 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"pt", "task":"translation_from", "metric":"bleu", "score":0.2429635608 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"pt", "task":"translation_from", "metric":"chrf", "score":0.473106986 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"pt", "task":"translation_to", "metric":"bleu", "score":0.2843181347 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"pt", "task":"translation_to", "metric":"chrf", "score":0.5735245279 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"ru", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"ru", "task":"mmlu", "metric":"accuracy", "score":0.0 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"ru", "task":"translation_from", "metric":"bleu", "score":0.1697780011 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"ru", "task":"translation_from", "metric":"chrf", "score":0.4361163813 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"ru", "task":"translation_to", "metric":"bleu", "score":0.3672218817 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"ru", "task":"translation_to", "metric":"chrf", "score":0.5564712622 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"sw", "task":"classification", "metric":"accuracy", "score":0.8 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"sw", "task":"mmlu", "metric":"accuracy", "score":0.0 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"sw", "task":"translation_from", "metric":"bleu", "score":0.188207488 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"sw", "task":"translation_from", "metric":"chrf", "score":0.4728711823 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"sw", "task":"translation_to", "metric":"bleu", "score":0.27482714 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"sw", "task":"translation_to", "metric":"chrf", "score":0.5603409861 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"te", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"te", "task":"mmlu", "metric":"accuracy", "score":0.0 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"te", "task":"translation_from", "metric":"bleu", "score":0.3095013641 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"te", "task":"translation_from", "metric":"chrf", "score":0.5252619118 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"te", "task":"translation_to", "metric":"bleu", "score":0.1798656271 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"te", "task":"translation_to", "metric":"chrf", "score":0.4011450412 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"ur", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"ur", "task":"translation_from", "metric":"bleu", "score":0.137296648 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"ur", "task":"translation_from", "metric":"chrf", "score":0.341408254 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"ur", "task":"translation_to", "metric":"bleu", "score":0.0595669677 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"ur", "task":"translation_to", "metric":"chrf", "score":0.201487497 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"zh", "task":"classification", "metric":"accuracy", "score":0.7 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"zh", "task":"mmlu", "metric":"accuracy", "score":0.1 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"zh", "task":"translation_from", "metric":"bleu", "score":0.1922618764 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"zh", "task":"translation_from", "metric":"chrf", "score":0.4517742384 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"zh", "task":"translation_to", "metric":"bleu", "score":0.2125812397 }, { "model":"google\/gemma-2-9b-it", "bcp_47":"zh", "task":"translation_to", "metric":"chrf", "score":0.2689557345 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"ar", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"ar", "task":"mmlu", "metric":"accuracy", "score":0.4 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"ar", "task":"translation_from", "metric":"bleu", "score":0.2891206499 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"ar", "task":"translation_from", "metric":"chrf", "score":0.5438550217 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"ar", "task":"translation_to", "metric":"bleu", "score":0.3184721364 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"ar", "task":"translation_to", "metric":"chrf", "score":0.5483731849 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"bn", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"bn", "task":"mmlu", "metric":"accuracy", "score":0.3 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"bn", "task":"translation_from", "metric":"bleu", "score":0.2907230812 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"bn", "task":"translation_from", "metric":"chrf", "score":0.5148223626 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"bn", "task":"translation_to", "metric":"bleu", "score":0.3473636391 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"bn", "task":"translation_to", "metric":"chrf", "score":0.5442574441 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"de", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"de", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"de", "task":"translation_from", "metric":"bleu", "score":0.3577876868 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"de", "task":"translation_from", "metric":"chrf", "score":0.5943423055 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"de", "task":"translation_to", "metric":"bleu", "score":0.5344280565 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"de", "task":"translation_to", "metric":"chrf", "score":0.7084649844 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"en", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"en", "task":"mmlu", "metric":"accuracy", "score":0.6 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"en", "task":"translation_from", "metric":"bleu", "score":0.452427177 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"en", "task":"translation_from", "metric":"chrf", "score":0.6525566656 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"en", "task":"translation_to", "metric":"bleu", "score":0.5467976399 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"en", "task":"translation_to", "metric":"chrf", "score":0.7780833183 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"es", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"es", "task":"mmlu", "metric":"accuracy", "score":0.6 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"es", "task":"translation_from", "metric":"bleu", "score":0.3554331718 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"es", "task":"translation_from", "metric":"chrf", "score":0.5997743406 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"es", "task":"translation_to", "metric":"bleu", "score":0.3847830842 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"es", "task":"translation_to", "metric":"chrf", "score":0.6191109047 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"fr", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"fr", "task":"mmlu", "metric":"accuracy", "score":0.3 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"fr", "task":"translation_from", "metric":"bleu", "score":0.2846092378 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"fr", "task":"translation_from", "metric":"chrf", "score":0.5655970541 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"fr", "task":"translation_to", "metric":"bleu", "score":0.5192984544 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"fr", "task":"translation_to", "metric":"chrf", "score":0.7020040834 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"hi", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"hi", "task":"mmlu", "metric":"accuracy", "score":0.5 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"hi", "task":"translation_from", "metric":"bleu", "score":0.3520691191 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"hi", "task":"translation_from", "metric":"chrf", "score":0.6035990708 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"hi", "task":"translation_to", "metric":"bleu", "score":0.3987037224 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"hi", "task":"translation_to", "metric":"chrf", "score":0.6195037668 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"id", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"id", "task":"mmlu", "metric":"accuracy", "score":0.3 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"id", "task":"translation_from", "metric":"bleu", "score":0.3377417704 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"id", "task":"translation_from", "metric":"chrf", "score":0.5674360496 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"id", "task":"translation_to", "metric":"bleu", "score":0.3534620252 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"id", "task":"translation_to", "metric":"chrf", "score":0.6680177029 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"ja", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"ja", "task":"mmlu", "metric":"accuracy", "score":0.4 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"ja", "task":"translation_from", "metric":"bleu", "score":0.2481856237 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"ja", "task":"translation_from", "metric":"chrf", "score":0.5180749152 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"ja", "task":"translation_to", "metric":"bleu", "score":0.242508046 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"ja", "task":"translation_to", "metric":"chrf", "score":0.4046420215 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"mr", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"mr", "task":"translation_from", "metric":"bleu", "score":0.3395095603 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"mr", "task":"translation_from", "metric":"chrf", "score":0.5877742809 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"mr", "task":"translation_to", "metric":"bleu", "score":0.2222923122 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"mr", "task":"translation_to", "metric":"chrf", "score":0.4572688692 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"pa", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"pa", "task":"translation_from", "metric":"bleu", "score":0.4164890636 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"pa", "task":"translation_from", "metric":"chrf", "score":0.6375470445 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"pa", "task":"translation_to", "metric":"bleu", "score":0.4166823661 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"pa", "task":"translation_to", "metric":"chrf", "score":0.575314128 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"pt", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"pt", "task":"mmlu", "metric":"accuracy", "score":0.6 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"pt", "task":"translation_from", "metric":"bleu", "score":0.340245547 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"pt", "task":"translation_from", "metric":"chrf", "score":0.5681284927 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"pt", "task":"translation_to", "metric":"bleu", "score":0.4702737577 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"pt", "task":"translation_to", "metric":"chrf", "score":0.6903236014 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"ru", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"ru", "task":"mmlu", "metric":"accuracy", "score":0.3 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"ru", "task":"translation_from", "metric":"bleu", "score":0.2862936285 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"ru", "task":"translation_from", "metric":"chrf", "score":0.5264436928 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"ru", "task":"translation_to", "metric":"bleu", "score":0.4800957551 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"ru", "task":"translation_to", "metric":"chrf", "score":0.6618495803 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"sw", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"sw", "task":"mmlu", "metric":"accuracy", "score":0.6 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"sw", "task":"translation_from", "metric":"bleu", "score":0.268709657 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"sw", "task":"translation_from", "metric":"chrf", "score":0.5306834056 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"sw", "task":"translation_to", "metric":"bleu", "score":0.3395981599 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"sw", "task":"translation_to", "metric":"chrf", "score":0.6130756934 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"te", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"te", "task":"mmlu", "metric":"accuracy", "score":0.2 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"te", "task":"translation_from", "metric":"bleu", "score":0.279412176 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"te", "task":"translation_from", "metric":"chrf", "score":0.5419647491 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"te", "task":"translation_to", "metric":"bleu", "score":0.259866454 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"te", "task":"translation_to", "metric":"chrf", "score":0.48543634 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"ur", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"ur", "task":"translation_from", "metric":"bleu", "score":0.2595582459 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"ur", "task":"translation_from", "metric":"chrf", "score":0.5043992681 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"ur", "task":"translation_to", "metric":"bleu", "score":0.2142625601 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"ur", "task":"translation_to", "metric":"chrf", "score":0.429749938 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"zh", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"zh", "task":"mmlu", "metric":"accuracy", "score":0.5 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"zh", "task":"translation_from", "metric":"bleu", "score":0.1900086584 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"zh", "task":"translation_from", "metric":"chrf", "score":0.4895930442 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"zh", "task":"translation_to", "metric":"bleu", "score":0.2395565562 }, { "model":"google\/gemma-3-27b-it", "bcp_47":"zh", "task":"translation_to", "metric":"chrf", "score":0.3237759485 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"ar", "task":"classification", "metric":"accuracy", "score":0.5 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"ar", "task":"mmlu", "metric":"accuracy", "score":0.4 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"ar", "task":"translation_from", "metric":"bleu", "score":0.0837265107 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"ar", "task":"translation_from", "metric":"chrf", "score":0.3012065838 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"ar", "task":"translation_to", "metric":"bleu", "score":0.1120670716 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"ar", "task":"translation_to", "metric":"chrf", "score":0.2771296913 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"bn", "task":"classification", "metric":"accuracy", "score":0.4 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"bn", "task":"mmlu", "metric":"accuracy", "score":0.3 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"bn", "task":"translation_from", "metric":"bleu", "score":0.021858254 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"bn", "task":"translation_from", "metric":"chrf", "score":0.2327499821 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"bn", "task":"translation_to", "metric":"bleu", "score":0.0253088472 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"bn", "task":"translation_to", "metric":"chrf", "score":0.1918662187 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"de", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"de", "task":"mmlu", "metric":"accuracy", "score":0.4 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"de", "task":"translation_from", "metric":"bleu", "score":0.1784955678 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"de", "task":"translation_from", "metric":"chrf", "score":0.4008799371 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"de", "task":"translation_to", "metric":"bleu", "score":0.1782663616 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"de", "task":"translation_to", "metric":"chrf", "score":0.4070825897 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"en", "task":"classification", "metric":"accuracy", "score":0.8 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"en", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"en", "task":"translation_from", "metric":"bleu", "score":0.2797104835 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"en", "task":"translation_from", "metric":"chrf", "score":0.4601471921 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"en", "task":"translation_to", "metric":"bleu", "score":0.3392404298 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"en", "task":"translation_to", "metric":"chrf", "score":0.5518674496 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"es", "task":"classification", "metric":"accuracy", "score":0.8 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"es", "task":"mmlu", "metric":"accuracy", "score":0.4 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"es", "task":"translation_from", "metric":"bleu", "score":0.1682956348 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"es", "task":"translation_from", "metric":"chrf", "score":0.3705606944 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"es", "task":"translation_to", "metric":"bleu", "score":0.1595479626 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"es", "task":"translation_to", "metric":"chrf", "score":0.4040956812 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"fr", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"fr", "task":"mmlu", "metric":"accuracy", "score":0.5 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"fr", "task":"translation_from", "metric":"bleu", "score":0.1362099506 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"fr", "task":"translation_from", "metric":"chrf", "score":0.4039646029 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"fr", "task":"translation_to", "metric":"bleu", "score":0.2151785904 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"fr", "task":"translation_to", "metric":"chrf", "score":0.4309035319 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"hi", "task":"classification", "metric":"accuracy", "score":0.6 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"hi", "task":"mmlu", "metric":"accuracy", "score":0.2 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"hi", "task":"translation_from", "metric":"bleu", "score":0.1658307051 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"hi", "task":"translation_from", "metric":"chrf", "score":0.3676480008 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"hi", "task":"translation_to", "metric":"bleu", "score":0.1711676323 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"hi", "task":"translation_to", "metric":"chrf", "score":0.360371738 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"id", "task":"classification", "metric":"accuracy", "score":0.7 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"id", "task":"mmlu", "metric":"accuracy", "score":0.3 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"id", "task":"translation_from", "metric":"bleu", "score":0.1399603895 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"id", "task":"translation_from", "metric":"chrf", "score":0.3604113675 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"id", "task":"translation_to", "metric":"bleu", "score":0.1312450274 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"id", "task":"translation_to", "metric":"chrf", "score":0.3988827234 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"ja", "task":"classification", "metric":"accuracy", "score":0.7 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"ja", "task":"mmlu", "metric":"accuracy", "score":0.1 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"ja", "task":"translation_from", "metric":"bleu", "score":0.1222242366 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"ja", "task":"translation_from", "metric":"chrf", "score":0.3415186168 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"ja", "task":"translation_to", "metric":"bleu", "score":0.1607198845 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"ja", "task":"translation_to", "metric":"chrf", "score":0.2706371796 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"mr", "task":"classification", "metric":"accuracy", "score":0.4 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"mr", "task":"translation_from", "metric":"bleu", "score":0.0411296617 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"mr", "task":"translation_from", "metric":"chrf", "score":0.2442292695 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"mr", "task":"translation_to", "metric":"bleu", "score":0.0482775033 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"mr", "task":"translation_to", "metric":"chrf", "score":0.2211003771 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"pa", "task":"classification", "metric":"accuracy", "score":0.0 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"pa", "task":"translation_from", "metric":"bleu", "score":0.2292007848 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"pa", "task":"translation_from", "metric":"chrf", "score":0.3716655897 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"pa", "task":"translation_to", "metric":"bleu", "score":0.2126702079 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"pa", "task":"translation_to", "metric":"chrf", "score":0.3248815955 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"pt", "task":"classification", "metric":"accuracy", "score":0.7 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"pt", "task":"mmlu", "metric":"accuracy", "score":0.4 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"pt", "task":"translation_from", "metric":"bleu", "score":0.1013445398 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"pt", "task":"translation_from", "metric":"chrf", "score":0.3063125264 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"pt", "task":"translation_to", "metric":"bleu", "score":0.2231857524 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"pt", "task":"translation_to", "metric":"chrf", "score":0.4401829864 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"ru", "task":"classification", "metric":"accuracy", "score":0.8 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"ru", "task":"mmlu", "metric":"accuracy", "score":0.3 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"ru", "task":"translation_from", "metric":"bleu", "score":0.1005466956 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"ru", "task":"translation_from", "metric":"chrf", "score":0.3356518748 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"ru", "task":"translation_to", "metric":"bleu", "score":0.1363185356 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"ru", "task":"translation_to", "metric":"chrf", "score":0.3769111636 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"sw", "task":"classification", "metric":"accuracy", "score":0.5 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"sw", "task":"mmlu", "metric":"accuracy", "score":0.3 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"sw", "task":"translation_from", "metric":"bleu", "score":0.0510060878 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"sw", "task":"translation_from", "metric":"chrf", "score":0.2132741272 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"sw", "task":"translation_to", "metric":"bleu", "score":0.0089146903 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"sw", "task":"translation_to", "metric":"chrf", "score":0.2314109768 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"te", "task":"classification", "metric":"accuracy", "score":0.2 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"te", "task":"mmlu", "metric":"accuracy", "score":0.1 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"te", "task":"translation_from", "metric":"bleu", "score":0.1096331511 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"te", "task":"translation_from", "metric":"chrf", "score":0.2888090685 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"te", "task":"translation_to", "metric":"bleu", "score":0.1206114883 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"te", "task":"translation_to", "metric":"chrf", "score":0.2597106436 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"ur", "task":"classification", "metric":"accuracy", "score":0.5 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"ur", "task":"translation_from", "metric":"bleu", "score":0.0891185343 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"ur", "task":"translation_from", "metric":"chrf", "score":0.281793335 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"ur", "task":"translation_to", "metric":"bleu", "score":0.0799576366 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"ur", "task":"translation_to", "metric":"chrf", "score":0.2661903898 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"zh", "task":"classification", "metric":"accuracy", "score":0.7 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"zh", "task":"mmlu", "metric":"accuracy", "score":0.3 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"zh", "task":"translation_from", "metric":"bleu", "score":0.1133138775 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"zh", "task":"translation_from", "metric":"chrf", "score":0.3754190494 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"zh", "task":"translation_to", "metric":"bleu", "score":0.1672636279 }, { "model":"gryphe\/mythomax-l2-13b", "bcp_47":"zh", "task":"translation_to", "metric":"chrf", "score":0.232325354 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"ar", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"ar", "task":"mmlu", "metric":"accuracy", "score":1.0 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"ar", "task":"translation_from", "metric":"bleu", "score":0.2609114367 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"ar", "task":"translation_from", "metric":"chrf", "score":0.5167379854 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"ar", "task":"translation_to", "metric":"bleu", "score":0.1932802581 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"ar", "task":"translation_to", "metric":"chrf", "score":0.4648835751 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"bn", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"bn", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"bn", "task":"translation_from", "metric":"bleu", "score":0.1938367121 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"bn", "task":"translation_from", "metric":"chrf", "score":0.4674774016 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"bn", "task":"translation_to", "metric":"bleu", "score":0.2073541352 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"bn", "task":"translation_to", "metric":"chrf", "score":0.4438396219 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"de", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"de", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"de", "task":"translation_from", "metric":"bleu", "score":0.2685056004 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"de", "task":"translation_from", "metric":"chrf", "score":0.506818165 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"de", "task":"translation_to", "metric":"bleu", "score":0.3801657831 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"de", "task":"translation_to", "metric":"chrf", "score":0.6056477234 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"en", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"en", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"en", "task":"translation_from", "metric":"bleu", "score":0.2216031518 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"en", "task":"translation_from", "metric":"chrf", "score":0.4453474399 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"en", "task":"translation_to", "metric":"bleu", "score":0.3322887566 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"en", "task":"translation_to", "metric":"chrf", "score":0.5824514758 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"es", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"es", "task":"mmlu", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"es", "task":"translation_from", "metric":"bleu", "score":0.1640783778 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"es", "task":"translation_from", "metric":"chrf", "score":0.434749516 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"es", "task":"translation_to", "metric":"bleu", "score":0.3126220052 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"es", "task":"translation_to", "metric":"chrf", "score":0.6038883227 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"fr", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"fr", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"fr", "task":"translation_from", "metric":"bleu", "score":0.2261588318 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"fr", "task":"translation_from", "metric":"chrf", "score":0.4944996319 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"fr", "task":"translation_to", "metric":"bleu", "score":0.4626001556 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"fr", "task":"translation_to", "metric":"chrf", "score":0.6614963779 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"hi", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"hi", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"hi", "task":"translation_from", "metric":"bleu", "score":0.3316295853 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"hi", "task":"translation_from", "metric":"chrf", "score":0.5594083443 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"hi", "task":"translation_to", "metric":"bleu", "score":0.365274938 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"hi", "task":"translation_to", "metric":"chrf", "score":0.588222932 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"id", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"id", "task":"mmlu", "metric":"accuracy", "score":1.0 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"id", "task":"translation_from", "metric":"bleu", "score":0.2236143729 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"id", "task":"translation_from", "metric":"chrf", "score":0.4912948296 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"id", "task":"translation_to", "metric":"bleu", "score":0.2910526755 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"id", "task":"translation_to", "metric":"chrf", "score":0.6005590773 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"ja", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"ja", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"ja", "task":"translation_from", "metric":"bleu", "score":0.2131020144 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"ja", "task":"translation_from", "metric":"chrf", "score":0.4845704057 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"ja", "task":"translation_to", "metric":"bleu", "score":0.1446650781 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"ja", "task":"translation_to", "metric":"chrf", "score":0.2292145443 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"mr", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"mr", "task":"translation_from", "metric":"bleu", "score":0.2152417217 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"mr", "task":"translation_from", "metric":"chrf", "score":0.4578207034 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"mr", "task":"translation_to", "metric":"bleu", "score":0.1654073391 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"mr", "task":"translation_to", "metric":"chrf", "score":0.3941079443 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"pa", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"pa", "task":"translation_from", "metric":"bleu", "score":0.1891048622 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"pa", "task":"translation_from", "metric":"chrf", "score":0.4233257981 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"pa", "task":"translation_to", "metric":"bleu", "score":0.2086330089 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"pa", "task":"translation_to", "metric":"chrf", "score":0.364326938 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"pt", "task":"classification", "metric":"accuracy", "score":0.8 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"pt", "task":"mmlu", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"pt", "task":"translation_from", "metric":"bleu", "score":0.2256776552 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"pt", "task":"translation_from", "metric":"chrf", "score":0.4793769886 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"pt", "task":"translation_to", "metric":"bleu", "score":0.3320822339 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"pt", "task":"translation_to", "metric":"chrf", "score":0.6132478102 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"ru", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"ru", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"ru", "task":"translation_from", "metric":"bleu", "score":0.1730354472 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"ru", "task":"translation_from", "metric":"chrf", "score":0.454951133 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"ru", "task":"translation_to", "metric":"bleu", "score":0.3073459183 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"ru", "task":"translation_to", "metric":"chrf", "score":0.554890569 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"sw", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"sw", "task":"mmlu", "metric":"accuracy", "score":0.6 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"sw", "task":"translation_from", "metric":"bleu", "score":0.1798653655 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"sw", "task":"translation_from", "metric":"chrf", "score":0.4290347271 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"sw", "task":"translation_to", "metric":"bleu", "score":0.1709237193 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"sw", "task":"translation_to", "metric":"chrf", "score":0.522080463 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"te", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"te", "task":"mmlu", "metric":"accuracy", "score":0.5 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"te", "task":"translation_from", "metric":"bleu", "score":0.2507091988 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"te", "task":"translation_from", "metric":"chrf", "score":0.5067409185 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"te", "task":"translation_to", "metric":"bleu", "score":0.2998213366 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"te", "task":"translation_to", "metric":"chrf", "score":0.5112516267 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"ur", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"ur", "task":"translation_from", "metric":"bleu", "score":0.194871957 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"ur", "task":"translation_from", "metric":"chrf", "score":0.4838690709 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"ur", "task":"translation_to", "metric":"bleu", "score":0.1908512613 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"ur", "task":"translation_to", "metric":"chrf", "score":0.3855814375 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"zh", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"zh", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"zh", "task":"translation_from", "metric":"bleu", "score":0.1869648376 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"zh", "task":"translation_from", "metric":"chrf", "score":0.4664675781 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"zh", "task":"translation_to", "metric":"bleu", "score":0.1696786467 }, { "model":"meta-llama\/llama-3-70b-instruct", "bcp_47":"zh", "task":"translation_to", "metric":"chrf", "score":0.2325846099 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"ar", "task":"classification", "metric":"accuracy", "score":0.8 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"ar", "task":"mmlu", "metric":"accuracy", "score":0.5 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"ar", "task":"translation_from", "metric":"bleu", "score":0.1934231192 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"ar", "task":"translation_from", "metric":"chrf", "score":0.4070435279 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"ar", "task":"translation_to", "metric":"bleu", "score":0.1705154843 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"ar", "task":"translation_to", "metric":"chrf", "score":0.4152198662 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"bn", "task":"classification", "metric":"accuracy", "score":0.8 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"bn", "task":"mmlu", "metric":"accuracy", "score":0.3 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"bn", "task":"translation_from", "metric":"bleu", "score":0.1488002659 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"bn", "task":"translation_from", "metric":"chrf", "score":0.3745722385 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"bn", "task":"translation_to", "metric":"bleu", "score":0.1902972944 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"bn", "task":"translation_to", "metric":"chrf", "score":0.3816229412 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"de", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"de", "task":"mmlu", "metric":"accuracy", "score":0.6 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"de", "task":"translation_from", "metric":"bleu", "score":0.1785474661 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"de", "task":"translation_from", "metric":"chrf", "score":0.4370657832 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"de", "task":"translation_to", "metric":"bleu", "score":0.2574134539 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"de", "task":"translation_to", "metric":"chrf", "score":0.5270318561 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"en", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"en", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"en", "task":"translation_from", "metric":"bleu", "score":0.3528640776 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"en", "task":"translation_from", "metric":"chrf", "score":0.5518750903 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"en", "task":"translation_to", "metric":"bleu", "score":0.4474952526 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"en", "task":"translation_to", "metric":"chrf", "score":0.6711323725 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"es", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"es", "task":"mmlu", "metric":"accuracy", "score":0.6 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"es", "task":"translation_from", "metric":"bleu", "score":0.1873108686 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"es", "task":"translation_from", "metric":"chrf", "score":0.4130794041 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"es", "task":"translation_to", "metric":"bleu", "score":0.233152279 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"es", "task":"translation_to", "metric":"chrf", "score":0.5457724764 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"fr", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"fr", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"fr", "task":"translation_from", "metric":"bleu", "score":0.1823885989 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"fr", "task":"translation_from", "metric":"chrf", "score":0.4281583879 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"fr", "task":"translation_to", "metric":"bleu", "score":0.3356151999 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"fr", "task":"translation_to", "metric":"chrf", "score":0.5339127213 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"hi", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"hi", "task":"mmlu", "metric":"accuracy", "score":0.5 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"hi", "task":"translation_from", "metric":"bleu", "score":0.2985177739 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"hi", "task":"translation_from", "metric":"chrf", "score":0.5122033781 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"hi", "task":"translation_to", "metric":"bleu", "score":0.2288990675 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"hi", "task":"translation_to", "metric":"chrf", "score":0.4641914485 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"id", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"id", "task":"mmlu", "metric":"accuracy", "score":0.5 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"id", "task":"translation_from", "metric":"bleu", "score":0.1513573791 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"id", "task":"translation_from", "metric":"chrf", "score":0.4126710531 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"id", "task":"translation_to", "metric":"bleu", "score":0.2153271782 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"id", "task":"translation_to", "metric":"chrf", "score":0.5281145734 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"ja", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"ja", "task":"mmlu", "metric":"accuracy", "score":0.6 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"ja", "task":"translation_from", "metric":"bleu", "score":0.1685400892 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"ja", "task":"translation_from", "metric":"chrf", "score":0.4152307643 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"ja", "task":"translation_to", "metric":"bleu", "score":0.133960866 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"ja", "task":"translation_to", "metric":"chrf", "score":0.2277435316 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"mr", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"mr", "task":"translation_from", "metric":"bleu", "score":0.1507564245 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"mr", "task":"translation_from", "metric":"chrf", "score":0.3990456964 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"mr", "task":"translation_to", "metric":"bleu", "score":0.1244036875 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"mr", "task":"translation_to", "metric":"chrf", "score":0.3211879294 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"pa", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"pa", "task":"translation_from", "metric":"bleu", "score":0.2304909323 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"pa", "task":"translation_from", "metric":"chrf", "score":0.4895854606 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"pa", "task":"translation_to", "metric":"bleu", "score":0.2411212371 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"pa", "task":"translation_to", "metric":"chrf", "score":0.4357494606 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"pt", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"pt", "task":"mmlu", "metric":"accuracy", "score":0.4 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"pt", "task":"translation_from", "metric":"bleu", "score":0.1707431977 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"pt", "task":"translation_from", "metric":"chrf", "score":0.41325613 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"pt", "task":"translation_to", "metric":"bleu", "score":0.3010483452 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"pt", "task":"translation_to", "metric":"chrf", "score":0.5494427262 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"ru", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"ru", "task":"mmlu", "metric":"accuracy", "score":0.6 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"ru", "task":"translation_from", "metric":"bleu", "score":0.1269147932 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"ru", "task":"translation_from", "metric":"chrf", "score":0.4169665509 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"ru", "task":"translation_to", "metric":"bleu", "score":0.2000369037 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"ru", "task":"translation_to", "metric":"chrf", "score":0.4178019923 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"sw", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"sw", "task":"mmlu", "metric":"accuracy", "score":0.2 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"sw", "task":"translation_from", "metric":"bleu", "score":0.1181160377 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"sw", "task":"translation_from", "metric":"chrf", "score":0.3506298599 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"sw", "task":"translation_to", "metric":"bleu", "score":0.080193685 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"sw", "task":"translation_to", "metric":"chrf", "score":0.431036918 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"te", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"te", "task":"mmlu", "metric":"accuracy", "score":0.2 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"te", "task":"translation_from", "metric":"bleu", "score":0.1477600607 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"te", "task":"translation_from", "metric":"chrf", "score":0.4004241369 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"te", "task":"translation_to", "metric":"bleu", "score":0.2008915859 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"te", "task":"translation_to", "metric":"chrf", "score":0.3876603494 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"ur", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"ur", "task":"translation_from", "metric":"bleu", "score":0.1383930526 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"ur", "task":"translation_from", "metric":"chrf", "score":0.3847225326 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"ur", "task":"translation_to", "metric":"bleu", "score":0.1092836008 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"ur", "task":"translation_to", "metric":"chrf", "score":0.2981134702 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"zh", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"zh", "task":"mmlu", "metric":"accuracy", "score":0.3 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"zh", "task":"translation_from", "metric":"bleu", "score":0.1188554471 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"zh", "task":"translation_from", "metric":"chrf", "score":0.393157353 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"zh", "task":"translation_to", "metric":"bleu", "score":0.1823790731 }, { "model":"meta-llama\/llama-3-8b-instruct", "bcp_47":"zh", "task":"translation_to", "metric":"chrf", "score":0.24906438 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"ar", "task":"classification", "metric":"accuracy", "score":0.8 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"ar", "task":"mmlu", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"ar", "task":"translation_from", "metric":"bleu", "score":0.2766513185 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"ar", "task":"translation_from", "metric":"chrf", "score":0.5528193738 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"ar", "task":"translation_to", "metric":"bleu", "score":0.3371708551 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"ar", "task":"translation_to", "metric":"chrf", "score":0.5607841978 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"bn", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"bn", "task":"mmlu", "metric":"accuracy", "score":0.6 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"bn", "task":"translation_from", "metric":"bleu", "score":0.2623045124 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"bn", "task":"translation_from", "metric":"chrf", "score":0.512895511 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"bn", "task":"translation_to", "metric":"bleu", "score":0.3328087961 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"bn", "task":"translation_to", "metric":"chrf", "score":0.5013967236 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"de", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"de", "task":"mmlu", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"de", "task":"translation_from", "metric":"bleu", "score":0.3059254014 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"de", "task":"translation_from", "metric":"chrf", "score":0.5403551155 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"de", "task":"translation_to", "metric":"bleu", "score":0.4588037752 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"de", "task":"translation_to", "metric":"chrf", "score":0.6692431614 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"en", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"en", "task":"mmlu", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"en", "task":"translation_from", "metric":"bleu", "score":0.5486108614 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"en", "task":"translation_from", "metric":"chrf", "score":0.6873139374 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"en", "task":"translation_to", "metric":"bleu", "score":0.691450825 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"en", "task":"translation_to", "metric":"chrf", "score":0.8407064328 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"es", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"es", "task":"mmlu", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"es", "task":"translation_from", "metric":"bleu", "score":0.3021494986 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"es", "task":"translation_from", "metric":"chrf", "score":0.5604145602 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"es", "task":"translation_to", "metric":"bleu", "score":0.3865320679 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"es", "task":"translation_to", "metric":"chrf", "score":0.6126903448 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"fr", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"fr", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"fr", "task":"translation_from", "metric":"bleu", "score":0.2924853239 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"fr", "task":"translation_from", "metric":"chrf", "score":0.5462367408 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"fr", "task":"translation_to", "metric":"bleu", "score":0.4963618411 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"fr", "task":"translation_to", "metric":"chrf", "score":0.682573515 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"hi", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"hi", "task":"mmlu", "metric":"accuracy", "score":1.0 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"hi", "task":"translation_from", "metric":"bleu", "score":0.3914590212 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"hi", "task":"translation_from", "metric":"chrf", "score":0.6128431348 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"hi", "task":"translation_to", "metric":"bleu", "score":0.4189427376 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"hi", "task":"translation_to", "metric":"chrf", "score":0.6262550579 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"id", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"id", "task":"mmlu", "metric":"accuracy", "score":1.0 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"id", "task":"translation_from", "metric":"bleu", "score":0.290274553 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"id", "task":"translation_from", "metric":"chrf", "score":0.5143960108 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"id", "task":"translation_to", "metric":"bleu", "score":0.3789023659 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"id", "task":"translation_to", "metric":"chrf", "score":0.6751523776 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"ja", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"ja", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"ja", "task":"translation_from", "metric":"bleu", "score":0.3306727326 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"ja", "task":"translation_from", "metric":"chrf", "score":0.5648258387 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"ja", "task":"translation_to", "metric":"bleu", "score":0.262512317 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"ja", "task":"translation_to", "metric":"chrf", "score":0.4036795798 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"mr", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"mr", "task":"translation_from", "metric":"bleu", "score":0.2987546297 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"mr", "task":"translation_from", "metric":"chrf", "score":0.526361525 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"mr", "task":"translation_to", "metric":"bleu", "score":0.1515083487 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"mr", "task":"translation_to", "metric":"chrf", "score":0.4332812085 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"pa", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"pa", "task":"translation_from", "metric":"bleu", "score":0.4479091606 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"pa", "task":"translation_from", "metric":"chrf", "score":0.6272058507 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"pa", "task":"translation_to", "metric":"bleu", "score":0.4155919737 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"pa", "task":"translation_to", "metric":"chrf", "score":0.5685427433 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"pt", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"pt", "task":"mmlu", "metric":"accuracy", "score":1.0 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"pt", "task":"translation_from", "metric":"bleu", "score":0.3183331223 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"pt", "task":"translation_from", "metric":"chrf", "score":0.5559144449 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"pt", "task":"translation_to", "metric":"bleu", "score":0.4500659682 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"pt", "task":"translation_to", "metric":"chrf", "score":0.6761551234 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"ru", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"ru", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"ru", "task":"translation_from", "metric":"bleu", "score":0.2051262499 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"ru", "task":"translation_from", "metric":"chrf", "score":0.4837372958 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"ru", "task":"translation_to", "metric":"bleu", "score":0.3699194641 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"ru", "task":"translation_to", "metric":"chrf", "score":0.5770913921 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"sw", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"sw", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"sw", "task":"translation_from", "metric":"bleu", "score":0.2983575506 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"sw", "task":"translation_from", "metric":"chrf", "score":0.5283269577 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"sw", "task":"translation_to", "metric":"bleu", "score":0.3087303367 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"sw", "task":"translation_to", "metric":"chrf", "score":0.6028401745 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"te", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"te", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"te", "task":"translation_from", "metric":"bleu", "score":0.357367421 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"te", "task":"translation_from", "metric":"chrf", "score":0.5659157865 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"te", "task":"translation_to", "metric":"bleu", "score":0.3515649071 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"te", "task":"translation_to", "metric":"chrf", "score":0.5645706042 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"ur", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"ur", "task":"translation_from", "metric":"bleu", "score":0.2584265792 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"ur", "task":"translation_from", "metric":"chrf", "score":0.5020697076 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"ur", "task":"translation_to", "metric":"bleu", "score":0.2624388601 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"ur", "task":"translation_to", "metric":"chrf", "score":0.4452300688 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"zh", "task":"classification", "metric":"accuracy", "score":1.0 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"zh", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"zh", "task":"translation_from", "metric":"bleu", "score":0.2619931658 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"zh", "task":"translation_from", "metric":"chrf", "score":0.5251502482 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"zh", "task":"translation_to", "metric":"bleu", "score":0.2725475868 }, { "model":"meta-llama\/llama-3.1-70b-instruct", "bcp_47":"zh", "task":"translation_to", "metric":"chrf", "score":0.3148065512 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"ar", "task":"classification", "metric":"accuracy", "score":0.8 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"ar", "task":"mmlu", "metric":"accuracy", "score":0.5 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"ar", "task":"translation_from", "metric":"bleu", "score":0.2166642565 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"ar", "task":"translation_from", "metric":"chrf", "score":0.4531623464 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"ar", "task":"translation_to", "metric":"bleu", "score":0.2433592841 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"ar", "task":"translation_to", "metric":"chrf", "score":0.3890038008 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"bn", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"bn", "task":"mmlu", "metric":"accuracy", "score":0.4 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"bn", "task":"translation_from", "metric":"bleu", "score":0.1749870589 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"bn", "task":"translation_from", "metric":"chrf", "score":0.3959743603 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"bn", "task":"translation_to", "metric":"bleu", "score":0.1992458221 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"bn", "task":"translation_to", "metric":"chrf", "score":0.3995820747 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"de", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"de", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"de", "task":"translation_from", "metric":"bleu", "score":0.2301502689 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"de", "task":"translation_from", "metric":"chrf", "score":0.4644650793 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"de", "task":"translation_to", "metric":"bleu", "score":0.2921772734 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"de", "task":"translation_to", "metric":"chrf", "score":0.4951314188 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"en", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"en", "task":"mmlu", "metric":"accuracy", "score":0.8 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"en", "task":"translation_from", "metric":"bleu", "score":0.4135052232 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"en", "task":"translation_from", "metric":"chrf", "score":0.5895253676 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"en", "task":"translation_to", "metric":"bleu", "score":0.5519762012 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"en", "task":"translation_to", "metric":"chrf", "score":0.7446058172 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"es", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"es", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"es", "task":"translation_from", "metric":"bleu", "score":0.2337331881 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"es", "task":"translation_from", "metric":"chrf", "score":0.4523774022 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"es", "task":"translation_to", "metric":"bleu", "score":0.2045157252 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"es", "task":"translation_to", "metric":"chrf", "score":0.4389018917 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"fr", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"fr", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"fr", "task":"translation_from", "metric":"bleu", "score":0.2381890937 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"fr", "task":"translation_from", "metric":"chrf", "score":0.4806457757 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"fr", "task":"translation_to", "metric":"bleu", "score":0.3213357404 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"fr", "task":"translation_to", "metric":"chrf", "score":0.5096591733 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"hi", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"hi", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"hi", "task":"translation_from", "metric":"bleu", "score":0.2951777856 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"hi", "task":"translation_from", "metric":"chrf", "score":0.5193991656 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"hi", "task":"translation_to", "metric":"bleu", "score":0.3052977087 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"hi", "task":"translation_to", "metric":"chrf", "score":0.5029439287 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"id", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"id", "task":"mmlu", "metric":"accuracy", "score":0.7 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"id", "task":"translation_from", "metric":"bleu", "score":0.1717243204 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"id", "task":"translation_from", "metric":"chrf", "score":0.4178336488 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"id", "task":"translation_to", "metric":"bleu", "score":0.1865357446 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"id", "task":"translation_to", "metric":"chrf", "score":0.429444403 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"ja", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"ja", "task":"mmlu", "metric":"accuracy", "score":0.4 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"ja", "task":"translation_from", "metric":"bleu", "score":0.1933631076 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"ja", "task":"translation_from", "metric":"chrf", "score":0.4686346426 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"ja", "task":"translation_to", "metric":"bleu", "score":0.1629637627 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"ja", "task":"translation_to", "metric":"chrf", "score":0.265082494 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"mr", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"mr", "task":"translation_from", "metric":"bleu", "score":0.144541816 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"mr", "task":"translation_from", "metric":"chrf", "score":0.3648764501 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"mr", "task":"translation_to", "metric":"bleu", "score":0.1047482644 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"mr", "task":"translation_to", "metric":"chrf", "score":0.2965277611 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"pa", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"pa", "task":"translation_from", "metric":"bleu", "score":0.3595710948 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"pa", "task":"translation_from", "metric":"chrf", "score":0.539538196 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"pa", "task":"translation_to", "metric":"bleu", "score":0.3110476538 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"pa", "task":"translation_to", "metric":"chrf", "score":0.4964416902 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"pt", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"pt", "task":"mmlu", "metric":"accuracy", "score":0.5 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"pt", "task":"translation_from", "metric":"bleu", "score":0.2073899393 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"pt", "task":"translation_from", "metric":"chrf", "score":0.4691838054 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"pt", "task":"translation_to", "metric":"bleu", "score":0.1956364634 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"pt", "task":"translation_to", "metric":"chrf", "score":0.3100521945 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"ru", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"ru", "task":"mmlu", "metric":"accuracy", "score":0.6 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"ru", "task":"translation_from", "metric":"bleu", "score":0.1671614266 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"ru", "task":"translation_from", "metric":"chrf", "score":0.4422244829 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"ru", "task":"translation_to", "metric":"bleu", "score":0.1893614933 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"ru", "task":"translation_to", "metric":"chrf", "score":0.4605763082 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"sw", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"sw", "task":"mmlu", "metric":"accuracy", "score":0.4 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"sw", "task":"translation_from", "metric":"bleu", "score":0.153544279 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"sw", "task":"translation_from", "metric":"chrf", "score":0.381670314 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"sw", "task":"translation_to", "metric":"bleu", "score":0.0470329064 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"sw", "task":"translation_to", "metric":"chrf", "score":0.3516233998 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"te", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"te", "task":"mmlu", "metric":"accuracy", "score":0.5 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"te", "task":"translation_from", "metric":"bleu", "score":0.2231414616 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"te", "task":"translation_from", "metric":"chrf", "score":0.4272936469 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"te", "task":"translation_to", "metric":"bleu", "score":0.2215396728 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"te", "task":"translation_to", "metric":"chrf", "score":0.4514044695 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"ur", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"ur", "task":"translation_from", "metric":"bleu", "score":0.142024307 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"ur", "task":"translation_from", "metric":"chrf", "score":0.3925303581 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"ur", "task":"translation_to", "metric":"bleu", "score":0.13811776 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"ur", "task":"translation_to", "metric":"chrf", "score":0.3521086255 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"zh", "task":"classification", "metric":"accuracy", "score":0.9 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"zh", "task":"mmlu", "metric":"accuracy", "score":0.5 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"zh", "task":"translation_from", "metric":"bleu", "score":0.2115579648 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"zh", "task":"translation_from", "metric":"chrf", "score":0.4634573062 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"zh", "task":"translation_to", "metric":"bleu", "score":0.1903215556 }, { "model":"meta-llama\/llama-3.1-8b-instruct", "bcp_47":"zh", "task":"translation_to", "metric":"chrf", "score":0.2429625775 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"ar", "task":"classification", "metric":"accuracy", "score":0.7 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"ar", "task":"mmlu", "metric":"accuracy", "score":0.5 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"ar", "task":"translation_from", "metric":"bleu", "score":0.0534412445 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"ar", "task":"translation_from", "metric":"chrf", "score":0.1423750168 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"ar", "task":"translation_to", "metric":"bleu", "score":0.0522677244 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"ar", "task":"translation_to", "metric":"chrf", "score":0.1317238701 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"bn", "task":"classification", "metric":"accuracy", "score":0.4 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"bn", "task":"mmlu", "metric":"accuracy", "score":0.3 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"bn", "task":"translation_from", "metric":"bleu", "score":0.0 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"bn", "task":"translation_from", "metric":"chrf", "score":0.0061046998 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"bn", "task":"translation_to", "metric":"bleu", "score":0.048395385 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"bn", "task":"translation_to", "metric":"chrf", "score":0.1624192131 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"de", "task":"classification", "metric":"accuracy", "score":0.7 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"de", "task":"mmlu", "metric":"accuracy", "score":0.1 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"de", "task":"translation_from", "metric":"bleu", "score":0.0365328036 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"de", "task":"translation_from", "metric":"chrf", "score":0.1594198405 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"de", "task":"translation_to", "metric":"bleu", "score":0.089863703 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"de", "task":"translation_to", "metric":"chrf", "score":0.2290725718 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"en", "task":"classification", "metric":"accuracy", "score":0.5 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"en", "task":"mmlu", "metric":"accuracy", "score":0.3 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"en", "task":"translation_from", "metric":"bleu", "score":0.1186358374 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"en", "task":"translation_from", "metric":"chrf", "score":0.2988653266 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"en", "task":"translation_to", "metric":"bleu", "score":0.0659782167 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"en", "task":"translation_to", "metric":"chrf", "score":0.1219080159 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"es", "task":"classification", "metric":"accuracy", "score":0.3 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"es", "task":"mmlu", "metric":"accuracy", "score":0.4 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"es", "task":"translation_from", "metric":"bleu", "score":0.0197183104 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"es", "task":"translation_from", "metric":"chrf", "score":0.1858500787 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"es", "task":"translation_to", "metric":"bleu", "score":0.1372446539 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"es", "task":"translation_to", "metric":"chrf", "score":0.1970334371 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"fr", "task":"classification", "metric":"accuracy", "score":0.5 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"fr", "task":"mmlu", "metric":"accuracy", "score":0.4 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"fr", "task":"translation_from", "metric":"bleu", "score":0.0225723033 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"fr", "task":"translation_from", "metric":"chrf", "score":0.2128372612 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"fr", "task":"translation_to", "metric":"bleu", "score":0.0900337367 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"fr", "task":"translation_to", "metric":"chrf", "score":0.2030113676 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"hi", "task":"classification", "metric":"accuracy", "score":0.5 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"hi", "task":"mmlu", "metric":"accuracy", "score":0.1 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"hi", "task":"translation_from", "metric":"bleu", "score":0.1275394125 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"hi", "task":"translation_from", "metric":"chrf", "score":0.2249822055 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"hi", "task":"translation_to", "metric":"bleu", "score":0.1164104087 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"hi", "task":"translation_to", "metric":"chrf", "score":0.2255759794 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"id", "task":"classification", "metric":"accuracy", "score":0.6 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"id", "task":"mmlu", "metric":"accuracy", "score":0.6 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"id", "task":"translation_from", "metric":"bleu", "score":0.0396919548 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"id", "task":"translation_from", "metric":"chrf", "score":0.193498077 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"id", "task":"translation_to", "metric":"bleu", "score":0.0463130015 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"id", "task":"translation_to", "metric":"chrf", "score":0.147263979 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"ja", "task":"classification", "metric":"accuracy", "score":0.5 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"ja", "task":"mmlu", "metric":"accuracy", "score":0.0 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"ja", "task":"translation_from", "metric":"bleu", "score":0.0196332917 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"ja", "task":"translation_from", "metric":"chrf", "score":0.0615297483 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"ja", "task":"translation_to", "metric":"bleu", "score":0.002077843 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"ja", "task":"translation_to", "metric":"chrf", "score":0.0228028425 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"mr", "task":"classification", "metric":"accuracy", "score":0.3 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"mr", "task":"translation_from", "metric":"bleu", "score":0.0103335448 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"mr", "task":"translation_from", "metric":"chrf", "score":0.0394434629 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"mr", "task":"translation_to", "metric":"bleu", "score":0.0093410183 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"mr", "task":"translation_to", "metric":"chrf", "score":0.0758287765 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"pa", "task":"classification", "metric":"accuracy", "score":0.3 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"pa", "task":"translation_from", "metric":"bleu", "score":0.1995473768 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"pa", "task":"translation_from", "metric":"chrf", "score":0.2187948253 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"pa", "task":"translation_to", "metric":"bleu", "score":0.2192906863 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"pa", "task":"translation_to", "metric":"chrf", "score":0.330676254 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"pt", "task":"classification", "metric":"accuracy", "score":0.2 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"pt", "task":"mmlu", "metric":"accuracy", "score":0.4 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"pt", "task":"translation_from", "metric":"bleu", "score":0.0506856305 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"pt", "task":"translation_from", "metric":"chrf", "score":0.2502940387 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"pt", "task":"translation_to", "metric":"bleu", "score":0.0801303171 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"pt", "task":"translation_to", "metric":"chrf", "score":0.1635269784 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"ru", "task":"classification", "metric":"accuracy", "score":0.6 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"ru", "task":"mmlu", "metric":"accuracy", "score":0.2 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"ru", "task":"translation_from", "metric":"bleu", "score":0.050859162 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"ru", "task":"translation_from", "metric":"chrf", "score":0.2002283949 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"ru", "task":"translation_to", "metric":"bleu", "score":0.0199756762 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"ru", "task":"translation_to", "metric":"chrf", "score":0.1055944134 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"sw", "task":"classification", "metric":"accuracy", "score":0.4 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"sw", "task":"mmlu", "metric":"accuracy", "score":0.3 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"sw", "task":"translation_from", "metric":"bleu", "score":0.001996605 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"sw", "task":"translation_from", "metric":"chrf", "score":0.1999016804 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"sw", "task":"translation_to", "metric":"bleu", "score":0.0453589274 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"sw", "task":"translation_to", "metric":"chrf", "score":0.1405820992 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"te", "task":"classification", "metric":"accuracy", "score":0.4 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"te", "task":"mmlu", "metric":"accuracy", "score":0.4 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"te", "task":"translation_from", "metric":"bleu", "score":0.1115247653 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"te", "task":"translation_from", "metric":"chrf", "score":0.1296059134 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"te", "task":"translation_to", "metric":"bleu", "score":0.1480453761 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"te", "task":"translation_to", "metric":"chrf", "score":0.2690624141 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"ur", "task":"classification", "metric":"accuracy", "score":0.4 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"ur", "task":"translation_from", "metric":"bleu", "score":0.0114376329 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"ur", "task":"translation_from", "metric":"chrf", "score":0.0233475148 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"ur", "task":"translation_to", "metric":"bleu", "score":0.0385720696 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"ur", "task":"translation_to", "metric":"chrf", "score":0.1423687352 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"zh", "task":"classification", "metric":"accuracy", "score":0.7 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"zh", "task":"mmlu", "metric":"accuracy", "score":0.1 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"zh", "task":"translation_from", "metric":"bleu", "score":0.0190942897 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"zh", "task":"translation_from", "metric":"chrf", "score":0.1108498114 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"zh", "task":"translation_to", "metric":"bleu", "score":0.0 }, { "model":"meta-llama\/llama-3.2-1b-instruct", "bcp_47":"zh", "task":"translation_to", "metric":"chrf", "score":0.0294450289 } ]