[ { "language_name": "English", "bcp_47": "en", "speakers": 1636485840, "scores": [ { "model": "openai/gpt-4o-mini", "mt_bleu": 0.5245466124037277, "mt_chrf": 65.25187717981981, "cls_acc": 0.5666666666666667, "mlm_chrf": 97.84704595784264, "asr_wer": 0, "overall_score": 0.7325519660144305 }, { "model": "meta-llama/llama-3.3-70b-instruct", "mt_bleu": 0.48750797044187216, "mt_chrf": 63.24229348441665, "cls_acc": 0.6, "mlm_chrf": 93.62602669879945, "asr_wer": 0, "overall_score": 0.7228944006107203 }, { "model": "mistralai/mistral-small-24b-instruct-2501", "mt_bleu": 0.486501959595472, "mt_chrf": 63.8187259254881, "cls_acc": 0.5333333333333333, "mlm_chrf": 79.91140615317198, "asr_wer": 0, "overall_score": 0.656878218039978 }, { "model": "google/gemini-2.0-flash-001", "mt_bleu": 0.6060954569411976, "mt_chrf": 71.2288943066563, "cls_acc": 0.8666666666666667, "mlm_chrf": 98.79868693366329, "asr_wer": 0, "overall_score": 0.8556474930232877 }, { "model": "microsoft/phi-4", "mt_bleu": 0.5199836121545649, "mt_chrf": 66.05410510011644, "cls_acc": 0.7, "mlm_chrf": 97.2290729316734, "asr_wer": 0, "overall_score": 0.7776105934392995 }, { "model": "elevenlabs/scribe_v1", "mt_bleu": 0, "mt_chrf": 0, "cls_acc": 0, "mlm_chrf": 0, "asr_wer": 0.3384224589434353, "overall_score": 0.0 }, { "model": "openai/whisper-large-v3-turbo", "mt_bleu": 0, "mt_chrf": 0, "cls_acc": 0, "mlm_chrf": 0, "asr_wer": 0.2670470041301468, "overall_score": 0.0 } ], "mt_bleu": 0.37494794450526203, "mt_chrf": 47.08512799949961, "cls_acc": 0.4666666666666667, "mlm_chrf": 66.77317695359297, "asr_wer": 0.08649563758194032, "overall_score": 0.5350832387325309, "commonvoice_hours": 2651.0, "commonvoice_locale": "en", "population": { "AC": 931, "AE": 4996040, "AG": 84434, "AI": 17186, "AQ": 300, "AR": 3183537, "AS": 47954, "AT": 6467398, "AU": 24447840, "AW": 2986, "BA": 1726016, "BB": 294560, "BD": 29277180, "BE": 6915213, "BG": 1741725, "BI": 6289, "BM": 66010, "BN": 7896, "BR": 16937280, "BS": 337721, "BT": 86055, "BV": 1, "BW": 1876956, "BZ": 399598, "CA": 32416926, "CC": 101, "CH": 5126434, "CK": 8574, "CL": 1727746, "CM": 10543100, "CN": 62731, "CP": 1, "CQ": 482, "CX": 1389, "CY": 924676, "CZ": 2889675, "DE": 51302208, "DG": 495, "DK": 5047693, "DM": 69788, "DO": 7980, "DZ": 3008103, "EE": 614310, "EG": 36443400, "ER": 3587908, "ES": 12003792, "ET": 46488590, "FI": 3900169, "FJ": 879816, "FK": 2814, "FM": 58389, "FR": 26460798, "GB": 64445878, "GD": 108570, "GG": 67052, "GH": 6161442, "GI": 23665, "GM": 869600, "GR": 5409621, "GS": 20, "GU": 153321, "GY": 750204, "HK": 3697454, "HM": 1, "HN": 40635, "HR": 2071598, "HU": 1954366, "IE": 5073039, "IL": 7374158, "IM": 90499, "IN": 251957100, "IO": 3500, "IQ": 13605445, "IT": 21216918, "JE": 96019, "JM": 2752399, "JO": 4869270, "KE": 10170301, "KI": 111796, "KN": 52745, "KY": 60705, "KZ": 2863785, "LB": 2187844, "LC": 149838, "LK": 2288920, "LR": 4210839, "LS": 531719, "LT": 1037955, "LU": 351893, "LV": 865366, "MA": 4978638, "MG": 4852026, "MH": 72463, "MO": 14133, "MP": 49890, "MS": 3492, "MT": 402395, "MU": 993146, "MV": 293928, "MW": 13353858, "MX": 16724500, "MY": 6856941, "NA": 184105, "NF": 1678, "NG": 113434840, "NL": 15552360, "NP": 909837, "NR": 9350, "NU": 1120, "NZ": 4826970, "PA": 545171, "PG": 3629730, "PH": 69875840, "PK": 116750500, "PL": 12633159, "PM": 187, "PN": 46, "PR": 1562644, "PT": 2781729, "PW": 1887, "RO": 6603899, "RW": 1906860, "SB": 685097, "SC": 36473, "SD": 27792576, "SE": 8774150, "SG": 5774984, "SH": 5425, "SI": 1240581, "SK": 1414556, "SL": 2318726, "SS": 2851524, "SX": 29816, "SZ": 883584, "TA": 272, "TC": 54807, "TH": 18623898, "TK": 1285, "TO": 29707, "TR": 13942975, "TT": 1063735, "TV": 1066, "TZ": 40401432, "UG": 1686867, "UM": 316, "US": 319333440, "VC": 97334, "VG": 36633, "VI": 79676, "VU": 247616, "WS": 4279, "YE": 2689596, "ZA": 17503716, "ZM": 2788256, "ZW": 6109446 }, "language_family": "Indo-European" }, { "language_name": "Chinese", "bcp_47": "zh", "speakers": 1304678914, "scores": [ { "model": "openai/gpt-4o-mini", "mt_bleu": 0.4887591978835982, "mt_chrf": 70.23584222732792, "cls_acc": 0.43333333333333335, "mlm_chrf": 91.54177259254774, "asr_wer": 0, "overall_score": 0.6837031605106967 }, { "model": "meta-llama/llama-3.3-70b-instruct", "mt_bleu": 0.38557580495281013, "mt_chrf": 61.11151378837755, "cls_acc": 0.5666666666666667, "mlm_chrf": 94.55849047452216, "asr_wer": 0, "overall_score": 0.7077889030985546 }, { "model": "mistralai/mistral-small-24b-instruct-2501", "mt_bleu": 0.48838341708028316, "mt_chrf": 71.48446350451079, "cls_acc": 0.7666666666666667, "mlm_chrf": 91.82889304045095, "asr_wer": 0, "overall_score": 0.799933410705428 }, { "model": "google/gemini-2.0-flash-001", "mt_bleu": 0.392664198976078, "mt_chrf": 58.52460212908651, "cls_acc": 0.8666666666666667, "mlm_chrf": 94.26439448464122, "asr_wer": 0, "overall_score": 0.7981855442679814 }, { "model": "microsoft/phi-4", "mt_bleu": 0.4892212172376435, "mt_chrf": 69.23023880157619, "cls_acc": 0.7, "mlm_chrf": 93.53779735205161, "asr_wer": 0, "overall_score": 0.7758934538454261 }, { "model": "elevenlabs/scribe_v1", "mt_bleu": 0, "mt_chrf": 0, "cls_acc": 0, "mlm_chrf": 0, "asr_wer": 1.0, "overall_score": 0.0 }, { "model": "openai/whisper-large-v3-turbo", "mt_bleu": 0, "mt_chrf": 0, "cls_acc": 0, "mlm_chrf": 0, "asr_wer": 1.0, "overall_score": 0.0 } ], "mt_bleu": 0.3206576908757733, "mt_chrf": 47.226665778697, "cls_acc": 0.47619047619047616, "mlm_chrf": 66.53304970631623, "asr_wer": 0.2857142857142857, "overall_score": 0.5379292103468696, "commonvoice_hours": 422.0, "commonvoice_locale": "zh-TW", "population": { "AU": 534796, "BN": 51093, "CA": 678494, "CN": 1254618000, "GB": 197283, "GF": 4988, "HK": 7249910, "ID": 2456639, "MN": 44352, "MO": 632892, "MY": 5550857, "PA": 5841, "PF": 23019, "PH": 797021, "SG": 4781438, "SR": 6705, "TH": 1241593, "TW": 22422850, "US": 2295209, "VN": 1085934 }, "language_family": "Sino-Tibetan" }, { "language_name": "Hindi", "bcp_47": "hi", "speakers": 546882144, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "mt_bleu": 0.32404902340686065, "mt_chrf": 53.54085104449268, "cls_acc": 0.5, "mlm_chrf": 96.17240172798218, "asr_wer": 0, "overall_score": 0.6657108425749162 }, { "model": "openai/whisper-large-v3-turbo", "mt_bleu": 0, "mt_chrf": 0, "cls_acc": 0, "mlm_chrf": 0, "asr_wer": 0.40237814140706235, "overall_score": 0.0 } ], "mt_bleu": 0.16202451170343032, "mt_chrf": 26.77042552224634, "cls_acc": 0.25, "mlm_chrf": 48.08620086399109, "asr_wer": 0.20118907070353118, "overall_score": 0.3328554212874581, "commonvoice_hours": 16.0, "commonvoice_locale": "hi-IN", "population": { "CA": 188470, "FJ": 411829, "IN": 545022990, "NP": 127377, "UG": 2206, "ZA": 1129272 }, "language_family": "Indo-European" }, { "language_name": "Spanish", "bcp_47": "es", "speakers": 493528077, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "mt_bleu": 0.31587937116142056, "mt_chrf": 52.142851262301726, "cls_acc": 0.5, "mlm_chrf": 96.92768852306384, "asr_wer": 0, "overall_score": 0.6635684659512185 }, { "model": "openai/whisper-large-v3-turbo", "mt_bleu": 0, "mt_chrf": 0, "cls_acc": 0, "mlm_chrf": 0, "asr_wer": 0.16641846898275137, "overall_score": 0.0 } ], "mt_bleu": 0.15793968558071028, "mt_chrf": 26.071425631150863, "cls_acc": 0.25, "mlm_chrf": 48.46384426153192, "asr_wer": 0.08320923449137568, "overall_score": 0.3317842329756093, "commonvoice_hours": 446.0, "commonvoice_locale": "es", "population": { "AD": 33110, "AR": 45479100, "BO": 7100339, "BR": 76218, "BZ": 111887, "CA": 603106, "CL": 17823064, "CO": 45648864, "CR": 4843090, "CU": 11059100, "CW": 5751, "DE": 4809582, "DO": 8189766, "EA": 147000, "EC": 16228704, "ES": 49515642, "FR": 8820266, "GB": 5260888, "GI": 14790, "GQ": 727475, "GT": 15952569, "HN": 7203565, "IC": 2056618, "MA": 23115, "MX": 106779500, "NI": 4838683, "PA": 2686915, "PE": 23297950, "PH": 33846110, "PR": 2774491, "PT": 1030270, "PY": 230134, "RO": 2130290, "SV": 5768179, "SX": 4823, "TT": 4110, "US": 31933344, "UY": 2981097, "VE": 23488572 }, "language_family": "Indo-European" }, { "language_name": "Arabic", "bcp_47": "ar", "speakers": 351664197, "scores": [ { "model": "meta-llama/llama-3.3-70b-instruct", "mt_bleu": 0.39547934933771334, "mt_chrf": 57.51652731936118, "cls_acc": 0.4666666666666667, "mlm_chrf": 94.97026443937914, "asr_wer": 0, "overall_score": 0.6638448614180232 }, { "model": "openai/whisper-large-v3-turbo", "mt_bleu": 0, "mt_chrf": 0, "cls_acc": 0, "mlm_chrf": 0, "asr_wer": 0.17249214841393012, "overall_score": 0.0 } ], "mt_bleu": 0.19773967466885667, "mt_chrf": 28.75826365968059, "cls_acc": 0.23333333333333334, "mlm_chrf": 47.48513221968957, "asr_wer": 0.08624607420696506, "overall_score": 0.3319224307090116, "commonvoice_hours": 91.0, "commonvoice_locale": "ar", "population": { "AE": 7793822, "BH": 1309350, "CA": 565412, "CM": 108206, "CY": 1267, "DJ": 67292, "DZ": 31799946, "EG": 97876560, "EH": 652271, "ER": 297979, "GB": 197283, "IL": 1735096, "IQ": 26433436, "IR": 1698466, "JO": 10820600, "KE": 24623, "KM": 558545, "KW": 2993710, "LB": 4703865, "LY": 5099000, "MA": 22048254, "ML": 175981, "MR": 3404658, "NE": 47822, "NG": 151960, "OM": 3778520, "PS": 4818260, "QA": 2175311, "SA": 34173500, "SD": 27792576, "SO": 3997414, "SS": 2851524, "SY": 15518720, "TD": 2869158, "TJ": 976, "TN": 10549080, "TR": 459298, "YE": 22114456 }, "language_family": "Afro-Asiatic" } ]