Spaces:
Running
Running
Commit
·
3900100
1
Parent(s):
d17cf2c
Update generation code to calculate dataset specific wer
Browse files
dashboard_data/device_map.json
CHANGED
|
@@ -11,6 +11,7 @@
|
|
| 11 |
"MacBookAir10,1": "Apple M1",
|
| 12 |
"iPad13,16": "iPad Air (5th generation)",
|
| 13 |
"iPad13,18": "iPad (10th generation)",
|
|
|
|
| 14 |
"iPad14,1": "iPad mini (6th generation)",
|
| 15 |
"iPad14,8": "iPad Air 11-inch (M2)",
|
| 16 |
"iPad15,3": "iPad Air 11-inch (M3)",
|
|
|
|
| 11 |
"MacBookAir10,1": "Apple M1",
|
| 12 |
"iPad13,16": "iPad Air (5th generation)",
|
| 13 |
"iPad13,18": "iPad (10th generation)",
|
| 14 |
+
"iPad13,6": "iPad Pro (11-inch) (3rd generation)",
|
| 15 |
"iPad14,1": "iPad mini (6th generation)",
|
| 16 |
"iPad14,8": "iPad Air 11-inch (M2)",
|
| 17 |
"iPad15,3": "iPad Air 11-inch (M3)",
|
dashboard_data/performance_data.json
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
dashboard_data/support_data_8c0acbd.csv
CHANGED
|
@@ -1,23 +1,23 @@
|
|
| 1 |
-
,Model,"Apple M2 Pro (Mac14,12)","Apple M3 Max (Mac15,10)","iPad Air (5th generation) (iPad13,16)","iPad mini (6th generation) (iPad14,1)","iPad Air 11-inch (M2) (iPad14,8)","iPad (A16) (iPad15,7)","iPad mini (A17 Pro) (iPad16,1)","iPad Pro 11-inch (M4) (iPad16,3)","iPhone 11 (iPhone12,1)","iPhone 12 mini (iPhone13,1)","iPhone 13 Pro (iPhone14,2)","iPhone 13 (iPhone14,5)","iPhone 14 (iPhone14,7)","iPhone 15 (iPhone15,4)","iPhone 15 Pro Max (iPhone16,2)","iPhone 16 (iPhone17,3)","iPhone 16e (iPhone17,5)"
|
| 2 |
-
distil-whisper_distil-large-v3,distil-whisper_distil-large-v3,✅ macOS 15.5,✅ macOS 15.5,Not Supported
|
| 3 |
-
distil-whisper_distil-large-v3_594MB,distil-whisper_distil-large-v3_594MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
|
| 4 |
-
distil-whisper_distil-large-v3_turbo,distil-whisper_distil-large-v3_turbo,✅ macOS 15.5,✅ macOS 15.5,Not Supported
|
| 5 |
-
distil-whisper_distil-large-v3_turbo_600MB,distil-whisper_distil-large-v3_turbo_600MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported
|
| 6 |
-
openai_whisper-base,openai_whisper-base,✅ macOS 15.5,✅ macOS 15.5,✅ iPadOS 18.4.1,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,✅ iOS 26.0,✅ iOS 18.4.1,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
|
| 7 |
-
openai_whisper-base.en,openai_whisper-base.en,✅ macOS 15.5,✅ macOS 15.5,✅ iPadOS 18.4.1,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,✅ iOS 26.0,✅ iOS 18.4.1,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
|
| 8 |
-
openai_whisper-large-v2,openai_whisper-large-v2,✅ macOS 15.5,✅ macOS 15.5,Not Supported
|
| 9 |
-
openai_whisper-large-v2_949MB,openai_whisper-large-v2_949MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,✅ iPadOS 18.4,✅ iPadOS 18.
|
| 10 |
-
openai_whisper-large-v2_turbo,openai_whisper-large-v2_turbo,✅ macOS 15.5,✅ macOS 15.5,Not Supported
|
| 11 |
-
openai_whisper-large-v2_turbo_955MB,openai_whisper-large-v2_turbo_955MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported
|
| 12 |
-
openai_whisper-large-v3,openai_whisper-large-v3,✅ macOS 15.5,✅ macOS 15.5,Not Supported
|
| 13 |
-
openai_whisper-large-v3-v20240930,openai_whisper-large-v3-v20240930,✅ macOS 15.5,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/Mac15%2C10_summary_2025-06-15T230300.json>macOS 15.5</a>,Not Supported
|
| 14 |
-
openai_whisper-large-v3-v20240930_626MB,openai_whisper-large-v3-v20240930_626MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 18.4,✅ iPadOS 18.4,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C3_summary_2025-06-16T220810.json>iPadOS 18.4.1</a>,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
|
| 15 |
-
openai_whisper-large-v3-v20240930_turbo,openai_whisper-large-v3-v20240930_turbo,✅ macOS 15.5,✅ macOS 15.5,Not Supported
|
| 16 |
-
openai_whisper-large-v3-v20240930_turbo_632MB,openai_whisper-large-v3-v20240930_turbo_632MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported
|
| 17 |
-
openai_whisper-large-v3_947MB,openai_whisper-large-v3_947MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,✅ iPadOS 18.4,✅ iPadOS 18.
|
| 18 |
-
openai_whisper-large-v3_turbo,openai_whisper-large-v3_turbo,✅ macOS 15.5,✅ macOS 15.5,Not Supported
|
| 19 |
-
openai_whisper-large-v3_turbo_954MB,openai_whisper-large-v3_turbo_954MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported
|
| 20 |
-
openai_whisper-small,openai_whisper-small,✅ macOS 15.5,✅ macOS 15.5,✅ iPadOS 18.4.1,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,✅ iOS 18.4.1,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
|
| 21 |
-
openai_whisper-small.en,openai_whisper-small.en,✅ macOS 15.5,✅ macOS 15.5,✅ iPadOS 18.4.1,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,✅ iOS 18.4.1,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
|
| 22 |
-
openai_whisper-tiny,openai_whisper-tiny,✅ macOS 15.5,✅ macOS 15.5,✅ iPadOS 18.4.1,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,✅ iOS 26.0,✅ iOS 18.4.1,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
|
| 23 |
-
openai_whisper-tiny.en,openai_whisper-tiny.en,✅ macOS 15.5,✅ macOS 15.5,✅ iPadOS 18.4.1,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,✅ iOS 26.0,✅ iOS 18.4.1,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
|
|
|
|
| 1 |
+
,Model,"Apple M2 Pro (Mac14,12)","Apple M3 Max (Mac15,10)","iPad Air (5th generation) (iPad13,16)","iPad Pro (11-inch) (3rd generation) (iPad13,6)","iPad mini (6th generation) (iPad14,1)","iPad Air 11-inch (M2) (iPad14,8)","iPad Air 11-inch (M3) (iPad15,3)","iPad (A16) (iPad15,7)","iPad mini (A17 Pro) (iPad16,1)","iPad Pro 11-inch (M4) (iPad16,3)","iPhone 11 (iPhone12,1)","iPhone 12 mini (iPhone13,1)","iPhone 13 Pro (iPhone14,2)","iPhone 13 (iPhone14,5)","iPhone 14 (iPhone14,7)","iPhone 15 (iPhone15,4)","iPhone 15 Pro Max (iPhone16,2)","iPhone 16 (iPhone17,3)","iPhone 16e (iPhone17,5)"
|
| 2 |
+
distil-whisper_distil-large-v3,distil-whisper_distil-large-v3,✅ macOS 15.5,✅ macOS 15.5,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,Not Supported,✅ iPadOS 18.5,✅ iPadOS 26.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C7_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C1_summary_2025-06-16T220810.json>iPadOS 18.4</a>,✅ iPadOS 18.4.1,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
|
| 3 |
+
distil-whisper_distil-large-v3_594MB,distil-whisper_distil-large-v3_594MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
|
| 4 |
+
distil-whisper_distil-large-v3_turbo,distil-whisper_distil-large-v3_turbo,✅ macOS 15.5,✅ macOS 15.5,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad14%2C8_summary_2025-06-16T192343.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C3_summary_2025-06-17T205025.json>iPadOS 26.0</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C7_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C1_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C3_summary_2025-06-16T220810.json>iPadOS 18.4.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
|
| 5 |
+
distil-whisper_distil-large-v3_turbo_600MB,distil-whisper_distil-large-v3_turbo_600MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
|
| 6 |
+
openai_whisper-base,openai_whisper-base,✅ macOS 15.5,✅ macOS 15.5,✅ iPadOS 18.4.1,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,✅ iOS 26.0,✅ iOS 18.4.1,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
|
| 7 |
+
openai_whisper-base.en,openai_whisper-base.en,✅ macOS 15.5,✅ macOS 15.5,✅ iPadOS 18.4.1,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,✅ iOS 26.0,✅ iOS 18.4.1,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
|
| 8 |
+
openai_whisper-large-v2,openai_whisper-large-v2,✅ macOS 15.5,✅ macOS 15.5,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,Not Supported,✅ iPadOS 18.5,✅ iPadOS 26.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C7_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C1_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C3_summary_2025-06-16T220810.json>iPadOS 18.4.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
|
| 9 |
+
openai_whisper-large-v2_949MB,openai_whisper-large-v2_949MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C3_summary_2025-06-17T205025.json>iPadOS 26.0</a>,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
|
| 10 |
+
openai_whisper-large-v2_turbo,openai_whisper-large-v2_turbo,✅ macOS 15.5,✅ macOS 15.5,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad14%2C8_summary_2025-06-16T192343.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C3_summary_2025-06-17T205025.json>iPadOS 26.0</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C7_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C1_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C3_summary_2025-06-16T220810.json>iPadOS 18.4.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
|
| 11 |
+
openai_whisper-large-v2_turbo_955MB,openai_whisper-large-v2_turbo_955MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
|
| 12 |
+
openai_whisper-large-v3,openai_whisper-large-v3,✅ macOS 15.5,✅ macOS 15.5,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad14%2C8_summary_2025-06-16T192343.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C3_summary_2025-06-17T205025.json>iPadOS 26.0</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C7_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C1_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C3_summary_2025-06-16T220810.json>iPadOS 18.4.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
|
| 13 |
+
openai_whisper-large-v3-v20240930,openai_whisper-large-v3-v20240930,✅ macOS 15.5,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/Mac15%2C10_summary_2025-06-15T230300.json>macOS 15.5</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,Not Supported,? iPadOS 18.5,? iPadOS 26.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C7_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C1_summary_2025-06-16T220810.json>iPadOS 18.4</a>,? iPadOS 18.4.1,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
|
| 14 |
+
openai_whisper-large-v3-v20240930_626MB,openai_whisper-large-v3-v20240930_626MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C3_summary_2025-06-16T220810.json>iPadOS 18.4.1</a>,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
|
| 15 |
+
openai_whisper-large-v3-v20240930_turbo,openai_whisper-large-v3-v20240930_turbo,✅ macOS 15.5,✅ macOS 15.5,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad14%2C8_summary_2025-06-16T192343.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C3_summary_2025-06-17T205025.json>iPadOS 26.0</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C7_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C1_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C3_summary_2025-06-16T220810.json>iPadOS 18.4.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
|
| 16 |
+
openai_whisper-large-v3-v20240930_turbo_632MB,openai_whisper-large-v3-v20240930_turbo_632MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPhone16%2C2_summary_2025-06-16T135523.json>iOS 18.5</a>,✅ iOS 18.4.1,✅ iOS 18.5
|
| 17 |
+
openai_whisper-large-v3_947MB,openai_whisper-large-v3_947MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C3_summary_2025-06-17T205025.json>iPadOS 26.0</a>,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
|
| 18 |
+
openai_whisper-large-v3_turbo,openai_whisper-large-v3_turbo,✅ macOS 15.5,✅ macOS 15.5,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad14%2C8_summary_2025-06-16T192343.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C3_summary_2025-06-17T205025.json>iPadOS 26.0</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C7_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C1_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C3_summary_2025-06-16T220810.json>iPadOS 18.4.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
|
| 19 |
+
openai_whisper-large-v3_turbo_954MB,openai_whisper-large-v3_turbo_954MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
|
| 20 |
+
openai_whisper-small,openai_whisper-small,✅ macOS 15.5,✅ macOS 15.5,✅ iPadOS 18.4.1,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,✅ iOS 18.4.1,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
|
| 21 |
+
openai_whisper-small.en,openai_whisper-small.en,✅ macOS 15.5,✅ macOS 15.5,✅ iPadOS 18.4.1,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,✅ iOS 18.4.1,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
|
| 22 |
+
openai_whisper-tiny,openai_whisper-tiny,✅ macOS 15.5,✅ macOS 15.5,✅ iPadOS 18.4.1,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,✅ iOS 26.0,✅ iOS 18.4.1,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
|
| 23 |
+
openai_whisper-tiny.en,openai_whisper-tiny.en,✅ macOS 15.5,✅ macOS 15.5,✅ iPadOS 18.4.1,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,✅ iOS 26.0,✅ iOS 18.4.1,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
|
performance_generate.py
CHANGED
|
@@ -131,6 +131,7 @@ def process_benchmark_file(file_path, dataset_dfs, results, releases):
|
|
| 131 |
}
|
| 132 |
results[key]["timestamp"] = timestamp
|
| 133 |
results[key]["average_wer"].append(wer_entry)
|
|
|
|
| 134 |
|
| 135 |
input_audio_seconds = test_info["timings"]["inputAudioSeconds"]
|
| 136 |
full_pipeline = test_info["timings"]["fullPipeline"]
|
|
@@ -283,6 +284,10 @@ def calculate_and_save_performance_results(
|
|
| 283 |
for dataset, tps_info in data["dataset_tokens_per_second"].items()
|
| 284 |
},
|
| 285 |
"average_wer": compute_average_wer(data["average_wer"]),
|
|
|
|
|
|
|
|
|
|
|
|
|
| 286 |
"qoi": round(mean(data["qoi"]), 2),
|
| 287 |
"commit_hash": data["commit_hash"],
|
| 288 |
"commit_timestamp": data["commit_timestamp"],
|
|
@@ -460,6 +465,7 @@ def main():
|
|
| 460 |
performance_results = defaultdict(
|
| 461 |
lambda: {
|
| 462 |
"average_wer": [],
|
|
|
|
| 463 |
"qoi": [],
|
| 464 |
"speed": {"inputAudioSeconds": 0, "fullPipeline": 0},
|
| 465 |
"tokens_per_second": {"totalDecodingLoops": 0, "fullPipeline": 0},
|
|
|
|
| 131 |
}
|
| 132 |
results[key]["timestamp"] = timestamp
|
| 133 |
results[key]["average_wer"].append(wer_entry)
|
| 134 |
+
results[key]["dataset_wer"][dataset_name].append(wer_entry)
|
| 135 |
|
| 136 |
input_audio_seconds = test_info["timings"]["inputAudioSeconds"]
|
| 137 |
full_pipeline = test_info["timings"]["fullPipeline"]
|
|
|
|
| 284 |
for dataset, tps_info in data["dataset_tokens_per_second"].items()
|
| 285 |
},
|
| 286 |
"average_wer": compute_average_wer(data["average_wer"]),
|
| 287 |
+
"dataset_average_wer": {
|
| 288 |
+
dataset: compute_average_wer(data["dataset_wer"][dataset])
|
| 289 |
+
for dataset in data["dataset_wer"]
|
| 290 |
+
},
|
| 291 |
"qoi": round(mean(data["qoi"]), 2),
|
| 292 |
"commit_hash": data["commit_hash"],
|
| 293 |
"commit_timestamp": data["commit_timestamp"],
|
|
|
|
| 465 |
performance_results = defaultdict(
|
| 466 |
lambda: {
|
| 467 |
"average_wer": [],
|
| 468 |
+
"dataset_wer": defaultdict(list),
|
| 469 |
"qoi": [],
|
| 470 |
"speed": {"inputAudioSeconds": 0, "fullPipeline": 0},
|
| 471 |
"tokens_per_second": {"totalDecodingLoops": 0, "fullPipeline": 0},
|