Spaces:

argmaxinc
/

whisperkit-benchmarks

Running

App Files Files Community

ardaatahan commited on 5 days ago

Commit

1871a8f

unverified ·

2 Parent(s): 4ac34ce 27d0e34

Merge pull request #7 from argmaxinc/SW-576-update-hf-benchmarks-webpage-with-measured-wer-instead-of-benchmarked-from-whisperkittools

Browse files

Files changed (26) hide show

.github/scripts/check_dataset_update.py +15 -10
Makefile +0 -5
README.md +1 -1
constants.py +8 -21
dashboard_data/Mac.json +241 -0
dashboard_data/config.json +222 -39
dashboard_data/iPad.json +206 -0
dashboard_data/iPhone.json +151 -0
dashboard_data/multilingual_confusion_matrices.json +0 -0
dashboard_data/performance_data.json +0 -0
dashboard_data/quality_data.json +1 -1
dashboard_data/support_data_11a1fab.csv +8 -8
dashboard_data/support_data_8c0acbd.csv +12 -12
dashboard_data/support_data_a9b92c4.csv +8 -8
dashboard_data/support_data_c814cae.csv +22 -22
dashboard_data/test_coverage_112a023.json +239 -0
dashboard_data/test_coverage_11a1fab.json +233 -0
dashboard_data/test_coverage_8c0acbd.json +234 -0
dashboard_data/test_coverage_a9b92c4.json +239 -0
dashboard_data/test_coverage_c814cae.json +233 -0
dashboard_data/test_coverage_ca49596.json +237 -0
main.py +327 -415
multilingual_generate.py +0 -133
performance_generate.py +492 -17
quality_generate.py +0 -186
utils.py +69 -41

.github/scripts/check_dataset_update.py CHANGED Viewed

@@ -1,8 +1,8 @@
 import json
 import os
 from github import Github
-from datetime import datetime, timedelta
 from huggingface_hub import HfApi, login
@@ -16,12 +16,17 @@ def check_dataset_updates(dataset_id):
     last_modified = dataset_info.lastModified.isoformat()
     current_sha = dataset_info.sha
-    repo_tree = api.list_repo_tree(repo_id=dataset_id, repo_type="dataset", path_in_repo="benchmark_data", recursive=False)
     cutoff_date = datetime.now(dataset_info.lastModified.tzinfo) - timedelta(weeks=6)
     commit_dates_hashes = [item.path.split("/")[-1] for item in repo_tree]
     new_commit_hashes = []
     for commit_date_hash in commit_dates_hashes:
         commit_date, commit_hash = commit_date_hash.split("_")
         commit_date = datetime.strptime(commit_date, "%Y-%m-%dT%H%M%S").replace(
@@ -42,10 +47,10 @@ def check_dataset_updates(dataset_id):
         except Exception as e:
             print(f"Error processing commit {commit_hash}: {str(e)}")
             continue
     # Sort by commit date
     commit_info.sort(key=lambda x: x[1])
     # Extract sorted commits and versions
     new_releases = [info[0] for info in commit_info]
     new_versions = [info[2] for info in commit_info]
@@ -94,15 +99,15 @@ def get_commit_version(repo, commit_hash):
     try:
         releases = list(repo.get_releases())
         releases.sort(key=lambda x: x.created_at)
         commit = repo.get_commit(commit_hash)
         commit_date = commit.commit.author.date
         for i, release in enumerate(releases):
             if commit_date <= release.created_at:
-                return releases[i].tag_name.lstrip('v')
-        return releases[-1].tag_name.lstrip('v')
     except Exception as e:
         print(f"Error processing commit {commit_hash}: {str(e)}")
         return None

 import json
 import os
+from datetime import datetime, timedelta
 from github import Github
 from huggingface_hub import HfApi, login
     last_modified = dataset_info.lastModified.isoformat()
     current_sha = dataset_info.sha
+    repo_tree = api.list_repo_tree(
+        repo_id=dataset_id,
+        repo_type="dataset",
+        path_in_repo="benchmark_data",
+        recursive=False,
+    )
     cutoff_date = datetime.now(dataset_info.lastModified.tzinfo) - timedelta(weeks=6)
     commit_dates_hashes = [item.path.split("/")[-1] for item in repo_tree]
     new_commit_hashes = []
     for commit_date_hash in commit_dates_hashes:
         commit_date, commit_hash = commit_date_hash.split("_")
         commit_date = datetime.strptime(commit_date, "%Y-%m-%dT%H%M%S").replace(
         except Exception as e:
             print(f"Error processing commit {commit_hash}: {str(e)}")
             continue
     # Sort by commit date
     commit_info.sort(key=lambda x: x[1])
     # Extract sorted commits and versions
     new_releases = [info[0] for info in commit_info]
     new_versions = [info[2] for info in commit_info]
     try:
         releases = list(repo.get_releases())
         releases.sort(key=lambda x: x.created_at)
         commit = repo.get_commit(commit_hash)
         commit_date = commit.commit.author.date
         for i, release in enumerate(releases):
             if commit_date <= release.created_at:
+                return releases[i].tag_name.lstrip("v")
+        return releases[-1].tag_name.lstrip("v")
     except Exception as e:
         print(f"Error processing commit {commit_hash}: {str(e)}")
         return None

Makefile CHANGED Viewed

@@ -4,12 +4,7 @@ format:
 	@pre-commit run --all-files
 use-huggingface-data:
-	@python multilingual_generate.py download
 	@python performance_generate.py download
-	@python quality_generate.py
 use-local-data:
 	@python performance_generate.py
-update-performance-data:
-	@python performance_generate.py download

 	@pre-commit run --all-files
 use-huggingface-data:
 	@python performance_generate.py download
 use-local-data:
 	@python performance_generate.py

README.md CHANGED Viewed

@@ -1,5 +1,5 @@
 ---
-title: WhisperKit Benchmarks
 emoji: 🏆
 colorFrom: green
 colorTo: indigo

 ---
+title: Argmax Open Source Regression Tests
 emoji: 🏆
 colorFrom: green
 colorTo: indigo

constants.py CHANGED Viewed

@@ -4,18 +4,12 @@ from iso639 import Lang
 BANNER_TEXT = """
 <div style="text-align: center;">
-    <h1><a href='https://github.com/argmaxinc/WhisperKit'>WhisperKit Benchmarks</a></h1>
 </div>
 """
-SHA_TO_VERSION = {
-    "a9b92c4": "0.9.1",
-    "112a023": "0.10.1",
-}
-INTRO_LABEL = """We present comprehensive benchmarks for WhisperKit, our on-device ASR solution, compared against a reference implementation. These benchmarks aim to help developers and enterprises make informed decisions when choosing optimized or compressed variants of machine learning models for production use. Show more."""
 INTRO_TEXT = """
@@ -30,7 +24,7 @@ Tokens per Second (⬆️): The number of output tokens generated per second. Hi
 Speed (⬆️): Input audio seconds transcribed per second. Higher is better.
 🎯 WhisperKit is evaluated across different datasets, with a focus on per-example no-regressions (QoI) and overall accuracy (WER).
-\n💻 Our benchmarks include:
 Reference: <a href='https://platform.openai.com/docs/guides/speech-to-text'>WhisperOpenAIAPI</a> (OpenAI's Whisper API)
 On-device: <a href='https://github.com/argmaxinc/WhisperKit'>WhisperKit</a> (various versions and optimizations)
@@ -40,11 +34,6 @@ On-device: <a href='https://github.com/argmaxinc/WhisperKit'>WhisperKit</a> (var
 <a href='https://huggingface.co/datasets/argmaxinc/librispeech'>LibriSpeech</a>: ~5 hours of short English audio clips
 <a href='https://huggingface.co/datasets/argmaxinc/earnings22'>Earnings22</a>: ~120 hours of English audio from earnings calls
-🌐 Multilingual Benchmarks:
-These benchmarks aim to demonstrate WhisperKit's capabilities across diverse languages, helping developers assess its suitability for multilingual applications.
-\nDataset:
-<a href='https://huggingface.co/datasets/argmaxinc/whisperkit-evals-multilingual'>Common Voice 17.0</a>: Short-form audio files (<30s/clip) for a maximum of 400 samples per language from Common Voice 17.0. Test set covers a wide range of languages to test model's versatility.
 \nMetrics:
 Average WER: Provides an overall measure of model performance across all languages.
 Language-specific WER: Allows for detailed analysis of model performance for each supported language.
@@ -59,7 +48,6 @@ Results are shown for both forced (correct language given as input) and unforced
 - <a href='https://github.com/argmaxinc/whisperkittools'>whisperkittools</a>
 - <a href='https://huggingface.co/datasets/argmaxinc/librispeech'>LibriSpeech</a>
 - <a href='https://huggingface.co/datasets/argmaxinc/earnings22'>Earnings22</a>
-- <a href='https://huggingface.co/datasets/argmaxinc/whisperkit-evals-multilingual'>Common Voice 17.0</a>
 - <a href='https://platform.openai.com/docs/guides/speech-to-text'>WhisperOpenAIAPI</a>
 """
@@ -69,7 +57,7 @@ METHODOLOGY_TEXT = dedent(
     # Methodology
     ## Overview
-    WhisperKit Benchmarks is the one-stop shop for on-device performance and quality testing of WhisperKit models across supported devices, OS versions and audio datasets.
     ## Metrics
@@ -77,16 +65,15 @@ METHODOLOGY_TEXT = dedent(
     - **Tok/s (Tokens per second)** (⬆️): Total number of text decoder forward passes divided by the end-to-end processing time.
         - This metric varies with input data given that the pace of speech changes the text decoder % of overall latency. This metric should not be confused with the reciprocal of the text decoder latency which is constant across input files.
     - **WER (Word Error Rate)** (⬇️): The ratio of words incorrectly transcribed when comparing the model's output to reference transcriptions, with lower values indicating better accuracy.
     - **QoI (Quality of Inference)** (⬆️): The ratio of examples where WhisperKit performs no worse than the reference model.
         - This metric does not capture improvements to the reference. It only measures potential regressions.
-    - **Multilingual results**: Separated into "language hinted" and "language predicted" categories to evaluate performance with and without prior knowledge of the input language.
     ## Data
     - **Short-form**: 5 hours of English audiobook clips with 30s/clip comprising the [librispeech test set](https://huggingface.co/datasets/argmaxinc/librispeech). Proxy for average streaming performance.
     - **Long-form**: 12 hours of earnings call recordings with ~1hr/clip in English with various accents. Built by randomly selecting 10% of the [earnings22 test set](https://huggingface.co/datasets/argmaxinc/earnings22-12hours). Proxy for average from-file performance.
     - Full datasets are used for English Quality tests and random 10-minute subsets are used for Performance tests.
-    - **Multilingual**: Max 400 samples per language with <30s/clip from [Common Voice 17.0 Test Set](https://huggingface.co/datasets/argmaxinc/common_voice_17_0-argmax_subset-400). Common Voice covers 77 of the 99 languages supported by Whisper.
     ## Performance Measurement
@@ -100,9 +87,8 @@ METHODOLOGY_TEXT = dedent(
     - Performance: Interactive filtering by model, device, OS, and performance metrics
     - Timeline: Visualizations of performance trends
-    - English Quality: English transcription quality on short- and long-form audio
-    - Multilingual Quality: Multilingual (77) transcription quality on short-form audio with and without language prediction
 	- Device Support: Matrix of supported device, OS and model version combinations. Unsupported combinations are marked with :warning:.
     - This methodology ensures a comprehensive and fair evaluation of speech recognition models supported by WhisperKit across a wide range of scenarios and use cases.
 """
 )
@@ -112,6 +98,7 @@ PERFORMANCE_TEXT = dedent(
     ## Metrics
     - **Speed factor** (⬆️): Computed as the ratio of input audio length to end-to-end WhisperKit latency for transcribing that audio. A speed factor of N means N seconds of input audio was transcribed in 1 second.
     - **Tok/s (Tokens per second)** (⬆️): Total number of text decoder forward passes divided by the end-to-end processing time.
     ## Data
@@ -141,7 +128,7 @@ COL_NAMES = {
     "device": "Device",
     "os": "OS",
     "english_wer": "English WER",
-    "multilingual_wer": "Multilingual WER",
 }

 BANNER_TEXT = """
 <div style="text-align: center;">
+    <h1><a href='https://github.com/argmaxinc/WhisperKit'>Argmax Open Source Regression Tests</a></h1>
 </div>
 """
+INTRO_LABEL = """We present comprehensive regression tests for WhisperKit, our on-device ASR solution. These tests aim to help developers and enterprises make informed decisions when choosing optimized or compressed variants of machine learning models for production use. Show more."""
 INTRO_TEXT = """
 Speed (⬆️): Input audio seconds transcribed per second. Higher is better.
 🎯 WhisperKit is evaluated across different datasets, with a focus on per-example no-regressions (QoI) and overall accuracy (WER).
+\n💻 Our regression tests include:
 Reference: <a href='https://platform.openai.com/docs/guides/speech-to-text'>WhisperOpenAIAPI</a> (OpenAI's Whisper API)
 On-device: <a href='https://github.com/argmaxinc/WhisperKit'>WhisperKit</a> (various versions and optimizations)
 <a href='https://huggingface.co/datasets/argmaxinc/librispeech'>LibriSpeech</a>: ~5 hours of short English audio clips
 <a href='https://huggingface.co/datasets/argmaxinc/earnings22'>Earnings22</a>: ~120 hours of English audio from earnings calls
 \nMetrics:
 Average WER: Provides an overall measure of model performance across all languages.
 Language-specific WER: Allows for detailed analysis of model performance for each supported language.
 - <a href='https://github.com/argmaxinc/whisperkittools'>whisperkittools</a>
 - <a href='https://huggingface.co/datasets/argmaxinc/librispeech'>LibriSpeech</a>
 - <a href='https://huggingface.co/datasets/argmaxinc/earnings22'>Earnings22</a>
 - <a href='https://platform.openai.com/docs/guides/speech-to-text'>WhisperOpenAIAPI</a>
 """
     # Methodology
     ## Overview
+    Argmax Open Source Regression Tests is the one-stop shop for on-device performance testing of WhisperKit models across supported devices, OS versions and audio datasets.
     ## Metrics
     - **Tok/s (Tokens per second)** (⬆️): Total number of text decoder forward passes divided by the end-to-end processing time.
         - This metric varies with input data given that the pace of speech changes the text decoder % of overall latency. This metric should not be confused with the reciprocal of the text decoder latency which is constant across input files.
     - **WER (Word Error Rate)** (⬇️): The ratio of words incorrectly transcribed when comparing the model's output to reference transcriptions, with lower values indicating better accuracy.
+    - **Parity**: The difference between measured WER and ground truth WER (Measured - Ground Truth). Values near zero indicate device performance matches expectations.
     - **QoI (Quality of Inference)** (⬆️): The ratio of examples where WhisperKit performs no worse than the reference model.
         - This metric does not capture improvements to the reference. It only measures potential regressions.
     ## Data
     - **Short-form**: 5 hours of English audiobook clips with 30s/clip comprising the [librispeech test set](https://huggingface.co/datasets/argmaxinc/librispeech). Proxy for average streaming performance.
     - **Long-form**: 12 hours of earnings call recordings with ~1hr/clip in English with various accents. Built by randomly selecting 10% of the [earnings22 test set](https://huggingface.co/datasets/argmaxinc/earnings22-12hours). Proxy for average from-file performance.
     - Full datasets are used for English Quality tests and random 10-minute subsets are used for Performance tests.
     ## Performance Measurement
     - Performance: Interactive filtering by model, device, OS, and performance metrics
     - Timeline: Visualizations of performance trends
 	- Device Support: Matrix of supported device, OS and model version combinations. Unsupported combinations are marked with :warning:.
+    - Test Coverage: Analysis of testing completeness across Apple devices, OS versions, and chips with coverage percentages.
     - This methodology ensures a comprehensive and fair evaluation of speech recognition models supported by WhisperKit across a wide range of scenarios and use cases.
 """
 )
     ## Metrics
     - **Speed factor** (⬆️): Computed as the ratio of input audio length to end-to-end WhisperKit latency for transcribing that audio. A speed factor of N means N seconds of input audio was transcribed in 1 second.
     - **Tok/s (Tokens per second)** (⬆️): Total number of text decoder forward passes divided by the end-to-end processing time.
+    - **Parity**: Difference between measured WER and ground truth WER (Measured - Ground Truth). Positive values indicate performance is worse than expected, negative values indicate better than expected performance.
     ## Data
     "device": "Device",
     "os": "OS",
     "english_wer": "English WER",
+    "parity": "Parity",
 }

dashboard_data/Mac.json ADDED Viewed

	@@ -0,0 +1,241 @@

+{
+    "date_generated": "2025-06-16T15:52:13.069236",
+    "xcode_version": "Developer",
+    "total_menu": {
+      "MacBook Pro 14\" (M4, 2024)": {
+        "sku": "Mac16,10",
+        "chip": "M4",
+        "ram": "16 GB"
+      },
+      "MacBook Pro 16\" (M4, 2024)": {
+        "sku": "Mac16,11",
+        "chip": "M4",
+        "ram": "16 GB"
+      },
+      "MacBook Pro 14\" (M4 Pro, 2024)": {
+        "sku": "Mac16,12",
+        "chip": "M4",
+        "ram": "24 GB"
+      },
+      "MacBook Pro 16\" (M4 Pro, 2024)": {
+        "sku": "Mac16,13",
+        "chip": "M4",
+        "ram": "24 GB"
+      },
+      "iMac 24\" (M4, 2024)": {
+        "sku": "Mac16,1",
+        "chip": "M4",
+        "ram": "16 GB"
+      },
+      "iMac 24\" (M4, 2024) - Higher Config": {
+        "sku": "Mac16,2",
+        "chip": "M4",
+        "ram": "24 GB"
+      },
+      "Mac mini (M4, 2024)": {
+        "sku": "Mac16,3",
+        "chip": "M4",
+        "ram": "16 GB"
+      },
+      "Mac mini (M4 Pro, 2024)": {
+        "sku": "Mac16,5",
+        "chip": "M4",
+        "ram": "24 GB"
+      },
+      "MacBook Air 13\" (M4, 2024)": {
+        "sku": "Mac16,6",
+        "chip": "M4",
+        "ram": "16 GB"
+      },
+      "MacBook Air 15\" (M4, 2024)": {
+        "sku": "Mac16,7",
+        "chip": "M4",
+        "ram": "16 GB"
+      },
+      "Mac Pro (M2 Ultra, 2023)": {
+        "sku": "Mac16,8",
+        "chip": "M2",
+        "ram": "64 GB"
+      },
+      "Mac Pro (M2 Ultra, 2023) - Higher Config": {
+        "sku": "Mac16,9",
+        "chip": "M2",
+        "ram": "128 GB"
+      },
+      "MacBook Pro 14\" (M3, 2023)": {
+        "sku": "Mac15,3",
+        "chip": "M3",
+        "ram": "16 GB"
+      },
+      "MacBook Pro 16\" (M3, 2023)": {
+        "sku": "Mac15,4",
+        "chip": "M3",
+        "ram": "16 GB"
+      },
+      "MacBook Pro 14\" (M3 Pro, 2023)": {
+        "sku": "Mac15,5",
+        "chip": "M3",
+        "ram": "24 GB"
+      },
+      "MacBook Pro 16\" (M3 Pro, 2023)": {
+        "sku": "Mac15,6",
+        "chip": "M3",
+        "ram": "24 GB"
+      },
+      "MacBook Pro 14\" (M3 Max, 2023)": {
+        "sku": "Mac15,7",
+        "chip": "M3",
+        "ram": "32 GB"
+      },
+      "MacBook Pro 16\" (M3 Max, 2023)": {
+        "sku": "Mac15,8",
+        "chip": "M3",
+        "ram": "32 GB"
+      },
+      "iMac 24\" (M3, 2023)": {
+        "sku": "Mac15,9",
+        "chip": "M3",
+        "ram": "16 GB"
+      },
+      "iMac 24\" (M3, 2023) - Higher Config": {
+        "sku": "Mac15,10",
+        "chip": "M3",
+        "ram": "24 GB"
+      },
+      "MacBook Air 13\" (M3, 2024)": {
+        "sku": "Mac15,11",
+        "chip": "M3",
+        "ram": "16 GB"
+      },
+      "MacBook Air 15\" (M3, 2024)": {
+        "sku": "Mac15,12",
+        "chip": "M3",
+        "ram": "16 GB"
+      },
+      "Mac mini (M3, 2024)": {
+        "sku": "Mac15,13",
+        "chip": "M3",
+        "ram": "16 GB"
+      },
+      "MacBook Air 13\" (M2, 2022)": {
+        "sku": "Mac14,2",
+        "chip": "M2",
+        "ram": "16 GB"
+      },
+      "MacBook Pro 13\" (M2, 2022)": {
+        "sku": "Mac14,3",
+        "chip": "M2",
+        "ram": "16 GB"
+      },
+      "Mac mini (M2, 2023)": {
+        "sku": "Mac14,4",
+        "chip": "M2",
+        "ram": "16 GB"
+      },
+      "MacBook Pro 14\" (M2, 2023)": {
+        "sku": "Mac14,5",
+        "chip": "M2",
+        "ram": "16 GB"
+      },
+      "MacBook Pro 16\" (M2, 2023)": {
+        "sku": "Mac14,6",
+        "chip": "M2",
+        "ram": "16 GB"
+      },
+      "MacBook Pro 14\" (M2 Pro, 2023)": {
+        "sku": "Mac14,7",
+        "chip": "M2",
+        "ram": "24 GB"
+      },
+      "MacBook Pro 16\" (M2 Pro, 2023)": {
+        "sku": "Mac14,8",
+        "chip": "M2",
+        "ram": "24 GB"
+      },
+      "MacBook Pro 14\" (M2 Max, 2023)": {
+        "sku": "Mac14,9",
+        "chip": "M2",
+        "ram": "32 GB"
+      },
+      "MacBook Pro 16\" (M2 Max, 2023)": {
+        "sku": "Mac14,10",
+        "chip": "M2",
+        "ram": "32 GB"
+      },
+      "Mac mini (M2 Pro, 2023)": {
+        "sku": "Mac14,12",
+        "chip": "M2",
+        "ram": "24 GB"
+      },
+      "Mac Studio (M2 Max, 2023)": {
+        "sku": "Mac14,13",
+        "chip": "M2",
+        "ram": "32 GB"
+      },
+      "Mac Studio (M2 Ultra, 2023)": {
+        "sku": "Mac14,14",
+        "chip": "M2",
+        "ram": "64 GB"
+      },
+      "MacBook Air 15\" (M2, 2023)": {
+        "sku": "Mac14,15",
+        "chip": "M2",
+        "ram": "16 GB"
+      },
+      "MacBook Pro 13\" (M1, 2020)": {
+        "sku": "MacBookPro17,1",
+        "chip": "M1",
+        "ram": "16 GB"
+      },
+      "MacBook Pro 14\" (M1 Pro, 2021)": {
+        "sku": "MacBookPro18,1",
+        "chip": "M1",
+        "ram": "24 GB"
+      },
+      "MacBook Pro 16\" (M1 Pro, 2021)": {
+        "sku": "MacBookPro18,2",
+        "chip": "M1",
+        "ram": "24 GB"
+      },
+      "MacBook Pro 14\" (M1 Max, 2021)": {
+        "sku": "MacBookPro18,3",
+        "chip": "M1",
+        "ram": "32 GB"
+      },
+      "MacBook Pro 16\" (M1 Max, 2021)": {
+        "sku": "MacBookPro18,4",
+        "chip": "M1",
+        "ram": "32 GB"
+      },
+      "MacBook Air 13\" (M1, 2020)": {
+        "sku": "MacBookAir10,1",
+        "chip": "M1",
+        "ram": "16 GB"
+      },
+      "Mac mini (M1, 2020)": {
+        "sku": "Macmini9,1",
+        "chip": "M1",
+        "ram": "16 GB"
+      },
+      "iMac 24\" (M1, 2021)": {
+        "sku": "iMac21,1",
+        "chip": "M1",
+        "ram": "16 GB"
+      },
+      "iMac 24\" (M1, 2021) - Higher Config": {
+        "sku": "iMac21,2",
+        "chip": "M1",
+        "ram": "24 GB"
+      },
+      "Mac Studio (M1 Max, 2022)": {
+        "sku": "Mac13,1",
+        "chip": "M1",
+        "ram": "32 GB"
+      },
+      "Mac Studio (M1 Ultra, 2022)": {
+        "sku": "Mac13,2",
+        "chip": "M1",
+        "ram": "64 GB"
+      }
+    }
+}

dashboard_data/config.json CHANGED Viewed

@@ -1,9 +1,43 @@
 {
-    "name": "whisperkit-coreml",
-    "version": "0.2",
     "device_support": [
         {
-            "identifiers": ["iPhone11", "iPhone12", "Watch7", "Watch8"],
             "models": {
                 "default": "openai_whisper-tiny",
                 "supported": [
@@ -15,27 +49,23 @@
             }
         },
         {
-            "identifiers": ["iPhone13", "iPad13,18", "iPad13,1"],
-            "models": {
-                "default": "openai_whisper-base",
-                "supported": [
-                    "openai_whisper-tiny",
-                    "openai_whisper-tiny.en",
-                    "openai_whisper-base",
-                    "openai_whisper-base.en",
-                    "openai_whisper-small",
-                    "openai_whisper-small.en"
-                ]
-            }
-        },
-        {
             "identifiers": [
-                "iPhone14",
-                "iPhone15",
-                "iPhone16",
-                "iPhone17",
-                "iPad14,1",
-                "iPad14,2"
             ],
             "models": {
                 "default": "openai_whisper-base",
@@ -46,6 +76,8 @@
                     "openai_whisper-base.en",
                     "openai_whisper-small",
                     "openai_whisper-small.en",
                     "openai_whisper-large-v2_949MB",
                     "openai_whisper-large-v2_turbo_955MB",
                     "openai_whisper-large-v3_947MB",
@@ -53,24 +85,38 @@
                     "distil-whisper_distil-large-v3_594MB",
                     "distil-whisper_distil-large-v3_turbo_600MB",
                     "openai_whisper-large-v3-v20240930_626MB",
-                    "openai_whisper-large-v3-v20240930_turbo_632MB"
                 ]
             }
         },
         {
             "identifiers": [
-                "Mac13",
-                "iMac21",
                 "MacBookAir10,1",
-                "MacBookPro17",
-                "MacBookPro18",
-                "Macmini9",
-                "iPad13,16",
                 "iPad13,4",
-                "iPad13,8"
             ],
             "models": {
-                "default": "openai_whisper-large-v3-v20240930",
                 "supported": [
                     "openai_whisper-tiny",
                     "openai_whisper-tiny.en",
@@ -78,6 +124,8 @@
                     "openai_whisper-base.en",
                     "openai_whisper-small",
                     "openai_whisper-small.en",
                     "openai_whisper-large-v2",
                     "openai_whisper-large-v2_949MB",
                     "openai_whisper-large-v3",
@@ -85,15 +133,51 @@
                     "distil-whisper_distil-large-v3",
                     "distil-whisper_distil-large-v3_594MB",
                     "openai_whisper-large-v3-v20240930",
-                    "openai_whisper-large-v3-v20240930_626MB"
                 ]
             }
         },
         {
             "identifiers": [
-                "Mac14",
-                "Mac15",
-                "Mac16",
                 "iPad14,3",
                 "iPad14,4",
                 "iPad14,5",
@@ -102,7 +186,14 @@
                 "iPad14,9",
                 "iPad14,10",
                 "iPad14,11",
-                "iPad16"
             ],
             "models": {
                 "default": "openai_whisper-large-v3-v20240930",
@@ -113,6 +204,8 @@
                     "openai_whisper-base.en",
                     "openai_whisper-small",
                     "openai_whisper-small.en",
                     "openai_whisper-large-v2",
                     "openai_whisper-large-v2_949MB",
                     "openai_whisper-large-v2_turbo",
@@ -128,9 +221,99 @@
                     "openai_whisper-large-v3-v20240930",
                     "openai_whisper-large-v3-v20240930_turbo",
                     "openai_whisper-large-v3-v20240930_626MB",
-                    "openai_whisper-large-v3-v20240930_turbo_632MB"
                 ]
             }
         }
-    ]
 }

 {
+    "name": "argmax-pro-sdk",
+    "version": "0.5",
     "device_support": [
         {
+            "chips": "A12, A13, S9, S10",
+            "identifiers": [
+                "iPhone11,2",
+                "iPhone11,3",
+                "iPhone11,4",
+                "iPhone11,5",
+                "iPhone11,6",
+                "iPhone11,8",
+                "iPhone12,1",
+                "iPhone12,2",
+                "iPhone12,3",
+                "iPhone12,4",
+                "iPhone12,5",
+                "iPhone12,8",
+                "iPad8,1",
+                "iPad8,2",
+                "iPad8,3",
+                "iPad8,4",
+                "iPad8,5",
+                "iPad8,6",
+                "iPad8,7",
+                "iPad8,8",
+                "iPad8,9",
+                "iPad8,10",
+                "iPad8,11",
+                "iPad8,12",
+                "iPad11,1",
+                "iPad11,2",
+                "iPad11,3",
+                "iPad11,4",
+                "iPad11,6",
+                "iPad11,7",
+                "iPad12,1",
+                "iPad12,2"
+            ],
             "models": {
                 "default": "openai_whisper-tiny",
                 "supported": [
             }
         },
         {
+            "chips": "A16, A17 Pro, A18, A18 Pro",
             "identifiers": [
+                "iPhone15,2",
+                "iPhone15,3",
+                "iPhone15,4",
+                "iPhone15,5",
+                "iPhone16,1",
+                "iPhone16,2",
+                "iPhone17,1",
+                "iPhone17,2",
+                "iPhone17,3",
+                "iPhone17,4",
+                "iPhone17,5",
+                "iPad15,7",
+                "iPad15,8",
+                "iPad16,1",
+                "iPad16,2"
             ],
             "models": {
                 "default": "openai_whisper-base",
                     "openai_whisper-base.en",
                     "openai_whisper-small",
                     "openai_whisper-small.en",
+                    "openai_whisper-small_216MB",
+                    "openai_whisper-small.en_217MB",
                     "openai_whisper-large-v2_949MB",
                     "openai_whisper-large-v2_turbo_955MB",
                     "openai_whisper-large-v3_947MB",
                     "distil-whisper_distil-large-v3_594MB",
                     "distil-whisper_distil-large-v3_turbo_600MB",
                     "openai_whisper-large-v3-v20240930_626MB",
+                    "openai_whisper-large-v3-v20240930_turbo_632MB",
+                    "nvidia_parakeet-v2_476MB"
                 ]
             }
         },
         {
+            "chips": "M1",
             "identifiers": [
+                "MacBookPro17,1",
+                "MacBookPro18,1",
+                "MacBookPro18,2",
+                "MacBookPro18,3",
+                "MacBookPro18,4",
                 "MacBookAir10,1",
+                "Macmini9,1",
+                "iMac21,1",
+                "iMac21,2",
+                "Mac13,1",
+                "Mac13,2",
                 "iPad13,4",
+                "iPad13,5",
+                "iPad13,6",
+                "iPad13,7",
+                "iPad13,8",
+                "iPad13,9",
+                "iPad13,10",
+                "iPad13,11",
+                "iPad13,16",
+                "iPad13,17"
             ],
             "models": {
+                "default": "openai_whisper-large-v3-v20240930_626MB",
                 "supported": [
                     "openai_whisper-tiny",
                     "openai_whisper-tiny.en",
                     "openai_whisper-base.en",
                     "openai_whisper-small",
                     "openai_whisper-small.en",
+                    "openai_whisper-small_216MB",
+                    "openai_whisper-small.en_217MB",
                     "openai_whisper-large-v2",
                     "openai_whisper-large-v2_949MB",
                     "openai_whisper-large-v3",
                     "distil-whisper_distil-large-v3",
                     "distil-whisper_distil-large-v3_594MB",
                     "openai_whisper-large-v3-v20240930",
+                    "openai_whisper-large-v3-v20240930_626MB",
+                    "nvidia_parakeet-v2_476MB",
+                    "nvidia_parakeet-v2"
                 ]
             }
         },
         {
+            "chips": "M2, M3, M4",
             "identifiers": [
+                "Mac14,2",
+                "Mac14,3",
+                "Mac14,4",
+                "Mac14,5",
+                "Mac14,6",
+                "Mac14,7",
+                "Mac14,8",
+                "Mac14,9",
+                "Mac14,10",
+                "Mac14,12",
+                "Mac14,13",
+                "Mac14,14",
+                "Mac14,15",
+                "Mac15,3",
+                "Mac15,4",
+                "Mac15,5",
+                "Mac15,6",
+                "Mac15,7",
+                "Mac15,8",
+                "Mac15,9",
+                "Mac15,10",
+                "Mac15,11",
+                "Mac15,12",
+                "Mac15,13",
+                "Mac16,1",
+                "Mac16,2",
+                "Mac16,3",
+                "Mac16,5",
+                "Mac16,6",
+                "Mac16,7",
+                "Mac16,8",
+                "Mac16,9",
+                "Mac16,10",
+                "Mac16,11",
+                "Mac16,12",
+                "Mac16,13",
                 "iPad14,3",
                 "iPad14,4",
                 "iPad14,5",
                 "iPad14,9",
                 "iPad14,10",
                 "iPad14,11",
+                "iPad15,3",
+                "iPad15,4",
+                "iPad15,5",
+                "iPad15,6",
+                "iPad16,3",
+                "iPad16,4",
+                "iPad16,5",
+                "iPad16,6"
             ],
             "models": {
                 "default": "openai_whisper-large-v3-v20240930",
                     "openai_whisper-base.en",
                     "openai_whisper-small",
                     "openai_whisper-small.en",
+                    "openai_whisper-small_216MB",
+                    "openai_whisper-small.en_217MB",
                     "openai_whisper-large-v2",
                     "openai_whisper-large-v2_949MB",
                     "openai_whisper-large-v2_turbo",
                     "openai_whisper-large-v3-v20240930",
                     "openai_whisper-large-v3-v20240930_turbo",
                     "openai_whisper-large-v3-v20240930_626MB",
+                    "openai_whisper-large-v3-v20240930_turbo_632MB",
+                    "nvidia_parakeet-v2_476MB",
+                    "nvidia_parakeet-v2"
+                ]
+            }
+        },
+        {
+            "chips": "A14",
+            "identifiers": [
+                "iPhone13,1",
+                "iPhone13,2",
+                "iPhone13,3",
+                "iPhone13,4",
+                "iPad13,1",
+                "iPad13,2",
+                "iPad13,18",
+                "iPad13,19"
+            ],
+            "models": {
+                "default": "openai_whisper-base",
+                "supported": [
+                    "openai_whisper-tiny",
+                    "openai_whisper-tiny.en",
+                    "openai_whisper-base",
+                    "openai_whisper-base.en",
+                    "openai_whisper-small",
+                    "openai_whisper-small.en",
+                    "openai_whisper-small_216MB",
+                    "openai_whisper-small.en_217MB",
+                    "nvidia_parakeet-v2_476MB"
+                ]
+            }
+        },
+        {
+            "chips": "A15",
+            "identifiers": [
+                "iPhone14,2",
+                "iPhone14,3",
+                "iPhone14,4",
+                "iPhone14,5",
+                "iPhone14,6",
+                "iPhone14,7",
+                "iPhone14,8",
+                "iPad14,1",
+                "iPad14,2"
+            ],
+            "models": {
+                "default": "openai_whisper-base",
+                "supported": [
+                    "openai_whisper-tiny",
+                    "openai_whisper-tiny.en",
+                    "openai_whisper-base",
+                    "openai_whisper-base.en",
+                    "openai_whisper-small",
+                    "openai_whisper-small.en",
+                    "openai_whisper-small_216MB",
+                    "openai_whisper-small.en_217MB",
+                    "openai_whisper-large-v2_949MB",
+                    "openai_whisper-large-v2_turbo_955MB",
+                    "openai_whisper-large-v3_947MB",
+                    "openai_whisper-large-v3_turbo_954MB",
+                    "distil-whisper_distil-large-v3_594MB",
+                    "distil-whisper_distil-large-v3_turbo_600MB",
+                    "openai_whisper-large-v3-v20240930_626MB",
+                    "openai_whisper-large-v3-v20240930_turbo_632MB",
+                    "nvidia_parakeet-v2_476MB"
                 ]
             }
         }
+    ],
+    "model_checksums": {
+        "distil-whisper_distil-large-v3": "9cd8271143b919402ae776c30b479565",
+        "distil-whisper_distil-large-v3_594MB": "ca532f45ddbf8a3d241132cc5cf41639",
+        "distil-whisper_distil-large-v3_turbo": "b8638452c6568dfe33a33bfcc2bc6aca",
+        "distil-whisper_distil-large-v3_turbo_600MB": "81746b4b1afbbb01a8ae9ea452460d88",
+        "openai_whisper-base.en": "fbcfd586f15e2952251b1d3257f18471",
+        "openai_whisper-base": "36e60501ad0f01c1a5719e83a1f63f20",
+        "openai_whisper-large-v2": "21b86c07318aeeef54598f15b7903979",
+        "openai_whisper-large-v2_949MB": "71bad4e1566749d1060eda42308d9fb4",
+        "openai_whisper-large-v2_turbo": "7734959b6550e7b5c2d732bf2b7acd23",
+        "openai_whisper-large-v2_turbo_955MB": "cb6411862a48ec75325572081f01e5b5",
+        "openai_whisper-large-v3-v20240930": "17ebd78ff7edfa59001b554e9cc4c021",
+        "openai_whisper-large-v3-v20240930_547MB": "c945dad68449ac3c78ecb2d561ac189d",
+        "openai_whisper-large-v3-v20240930_626MB": "578fe5a07f4eb7e4187c920bca571aa5",
+        "openai_whisper-large-v3-v20240930_turbo": "dfbf09ab741af1d5400ddbd07bb37dad",
+        "openai_whisper-large-v3-v20240930_turbo_632MB": "33954440dbd785ca1828afe25514f5a5",
+        "openai_whisper-large-v3": "a6f24dc72785722e9cea89e227856dfe",
+        "openai_whisper-large-v3_947MB": "ef6b0e9622a046ce2361b4c72307877f",
+        "openai_whisper-large-v3_turbo": "c550fbdea70c5784d322c0a427f8b5cd",
+        "openai_whisper-large-v3_turbo_954MB": "e639c4bb98d905064ef5dd38757dd9d1",
+        "openai_whisper-small.en": "38efe6a00706bbdb995795c67a836e5e",
+        "openai_whisper-small": "f1d21adb950bc9be5d5343bcdeccd23b",
+        "openai_whisper-tiny.en": "e1183fd55448923b1ce43a2da67aa21f",
+        "openai_whisper-tiny": "7147518a3d68ddbea0691e04cfffa4ff"
+    }
 }

dashboard_data/iPad.json ADDED Viewed

	@@ -0,0 +1,206 @@

+{
+    "date_generated": "2025-06-23T18:22:05.764459",
+    "xcode_version": "Version 26.0 (17A5241e)",
+    "total_menu": {
+      "iPad Pro (11-inch) (2nd generation)": {
+        "sku": [
+          "iPad8,9",
+          "iPad8,10"
+        ],
+        "chip": "A12Z",
+        "ram": "6 GB"
+      },
+      "iPad Pro (12.9-inch) (3rd generation)": {
+        "sku": [
+          "iPad8,5",
+          "iPad8,6",
+          "iPad8,7",
+          "iPad8,8"
+        ],
+        "chip": "A12X",
+        "ram": "4 GB"
+      },
+      "iPad Pro (11-inch)": {
+        "sku": [
+          "iPad8,1",
+          "iPad8,2",
+          "iPad8,3",
+          "iPad8,4"
+        ],
+        "chip": "A12X",
+        "ram": "4 GB"
+      },
+      "iPad Pro (12.9-inch) (4th generation)": {
+        "sku": [
+          "iPad8,11",
+          "iPad8,12"
+        ],
+        "chip": "A12Z",
+        "ram": "6 GB"
+      },
+      "iPad Pro 13-inch (M4)": {
+        "sku": [
+          "iPad16,5",
+          "iPad16,6"
+        ],
+        "chip": "M4",
+        "ram": "8 GB"
+      },
+      "iPad Pro 11-inch (M4)": {
+        "sku": [
+          "iPad16,3",
+          "iPad16,4"
+        ],
+        "chip": "M4",
+        "ram": "8 GB"
+      },
+      "iPad mini (A17 Pro)": {
+        "sku": [
+          "iPad16,1",
+          "iPad16,2"
+        ],
+        "chip": "A17 Pro",
+        "ram": "8 GB"
+      },
+      "iPad (A16)": {
+        "sku": [
+          "iPad15,7",
+          "iPad15,8"
+        ],
+        "chip": "A16",
+        "ram": "8 GB"
+      },
+      "iPad Air 13-inch (M3)": {
+        "sku": [
+          "iPad15,5",
+          "iPad15,6"
+        ],
+        "chip": "M3",
+        "ram": "8 GB"
+      },
+      "iPad Air 11-inch (M3)": {
+        "sku": [
+          "iPad15,3",
+          "iPad15,4"
+        ],
+        "chip": "M3",
+        "ram": "8 GB"
+      },
+      "iPad Air 11-inch (M2)": {
+        "sku": [
+          "iPad14,8",
+          "iPad14,9"
+        ],
+        "chip": "M2",
+        "ram": "8 GB"
+      },
+      "iPad Pro (12.9-inch) (6th generation)": {
+        "sku": [
+          "iPad14,5",
+          "iPad14,6"
+        ],
+        "chip": "M2",
+        "ram": "8 GB"
+      },
+      "iPad Pro (11-inch) (4th generation)": {
+        "sku": [
+          "iPad14,3",
+          "iPad14,4"
+        ],
+        "chip": "M2",
+        "ram": "8 GB"
+      },
+      "iPad mini (6th generation)": {
+        "sku": [
+          "iPad14,1",
+          "iPad14,2"
+        ],
+        "chip": "A15",
+        "ram": "4 GB"
+      },
+      "iPad Air 13-inch (M2)": {
+        "sku": [
+          "iPad14,10",
+          "iPad14,11"
+        ],
+        "chip": "M2",
+        "ram": "8 GB"
+      },
+      "iPad Pro (12.9-inch) (5th generation)": {
+        "sku": [
+          "iPad13,8",
+          "iPad13,9",
+          "iPad13,10",
+          "iPad13,11"
+        ],
+        "chip": "M1",
+        "ram": "8 GB"
+      },
+      "iPad Pro (11-inch) (3rd generation)": {
+        "sku": [
+          "iPad13,4",
+          "iPad13,5",
+          "iPad13,6",
+          "iPad13,7"
+        ],
+        "chip": "M1",
+        "ram": "8 GB"
+      },
+      "iPad Air (4th generation)": {
+        "sku": [
+          "iPad13,1",
+          "iPad13,2"
+        ],
+        "chip": "A14",
+        "ram": "4 GB"
+      },
+      "iPad (10th generation)": {
+        "sku": [
+          "iPad13,18",
+          "iPad13,19"
+        ],
+        "chip": "A14",
+        "ram": "4 GB"
+      },
+      "iPad Air (5th generation)": {
+        "sku": [
+          "iPad13,16",
+          "iPad13,17"
+        ],
+        "chip": "M1",
+        "ram": "8 GB"
+      },
+      "iPad (9th generation)": {
+        "sku": [
+          "iPad12,1",
+          "iPad12,2"
+        ],
+        "chip": "A13",
+        "ram": "3 GB"
+      },
+      "iPad (8th generation)": {
+        "sku": [
+          "iPad11,6",
+          "iPad11,7"
+        ],
+        "chip": "A12",
+        "ram": "3 GB"
+      },
+      "iPad Air (3rd generation)": {
+        "sku": [
+          "iPad11,3",
+          "iPad11,4"
+        ],
+        "chip": "A12",
+        "ram": "3 GB"
+      },
+      "iPad mini (5th generation)": {
+        "sku": [
+          "iPad11,1",
+          "iPad11,2"
+        ],
+        "chip": "A12",
+        "ram": "3 GB"
+      }
+    }
+}

dashboard_data/iPhone.json ADDED Viewed

	@@ -0,0 +1,151 @@

+{
+    "date_generated": "2025-06-16T15:52:13.069236",
+    "xcode_version": "Developer",
+    "total_menu": {
+      "iPhone 16e": {
+        "sku": "iPhone17,5",
+        "chip": "A18",
+        "ram": "8 GB"
+      },
+      "iPhone 16 Plus": {
+        "sku": "iPhone17,4",
+        "chip": "A18",
+        "ram": "8 GB"
+      },
+      "iPhone 16": {
+        "sku": "iPhone17,3",
+        "chip": "A18",
+        "ram": "8 GB"
+      },
+      "iPhone 16 Pro Max": {
+        "sku": "iPhone17,2",
+        "chip": "A18 Pro",
+        "ram": "8 GB"
+      },
+      "iPhone 16 Pro": {
+        "sku": "iPhone17,1",
+        "chip": "A18 Pro",
+        "ram": "8 GB"
+      },
+      "iPhone 15 Pro Max": {
+        "sku": "iPhone16,2",
+        "chip": "A17 Pro",
+        "ram": "8 GB"
+      },
+      "iPhone 15 Pro": {
+        "sku": "iPhone16,1",
+        "chip": "A17 Pro",
+        "ram": "8 GB"
+      },
+      "iPhone 15 Plus": {
+        "sku": "iPhone15,5",
+        "chip": "A16",
+        "ram": "6 GB"
+      },
+      "iPhone 15": {
+        "sku": "iPhone15,4",
+        "chip": "A16",
+        "ram": "6 GB"
+      },
+      "iPhone 14 Pro Max": {
+        "sku": "iPhone15,3",
+        "chip": "A16",
+        "ram": "6 GB"
+      },
+      "iPhone 14 Pro": {
+        "sku": "iPhone15,2",
+        "chip": "A16",
+        "ram": "6 GB"
+      },
+      "iPhone 14 Plus": {
+        "sku": "iPhone14,8",
+        "chip": "A15",
+        "ram": "6 GB"
+      },
+      "iPhone 14": {
+        "sku": "iPhone14,7",
+        "chip": "A15",
+        "ram": "6 GB"
+      },
+      "iPhone SE (3rd generation)": {
+        "sku": "iPhone14,6",
+        "chip": "A15",
+        "ram": "4 GB"
+      },
+      "iPhone 13": {
+        "sku": "iPhone14,5",
+        "chip": "A15",
+        "ram": "4 GB"
+      },
+      "iPhone 13 mini": {
+        "sku": "iPhone14,4",
+        "chip": "A15",
+        "ram": "4 GB"
+      },
+      "iPhone 13 Pro Max": {
+        "sku": "iPhone14,3",
+        "chip": "A15",
+        "ram": "6 GB"
+      },
+      "iPhone 13 Pro": {
+        "sku": "iPhone14,2",
+        "chip": "A15",
+        "ram": "6 GB"
+      },
+      "iPhone 12 Pro Max": {
+        "sku": "iPhone13,4",
+        "chip": "A14",
+        "ram": "6 GB"
+      },
+      "iPhone 12 Pro": {
+        "sku": "iPhone13,3",
+        "chip": "A14",
+        "ram": "6 GB"
+      },
+      "iPhone 12": {
+        "sku": "iPhone13,2",
+        "chip": "A14",
+        "ram": "4 GB"
+      },
+      "iPhone 12 mini": {
+        "sku": "iPhone13,1",
+        "chip": "A14",
+        "ram": "4 GB"
+      },
+      "iPhone SE (2nd generation)": {
+        "sku": "iPhone12,8",
+        "chip": "A13",
+        "ram": "3 GB"
+      },
+      "iPhone 11 Pro Max": {
+        "sku": "iPhone12,5",
+        "chip": "A13",
+        "ram": "4 GB"
+      },
+      "iPhone 11 Pro": {
+        "sku": "iPhone12,3",
+        "chip": "A13",
+        "ram": "4 GB"
+      },
+      "iPhone 11": {
+        "sku": "iPhone12,1",
+        "chip": "A13",
+        "ram": "4 GB"
+      },
+      "iPhone XR": {
+        "sku": "iPhone11,8",
+        "chip": "A12",
+        "ram": "3 GB"
+      },
+      "iPhone XS Max": {
+        "sku": "iPhone11,4",
+        "chip": "A12",
+        "ram": "4 GB"
+      },
+      "iPhone XS": {
+        "sku": "iPhone11,2",
+        "chip": "A12",
+        "ram": "4 GB"
+      }
+    }
+}

dashboard_data/multilingual_confusion_matrices.json DELETED Viewed

The diff for this file is too large to render. See raw diff

dashboard_data/performance_data.json CHANGED Viewed

The diff for this file is too large to render. See raw diff

dashboard_data/quality_data.json CHANGED Viewed

@@ -20,4 +20,4 @@
 {"model": "openai/whisper-small.en", "timestamp": "2024-10-18_15:39:48_GMT-0400", "average_wer": 7.85, "dataset_wer": {"librispeech": 2.88, "earnings22-12hours": 12.82}, "qoi": 0.86}
 {"model": "distil-whisper/distil-large-v3/turbo", "timestamp": "2024-10-20_12:45:20_GMT-0700", "average_wer": 7.2, "dataset_wer": {"librispeech": 2.35, "earnings22-12hours": 12.05}, "qoi": 0.9}
 {"model": "openai/whisper-base", "timestamp": "2024-10-18_20:25:50_GMT-0700", "average_wer": 10.67, "dataset_wer": {"librispeech": 4.94, "earnings22-12hours": 16.4}, "qoi": 0.67}
-{"model": "openai/whisper-large-v3/turbo", "timestamp": "2024-10-20_16:58:25_GMT-0400", "average_wer": 6.86, "dataset_wer": {"librispeech": 1.97, "earnings22-12hours": 11.74}, "qoi": 0.95}

 {"model": "openai/whisper-small.en", "timestamp": "2024-10-18_15:39:48_GMT-0400", "average_wer": 7.85, "dataset_wer": {"librispeech": 2.88, "earnings22-12hours": 12.82}, "qoi": 0.86}
 {"model": "distil-whisper/distil-large-v3/turbo", "timestamp": "2024-10-20_12:45:20_GMT-0700", "average_wer": 7.2, "dataset_wer": {"librispeech": 2.35, "earnings22-12hours": 12.05}, "qoi": 0.9}
 {"model": "openai/whisper-base", "timestamp": "2024-10-18_20:25:50_GMT-0700", "average_wer": 10.67, "dataset_wer": {"librispeech": 4.94, "earnings22-12hours": 16.4}, "qoi": 0.67}
+{"model": "openai/whisper-large-v3/turbo", "timestamp": "2024-10-20_16:58:25_GMT-0400", "average_wer": 6.86, "dataset_wer": {"librispeech": 1.97, "earnings22-12hours": 11.74}, "qoi": 0.95}

dashboard_data/support_data_11a1fab.csv CHANGED Viewed

@@ -1,21 +1,21 @@
 ,Model,"Apple M4 (Mac16,10)","iPad (10th generation) (iPad13,18)","iPad Air 11-inch (M3) (iPad15,3)","iPad (A16) (iPad15,7)","iPad mini (A17 Pro) (iPad16,1)"
-distil-whisper_distil-large-v3,distil-whisper_distil-large-v3,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad15%2C7_summary_2025-04-16T220030.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad16%2C1_summary_2025-04-16T220030.json>iPadOS 18.4</a>
 distil-whisper_distil-large-v3_594MB,distil-whisper_distil-large-v3_594MB,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4
-distil-whisper_distil-large-v3_turbo,distil-whisper_distil-large-v3_turbo,✅ macOS 15.4,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad15%2C3_summary_2025-04-16T013539.json>iPadOS 18.3.2</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad15%2C7_summary_2025-04-16T220030.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad16%2C1_summary_2025-04-16T220030.json>iPadOS 18.4</a>
 distil-whisper_distil-large-v3_turbo_600MB,distil-whisper_distil-large-v3_turbo_600MB,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4
 openai_whisper-base,openai_whisper-base,✅ macOS 15.4,✅ iPadOS 18.4,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4
 openai_whisper-base.en,openai_whisper-base.en,✅ macOS 15.4,✅ iPadOS 18.4,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4
-openai_whisper-large-v2,openai_whisper-large-v2,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad15%2C7_summary_2025-04-16T220030.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad16%2C1_summary_2025-04-16T220030.json>iPadOS 18.4</a>
 openai_whisper-large-v2_949MB,openai_whisper-large-v2_949MB,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4
-openai_whisper-large-v2_turbo,openai_whisper-large-v2_turbo,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad15%2C7_summary_2025-04-16T220030.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad16%2C1_summary_2025-04-16T220030.json>iPadOS 18.4</a>
 openai_whisper-large-v2_turbo_955MB,openai_whisper-large-v2_turbo_955MB,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4
-openai_whisper-large-v3,openai_whisper-large-v3,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad15%2C7_summary_2025-04-16T220030.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad16%2C1_summary_2025-04-16T220030.json>iPadOS 18.4</a>
-openai_whisper-large-v3-v20240930,openai_whisper-large-v3-v20240930,✅ macOS 15.4,Not Supported,? iPadOS 18.3.2,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad15%2C7_summary_2025-04-16T220030.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad16%2C1_summary_2025-04-16T220030.json>iPadOS 18.4</a>
 openai_whisper-large-v3-v20240930_626MB,openai_whisper-large-v3-v20240930_626MB,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4
-openai_whisper-large-v3-v20240930_turbo,openai_whisper-large-v3-v20240930_turbo,✅ macOS 15.4,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad15%2C3_summary_2025-04-16T013539.json>iPadOS 18.3.2</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad15%2C7_summary_2025-04-16T220030.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad16%2C1_summary_2025-04-16T220030.json>iPadOS 18.4</a>
 openai_whisper-large-v3-v20240930_turbo_632MB,openai_whisper-large-v3-v20240930_turbo_632MB,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4
 openai_whisper-large-v3_947MB,openai_whisper-large-v3_947MB,✅ macOS 15.4,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad15%2C3_summary_2025-04-16T013539.json>iPadOS 18.3.2</a>,✅ iPadOS 18.4,✅ iPadOS 18.4
-openai_whisper-large-v3_turbo,openai_whisper-large-v3_turbo,✅ macOS 15.4,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad15%2C3_summary_2025-04-16T013539.json>iPadOS 18.3.2</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad15%2C7_summary_2025-04-16T220030.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad16%2C1_summary_2025-04-16T220030.json>iPadOS 18.4</a>
 openai_whisper-large-v3_turbo_954MB,openai_whisper-large-v3_turbo_954MB,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4
 openai_whisper-small,openai_whisper-small,✅ macOS 15.4,✅ iPadOS 18.4,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4
 openai_whisper-small.en,openai_whisper-small.en,✅ macOS 15.4,✅ iPadOS 18.4,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4

 ,Model,"Apple M4 (Mac16,10)","iPad (10th generation) (iPad13,18)","iPad Air 11-inch (M3) (iPad15,3)","iPad (A16) (iPad15,7)","iPad mini (A17 Pro) (iPad16,1)"
+distil-whisper_distil-large-v3,distil-whisper_distil-large-v3,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,Not Supported,Not Supported
 distil-whisper_distil-large-v3_594MB,distil-whisper_distil-large-v3_594MB,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4
+distil-whisper_distil-large-v3_turbo,distil-whisper_distil-large-v3_turbo,✅ macOS 15.4,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad15%2C3_summary_2025-04-16T013539.json>iPadOS 18.3.2</a>,Not Supported,Not Supported
 distil-whisper_distil-large-v3_turbo_600MB,distil-whisper_distil-large-v3_turbo_600MB,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4
 openai_whisper-base,openai_whisper-base,✅ macOS 15.4,✅ iPadOS 18.4,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4
 openai_whisper-base.en,openai_whisper-base.en,✅ macOS 15.4,✅ iPadOS 18.4,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4
+openai_whisper-large-v2,openai_whisper-large-v2,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,Not Supported,Not Supported
 openai_whisper-large-v2_949MB,openai_whisper-large-v2_949MB,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4
+openai_whisper-large-v2_turbo,openai_whisper-large-v2_turbo,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,Not Supported,Not Supported
 openai_whisper-large-v2_turbo_955MB,openai_whisper-large-v2_turbo_955MB,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4
+openai_whisper-large-v3,openai_whisper-large-v3,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,Not Supported,Not Supported
+openai_whisper-large-v3-v20240930,openai_whisper-large-v3-v20240930,✅ macOS 15.4,Not Supported,? iPadOS 18.3.2,Not Supported,Not Supported
 openai_whisper-large-v3-v20240930_626MB,openai_whisper-large-v3-v20240930_626MB,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4
+openai_whisper-large-v3-v20240930_turbo,openai_whisper-large-v3-v20240930_turbo,✅ macOS 15.4,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad15%2C3_summary_2025-04-16T013539.json>iPadOS 18.3.2</a>,Not Supported,Not Supported
 openai_whisper-large-v3-v20240930_turbo_632MB,openai_whisper-large-v3-v20240930_turbo_632MB,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4
 openai_whisper-large-v3_947MB,openai_whisper-large-v3_947MB,✅ macOS 15.4,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad15%2C3_summary_2025-04-16T013539.json>iPadOS 18.3.2</a>,✅ iPadOS 18.4,✅ iPadOS 18.4
+openai_whisper-large-v3_turbo,openai_whisper-large-v3_turbo,✅ macOS 15.4,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad15%2C3_summary_2025-04-16T013539.json>iPadOS 18.3.2</a>,Not Supported,Not Supported
 openai_whisper-large-v3_turbo_954MB,openai_whisper-large-v3_turbo_954MB,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4
 openai_whisper-small,openai_whisper-small,✅ macOS 15.4,✅ iPadOS 18.4,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4
 openai_whisper-small.en,openai_whisper-small.en,✅ macOS 15.4,✅ iPadOS 18.4,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4

dashboard_data/support_data_8c0acbd.csv CHANGED Viewed

@@ -1,22 +1,22 @@
 ,Model,"Apple M2 Pro (Mac14,12)","Apple M3 Max (Mac15,10)","iPad Air (5th generation) (iPad13,16)","iPad Pro (11-inch) (3rd generation) (iPad13,6)","iPad mini (6th generation) (iPad14,1)","iPad Air 11-inch (M2) (iPad14,8)","iPad Air 11-inch (M3) (iPad15,3)","iPad (A16) (iPad15,7)","iPad mini (A17 Pro) (iPad16,1)","iPad Pro 11-inch (M4) (iPad16,3)","iPhone 11 (iPhone12,1)","iPhone 12 mini (iPhone13,1)","iPhone 13 Pro (iPhone14,2)","iPhone 13 (iPhone14,5)","iPhone 14 (iPhone14,7)","iPhone 15 (iPhone15,4)","iPhone 15 Pro Max (iPhone16,2)","iPhone 16 (iPhone17,3)","iPhone 16e (iPhone17,5)"
-distil-whisper_distil-large-v3,distil-whisper_distil-large-v3,✅ macOS 15.5,✅ macOS 15.5,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,Not Supported,✅ iPadOS 18.5,✅ iPadOS 26.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C7_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C1_summary_2025-06-16T220810.json>iPadOS 18.4</a>,✅ iPadOS 18.4.1,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 distil-whisper_distil-large-v3_594MB,distil-whisper_distil-large-v3_594MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
-distil-whisper_distil-large-v3_turbo,distil-whisper_distil-large-v3_turbo,✅ macOS 15.5,✅ macOS 15.5,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad14%2C8_summary_2025-06-16T192343.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C3_summary_2025-06-17T205025.json>iPadOS 26.0</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C7_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C1_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C3_summary_2025-06-16T220810.json>iPadOS 18.4.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
-distil-whisper_distil-large-v3_turbo_600MB,distil-whisper_distil-large-v3_turbo_600MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
 openai_whisper-base,openai_whisper-base,✅ macOS 15.5,✅ macOS 15.5,✅ iPadOS 18.4.1,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,✅ iOS 26.0,✅ iOS 18.4.1,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
 openai_whisper-base.en,openai_whisper-base.en,✅ macOS 15.5,✅ macOS 15.5,✅ iPadOS 18.4.1,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,✅ iOS 26.0,✅ iOS 18.4.1,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
-openai_whisper-large-v2,openai_whisper-large-v2,✅ macOS 15.5,✅ macOS 15.5,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,Not Supported,✅ iPadOS 18.5,✅ iPadOS 26.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C7_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C1_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C3_summary_2025-06-16T220810.json>iPadOS 18.4.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 openai_whisper-large-v2_949MB,openai_whisper-large-v2_949MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C3_summary_2025-06-17T205025.json>iPadOS 26.0</a>,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
-openai_whisper-large-v2_turbo,openai_whisper-large-v2_turbo,✅ macOS 15.5,✅ macOS 15.5,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad14%2C8_summary_2025-06-16T192343.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C3_summary_2025-06-17T205025.json>iPadOS 26.0</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C7_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C1_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C3_summary_2025-06-16T220810.json>iPadOS 18.4.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
-openai_whisper-large-v2_turbo_955MB,openai_whisper-large-v2_turbo_955MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
-openai_whisper-large-v3,openai_whisper-large-v3,✅ macOS 15.5,✅ macOS 15.5,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad14%2C8_summary_2025-06-16T192343.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C3_summary_2025-06-17T205025.json>iPadOS 26.0</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C7_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C1_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C3_summary_2025-06-16T220810.json>iPadOS 18.4.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
-openai_whisper-large-v3-v20240930,openai_whisper-large-v3-v20240930,✅ macOS 15.5,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/Mac15%2C10_summary_2025-06-15T230300.json>macOS 15.5</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,Not Supported,? iPadOS 18.5,? iPadOS 26.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C7_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C1_summary_2025-06-16T220810.json>iPadOS 18.4</a>,? iPadOS 18.4.1,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 openai_whisper-large-v3-v20240930_626MB,openai_whisper-large-v3-v20240930_626MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C3_summary_2025-06-16T220810.json>iPadOS 18.4.1</a>,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
-openai_whisper-large-v3-v20240930_turbo,openai_whisper-large-v3-v20240930_turbo,✅ macOS 15.5,✅ macOS 15.5,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad14%2C8_summary_2025-06-16T192343.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C3_summary_2025-06-17T205025.json>iPadOS 26.0</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C7_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C1_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C3_summary_2025-06-16T220810.json>iPadOS 18.4.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
-openai_whisper-large-v3-v20240930_turbo_632MB,openai_whisper-large-v3-v20240930_turbo_632MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPhone16%2C2_summary_2025-06-16T135523.json>iOS 18.5</a>,✅ iOS 18.4.1,✅ iOS 18.5
 openai_whisper-large-v3_947MB,openai_whisper-large-v3_947MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C3_summary_2025-06-17T205025.json>iPadOS 26.0</a>,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
-openai_whisper-large-v3_turbo,openai_whisper-large-v3_turbo,✅ macOS 15.5,✅ macOS 15.5,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad14%2C8_summary_2025-06-16T192343.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C3_summary_2025-06-17T205025.json>iPadOS 26.0</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C7_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C1_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C3_summary_2025-06-16T220810.json>iPadOS 18.4.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
-openai_whisper-large-v3_turbo_954MB,openai_whisper-large-v3_turbo_954MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
 openai_whisper-small,openai_whisper-small,✅ macOS 15.5,✅ macOS 15.5,✅ iPadOS 18.4.1,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,✅ iOS 18.4.1,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
 openai_whisper-small.en,openai_whisper-small.en,✅ macOS 15.5,✅ macOS 15.5,✅ iPadOS 18.4.1,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,✅ iOS 18.4.1,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
 openai_whisper-tiny,openai_whisper-tiny,✅ macOS 15.5,✅ macOS 15.5,✅ iPadOS 18.4.1,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,✅ iOS 26.0,✅ iOS 18.4.1,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5

 ,Model,"Apple M2 Pro (Mac14,12)","Apple M3 Max (Mac15,10)","iPad Air (5th generation) (iPad13,16)","iPad Pro (11-inch) (3rd generation) (iPad13,6)","iPad mini (6th generation) (iPad14,1)","iPad Air 11-inch (M2) (iPad14,8)","iPad Air 11-inch (M3) (iPad15,3)","iPad (A16) (iPad15,7)","iPad mini (A17 Pro) (iPad16,1)","iPad Pro 11-inch (M4) (iPad16,3)","iPhone 11 (iPhone12,1)","iPhone 12 mini (iPhone13,1)","iPhone 13 Pro (iPhone14,2)","iPhone 13 (iPhone14,5)","iPhone 14 (iPhone14,7)","iPhone 15 (iPhone15,4)","iPhone 15 Pro Max (iPhone16,2)","iPhone 16 (iPhone17,3)","iPhone 16e (iPhone17,5)"
+distil-whisper_distil-large-v3,distil-whisper_distil-large-v3,✅ macOS 15.5,✅ macOS 15.5,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,Not Supported,✅ iPadOS 18.5,✅ iPadOS 26.0,Not Supported,Not Supported,✅ iPadOS 18.4.1,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 distil-whisper_distil-large-v3_594MB,distil-whisper_distil-large-v3_594MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
+distil-whisper_distil-large-v3_turbo,distil-whisper_distil-large-v3_turbo,✅ macOS 15.5,✅ macOS 15.5,Not Supported,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad14%2C8_summary_2025-06-16T192343.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C3_summary_2025-06-17T205025.json>iPadOS 26.0</a>,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C3_summary_2025-06-16T220810.json>iPadOS 18.4.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
+distil-whisper_distil-large-v3_turbo_600MB,distil-whisper_distil-large-v3_turbo_600MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,Not Supported,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
 openai_whisper-base,openai_whisper-base,✅ macOS 15.5,✅ macOS 15.5,✅ iPadOS 18.4.1,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,✅ iOS 26.0,✅ iOS 18.4.1,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
 openai_whisper-base.en,openai_whisper-base.en,✅ macOS 15.5,✅ macOS 15.5,✅ iPadOS 18.4.1,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,✅ iOS 26.0,✅ iOS 18.4.1,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
+openai_whisper-large-v2,openai_whisper-large-v2,✅ macOS 15.5,✅ macOS 15.5,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,Not Supported,✅ iPadOS 18.5,✅ iPadOS 26.0,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C3_summary_2025-06-16T220810.json>iPadOS 18.4.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 openai_whisper-large-v2_949MB,openai_whisper-large-v2_949MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C3_summary_2025-06-17T205025.json>iPadOS 26.0</a>,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
+openai_whisper-large-v2_turbo,openai_whisper-large-v2_turbo,✅ macOS 15.5,✅ macOS 15.5,Not Supported,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad14%2C8_summary_2025-06-16T192343.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C3_summary_2025-06-17T205025.json>iPadOS 26.0</a>,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C3_summary_2025-06-16T220810.json>iPadOS 18.4.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
+openai_whisper-large-v2_turbo_955MB,openai_whisper-large-v2_turbo_955MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,Not Supported,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
+openai_whisper-large-v3,openai_whisper-large-v3,✅ macOS 15.5,✅ macOS 15.5,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad14%2C8_summary_2025-06-16T192343.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C3_summary_2025-06-17T205025.json>iPadOS 26.0</a>,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C3_summary_2025-06-16T220810.json>iPadOS 18.4.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
+openai_whisper-large-v3-v20240930,openai_whisper-large-v3-v20240930,✅ macOS 15.5,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/Mac15%2C10_summary_2025-06-15T230300.json>macOS 15.5</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,Not Supported,? iPadOS 18.5,? iPadOS 26.0,Not Supported,Not Supported,? iPadOS 18.4.1,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 openai_whisper-large-v3-v20240930_626MB,openai_whisper-large-v3-v20240930_626MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C3_summary_2025-06-16T220810.json>iPadOS 18.4.1</a>,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
+openai_whisper-large-v3-v20240930_turbo,openai_whisper-large-v3-v20240930_turbo,✅ macOS 15.5,✅ macOS 15.5,Not Supported,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad14%2C8_summary_2025-06-16T192343.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C3_summary_2025-06-17T205025.json>iPadOS 26.0</a>,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C3_summary_2025-06-16T220810.json>iPadOS 18.4.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
+openai_whisper-large-v3-v20240930_turbo_632MB,openai_whisper-large-v3-v20240930_turbo_632MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,Not Supported,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPhone16%2C2_summary_2025-06-16T135523.json>iOS 18.5</a>,✅ iOS 18.4.1,✅ iOS 18.5
 openai_whisper-large-v3_947MB,openai_whisper-large-v3_947MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C3_summary_2025-06-17T205025.json>iPadOS 26.0</a>,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
+openai_whisper-large-v3_turbo,openai_whisper-large-v3_turbo,✅ macOS 15.5,✅ macOS 15.5,Not Supported,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad14%2C8_summary_2025-06-16T192343.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C3_summary_2025-06-17T205025.json>iPadOS 26.0</a>,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C3_summary_2025-06-16T220810.json>iPadOS 18.4.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
+openai_whisper-large-v3_turbo_954MB,openai_whisper-large-v3_turbo_954MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,Not Supported,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
 openai_whisper-small,openai_whisper-small,✅ macOS 15.5,✅ macOS 15.5,✅ iPadOS 18.4.1,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,✅ iOS 18.4.1,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
 openai_whisper-small.en,openai_whisper-small.en,✅ macOS 15.5,✅ macOS 15.5,✅ iPadOS 18.4.1,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,✅ iOS 18.4.1,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
 openai_whisper-tiny,openai_whisper-tiny,✅ macOS 15.5,✅ macOS 15.5,✅ iPadOS 18.4.1,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,✅ iOS 26.0,✅ iOS 18.4.1,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5

dashboard_data/support_data_a9b92c4.csv CHANGED Viewed

@@ -1,21 +1,21 @@
 ,Model,"Apple M2 Pro (Mac14,12)","Apple M2 Ultra (Mac14,14)","Apple M2 (Mac14,2)","Apple M3 Max (Mac15,10)","Apple M3 Pro (Mac15,6)","Apple M3 Max (Mac15,9)","Apple M4 (Mac16,10)","Apple M4 (Mac16,3)","Apple M1 (MacBookAir10,1)","iPad Air (5th generation) (iPad13,16)","iPad Air 11-inch (M2) (iPad14,8)","iPad mini (A17 Pro) (iPad16,1)","iPad Pro 11-inch (M4) (iPad16,3)","iPhone 11 (iPhone12,1)","iPhone 12 mini (iPhone13,1)","iPhone 12 (iPhone13,2)","iPhone 12 Pro Max (iPhone13,4)","iPhone 13 Pro (iPhone14,2)","iPhone 13 (iPhone14,5)","iPhone 14 (iPhone14,7)","iPhone 15 Pro (iPhone16,1)","iPhone 15 Pro Max (iPhone16,2)","iPhone 16 Pro (iPhone17,1)","iPhone 16 (iPhone17,3)","iPhone 16 Plus (iPhone17,4)"
-distil-whisper_distil-large-v3,distil-whisper_distil-large-v3,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/MacBookAir10%2C1_summary_2024-10-30T012535.json>macOS 15.1</a>,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 distil-whisper_distil-large-v3_594MB,distil-whisper_distil-large-v3_594MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
-distil-whisper_distil-large-v3_turbo,distil-whisper_distil-large-v3_turbo,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad14%2C8_summary_2024-10-30T073931.json>iPadOS 17.6.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C1_summary_2024-10-29T234233.json>iPadOS 18.0.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C3_summary_2024-10-30T073931.json>iPadOS 18.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 distil-whisper_distil-large-v3_turbo_600MB,distil-whisper_distil-large-v3_turbo_600MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
 openai_whisper-base,openai_whisper-base,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,✅ iPadOS 18.1,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,✅ iOS 17.6.1,✅ iOS 17.7,✅ iOS 17.2.1,✅ iOS 18.1,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
 openai_whisper-base.en,openai_whisper-base.en,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,✅ iPadOS 18.1,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,✅ iOS 17.6.1,✅ iOS 17.7,✅ iOS 17.2.1,✅ iOS 18.1,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
-openai_whisper-large-v2,openai_whisper-large-v2,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/MacBookAir10%2C1_summary_2024-10-30T012535.json>macOS 15.1</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad14%2C8_summary_2024-10-30T073931.json>iPadOS 17.6.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C1_summary_2024-10-29T234233.json>iPadOS 18.0.1</a>,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 openai_whisper-large-v2_949MB,openai_whisper-large-v2_949MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C5_summary_2024-10-30T073931.json>iOS 17.3</a>,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
-openai_whisper-large-v2_turbo,openai_whisper-large-v2_turbo,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad14%2C8_summary_2024-10-30T073931.json>iPadOS 17.6.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C1_summary_2024-10-29T234233.json>iPadOS 18.0.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C3_summary_2024-10-30T073931.json>iPadOS 18.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 openai_whisper-large-v2_turbo_955MB,openai_whisper-large-v2_turbo_955MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C5_summary_2024-10-30T073931.json>iOS 17.3</a>,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
-openai_whisper-large-v3,openai_whisper-large-v3,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/MacBookAir10%2C1_summary_2024-10-30T012535.json>macOS 15.1</a>,Not Supported,✅ iPadOS 17.6.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C1_summary_2024-10-29T234233.json>iPadOS 18.0.1</a>,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
-openai_whisper-large-v3-v20240930,openai_whisper-large-v3-v20240930,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/MacBookAir10%2C1_summary_2024-10-30T012535.json>macOS 15.1</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad14%2C8_summary_2024-10-30T073931.json>iPadOS 17.6.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C1_summary_2024-10-29T234233.json>iPadOS 18.0.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C3_summary_2024-10-30T073931.json>iPadOS 18.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 openai_whisper-large-v3-v20240930_626MB,openai_whisper-large-v3-v20240930_626MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
-openai_whisper-large-v3-v20240930_turbo,openai_whisper-large-v3-v20240930_turbo,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,✅ iPadOS 17.6.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C1_summary_2024-10-29T234233.json>iPadOS 18.0.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C3_summary_2024-10-30T073931.json>iPadOS 18.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 openai_whisper-large-v3-v20240930_turbo_632MB,openai_whisper-large-v3-v20240930_turbo_632MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C5_summary_2024-10-30T073931.json>iOS 17.3</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C7_summary_2024-10-30T073931.json>iOS 17.3</a>,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone17%2C4_summary_2024-10-30T132649.json>iOS 18.0.1</a>
 openai_whisper-large-v3_947MB,openai_whisper-large-v3_947MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C5_summary_2024-10-30T073931.json>iOS 17.3</a>,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
-openai_whisper-large-v3_turbo,openai_whisper-large-v3_turbo,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad14%2C8_summary_2024-10-30T073931.json>iPadOS 17.6.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C1_summary_2024-10-29T234233.json>iPadOS 18.0.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C3_summary_2024-10-30T073931.json>iPadOS 18.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 openai_whisper-large-v3_turbo_954MB,openai_whisper-large-v3_turbo_954MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C5_summary_2024-10-30T073931.json>iOS 17.3</a>,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone17%2C4_summary_2024-10-30T132649.json>iOS 18.0.1</a>
 openai_whisper-small,openai_whisper-small,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,✅ iPadOS 18.1,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone13%2C1_summary_2024-10-30T022739.json>iOS 17.7</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone13%2C2_summary_2024-10-30T152729.json>iOS 17.2.1</a>,✅ iOS 18.1,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
 openai_whisper-small.en,openai_whisper-small.en,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,✅ iPadOS 18.1,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone13%2C1_summary_2024-10-30T022739.json>iOS 17.7</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone13%2C2_summary_2024-10-30T152729.json>iOS 17.2.1</a>,✅ iOS 18.1,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1

 ,Model,"Apple M2 Pro (Mac14,12)","Apple M2 Ultra (Mac14,14)","Apple M2 (Mac14,2)","Apple M3 Max (Mac15,10)","Apple M3 Pro (Mac15,6)","Apple M3 Max (Mac15,9)","Apple M4 (Mac16,10)","Apple M4 (Mac16,3)","Apple M1 (MacBookAir10,1)","iPad Air (5th generation) (iPad13,16)","iPad Air 11-inch (M2) (iPad14,8)","iPad mini (A17 Pro) (iPad16,1)","iPad Pro 11-inch (M4) (iPad16,3)","iPhone 11 (iPhone12,1)","iPhone 12 mini (iPhone13,1)","iPhone 12 (iPhone13,2)","iPhone 12 Pro Max (iPhone13,4)","iPhone 13 Pro (iPhone14,2)","iPhone 13 (iPhone14,5)","iPhone 14 (iPhone14,7)","iPhone 15 Pro (iPhone16,1)","iPhone 15 Pro Max (iPhone16,2)","iPhone 16 Pro (iPhone17,1)","iPhone 16 (iPhone17,3)","iPhone 16 Plus (iPhone17,4)"
+distil-whisper_distil-large-v3,distil-whisper_distil-large-v3,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/MacBookAir10%2C1_summary_2024-10-30T012535.json>macOS 15.1</a>,Not Supported,✅ iPadOS 17.6.1,Not Supported,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 distil-whisper_distil-large-v3_594MB,distil-whisper_distil-large-v3_594MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
+distil-whisper_distil-large-v3_turbo,distil-whisper_distil-large-v3_turbo,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad14%2C8_summary_2024-10-30T073931.json>iPadOS 17.6.1</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C3_summary_2024-10-30T073931.json>iPadOS 18.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 distil-whisper_distil-large-v3_turbo_600MB,distil-whisper_distil-large-v3_turbo_600MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
 openai_whisper-base,openai_whisper-base,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,✅ iPadOS 18.1,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,✅ iOS 17.6.1,✅ iOS 17.7,✅ iOS 17.2.1,✅ iOS 18.1,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
 openai_whisper-base.en,openai_whisper-base.en,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,✅ iPadOS 18.1,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,✅ iOS 17.6.1,✅ iOS 17.7,✅ iOS 17.2.1,✅ iOS 18.1,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
+openai_whisper-large-v2,openai_whisper-large-v2,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/MacBookAir10%2C1_summary_2024-10-30T012535.json>macOS 15.1</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad14%2C8_summary_2024-10-30T073931.json>iPadOS 17.6.1</a>,Not Supported,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 openai_whisper-large-v2_949MB,openai_whisper-large-v2_949MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C5_summary_2024-10-30T073931.json>iOS 17.3</a>,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
+openai_whisper-large-v2_turbo,openai_whisper-large-v2_turbo,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad14%2C8_summary_2024-10-30T073931.json>iPadOS 17.6.1</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C3_summary_2024-10-30T073931.json>iPadOS 18.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 openai_whisper-large-v2_turbo_955MB,openai_whisper-large-v2_turbo_955MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C5_summary_2024-10-30T073931.json>iOS 17.3</a>,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
+openai_whisper-large-v3,openai_whisper-large-v3,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/MacBookAir10%2C1_summary_2024-10-30T012535.json>macOS 15.1</a>,Not Supported,✅ iPadOS 17.6.1,Not Supported,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
+openai_whisper-large-v3-v20240930,openai_whisper-large-v3-v20240930,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/MacBookAir10%2C1_summary_2024-10-30T012535.json>macOS 15.1</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad14%2C8_summary_2024-10-30T073931.json>iPadOS 17.6.1</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C3_summary_2024-10-30T073931.json>iPadOS 18.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 openai_whisper-large-v3-v20240930_626MB,openai_whisper-large-v3-v20240930_626MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
+openai_whisper-large-v3-v20240930_turbo,openai_whisper-large-v3-v20240930_turbo,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,✅ iPadOS 17.6.1,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C3_summary_2024-10-30T073931.json>iPadOS 18.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 openai_whisper-large-v3-v20240930_turbo_632MB,openai_whisper-large-v3-v20240930_turbo_632MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C5_summary_2024-10-30T073931.json>iOS 17.3</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C7_summary_2024-10-30T073931.json>iOS 17.3</a>,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone17%2C4_summary_2024-10-30T132649.json>iOS 18.0.1</a>
 openai_whisper-large-v3_947MB,openai_whisper-large-v3_947MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C5_summary_2024-10-30T073931.json>iOS 17.3</a>,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
+openai_whisper-large-v3_turbo,openai_whisper-large-v3_turbo,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad14%2C8_summary_2024-10-30T073931.json>iPadOS 17.6.1</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C3_summary_2024-10-30T073931.json>iPadOS 18.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 openai_whisper-large-v3_turbo_954MB,openai_whisper-large-v3_turbo_954MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C5_summary_2024-10-30T073931.json>iOS 17.3</a>,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone17%2C4_summary_2024-10-30T132649.json>iOS 18.0.1</a>
 openai_whisper-small,openai_whisper-small,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,✅ iPadOS 18.1,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone13%2C1_summary_2024-10-30T022739.json>iOS 17.7</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone13%2C2_summary_2024-10-30T152729.json>iOS 17.2.1</a>,✅ iOS 18.1,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
 openai_whisper-small.en,openai_whisper-small.en,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,✅ iPadOS 18.1,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone13%2C1_summary_2024-10-30T022739.json>iOS 17.7</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone13%2C2_summary_2024-10-30T152729.json>iOS 17.2.1</a>,✅ iOS 18.1,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1

dashboard_data/support_data_c814cae.csv CHANGED Viewed

@@ -1,23 +1,23 @@
 ,Model,"Apple M2 (Mac14,3)","Apple M4 (Mac16,10)","Apple M1 (Macmini9,1)","iPad Pro (11-inch) (3rd generation) (iPad13,6)","iPad Air 11-inch (M2) (iPad14,8)","iPad Air 13-inch (M3) (iPad15,5)","iPad (A16) (iPad15,7)","iPad mini (A17 Pro) (iPad16,1)","iPad Pro 11-inch (M4) (iPad16,3)","iPhone 15 (iPhone15,4)","iPhone 16 Pro (iPhone17,1)","iPhone 16e (iPhone17,5)"
-distil-whisper_distil-large-v3,distil-whisper_distil-large-v3,✅ macOS 26.0,✅ macOS 26.0<p>✅ macOS 15.5</p>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/Macmini9%2C1_summary_2025-08-29T100943.json>macOS 26.0</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad13%2C6_summary_2025-08-04T202532.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a><p>? iPadOS 18.6</p>,? iPadOS 18.6,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad15%2C7_summary_2025-08-01T235644.json>iPadOS 18.6</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C1_summary_2025-08-04T202532.json>iPadOS 18.5</a>,✅ iPadOS 26.0<p>? iPadOS 18.4.1</p>,Not Supported,Not Supported,Not Supported
-distil-whisper_distil-large-v3_594MB,distil-whisper_distil-large-v3_594MB,✅ macOS 26.0,✅ macOS 26.0<p>✅ macOS 15.5</p>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/Macmini9%2C1_summary_2025-08-29T100943.json>macOS 26.0</a>,✅ iPadOS 18.5,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a><p>✅ iPadOS 18.6</p>,✅ iPadOS 18.6,✅ iPadOS 18.6,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C1_summary_2025-08-04T202532.json>iPadOS 18.5</a>,✅ iPadOS 26.0<p>✅ iPadOS 18.4.1</p>,✅ iOS 26.0,✅ iOS 26.0,✅ iOS 18.5
-distil-whisper_distil-large-v3_turbo,distil-whisper_distil-large-v3_turbo,✅ macOS 26.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/Mac16%2C10_summary_2025-08-29T104851.json>macOS 26.0</a><p>✅ macOS 15.5</p>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad13%2C6_summary_2025-08-04T202532.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a><p>? iPadOS 18.6</p>,? iPadOS 18.6,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad15%2C7_summary_2025-08-01T235644.json>iPadOS 18.6</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C1_summary_2025-08-04T202532.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C3_summary_2025-08-27T211253.json>iPadOS 26.0</a><p>? iPadOS 18.4.1</p>,Not Supported,Not Supported,Not Supported
-distil-whisper_distil-large-v3_turbo_600MB,distil-whisper_distil-large-v3_turbo_600MB,✅ macOS 26.0,✅ macOS 26.0<p>✅ macOS 15.5</p>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad13%2C6_summary_2025-08-04T202532.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a><p>? iPadOS 18.6</p>,? iPadOS 18.6,? iPadOS 18.6,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C1_summary_2025-08-04T202532.json>iPadOS 18.5</a>,✅ iPadOS 26.0<p>? iPadOS 18.4.1</p>,✅ iOS 26.0,✅ iOS 26.0,✅ iOS 18.5
-openai_whisper-base,openai_whisper-base,✅ macOS 26.0,✅ macOS 26.0<p>✅ macOS 15.5</p>,✅ macOS 26.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad13%2C6_summary_2025-08-04T202532.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a><p>✅ iPadOS 18.6</p>,✅ iPadOS 18.6,✅ iPadOS 18.6,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C1_summary_2025-08-04T202532.json>iPadOS 18.5</a>,✅ iPadOS 26.0<p>✅ iPadOS 18.4.1</p>,✅ iOS 26.0,✅ iOS 26.0,✅ iOS 18.5
-openai_whisper-base.en,openai_whisper-base.en,✅ macOS 26.0,✅ macOS 26.0<p>✅ macOS 15.5</p>,✅ macOS 26.0,✅ iPadOS 18.5,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a><p>✅ iPadOS 18.6</p>,✅ iPadOS 18.6,✅ iPadOS 18.6,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C1_summary_2025-08-04T202532.json>iPadOS 18.5</a>,✅ iPadOS 26.0<p>✅ iPadOS 18.4.1</p>,✅ iOS 26.0,✅ iOS 26.0,✅ iOS 18.5
-openai_whisper-large-v2,openai_whisper-large-v2,✅ macOS 26.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/Mac16%2C10_summary_2025-08-29T104851.json>macOS 26.0</a><p>✅ macOS 15.5</p>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/Macmini9%2C1_summary_2025-08-29T100943.json>macOS 26.0</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad13%2C6_summary_2025-08-04T202532.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a><p>✅ iPadOS 18.6</p>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad15%2C5_summary_2025-08-01T235644.json>iPadOS 18.6</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad15%2C7_summary_2025-08-01T235644.json>iPadOS 18.6</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C1_summary_2025-08-04T202532.json>iPadOS 18.5</a>,✅ iPadOS 26.0<p>✅ iPadOS 18.4.1</p>,Not Supported,Not Supported,Not Supported
-openai_whisper-large-v2_949MB,openai_whisper-large-v2_949MB,✅ macOS 26.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/Mac16%2C10_summary_2025-08-29T104851.json>macOS 26.0</a><p>✅ macOS 15.5</p>,✅ macOS 26.0,✅ iPadOS 18.5,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a><p>? iPadOS 18.6</p>,? iPadOS 18.6,? iPadOS 18.6,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C1_summary_2025-08-04T202532.json>iPadOS 18.5</a>,✅ iPadOS 26.0<p>? iPadOS 18.4.1</p>,✅ iOS 26.0,✅ iOS 26.0,✅ iOS 18.5
-openai_whisper-large-v2_turbo,openai_whisper-large-v2_turbo,✅ macOS 26.0,✅ macOS 26.0<p>✅ macOS 15.5</p>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad13%2C6_summary_2025-08-04T202532.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a><p>? iPadOS 18.6</p>,? iPadOS 18.6,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad15%2C7_summary_2025-08-01T235644.json>iPadOS 18.6</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C1_summary_2025-08-04T202532.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C3_summary_2025-08-27T211253.json>iPadOS 26.0</a><p>? iPadOS 18.4.1</p>,Not Supported,Not Supported,Not Supported
-openai_whisper-large-v2_turbo_955MB,openai_whisper-large-v2_turbo_955MB,✅ macOS 26.0,✅ macOS 26.0<p>✅ macOS 15.5</p>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad13%2C6_summary_2025-08-04T202532.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a><p>? iPadOS 18.6</p>,? iPadOS 18.6,? iPadOS 18.6,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C1_summary_2025-08-04T202532.json>iPadOS 18.5</a>,✅ iPadOS 26.0<p>✅ iPadOS 18.4.1</p>,✅ iOS 26.0,✅ iOS 26.0,✅ iOS 18.5
-openai_whisper-large-v3,openai_whisper-large-v3,✅ macOS 26.0,✅ macOS 26.0<p>✅ macOS 15.5</p>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/Macmini9%2C1_summary_2025-08-29T100943.json>macOS 26.0</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad13%2C6_summary_2025-08-04T202532.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a><p>? iPadOS 18.6</p>,? iPadOS 18.6,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad15%2C7_summary_2025-08-01T235644.json>iPadOS 18.6</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C1_summary_2025-08-04T202532.json>iPadOS 18.5</a>,✅ iPadOS 26.0<p>? iPadOS 18.4.1</p>,Not Supported,Not Supported,Not Supported
-openai_whisper-large-v3-v20240930,openai_whisper-large-v3-v20240930,✅ macOS 26.0,✅ macOS 26.0<p>✅ macOS 15.5</p>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/Macmini9%2C1_summary_2025-08-29T100943.json>macOS 26.0</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad13%2C6_summary_2025-08-04T202532.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a><p>? iPadOS 18.6</p>,? iPadOS 18.6,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad15%2C7_summary_2025-08-01T235644.json>iPadOS 18.6</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C1_summary_2025-08-04T202532.json>iPadOS 18.5</a>,? iPadOS 26.0<p>? iPadOS 18.4.1</p>,Not Supported,Not Supported,Not Supported
-openai_whisper-large-v3-v20240930_626MB,openai_whisper-large-v3-v20240930_626MB,✅ macOS 26.0,✅ macOS 26.0<p>✅ macOS 15.5</p>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/Macmini9%2C1_summary_2025-08-29T100943.json>macOS 26.0</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad13%2C6_summary_2025-08-04T202532.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a><p>✅ iPadOS 18.6</p>,✅ iPadOS 18.6,✅ iPadOS 18.6,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C1_summary_2025-08-04T202532.json>iPadOS 18.5</a>,✅ iPadOS 26.0<p>✅ iPadOS 18.4.1</p>,✅ iOS 26.0,✅ iOS 26.0,✅ iOS 18.5
-openai_whisper-large-v3-v20240930_turbo,openai_whisper-large-v3-v20240930_turbo,✅ macOS 26.0,✅ macOS 26.0<p>✅ macOS 15.5</p>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad13%2C6_summary_2025-08-04T202532.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a><p>? iPadOS 18.6</p>,? iPadOS 18.6,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad15%2C7_summary_2025-08-01T235644.json>iPadOS 18.6</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C1_summary_2025-08-04T202532.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C3_summary_2025-08-27T211253.json>iPadOS 26.0</a><p>? iPadOS 18.4.1</p>,Not Supported,Not Supported,Not Supported
-openai_whisper-large-v3-v20240930_turbo_632MB,openai_whisper-large-v3-v20240930_turbo_632MB,✅ macOS 26.0,✅ macOS 26.0<p>✅ macOS 15.5</p>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad13%2C6_summary_2025-08-04T202532.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a><p>? iPadOS 18.6</p>,? iPadOS 18.6,? iPadOS 18.6,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C1_summary_2025-08-04T202532.json>iPadOS 18.5</a>,✅ iPadOS 26.0<p>? iPadOS 18.4.1</p>,✅ iOS 26.0,✅ iOS 26.0,✅ iOS 18.5
-openai_whisper-large-v3_947MB,openai_whisper-large-v3_947MB,✅ macOS 26.0,✅ macOS 26.0<p>✅ macOS 15.5</p>,✅ macOS 26.0,✅ iPadOS 18.5,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a><p>? iPadOS 18.6</p>,? iPadOS 18.6,? iPadOS 18.6,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C1_summary_2025-08-04T202532.json>iPadOS 18.5</a>,✅ iPadOS 26.0<p>? iPadOS 18.4.1</p>,✅ iOS 26.0,✅ iOS 26.0,✅ iOS 18.5
-openai_whisper-large-v3_turbo,openai_whisper-large-v3_turbo,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/Mac14%2C3_summary_2025-08-27T130519.json>macOS 26.0</a>,✅ macOS 26.0<p>✅ macOS 15.5</p>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad13%2C6_summary_2025-08-04T202532.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a><p>? iPadOS 18.6</p>,? iPadOS 18.6,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad15%2C7_summary_2025-08-01T235644.json>iPadOS 18.6</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C1_summary_2025-08-04T202532.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C3_summary_2025-08-27T211253.json>iPadOS 26.0</a><p>? iPadOS 18.4.1</p>,Not Supported,Not Supported,Not Supported
-openai_whisper-large-v3_turbo_954MB,openai_whisper-large-v3_turbo_954MB,✅ macOS 26.0,✅ macOS 26.0<p>✅ macOS 15.5</p>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad13%2C6_summary_2025-08-04T202532.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a><p>? iPadOS 18.6</p>,? iPadOS 18.6,? iPadOS 18.6,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C1_summary_2025-08-04T202532.json>iPadOS 18.5</a>,✅ iPadOS 26.0<p>? iPadOS 18.4.1</p>,✅ iOS 26.0,✅ iOS 26.0,✅ iOS 18.5
-openai_whisper-small,openai_whisper-small,✅ macOS 26.0,✅ macOS 26.0<p>✅ macOS 15.5</p>,✅ macOS 26.0,✅ iPadOS 18.5,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a><p>✅ iPadOS 18.6</p>,✅ iPadOS 18.6,✅ iPadOS 18.6,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C1_summary_2025-08-04T202532.json>iPadOS 18.5</a>,✅ iPadOS 26.0<p>✅ iPadOS 18.4.1</p>,✅ iOS 26.0,✅ iOS 26.0,✅ iOS 18.5
-openai_whisper-small.en,openai_whisper-small.en,✅ macOS 26.0,✅ macOS 26.0<p>✅ macOS 15.5</p>,✅ macOS 26.0,✅ iPadOS 18.5,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a><p>✅ iPadOS 18.6</p>,✅ iPadOS 18.6,✅ iPadOS 18.6,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C1_summary_2025-08-04T202532.json>iPadOS 18.5</a>,✅ iPadOS 26.0<p>✅ iPadOS 18.4.1</p>,✅ iOS 26.0,✅ iOS 26.0,✅ iOS 18.5
-openai_whisper-tiny,openai_whisper-tiny,✅ macOS 26.0,✅ macOS 26.0<p>✅ macOS 15.5</p>,✅ macOS 26.0,✅ iPadOS 18.5,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a><p>✅ iPadOS 18.6</p>,✅ iPadOS 18.6,✅ iPadOS 18.6,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C1_summary_2025-08-04T202532.json>iPadOS 18.5</a>,✅ iPadOS 26.0<p>✅ iPadOS 18.4.1</p>,✅ iOS 26.0,✅ iOS 26.0,✅ iOS 18.5
-openai_whisper-tiny.en,openai_whisper-tiny.en,✅ macOS 26.0,✅ macOS 26.0<p>✅ macOS 15.5</p>,✅ macOS 26.0,✅ iPadOS 18.5,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a><p>✅ iPadOS 18.6</p>,✅ iPadOS 18.6,✅ iPadOS 18.6,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C1_summary_2025-08-04T202532.json>iPadOS 18.5</a>,✅ iPadOS 26.0<p>✅ iPadOS 18.4.1</p>,✅ iOS 26.0,✅ iOS 26.0,✅ iOS 18.5

 ,Model,"Apple M2 (Mac14,3)","Apple M4 (Mac16,10)","Apple M1 (Macmini9,1)","iPad Pro (11-inch) (3rd generation) (iPad13,6)","iPad Air 11-inch (M2) (iPad14,8)","iPad Air 13-inch (M3) (iPad15,5)","iPad (A16) (iPad15,7)","iPad mini (A17 Pro) (iPad16,1)","iPad Pro 11-inch (M4) (iPad16,3)","iPhone 15 (iPhone15,4)","iPhone 16 Pro (iPhone17,1)","iPhone 16e (iPhone17,5)"
+distil-whisper_distil-large-v3,distil-whisper_distil-large-v3,✅ macOS 26.0,✅ macOS 26.0<p>✅ macOS 15.5</p>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/Macmini9%2C1_summary_2025-08-29T100943.json>macOS 26.0</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad13%2C6_summary_2025-08-04T202532.json>iPadOS 18.5</a>,? iPadOS 18.6<p>⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a></p>,? iPadOS 18.6,Not Supported,Not Supported,✅ iPadOS 26.0<p>? iPadOS 18.4.1</p>,Not Supported,Not Supported,Not Supported
+distil-whisper_distil-large-v3_594MB,distil-whisper_distil-large-v3_594MB,✅ macOS 26.0,✅ macOS 26.0<p>✅ macOS 15.5</p>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/Macmini9%2C1_summary_2025-08-29T100943.json>macOS 26.0</a>,✅ iPadOS 18.5,✅ iPadOS 18.6<p>⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a></p>,✅ iPadOS 18.6,✅ iPadOS 18.6,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C1_summary_2025-08-04T202532.json>iPadOS 18.5</a>,✅ iPadOS 26.0<p>✅ iPadOS 18.4.1</p>,✅ iOS 26.0,✅ iOS 26.0,✅ iOS 18.5
+distil-whisper_distil-large-v3_turbo,distil-whisper_distil-large-v3_turbo,✅ macOS 26.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/Mac16%2C10_summary_2025-08-29T104851.json>macOS 26.0</a><p>✅ macOS 15.5</p>,Not Supported,Not Supported,? iPadOS 18.6<p>⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a></p>,? iPadOS 18.6,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C3_summary_2025-08-27T211253.json>iPadOS 26.0</a><p>? iPadOS 18.4.1</p>,Not Supported,Not Supported,Not Supported
+distil-whisper_distil-large-v3_turbo_600MB,distil-whisper_distil-large-v3_turbo_600MB,✅ macOS 26.0,✅ macOS 26.0<p>✅ macOS 15.5</p>,Not Supported,Not Supported,? iPadOS 18.6<p>⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a></p>,? iPadOS 18.6,? iPadOS 18.6,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C1_summary_2025-08-04T202532.json>iPadOS 18.5</a>,✅ iPadOS 26.0<p>? iPadOS 18.4.1</p>,✅ iOS 26.0,✅ iOS 26.0,✅ iOS 18.5
+openai_whisper-base,openai_whisper-base,✅ macOS 26.0,✅ macOS 26.0<p>✅ macOS 15.5</p>,✅ macOS 26.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad13%2C6_summary_2025-08-04T202532.json>iPadOS 18.5</a>,✅ iPadOS 18.6<p>⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a></p>,✅ iPadOS 18.6,✅ iPadOS 18.6,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C1_summary_2025-08-04T202532.json>iPadOS 18.5</a>,✅ iPadOS 26.0<p>✅ iPadOS 18.4.1</p>,✅ iOS 26.0,✅ iOS 26.0,✅ iOS 18.5
+openai_whisper-base.en,openai_whisper-base.en,✅ macOS 26.0,✅ macOS 26.0<p>✅ macOS 15.5</p>,✅ macOS 26.0,✅ iPadOS 18.5,✅ iPadOS 18.6<p>⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a></p>,✅ iPadOS 18.6,✅ iPadOS 18.6,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C1_summary_2025-08-04T202532.json>iPadOS 18.5</a>,✅ iPadOS 26.0<p>✅ iPadOS 18.4.1</p>,✅ iOS 26.0,✅ iOS 26.0,✅ iOS 18.5
+openai_whisper-large-v2,openai_whisper-large-v2,✅ macOS 26.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/Mac16%2C10_summary_2025-08-29T104851.json>macOS 26.0</a><p>✅ macOS 15.5</p>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/Macmini9%2C1_summary_2025-08-29T100943.json>macOS 26.0</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad13%2C6_summary_2025-08-04T202532.json>iPadOS 18.5</a>,✅ iPadOS 18.6<p>⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a></p>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad15%2C5_summary_2025-08-01T235644.json>iPadOS 18.6</a>,Not Supported,Not Supported,✅ iPadOS 26.0<p>✅ iPadOS 18.4.1</p>,Not Supported,Not Supported,Not Supported
+openai_whisper-large-v2_949MB,openai_whisper-large-v2_949MB,✅ macOS 26.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/Mac16%2C10_summary_2025-08-29T104851.json>macOS 26.0</a><p>✅ macOS 15.5</p>,✅ macOS 26.0,✅ iPadOS 18.5,? iPadOS 18.6<p>⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a></p>,? iPadOS 18.6,? iPadOS 18.6,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C1_summary_2025-08-04T202532.json>iPadOS 18.5</a>,✅ iPadOS 26.0<p>? iPadOS 18.4.1</p>,✅ iOS 26.0,✅ iOS 26.0,✅ iOS 18.5
+openai_whisper-large-v2_turbo,openai_whisper-large-v2_turbo,✅ macOS 26.0,✅ macOS 26.0<p>✅ macOS 15.5</p>,Not Supported,Not Supported,? iPadOS 18.6<p>⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a></p>,? iPadOS 18.6,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C3_summary_2025-08-27T211253.json>iPadOS 26.0</a><p>? iPadOS 18.4.1</p>,Not Supported,Not Supported,Not Supported
+openai_whisper-large-v2_turbo_955MB,openai_whisper-large-v2_turbo_955MB,✅ macOS 26.0,✅ macOS 26.0<p>✅ macOS 15.5</p>,Not Supported,Not Supported,? iPadOS 18.6<p>⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a></p>,? iPadOS 18.6,? iPadOS 18.6,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C1_summary_2025-08-04T202532.json>iPadOS 18.5</a>,✅ iPadOS 26.0<p>✅ iPadOS 18.4.1</p>,✅ iOS 26.0,✅ iOS 26.0,✅ iOS 18.5
+openai_whisper-large-v3,openai_whisper-large-v3,✅ macOS 26.0,✅ macOS 26.0<p>✅ macOS 15.5</p>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/Macmini9%2C1_summary_2025-08-29T100943.json>macOS 26.0</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad13%2C6_summary_2025-08-04T202532.json>iPadOS 18.5</a>,? iPadOS 18.6<p>⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a></p>,? iPadOS 18.6,Not Supported,Not Supported,✅ iPadOS 26.0<p>? iPadOS 18.4.1</p>,Not Supported,Not Supported,Not Supported
+openai_whisper-large-v3-v20240930,openai_whisper-large-v3-v20240930,✅ macOS 26.0,✅ macOS 26.0<p>✅ macOS 15.5</p>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/Macmini9%2C1_summary_2025-08-29T100943.json>macOS 26.0</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad13%2C6_summary_2025-08-04T202532.json>iPadOS 18.5</a>,? iPadOS 18.6<p>⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a></p>,? iPadOS 18.6,Not Supported,Not Supported,? iPadOS 26.0<p>? iPadOS 18.4.1</p>,Not Supported,Not Supported,Not Supported
+openai_whisper-large-v3-v20240930_626MB,openai_whisper-large-v3-v20240930_626MB,✅ macOS 26.0,✅ macOS 26.0<p>✅ macOS 15.5</p>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/Macmini9%2C1_summary_2025-08-29T100943.json>macOS 26.0</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad13%2C6_summary_2025-08-04T202532.json>iPadOS 18.5</a>,✅ iPadOS 18.6<p>⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a></p>,✅ iPadOS 18.6,✅ iPadOS 18.6,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C1_summary_2025-08-04T202532.json>iPadOS 18.5</a>,✅ iPadOS 26.0<p>✅ iPadOS 18.4.1</p>,✅ iOS 26.0,✅ iOS 26.0,✅ iOS 18.5
+openai_whisper-large-v3-v20240930_turbo,openai_whisper-large-v3-v20240930_turbo,✅ macOS 26.0,✅ macOS 26.0<p>✅ macOS 15.5</p>,Not Supported,Not Supported,? iPadOS 18.6<p>⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a></p>,? iPadOS 18.6,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C3_summary_2025-08-27T211253.json>iPadOS 26.0</a><p>? iPadOS 18.4.1</p>,Not Supported,Not Supported,Not Supported
+openai_whisper-large-v3-v20240930_turbo_632MB,openai_whisper-large-v3-v20240930_turbo_632MB,✅ macOS 26.0,✅ macOS 26.0<p>✅ macOS 15.5</p>,Not Supported,Not Supported,? iPadOS 18.6<p>⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a></p>,? iPadOS 18.6,? iPadOS 18.6,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C1_summary_2025-08-04T202532.json>iPadOS 18.5</a>,✅ iPadOS 26.0<p>? iPadOS 18.4.1</p>,✅ iOS 26.0,✅ iOS 26.0,✅ iOS 18.5
+openai_whisper-large-v3_947MB,openai_whisper-large-v3_947MB,✅ macOS 26.0,✅ macOS 26.0<p>✅ macOS 15.5</p>,✅ macOS 26.0,✅ iPadOS 18.5,? iPadOS 18.6<p>⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a></p>,? iPadOS 18.6,? iPadOS 18.6,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C1_summary_2025-08-04T202532.json>iPadOS 18.5</a>,✅ iPadOS 26.0<p>? iPadOS 18.4.1</p>,✅ iOS 26.0,✅ iOS 26.0,✅ iOS 18.5
+openai_whisper-large-v3_turbo,openai_whisper-large-v3_turbo,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/Mac14%2C3_summary_2025-08-27T130519.json>macOS 26.0</a>,✅ macOS 26.0<p>✅ macOS 15.5</p>,Not Supported,Not Supported,? iPadOS 18.6<p>⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a></p>,? iPadOS 18.6,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C3_summary_2025-08-27T211253.json>iPadOS 26.0</a><p>? iPadOS 18.4.1</p>,Not Supported,Not Supported,Not Supported
+openai_whisper-large-v3_turbo_954MB,openai_whisper-large-v3_turbo_954MB,✅ macOS 26.0,✅ macOS 26.0<p>✅ macOS 15.5</p>,Not Supported,Not Supported,? iPadOS 18.6<p>⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a></p>,? iPadOS 18.6,? iPadOS 18.6,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C1_summary_2025-08-04T202532.json>iPadOS 18.5</a>,✅ iPadOS 26.0<p>? iPadOS 18.4.1</p>,✅ iOS 26.0,✅ iOS 26.0,✅ iOS 18.5
+openai_whisper-small,openai_whisper-small,✅ macOS 26.0,✅ macOS 26.0<p>✅ macOS 15.5</p>,✅ macOS 26.0,✅ iPadOS 18.5,✅ iPadOS 18.6<p>⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a></p>,✅ iPadOS 18.6,✅ iPadOS 18.6,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C1_summary_2025-08-04T202532.json>iPadOS 18.5</a>,✅ iPadOS 26.0<p>✅ iPadOS 18.4.1</p>,✅ iOS 26.0,✅ iOS 26.0,✅ iOS 18.5
+openai_whisper-small.en,openai_whisper-small.en,✅ macOS 26.0,✅ macOS 26.0<p>✅ macOS 15.5</p>,✅ macOS 26.0,✅ iPadOS 18.5,✅ iPadOS 18.6<p>⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a></p>,✅ iPadOS 18.6,✅ iPadOS 18.6,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C1_summary_2025-08-04T202532.json>iPadOS 18.5</a>,✅ iPadOS 26.0<p>✅ iPadOS 18.4.1</p>,✅ iOS 26.0,✅ iOS 26.0,✅ iOS 18.5
+openai_whisper-tiny,openai_whisper-tiny,✅ macOS 26.0,✅ macOS 26.0<p>✅ macOS 15.5</p>,✅ macOS 26.0,✅ iPadOS 18.5,✅ iPadOS 18.6<p>⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a></p>,✅ iPadOS 18.6,✅ iPadOS 18.6,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C1_summary_2025-08-04T202532.json>iPadOS 18.5</a>,✅ iPadOS 26.0<p>✅ iPadOS 18.4.1</p>,✅ iOS 26.0,✅ iOS 26.0,✅ iOS 18.5
+openai_whisper-tiny.en,openai_whisper-tiny.en,✅ macOS 26.0,✅ macOS 26.0<p>✅ macOS 15.5</p>,✅ macOS 26.0,✅ iPadOS 18.5,✅ iPadOS 18.6<p>⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad14%2C8_summary_2025-08-04T202532.json>iPadOS 18.4.1</a></p>,✅ iPadOS 18.6,✅ iPadOS 18.6,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-07-31T175755_c814cae/iPad16%2C1_summary_2025-08-04T202532.json>iPadOS 18.5</a>,✅ iPadOS 26.0<p>✅ iPadOS 18.4.1</p>,✅ iOS 26.0,✅ iOS 26.0,✅ iOS 18.5

dashboard_data/test_coverage_112a023.json ADDED Viewed

	@@ -0,0 +1,239 @@

+{
+  "commit_hash": "112a023",
+  "total_devices": 137,
+  "tested_devices": 49,
+  "skipped_devices": 88,
+  "coverage_percentage": 35.76642335766424,
+  "tested_device_list": [
+    "iPhone14,6",
+    "Mac16,8",
+    "Mac15,3",
+    "Mac15,13",
+    "Mac14,5",
+    "Mac15,6",
+    "iPhone12,8",
+    "iPhone14,8",
+    "Mac15,8",
+    "Mac14,9",
+    "iPad14,4",
+    "Mac15,4",
+    "iPad16,4",
+    "Mac15,7",
+    "Mac14,7",
+    "Mac15,10",
+    "Mac14,8",
+    "iPhone14,5",
+    "Mac14,6",
+    "Mac14,12",
+    "Mac14,10",
+    "iPhone12,5",
+    "iPhone12,1",
+    "Mac15,12",
+    "iPad14,5",
+    "iPhone12,3",
+    "iPad14,11",
+    "iPad14,3",
+    "Mac14,4",
+    "iPhone14,2",
+    "Mac15,11",
+    "iPhone14,7",
+    "iPad14,8",
+    "iPad14,6",
+    "iPad16,5",
+    "Mac14,3",
+    "iPhone14,4",
+    "iPad14,9",
+    "Mac15,9",
+    "Mac15,5",
+    "Mac14,2",
+    "iPad16,6",
+    "Mac14,13",
+    "Mac14,14",
+    "iPad14,10",
+    "Mac16,9",
+    "iPhone14,3",
+    "Mac14,15",
+    "iPad16,3"
+  ],
+  "skipped_device_list": [
+    "Mac16,12",
+    "MacBookPro18,1",
+    "iPad8,3",
+    "iPad8,10",
+    "iPad14,2",
+    "iPad13,16",
+    "iPhone15,2",
+    "iPad13,18",
+    "iPad13,17",
+    "iPhone16,2",
+    "iPad15,8",
+    "iPad8,7",
+    "iPhone11,4",
+    "MacBookAir10,1",
+    "iPad12,1",
+    "iPhone13,3",
+    "Mac16,6",
+    "iPhone11,2",
+    "MacBookPro18,2",
+    "iPad8,1",
+    "iPhone11,8",
+    "iPad15,6",
+    "iPhone12,4",
+    "iPhone17,3",
+    "iPad8,2",
+    "iPad8,8",
+    "iPhone15,5",
+    "iPad16,1",
+    "Mac16,3",
+    "iPhone11,5",
+    "MacBookPro18,3",
+    "iPad14,1",
+    "Mac16,1",
+    "MacBookPro18,4",
+    "iMac21,1",
+    "iPad11,2",
+    "iMac21,2",
+    "iPad13,8",
+    "iPhone13,2",
+    "iPad8,9",
+    "Mac16,5",
+    "iPad12,2",
+    "iPhone16,1",
+    "iPad11,3",
+    "Mac16,13",
+    "iPhone12,2",
+    "iPad11,1",
+    "iPad13,1",
+    "iPad15,5",
+    "iPhone11,3",
+    "Mac16,2",
+    "iPhone13,4",
+    "iPhone17,1",
+    "iPhone15,3",
+    "iPad8,6",
+    "iPad16,2",
+    "iPad13,5",
+    "iPad11,7",
+    "iPad13,4",
+    "iPhone17,5",
+    "iPad13,9",
+    "Mac16,11",
+    "Macmini9,1",
+    "iPad13,11",
+    "iPad15,4",
+    "iPad13,10",
+    "iPad8,11",
+    "Mac13,2",
+    "iPhone17,2",
+    "iPad11,6",
+    "Mac16,10",
+    "iPhone13,1",
+    "iPhone17,4",
+    "MacBookPro17,1",
+    "iPad13,2",
+    "iPad8,12",
+    "iPad15,7",
+    "iPad13,19",
+    "Mac13,1",
+    "iPad8,4",
+    "iPad15,3",
+    "Mac16,7",
+    "iPhone15,4",
+    "iPad8,5",
+    "iPhone11,6",
+    "iPad13,7",
+    "iPad13,6",
+    "iPad11,4"
+  ],
+  "tested_os_versions": [
+    "macOS_15.0.1",
+    "iPadOS_17.6.1",
+    "iOS_17.7.1",
+    "macOS_15.2",
+    "iPadOS_18.1",
+    "macOS_15.2.0",
+    "iOS_17.6.1",
+    "macOS_15.1.1"
+  ],
+  "has_target_os_coverage": false,
+  "covered_target_versions": [
+    "macOS 15",
+    "iOS 17",
+    "iOS 18"
+  ],
+  "missing_target_versions": [
+    "macOS 14",
+    "macOS 26",
+    "iOS 26"
+  ],
+  "has_target_chip_coverage": false,
+  "platform_chip_coverage": {
+    "iPad": {
+      "total_chips": 8,
+      "tested_chips": 2,
+      "coverage_percentage": 25.0,
+      "covered_chips": [
+        "M2",
+        "M4"
+      ],
+      "missing_chips": [
+        "A14",
+        "A15",
+        "A16",
+        "A17 Pro",
+        "M1",
+        "M3"
+      ]
+    },
+    "iPhone": {
+      "total_chips": 6,
+      "tested_chips": 2,
+      "coverage_percentage": 33.33333333333333,
+      "covered_chips": [
+        "A13",
+        "A15"
+      ],
+      "missing_chips": [
+        "A14",
+        "A16",
+        "A17 Pro",
+        "A18",
+        "A18 Pro"
+      ]
+    },
+    "Mac": {
+      "total_chips": 4,
+      "tested_chips": 2,
+      "coverage_percentage": 50.0,
+      "covered_chips": [
+        "M2",
+        "M3"
+      ],
+      "missing_chips": [
+        "M1",
+        "M4"
+      ]
+    }
+  },
+  "missing_target_chips": {
+    "iPad": [
+      "A14",
+      "A15",
+      "A16",
+      "A17 Pro",
+      "M1",
+      "M3"
+    ],
+    "iPhone": [
+      "A14",
+      "A16",
+      "A17 Pro",
+      "A18",
+      "A18 Pro"
+    ],
+    "Mac": [
+      "M1",
+      "M4"
+    ]
+  }
+}

dashboard_data/test_coverage_11a1fab.json ADDED Viewed

	@@ -0,0 +1,233 @@

+{
+  "commit_hash": "11a1fab",
+  "total_devices": 137,
+  "tested_devices": 22,
+  "skipped_devices": 115,
+  "coverage_percentage": 16.05839416058394,
+  "tested_device_list": [
+    "iPad15,8",
+    "Mac16,1",
+    "Mac16,12",
+    "iPad15,6",
+    "Mac16,10",
+    "iPad15,4",
+    "Mac16,13",
+    "iPad15,3",
+    "Mac16,6",
+    "Mac16,7",
+    "iPad13,1",
+    "Mac16,11",
+    "iPad15,5",
+    "Mac16,2",
+    "iPad13,2",
+    "iPad16,1",
+    "Mac16,3",
+    "iPad16,2",
+    "iPad15,7",
+    "iPad13,18",
+    "iPad13,19",
+    "Mac16,5"
+  ],
+  "skipped_device_list": [
+    "iPhone14,6",
+    "MacBookPro18,1",
+    "Mac14,5",
+    "Mac15,6",
+    "iPad8,3",
+    "iPad8,10",
+    "iPad16,4",
+    "Mac14,7",
+    "Mac14,8",
+    "Mac15,10",
+    "iPad14,2",
+    "iPad13,16",
+    "iPhone15,2",
+    "iPad13,17",
+    "iPhone16,2",
+    "Mac14,6",
+    "iPad8,7",
+    "iPhone11,4",
+    "MacBookAir10,1",
+    "iPad12,1",
+    "iPad14,3",
+    "Mac14,4",
+    "Mac15,11",
+    "iPhone13,3",
+    "iPad14,9",
+    "iPhone14,4",
+    "Mac15,9",
+    "iPhone11,2",
+    "MacBookPro18,2",
+    "iPad8,1",
+    "iPhone11,8",
+    "Mac14,15",
+    "iPhone12,4",
+    "iPhone17,3",
+    "iPad8,2",
+    "iPad8,8",
+    "Mac15,8",
+    "iPhone14,5",
+    "iPhone15,5",
+    "iPhone11,5",
+    "MacBookPro18,3",
+    "iPad14,1",
+    "iPhone12,1",
+    "iPad14,5",
+    "MacBookPro18,4",
+    "iMac21,1",
+    "iPad11,2",
+    "iPhone14,2",
+    "iPad14,8",
+    "iMac21,2",
+    "iPad13,8",
+    "iPhone13,2",
+    "iPad8,9",
+    "Mac14,14",
+    "iPad12,2",
+    "Mac16,8",
+    "iPhone16,1",
+    "iPad11,3",
+    "iPhone12,8",
+    "iPhone12,2",
+    "iPad11,1",
+    "iPhone14,8",
+    "Mac14,9",
+    "iPhone11,3",
+    "iPad14,4",
+    "iPhone13,4",
+    "iPhone17,1",
+    "iPad8,6",
+    "iPhone15,3",
+    "iPad13,5",
+    "Mac14,12",
+    "iPhone12,5",
+    "Mac15,12",
+    "iPad11,7",
+    "iPhone12,3",
+    "iPad13,4",
+    "iPhone17,5",
+    "iPad14,6",
+    "iPad13,9",
+    "Mac15,5",
+    "Mac14,2",
+    "Macmini9,1",
+    "iPad16,6",
+    "iPad13,11",
+    "iPad13,10",
+    "iPad8,11",
+    "Mac13,2",
+    "iPhone17,2",
+    "iPad11,6",
+    "Mac15,3",
+    "Mac15,13",
+    "iPhone13,1",
+    "iPhone17,4",
+    "MacBookPro17,1",
+    "Mac15,4",
+    "Mac15,7",
+    "iPad8,12",
+    "Mac13,1",
+    "Mac14,10",
+    "iPad8,4",
+    "iPad14,11",
+    "iPad16,5",
+    "iPhone14,7",
+    "Mac14,3",
+    "iPhone15,4",
+    "iPad8,5",
+    "iPhone11,6",
+    "iPad13,7",
+    "Mac14,13",
+    "iPad13,6",
+    "iPad14,10",
+    "Mac16,9",
+    "iPhone14,3",
+    "iPad11,4",
+    "iPad16,3"
+  ],
+  "tested_os_versions": [
+    "macOS_15.4.0",
+    "iPadOS_18.4",
+    "iPadOS_18.3.2",
+    "macOS_15.4"
+  ],
+  "has_target_os_coverage": false,
+  "covered_target_versions": [
+    "macOS 15",
+    "iOS 18"
+  ],
+  "missing_target_versions": [
+    "macOS 14",
+    "macOS 26",
+    "iOS 17",
+    "iOS 26"
+  ],
+  "has_target_chip_coverage": false,
+  "platform_chip_coverage": {
+    "iPad": {
+      "total_chips": 8,
+      "tested_chips": 4,
+      "coverage_percentage": 50.0,
+      "covered_chips": [
+        "A14",
+        "A16",
+        "A17 Pro",
+        "M3"
+      ],
+      "missing_chips": [
+        "A15",
+        "M1",
+        "M2",
+        "M4"
+      ]
+    },
+    "iPhone": {
+      "total_chips": 6,
+      "tested_chips": 0,
+      "coverage_percentage": 0.0,
+      "covered_chips": [],
+      "missing_chips": [
+        "A14",
+        "A15",
+        "A16",
+        "A17 Pro",
+        "A18",
+        "A18 Pro"
+      ]
+    },
+    "Mac": {
+      "total_chips": 4,
+      "tested_chips": 1,
+      "coverage_percentage": 25.0,
+      "covered_chips": [
+        "M4"
+      ],
+      "missing_chips": [
+        "M1",
+        "M2",
+        "M3"
+      ]
+    }
+  },
+  "missing_target_chips": {
+    "iPad": [
+      "A15",
+      "M1",
+      "M2",
+      "M4"
+    ],
+    "iPhone": [
+      "A14",
+      "A15",
+      "A16",
+      "A17 Pro",
+      "A18",
+      "A18 Pro"
+    ],
+    "Mac": [
+      "M1",
+      "M2",
+      "M3"
+    ]
+  }
+}

dashboard_data/test_coverage_8c0acbd.json ADDED Viewed

	@@ -0,0 +1,234 @@

+{
+  "commit_hash": "8c0acbd",
+  "total_devices": 137,
+  "tested_devices": 82,
+  "skipped_devices": 55,
+  "coverage_percentage": 59.854014598540154,
+  "tested_device_list": [
+    "iPhone14,6",
+    "Mac16,8",
+    "iPhone16,1",
+    "Mac14,5",
+    "Mac15,6",
+    "iPhone12,8",
+    "iPad15,5",
+    "iPhone14,8",
+    "Mac14,9",
+    "iPad14,4",
+    "iPad16,4",
+    "Mac14,7",
+    "Mac14,8",
+    "Mac15,10",
+    "iPhone13,4",
+    "iPad14,2",
+    "iPad13,5",
+    "iPhone15,3",
+    "iPad13,16",
+    "iPad16,2",
+    "iPhone15,2",
+    "iPad13,17",
+    "iPhone16,2",
+    "Mac14,12",
+    "Mac14,6",
+    "iPhone12,5",
+    "iPad15,8",
+    "Mac15,12",
+    "iPhone12,3",
+    "iPad14,3",
+    "Mac14,4",
+    "Mac15,11",
+    "iPhone17,5",
+    "iPad13,4",
+    "iPhone13,3",
+    "iPad14,6",
+    "iPhone14,4",
+    "iPad14,9",
+    "Mac15,9",
+    "iPad13,9",
+    "Mac15,5",
+    "Mac14,2",
+    "iPad16,6",
+    "iPad13,11",
+    "iPad13,10",
+    "iPad15,4",
+    "Mac14,15",
+    "Mac15,3",
+    "Mac15,13",
+    "iPad15,6",
+    "iPhone13,1",
+    "iPhone17,4",
+    "iPhone17,3",
+    "Mac15,8",
+    "Mac15,4",
+    "Mac15,7",
+    "iPhone14,5",
+    "iPhone15,5",
+    "iPad16,1",
+    "iPad15,7",
+    "Mac14,10",
+    "iPad14,1",
+    "iPhone12,1",
+    "iPad14,5",
+    "iPad14,11",
+    "iPhone14,2",
+    "iPad16,5",
+    "iPhone14,7",
+    "iPad14,8",
+    "iPad15,3",
+    "Mac14,3",
+    "iPhone15,4",
+    "iPhone13,2",
+    "iPad13,8",
+    "iPad13,7",
+    "Mac14,13",
+    "Mac14,14",
+    "iPad13,6",
+    "iPad14,10",
+    "iPhone14,3",
+    "Mac16,9",
+    "iPad16,3"
+  ],
+  "skipped_device_list": [
+    "Mac13,2",
+    "iPhone17,2",
+    "Mac16,12",
+    "MacBookPro18,1",
+    "iPad11,6",
+    "iPhone12,4",
+    "iPad11,3",
+    "Mac16,10",
+    "Mac16,13",
+    "iPad8,2",
+    "iPhone12,2",
+    "iPad11,1",
+    "iPad13,1",
+    "iPad8,3",
+    "iPad8,8",
+    "MacBookPro17,1",
+    "iPad8,10",
+    "iPhone11,3",
+    "Mac16,2",
+    "iPad13,2",
+    "iPhone17,1",
+    "Mac16,3",
+    "iPad8,6",
+    "iPhone11,5",
+    "MacBookPro18,3",
+    "iPad8,12",
+    "iPad13,18",
+    "iPad13,19",
+    "Mac13,1",
+    "iPad8,7",
+    "iPhone11,4",
+    "Mac16,1",
+    "MacBookAir10,1",
+    "iPad11,7",
+    "iPad8,4",
+    "MacBookPro18,4",
+    "iMac21,1",
+    "iPad12,1",
+    "iPad11,2",
+    "Mac16,6",
+    "iMac21,2",
+    "Mac16,7",
+    "iPad8,5",
+    "Mac16,11",
+    "iPhone11,2",
+    "iPhone11,6",
+    "MacBookPro18,2",
+    "Macmini9,1",
+    "iPad8,1",
+    "iPad8,11",
+    "iPhone11,8",
+    "iPad8,9",
+    "Mac16,5",
+    "iPad11,4",
+    "iPad12,2"
+  ],
+  "tested_os_versions": [
+    "iPadOS_19.0",
+    "iOS_18.1",
+    "macOS_15.5",
+    "iPadOS_18.4.1",
+    "macOS_15.5.0",
+    "iPadOS_26.0",
+    "iOS_26.0",
+    "iPadOS_18.5",
+    "iOS_19.0",
+    "iOS_18.5",
+    "iPadOS_18.4",
+    "iOS_18.4.1"
+  ],
+  "has_target_os_coverage": false,
+  "covered_target_versions": [
+    "macOS 15",
+    "iOS 18",
+    "iOS 26"
+  ],
+  "missing_target_versions": [
+    "macOS 14",
+    "macOS 26",
+    "iOS 17"
+  ],
+  "has_target_chip_coverage": false,
+  "platform_chip_coverage": {
+    "iPad": {
+      "total_chips": 8,
+      "tested_chips": 7,
+      "coverage_percentage": 87.5,
+      "covered_chips": [
+        "A15",
+        "A16",
+        "A17 Pro",
+        "M1",
+        "M2",
+        "M3",
+        "M4"
+      ],
+      "missing_chips": [
+        "A14"
+      ]
+    },
+    "iPhone": {
+      "total_chips": 6,
+      "tested_chips": 6,
+      "coverage_percentage": 100.0,
+      "covered_chips": [
+        "A13",
+        "A14",
+        "A15",
+        "A16",
+        "A17 Pro",
+        "A18"
+      ],
+      "missing_chips": [
+        "A18 Pro"
+      ]
+    },
+    "Mac": {
+      "total_chips": 4,
+      "tested_chips": 2,
+      "coverage_percentage": 50.0,
+      "covered_chips": [
+        "M2",
+        "M3"
+      ],
+      "missing_chips": [
+        "M1",
+        "M4"
+      ]
+    }
+  },
+  "missing_target_chips": {
+    "iPad": [
+      "A14"
+    ],
+    "iPhone": [
+      "A18 Pro"
+    ],
+    "Mac": [
+      "M1",
+      "M4"
+    ]
+  }
+}

dashboard_data/test_coverage_a9b92c4.json ADDED Viewed

	@@ -0,0 +1,239 @@

+{
+  "commit_hash": "a9b92c4",
+  "total_devices": 137,
+  "tested_devices": 93,
+  "skipped_devices": 44,
+  "coverage_percentage": 67.88321167883211,
+  "tested_device_list": [
+    "iPhone14,6",
+    "Mac16,8",
+    "iPhone16,1",
+    "Mac16,12",
+    "MacBookPro18,1",
+    "Mac14,5",
+    "Mac15,6",
+    "Mac16,13",
+    "iPhone12,8",
+    "iPhone14,8",
+    "Mac14,9",
+    "iPad14,4",
+    "Mac16,2",
+    "iPad16,4",
+    "Mac14,7",
+    "Mac14,8",
+    "Mac15,10",
+    "iPhone13,4",
+    "iPhone17,1",
+    "iPad13,5",
+    "iPad13,16",
+    "iPad16,2",
+    "iPad13,17",
+    "iPhone16,2",
+    "Mac14,12",
+    "Mac14,6",
+    "iPhone12,5",
+    "Mac15,12",
+    "MacBookAir10,1",
+    "iPhone12,3",
+    "iPad14,3",
+    "Mac14,4",
+    "Mac15,11",
+    "iPhone17,5",
+    "iPad13,4",
+    "iPhone13,3",
+    "iPad14,6",
+    "Mac16,6",
+    "iPhone14,4",
+    "iPad14,9",
+    "Mac15,9",
+    "Mac16,11",
+    "iPad13,9",
+    "Mac15,5",
+    "MacBookPro18,2",
+    "Mac14,2",
+    "Macmini9,1",
+    "iPad16,6",
+    "iPad13,11",
+    "iPad13,10",
+    "Mac14,15",
+    "Mac13,2",
+    "iPhone17,2",
+    "Mac15,3",
+    "Mac15,13",
+    "Mac16,10",
+    "iPhone13,1",
+    "iPhone17,4",
+    "iPhone17,3",
+    "MacBookPro17,1",
+    "Mac15,8",
+    "Mac15,4",
+    "Mac15,7",
+    "iPhone14,5",
+    "iPad16,1",
+    "Mac16,3",
+    "MacBookPro18,3",
+    "Mac13,1",
+    "Mac14,10",
+    "iPhone12,1",
+    "Mac16,1",
+    "iPad14,5",
+    "MacBookPro18,4",
+    "iMac21,1",
+    "iPad14,11",
+    "iPhone14,2",
+    "iPad16,5",
+    "iPhone14,7",
+    "iPad14,8",
+    "Mac14,3",
+    "Mac16,7",
+    "iMac21,2",
+    "iPhone13,2",
+    "iPad13,8",
+    "iPad13,7",
+    "Mac14,13",
+    "Mac14,14",
+    "iPad13,6",
+    "iPad14,10",
+    "iPhone14,3",
+    "Mac16,9",
+    "Mac16,5",
+    "iPad16,3"
+  ],
+  "skipped_device_list": [
+    "iPad15,6",
+    "iPad11,6",
+    "iPhone12,4",
+    "iPad11,3",
+    "iPad15,4",
+    "iPad8,2",
+    "iPhone12,2",
+    "iPad11,1",
+    "iPad13,1",
+    "iPad8,3",
+    "iPad8,8",
+    "iPad15,5",
+    "iPad8,10",
+    "iPhone11,3",
+    "iPhone15,5",
+    "iPad13,2",
+    "iPad14,2",
+    "iPad8,6",
+    "iPhone15,3",
+    "iPhone11,5",
+    "iPhone15,2",
+    "iPad8,12",
+    "iPad13,18",
+    "iPad15,7",
+    "iPad13,19",
+    "iPad14,1",
+    "iPad15,8",
+    "iPad8,7",
+    "iPhone11,4",
+    "iPad11,7",
+    "iPad8,4",
+    "iPad12,1",
+    "iPad11,2",
+    "iPad15,3",
+    "iPhone15,4",
+    "iPad8,5",
+    "iPhone11,2",
+    "iPhone11,6",
+    "iPad8,1",
+    "iPhone11,8",
+    "iPad8,9",
+    "iPad8,11",
+    "iPad11,4",
+    "iPad12,2"
+  ],
+  "tested_os_versions": [
+    "macOS_15.0.1",
+    "iOS_18.1",
+    "iPadOS_17.6.1",
+    "iPadOS_18.0.1",
+    "macOS_15.2",
+    "macOS_15.0",
+    "iPadOS_18.1",
+    "iOS_18.2",
+    "macOS_15.2.0",
+    "iOS_17.6.1",
+    "macOS_15.1",
+    "iOS_18.0.1",
+    "macOS_15.0.0",
+    "iOS_17.2.1",
+    "iOS_17.7",
+    "macOS_15.1.0",
+    "iOS_18.0",
+    "iOS_17.3"
+  ],
+  "has_target_os_coverage": false,
+  "covered_target_versions": [
+    "macOS 15",
+    "iOS 17",
+    "iOS 18"
+  ],
+  "missing_target_versions": [
+    "macOS 14",
+    "macOS 26",
+    "iOS 26"
+  ],
+  "has_target_chip_coverage": false,
+  "platform_chip_coverage": {
+    "iPad": {
+      "total_chips": 8,
+      "tested_chips": 4,
+      "coverage_percentage": 50.0,
+      "covered_chips": [
+        "A17 Pro",
+        "M1",
+        "M2",
+        "M4"
+      ],
+      "missing_chips": [
+        "A14",
+        "A15",
+        "A16",
+        "M3"
+      ]
+    },
+    "iPhone": {
+      "total_chips": 6,
+      "tested_chips": 6,
+      "coverage_percentage": 100.0,
+      "covered_chips": [
+        "A13",
+        "A14",
+        "A15",
+        "A17 Pro",
+        "A18",
+        "A18 Pro"
+      ],
+      "missing_chips": [
+        "A16"
+      ]
+    },
+    "Mac": {
+      "total_chips": 4,
+      "tested_chips": 4,
+      "coverage_percentage": 100.0,
+      "covered_chips": [
+        "M1",
+        "M2",
+        "M3",
+        "M4"
+      ],
+      "missing_chips": []
+    }
+  },
+  "missing_target_chips": {
+    "iPad": [
+      "A14",
+      "A15",
+      "A16",
+      "M3"
+    ],
+    "iPhone": [
+      "A16"
+    ],
+    "Mac": []
+  }
+}

dashboard_data/test_coverage_c814cae.json ADDED Viewed

	@@ -0,0 +1,233 @@

+{
+  "commit_hash": "c814cae",
+  "total_devices": 137,
+  "tested_devices": 75,
+  "skipped_devices": 62,
+  "coverage_percentage": 54.74452554744526,
+  "tested_device_list": [
+    "Mac13,2",
+    "Mac16,8",
+    "iPhone17,2",
+    "Mac16,12",
+    "MacBookPro18,1",
+    "Mac14,5",
+    "iPad15,6",
+    "Mac16,10",
+    "iPad15,4",
+    "Mac16,13",
+    "iPhone17,4",
+    "iPhone17,3",
+    "iPad15,5",
+    "MacBookPro17,1",
+    "Mac14,9",
+    "iPad14,4",
+    "Mac16,2",
+    "iPad16,4",
+    "Mac14,7",
+    "Mac14,8",
+    "iPhone15,5",
+    "iPad16,1",
+    "iPhone17,1",
+    "Mac16,3",
+    "iPad13,5",
+    "MacBookPro18,3",
+    "iPad15,7",
+    "iPad16,2",
+    "iPad13,16",
+    "iPhone15,2",
+    "iPad13,17",
+    "iPhone15,3",
+    "Mac13,1",
+    "Mac14,12",
+    "Mac14,6",
+    "Mac14,10",
+    "iPad15,8",
+    "Mac16,1",
+    "MacBookAir10,1",
+    "iPad14,5",
+    "MacBookPro18,4",
+    "iMac21,1",
+    "iPad14,11",
+    "iPad14,3",
+    "Mac14,4",
+    "iPad16,5",
+    "iPad13,4",
+    "iPhone17,5",
+    "iPad14,8",
+    "iPad14,6",
+    "iPad15,3",
+    "Mac16,6",
+    "Mac14,3",
+    "iMac21,2",
+    "iPhone15,4",
+    "Mac16,7",
+    "iPad14,9",
+    "Mac16,11",
+    "iPad13,10",
+    "iPad13,9",
+    "MacBookPro18,2",
+    "Mac14,2",
+    "Macmini9,1",
+    "iPad13,8",
+    "iPad13,7",
+    "Mac14,13",
+    "iPad16,6",
+    "Mac14,14",
+    "iPad13,6",
+    "iPad14,10",
+    "Mac16,9",
+    "iPad13,11",
+    "Mac16,5",
+    "Mac14,15",
+    "iPad16,3"
+  ],
+  "skipped_device_list": [
+    "iPhone14,6",
+    "iPhone16,1",
+    "iPad11,6",
+    "Mac15,3",
+    "Mac15,13",
+    "iPhone12,4",
+    "iPad11,3",
+    "iPhone13,1",
+    "Mac15,6",
+    "iPhone12,8",
+    "iPad8,2",
+    "iPhone12,2",
+    "iPad11,1",
+    "iPad13,1",
+    "iPad8,3",
+    "iPad8,8",
+    "Mac15,8",
+    "iPhone14,8",
+    "iPad8,10",
+    "iPhone11,3",
+    "Mac15,4",
+    "Mac15,7",
+    "Mac15,10",
+    "iPhone14,5",
+    "iPhone13,4",
+    "iPad13,2",
+    "iPad14,2",
+    "iPad8,6",
+    "iPhone11,5",
+    "iPad8,12",
+    "iPad13,18",
+    "iPhone16,2",
+    "iPad13,19",
+    "iPhone12,5",
+    "iPad14,1",
+    "iPhone12,1",
+    "iPad8,7",
+    "iPhone11,4",
+    "Mac15,12",
+    "iPad11,7",
+    "iPad8,4",
+    "iPhone12,3",
+    "iPad12,1",
+    "iPad11,2",
+    "iPhone14,2",
+    "Mac15,11",
+    "iPhone14,7",
+    "iPhone13,3",
+    "iPhone14,4",
+    "Mac15,9",
+    "iPad8,5",
+    "iPhone11,2",
+    "Mac15,5",
+    "iPhone11,6",
+    "iPhone13,2",
+    "iPad8,1",
+    "iPhone11,8",
+    "iPad8,9",
+    "iPhone14,3",
+    "iPad8,11",
+    "iPad11,4",
+    "iPad12,2"
+  ],
+  "tested_os_versions": [
+    "macOS_15.5",
+    "iPadOS_18.4.1",
+    "iOS_26.0",
+    "macOS_15.5.0",
+    "iPadOS_26.0",
+    "macOS_26.0",
+    "iPadOS_18.6",
+    "iPadOS_18.5",
+    "macOS_26.0.0",
+    "iOS_18.5"
+  ],
+  "has_target_os_coverage": false,
+  "covered_target_versions": [
+    "macOS 15",
+    "macOS 26",
+    "iOS 18",
+    "iOS 26"
+  ],
+  "missing_target_versions": [
+    "macOS 14",
+    "iOS 17"
+  ],
+  "has_target_chip_coverage": false,
+  "platform_chip_coverage": {
+    "iPad": {
+      "total_chips": 8,
+      "tested_chips": 6,
+      "coverage_percentage": 75.0,
+      "covered_chips": [
+        "A16",
+        "A17 Pro",
+        "M1",
+        "M2",
+        "M3",
+        "M4"
+      ],
+      "missing_chips": [
+        "A14",
+        "A15"
+      ]
+    },
+    "iPhone": {
+      "total_chips": 6,
+      "tested_chips": 3,
+      "coverage_percentage": 50.0,
+      "covered_chips": [
+        "A16",
+        "A18",
+        "A18 Pro"
+      ],
+      "missing_chips": [
+        "A14",
+        "A15",
+        "A17 Pro"
+      ]
+    },
+    "Mac": {
+      "total_chips": 4,
+      "tested_chips": 3,
+      "coverage_percentage": 75.0,
+      "covered_chips": [
+        "M1",
+        "M2",
+        "M4"
+      ],
+      "missing_chips": [
+        "M3"
+      ]
+    }
+  },
+  "missing_target_chips": {
+    "iPad": [
+      "A14",
+      "A15"
+    ],
+    "iPhone": [
+      "A14",
+      "A15",
+      "A17 Pro"
+    ],
+    "Mac": [
+      "M3"
+    ]
+  }
+}

dashboard_data/test_coverage_ca49596.json ADDED Viewed

	@@ -0,0 +1,237 @@

+{
+  "commit_hash": "ca49596",
+  "total_devices": 137,
+  "tested_devices": 63,
+  "skipped_devices": 74,
+  "coverage_percentage": 45.98540145985402,
+  "tested_device_list": [
+    "iPhone14,6",
+    "Mac16,8",
+    "Mac15,3",
+    "Mac15,13",
+    "Mac16,12",
+    "iPad15,6",
+    "Mac16,10",
+    "Mac14,5",
+    "iPad15,4",
+    "Mac15,6",
+    "Mac16,13",
+    "iPhone12,8",
+    "iPad15,5",
+    "iPhone14,8",
+    "Mac15,8",
+    "Mac14,9",
+    "iPad14,4",
+    "Mac15,4",
+    "Mac16,2",
+    "Mac15,7",
+    "Mac14,7",
+    "Mac15,10",
+    "iPhone14,5",
+    "Mac14,8",
+    "iPad16,4",
+    "Mac16,3",
+    "Mac14,12",
+    "Mac14,6",
+    "Mac14,10",
+    "iPhone12,5",
+    "iPhone12,1",
+    "Mac15,12",
+    "Mac16,1",
+    "iPad14,5",
+    "iPhone12,3",
+    "iPad14,11",
+    "iPad14,3",
+    "iPhone14,2",
+    "Mac14,4",
+    "Mac15,11",
+    "iPhone14,7",
+    "iPad14,8",
+    "iPad14,6",
+    "iPad15,3",
+    "iPad16,5",
+    "Mac16,6",
+    "Mac14,3",
+    "Mac16,7",
+    "iPhone14,4",
+    "Mac15,9",
+    "iPad14,9",
+    "Mac16,11",
+    "Mac15,5",
+    "Mac14,2",
+    "iPad16,6",
+    "Mac14,13",
+    "Mac14,14",
+    "iPad14,10",
+    "iPhone14,3",
+    "Mac16,9",
+    "Mac16,5",
+    "Mac14,15",
+    "iPad16,3"
+  ],
+  "skipped_device_list": [
+    "Mac13,2",
+    "iPhone16,1",
+    "iPhone17,2",
+    "iPad11,6",
+    "MacBookPro18,1",
+    "iPhone12,4",
+    "iPad11,3",
+    "iPhone13,1",
+    "iPhone17,4",
+    "iPhone17,3",
+    "iPad8,2",
+    "iPhone12,2",
+    "iPad11,1",
+    "iPad13,1",
+    "iPad8,3",
+    "iPad8,8",
+    "MacBookPro17,1",
+    "iPad8,10",
+    "iPhone11,3",
+    "iPhone13,4",
+    "iPhone15,5",
+    "iPad13,2",
+    "iPad14,2",
+    "iPad16,1",
+    "iPhone17,1",
+    "iPad13,16",
+    "iPhone11,5",
+    "iPhone15,2",
+    "MacBookPro18,3",
+    "iPad13,18",
+    "iPad8,6",
+    "iPad13,17",
+    "iPhone16,2",
+    "iPhone15,3",
+    "iPad16,2",
+    "iPad8,12",
+    "iPad15,7",
+    "iPad14,1",
+    "iPad13,19",
+    "Mac13,1",
+    "iPad15,8",
+    "iPad8,7",
+    "iPhone11,4",
+    "MacBookAir10,1",
+    "iPad11,7",
+    "iPad8,4",
+    "MacBookPro18,4",
+    "iMac21,1",
+    "iPad12,1",
+    "iPad11,2",
+    "iPad13,4",
+    "iPhone17,5",
+    "iPhone13,3",
+    "iMac21,2",
+    "iPhone15,4",
+    "iPad8,5",
+    "iPad13,9",
+    "iPhone11,2",
+    "iPhone11,6",
+    "MacBookPro18,2",
+    "Macmini9,1",
+    "iPad13,8",
+    "iPhone13,2",
+    "iPad8,1",
+    "iPad8,11",
+    "iPhone11,8",
+    "iPad8,9",
+    "iPad13,5",
+    "iPad13,7",
+    "iPad13,6",
+    "iPad13,11",
+    "iPad13,10",
+    "iPad11,4",
+    "iPad12,2"
+  ],
+  "tested_os_versions": [
+    "iOS_18.1",
+    "macOS_15.3.0",
+    "macOS_15.4.0",
+    "iPadOS_18.3.2",
+    "iOS_17.6.1",
+    "macOS_15.3",
+    "macOS_15.4",
+    "iPadOS_18.4"
+  ],
+  "has_target_os_coverage": false,
+  "covered_target_versions": [
+    "macOS 15",
+    "iOS 17",
+    "iOS 18"
+  ],
+  "missing_target_versions": [
+    "macOS 14",
+    "macOS 26",
+    "iOS 26"
+  ],
+  "has_target_chip_coverage": false,
+  "platform_chip_coverage": {
+    "iPad": {
+      "total_chips": 8,
+      "tested_chips": 3,
+      "coverage_percentage": 37.5,
+      "covered_chips": [
+        "M2",
+        "M3",
+        "M4"
+      ],
+      "missing_chips": [
+        "A14",
+        "A15",
+        "A16",
+        "A17 Pro",
+        "M1"
+      ]
+    },
+    "iPhone": {
+      "total_chips": 6,
+      "tested_chips": 2,
+      "coverage_percentage": 33.33333333333333,
+      "covered_chips": [
+        "A13",
+        "A15"
+      ],
+      "missing_chips": [
+        "A14",
+        "A16",
+        "A17 Pro",
+        "A18",
+        "A18 Pro"
+      ]
+    },
+    "Mac": {
+      "total_chips": 4,
+      "tested_chips": 3,
+      "coverage_percentage": 75.0,
+      "covered_chips": [
+        "M2",
+        "M3",
+        "M4"
+      ],
+      "missing_chips": [
+        "M1"
+      ]
+    }
+  },
+  "missing_target_chips": {
+    "iPad": [
+      "A14",
+      "A15",
+      "A16",
+      "A17 Pro",
+      "M1"
+    ],
+    "iPhone": [
+      "A14",
+      "A16",
+      "A17 Pro",
+      "A18",
+      "A18 Pro"
+    ],
+    "Mac": [
+      "M1"
+    ]
+  }
+}

main.py CHANGED Viewed

@@ -23,24 +23,17 @@ from constants import (
     CITATION_BUTTON_TEXT,
     COL_NAMES,
     HEADER,
-    LANGUAGE_MAP,
     METHODOLOGY_TEXT,
     PERFORMANCE_TEXT,
-    QUALITY_TEXT,
-    # SHA_TO_VERSION,
 )
 from utils import (
     add_datasets_to_performance_columns,
-    add_datasets_to_quality_columns,
-    create_confusion_matrix_plot,
     create_initial_performance_column_dict,
-    create_initial_quality_column_dict,
     css,
     fields,
     get_os_name_and_version,
-    make_dataset_wer_clickable_link,
     make_model_name_clickable_link,
-    make_multilingual_model_clickable_link,
     plot_metric,
     read_json_line_by_line,
 )
@@ -61,16 +54,18 @@ local_dir = ""
 # Load benchmark data from JSON files
 PERFORMANCE_DATA = read_json_line_by_line("dashboard_data/performance_data.json")
-QUALITY_DATA = read_json_line_by_line("dashboard_data/quality_data.json")
 with open("dashboard_data/version.json", "r") as file:
     VERSION_DATA = json.load(file)
 SHA_TO_VERSION = {
-    VERSION_DATA["releases"][i]: VERSION_DATA["versions"][i] for i in range(len(VERSION_DATA["versions"]))
 }
-# Convert JSON data to pandas DataFrames
-quality_df = pd.json_normalize(QUALITY_DATA)
 benchmark_df = pd.json_normalize(PERFORMANCE_DATA)
 releases = VERSION_DATA["releases"]
@@ -78,55 +73,9 @@ releases = VERSION_DATA["releases"]
 benchmark_df["timestamp"] = pd.to_datetime(benchmark_df["timestamp"]).dt.tz_localize(
     None
 )
-benchmark_df["timestamp"] = pd.to_datetime(benchmark_df["timestamp"]).dt.tz_localize(
-    None
-)
-# First create a temporary column for model length
-sorted_quality_df = (
-    quality_df.assign(model_len=quality_df["model"].str.len())
-    .sort_values(
-        by=["model_len", "model", "timestamp"],
-        ascending=[True, True, False],
-    )
-    .drop(columns=["model_len"])
-    .drop_duplicates(subset=["model"], keep="first")
-    .reset_index(drop=True)
-)
-multilingual_df = pd.read_csv("dashboard_data/multilingual_results.csv")
-multilingual_models_df = multilingual_df[["Model"]].drop_duplicates()
-multilingual_models_buttons = []
-for model in multilingual_models_df["Model"]:
-    elem_id = (
-        f"{model}".replace(" ", "_").replace('"', "").replace("'", "").replace(",", "")
-    )
-    multilingual_models_buttons.append(
-        gr.Button(value=model, elem_id=elem_id, visible=False)
-    )
-multilingual_models_df["Model"] = multilingual_models_df["Model"].apply(
-    lambda x: make_multilingual_model_clickable_link(x)
-)
-with open("dashboard_data/multilingual_confusion_matrices.json", "r") as file:
-    confusion_matrix_map = dict(json.load(file))
-# Create a mapping of model to average WER
-model_to_english_wer = dict(zip(sorted_quality_df["model"], sorted_quality_df["average_wer"]))
-model_to_multilingual_wer = dict(
-    zip(multilingual_df["Model"], multilingual_df["Average WER"])
-)
-# Add English WER and Multilingual WER to performance_df
-benchmark_df["english_wer"] = benchmark_df["model"].map(model_to_english_wer)
-benchmark_df["multilingual_wer"] = benchmark_df["model"].map(model_to_multilingual_wer)
-benchmark_df.fillna({"multilingual_wer": "N/A"}, inplace=True) # Mark all untested models as N/A
-# Mark English-only models
-english_only_mask = benchmark_df["model"].str.contains(r"\.en$|distil-whisper", case=False, na=False)
-benchmark_df.loc[english_only_mask, "multilingual_wer"] = "English-only model"
-benchmark_df["multilingual_wer"] = benchmark_df["multilingual_wer"].astype(str)
 sorted_performance_df = (
     benchmark_df.assign(model_len=benchmark_df["model"].str.len())
@@ -140,9 +89,6 @@ sorted_performance_df = (
 )
 # Identify dataset-specific columns
-dataset_wer_columns = [
-    col for col in sorted_quality_df.columns if col.startswith("dataset_wer.")
-]
 dataset_speed_columns = [
     col for col in sorted_performance_df.columns if col.startswith("dataset_speed.")
 ]
@@ -153,46 +99,36 @@ dataset_toks_columns = [
 ]
 # Extract dataset names
-QUALITY_DATASETS = [col.split(".")[-1] for col in dataset_wer_columns]
 PERFORMANCE_DATASETS = [col.split(".")[-1] for col in dataset_speed_columns]
 # Prepare DataFrames for display
-model_df = sorted_quality_df[
-    ["model", "average_wer", "qoi", "timestamp"] + dataset_wer_columns
-]
 performance_df = sorted_performance_df[
     [
         "model",
         "device",
         "os",
         "english_wer",
-        "multilingual_wer",
         "qoi",
         "speed",
         "tokens_per_second",
         "timestamp",
-        "commit_hash"
     ]
     + dataset_speed_columns
     + dataset_toks_columns
 ].copy()
 # Rename columns for clarity
 performance_df = performance_df.rename(
     lambda x: COL_NAMES[x] if x in COL_NAMES else x, axis="columns"
 )
-model_df = model_df.rename(
-    lambda x: COL_NAMES[x] if x in COL_NAMES else x, axis="columns"
-)
 # Process dataset-specific columns
-for col in dataset_wer_columns:
-    dataset_name = col.split(".")[-1]
-    model_df = model_df.rename(columns={col: dataset_name})
-    model_df[dataset_name] = model_df.apply(
-        lambda x: make_dataset_wer_clickable_link(x, dataset_name), axis=1
-    )
 for col in dataset_speed_columns:
     dataset_name = col.split(".")[-1]
     performance_df = performance_df.rename(
@@ -210,9 +146,7 @@ for col in dataset_toks_columns:
     )
 # Process model names for display
-model_df["model_raw"] = model_df["Model"].copy()
 performance_df["model_raw"] = performance_df["Model"].copy()
-model_df["Model"] = model_df["Model"].apply(lambda x: make_model_name_clickable_link(x))
 performance_df["Model"] = performance_df["Model"].apply(
     lambda x: make_model_name_clickable_link(x)
 )
@@ -220,49 +154,42 @@ performance_df["Model"] = performance_df["Model"].apply(
 # Extract unique devices and OS versions
 initial_release_df = benchmark_df[benchmark_df["commit_hash"] == releases[-1]]
 PERFORMANCE_DEVICES = initial_release_df["device"].unique().tolist()
-PERFORMANCE_OS = initial_release_df["os"].apply(get_os_name_and_version).unique().tolist()
 PERFORMANCE_OS.sort()
 # Create initial column dictionaries and update with dataset information
 initial_performance_column_dict = create_initial_performance_column_dict()
-initial_quality_column_dict = create_initial_quality_column_dict()
 performance_column_info = add_datasets_to_performance_columns(
     initial_performance_column_dict, PERFORMANCE_DATASETS
 )
-quality_column_info = add_datasets_to_quality_columns(
-    initial_quality_column_dict, QUALITY_DATASETS
-)
 # Unpack the returned dictionaries
 updated_performance_column_dict = performance_column_info["column_dict"]
-updated_quality_column_dict = quality_column_info["column_dict"]
 PerformanceAutoEvalColumn = performance_column_info["AutoEvalColumn"]
-QualityAutoEvalColumn = quality_column_info["AutoEvalColumn"]
 # Define column sets for different views
 PERFORMANCE_COLS = performance_column_info["COLS"]
-QUALITY_COLS = quality_column_info["COLS"]
 PERFORMANCE_TYPES = performance_column_info["TYPES"]
-QUALITY_TYPES = quality_column_info["TYPES"]
 PERFORMANCE_ALWAYS_HERE_COLS = performance_column_info["ALWAYS_HERE_COLS"]
-QUALITY_ALWAYS_HERE_COLS = quality_column_info["ALWAYS_HERE_COLS"]
 PERFORMANCE_TOGGLE_COLS = performance_column_info["TOGGLE_COLS"]
-QUALITY_TOGGLE_COLS = quality_column_info["TOGGLE_COLS"]
 PERFORMANCE_SELECTED_COLS = performance_column_info["SELECTED_COLS"]
-QUALITY_SELECTED_COLS = quality_column_info["SELECTED_COLS"]
 def get_release_devices(release):
     """
     Get the list of devices for a specific release.
     :param release: Selected release hash
     :return: List of devices available in the release
     """
     release_df = benchmark_df[benchmark_df["commit_hash"] == release]
     return release_df["device"].unique().tolist()
 def performance_filter(
     df,
     columns,
@@ -367,95 +294,44 @@ def performance_filter(
     return filtered_df
-def quality_filter(df, columns, model_query, wer_slider, qoi_slider, exclude_models):
-    """
-    Filters the quality DataFrame based on specified criteria.
-    :param df: The DataFrame to be filtered.
-    :param columns: The columns to be included in the filtered DataFrame.
-    :param model_query: The query string to filter the 'Model' column.
-    :param wer_slider: The range of values to filter the 'Average WER' column.
-    :param qoi_slider: The range of values to filter the 'QoI' column.
-    :param exclude_models: Models to exclude from the results.
-    :return: The filtered DataFrame.
-    """
-    # Select columns based on input and always-present columns
-    filtered_df = df[
-        QUALITY_ALWAYS_HERE_COLS
-        + [c for c in QUALITY_COLS if c in df.columns and c in columns]
-    ]
-    # Filter models based on query
-    if model_query:
-        filtered_df = filtered_df[
-            filtered_df["Model"].str.contains(
-                "|".join(q.strip() for q in model_query.split(";")), case=False
-            )
-        ]
-    # Exclude specified models
-    if exclude_models:
-        exclude_list = [m.strip() for m in exclude_models.split(";")]
-        filtered_df = filtered_df[
-            ~filtered_df["Model"].str.contains("|".join(exclude_list), case=False)
-        ]
-    # Apply WER and QoI filters
-    min_wer_slider, max_wer_slider = wer_slider
-    min_qoi_slider, max_qoi_slider = qoi_slider
-    if "Average WER" in filtered_df.columns:
-        filtered_df = filtered_df[
-            (filtered_df["Average WER"] >= min_wer_slider)
-            & (filtered_df["Average WER"] <= max_wer_slider)
-        ]
-    if "QoI" in filtered_df.columns:
-        filtered_df = filtered_df[
-            (filtered_df["QoI"] >= min_qoi_slider)
-            & (filtered_df["QoI"] <= max_qoi_slider)
-        ]
-    return filtered_df
 def update_performance_filters(release):
     """
     Updates the performance filters (devices and OS) based on the selected release.
     :param release: Selected release hash
     :return: Tuple containing updated device and OS choices
     """
     # Filter benchmark data for the selected release
     release_df = benchmark_df[benchmark_df["commit_hash"] == release]
     # Get unique devices and OS versions for this release
     release_devices = release_df["device"].unique().tolist()
     release_os = release_df["os"].apply(get_os_name_and_version).unique().tolist()
     release_os.sort()
     return (
         gr.update(choices=release_devices, value=release_devices),
-        gr.update(choices=release_os, value=release_os)
     )
 def update_support_table(release):
     """
     Updates the support table and its column configuration for a given release.
     :param release: Selected release hash
     :return: Tuple containing (updated DataFrame, updated column choices, updated column values)
     """
     # Load new support data
     support_data = pd.read_csv(f"dashboard_data/support_data_{release[:7]}.csv")
     support_data.set_index(support_data.columns[0], inplace=True)
     # Process model names
-    support_data["Model"] = support_data["Model"].apply(
-        lambda x: x.replace("_", "/")
-    )
     support_data["Model"] = support_data["Model"].apply(
         lambda x: make_model_name_clickable_link(x)
     )
     # Sort by model name length
     support_data = (
         support_data.assign(model_len=support_data["Model"].str.len())
@@ -465,14 +341,14 @@ def update_support_table(release):
         )
         .drop(columns=["model_len"])
     )
     # Get new columns (excluding 'Model')
     new_columns = support_data.columns.tolist()[1:]
     return (
         gr.update(value=support_data, datatype=["html" for _ in support_data.columns]),
         gr.update(choices=new_columns, value=new_columns),
-        gr.update(value=support_data)
     )
@@ -481,96 +357,6 @@ text_diff_elems = []
 tabs = gr.Tabs(elem_id="tab-elems")
-def update_multilingual_results(selected_model):
-    """
-    Updates the multilingual results display based on the selected model.
-    This function processes the multilingual data for the chosen model,
-    calculates average WER for different scenarios (language hinted vs. predicted),
-    and prepares language-specific WER data for display.
-    :param selected_model: The name of the selected model
-    :return: A list containing updated components for the Gradio interface
-    """
-    if selected_model is None:
-        return "# Select a model from the dropdown to view results."
-    # Filter data for the selected model
-    model_data = multilingual_df[multilingual_df["Model"] == selected_model]
-    if model_data.empty:
-        return f"# No data available for model: {selected_model}"
-    # Separate data for forced and not forced scenarios
-    forced_data = model_data[model_data["Forced Tokens"] == True]
-    not_forced_data = model_data[model_data["Forced Tokens"] == False]
-    result_text = f"# Model: {selected_model}\n\n"
-    # Prepare average WER data
-    average_wer_data = []
-    if not forced_data.empty:
-        average_wer_data.append(
-            {
-                "Scenario": "Language Hinted",
-                "Average WER": forced_data.iloc[0]["Average WER"],
-            }
-        )
-    if not not_forced_data.empty:
-        average_wer_data.append(
-            {
-                "Scenario": "Language Predicted",
-                "Average WER": not_forced_data.iloc[0]["Average WER"],
-            }
-        )
-    average_wer_df = pd.DataFrame(average_wer_data)
-    average_wer_df["Average WER"] = average_wer_df["Average WER"].apply(
-        lambda x: round(x, 2)
-    )
-    # Prepare language-specific WER data
-    lang_columns = [col for col in model_data.columns if col.startswith("WER_")]
-    lang_wer_data = []
-    for column in lang_columns:
-        lang = column.split("_")[1]
-        forced_wer = forced_data[column].iloc[0] if not forced_data.empty else None
-        not_forced_wer = (
-            not_forced_data[column].iloc[0] if not not_forced_data.empty else None
-        )
-        if forced_wer is not None or not_forced_wer is not None:
-            lang_wer_data.append(
-                {
-                    "Language": LANGUAGE_MAP[lang],
-                    "Language Hinted WER": round(forced_wer, 2)
-                    if forced_wer is not None
-                    else "N/A",
-                    "Language Predicted WER": round(not_forced_wer, 2)
-                    if not_forced_wer is not None
-                    else "N/A",
-                }
-            )
-    lang_wer_df = pd.DataFrame(lang_wer_data)
-    lang_wer_df = lang_wer_df.fillna("No Data")
-    # Create confusion matrix plot for unforced scenario
-    unforced_plot = None
-    if selected_model in confusion_matrix_map:
-        if "not_forced" in confusion_matrix_map[selected_model]:
-            unforced_plot = create_confusion_matrix_plot(
-                confusion_matrix_map[selected_model]["not_forced"]["matrix"],
-                confusion_matrix_map[selected_model]["not_forced"]["labels"],
-                False,
-            )
-    # Return updated components for Gradio interface
-    return [
-        gr.update(value=result_text),
-        gr.update(visible=True, value=average_wer_df),
-        gr.update(visible=True, value=lang_wer_df),
-        gr.update(visible=unforced_plot is not None, value=unforced_plot),
-    ]
 font = [
     "Zwizz Regular",  # Local font
     "IBM Plex Mono",  # Monospace font
@@ -579,6 +365,10 @@ font = [
     "sans-serif",
 ]
 # Define the Gradio interface
 with gr.Blocks(css=css, theme=gr.themes.Base(font=font)) as demo:
     # Add header and banner to the interface
@@ -586,7 +376,9 @@ with gr.Blocks(css=css, theme=gr.themes.Base(font=font)) as demo:
     gr.HTML(BANNER_TEXT, elem_classes="markdown-text")
     gr.Markdown("### Release")
     release_dropdown = gr.Dropdown(
-        choices=[(f"{release} v{SHA_TO_VERSION[release]}", release) for release in releases],
         label="Select Release",
         value=releases[-1] if releases else None,
         elem_id="release-dropdown",
@@ -596,7 +388,7 @@ with gr.Blocks(css=css, theme=gr.themes.Base(font=font)) as demo:
     # Create tabs for different sections of the dashboard
     with tabs.render():
         # Performance Tab
-        with gr.TabItem("Performance", elem_id="benchmark", id=0):
             with gr.Row():
                 with gr.Column(scale=1):
                     with gr.Row():
@@ -760,7 +552,9 @@ with gr.Blocks(css=css, theme=gr.themes.Base(font=font)) as demo:
                     with gr.Row():
                         gr.Markdown(PERFORMANCE_TEXT, elem_classes="markdown-text")
             with gr.Row():
-                initial_df = performance_df[performance_df["commit_hash"] == releases[-1]]
                 leaderboard_df = gr.components.Dataframe(
                     value=initial_df[
                         PERFORMANCE_ALWAYS_HERE_COLS + performance_shown_columns.value
@@ -837,113 +631,13 @@ with gr.Blocks(css=css, theme=gr.themes.Base(font=font)) as demo:
                     fn=update_performance_filters,
                     inputs=[release_dropdown],
                     outputs=[performance_shown_devices, performance_shown_os],
-                    queue=False
                 ).then(
                     fn=performance_filter,
                     inputs=performance_filter_inputs,
-                    outputs=filter_output
-                )
-        # English Quality Tab
-        with gr.TabItem("English Quality", elem_id="timeline", id=1):
-            with gr.Row():
-                with gr.Column(scale=1):
-                    with gr.Row():
-                        with gr.Column(scale=6, elem_classes="filter_models_column"):
-                            filter_quality_models = gr.Textbox(
-                                placeholder="🔍 Filter Model (separate multiple queries with ';')",
-                                label="Filter Models",
-                            )
-                        with gr.Column(scale=4, elem_classes="exclude_models_column"):
-                            exclude_quality_models = gr.Textbox(
-                                placeholder="🔍 Exclude Model",
-                                label="Exclude Model",
-                            )
-                    with gr.Row():
-                        with gr.Accordion("See All Columns", open=False):
-                            quality_shown_columns = gr.CheckboxGroup(
-                                choices=QUALITY_TOGGLE_COLS,
-                                value=QUALITY_SELECTED_COLS,
-                                label="Toggle Columns",
-                                elem_id="column-select",
-                                interactive=True,
-                            )
-                with gr.Column(scale=1):
-                    with gr.Accordion("See Quality Filters"):
-                        with gr.Row():
-                            with gr.Row():
-                                quality_min_avg_wer, quality_max_avg_wer = (
-                                    floor(min(model_df["Average WER"])),
-                                    ceil(max(model_df["Average WER"])) + 1,
-                                )
-                                wer_slider = RangeSlider(
-                                    value=[quality_min_avg_wer, quality_max_avg_wer],
-                                    minimum=quality_min_avg_wer,
-                                    maximum=quality_max_avg_wer,
-                                    label="Average WER",
-                                )
-                            with gr.Row():
-                                quality_min_qoi, quality_max_qoi = floor(
-                                    min(model_df["QoI"])
-                                ), ceil(max(model_df["QoI"] + 1))
-                                qoi_slider = RangeSlider(
-                                    value=[quality_min_qoi, quality_max_qoi],
-                                    minimum=quality_min_qoi,
-                                    maximum=quality_max_qoi,
-                                    label="QoI",
-                                )
-                    with gr.Row():
-                        gr.Markdown(QUALITY_TEXT)
-            with gr.Row():
-                quality_leaderboard_df = gr.components.Dataframe(
-                    value=model_df[
-                        QUALITY_ALWAYS_HERE_COLS + quality_shown_columns.value
-                    ],
-                    headers=[QUALITY_ALWAYS_HERE_COLS + quality_shown_columns.value],
-                    datatype=[
-                        c.type
-                        for c in fields(QualityAutoEvalColumn)
-                        if c.name in QUALITY_COLS
-                    ],
-                    elem_id="leaderboard-table",
-                    elem_classes="large-table",
-                    interactive=False,
                 )
-                # Copy of the leaderboard dataframe to apply filters to
-                hidden_quality_leaderboard_df = gr.components.Dataframe(
-                    value=model_df,
-                    headers=QUALITY_COLS,
-                    datatype=[
-                        c.type
-                        for c in fields(QualityAutoEvalColumn)
-                        if c.name in QUALITY_COLS
-                    ],
-                    visible=False,
-                )
-                # Inputs for the dataframe filter function
-                filter_inputs = [
-                    hidden_quality_leaderboard_df,
-                    quality_shown_columns,
-                    filter_quality_models,
-                    wer_slider,
-                    qoi_slider,
-                    exclude_quality_models,
-                ]
-                filter_output = quality_leaderboard_df
-                filter_quality_models.change(
-                    quality_filter, filter_inputs, filter_output
-                )
-                exclude_quality_models.change(
-                    quality_filter, filter_inputs, filter_output
-                )
-                quality_shown_columns.change(
-                    quality_filter, filter_inputs, filter_output
-                )
-                wer_slider.change(quality_filter, filter_inputs, filter_output)
-                qoi_slider.change(quality_filter, filter_inputs, filter_output)
         # Timeline Tab
         with gr.TabItem("Timeline", elem_id="timeline", id=4):
             # Create subtabs for different metrics
@@ -1204,59 +898,27 @@ with gr.Blocks(css=css, theme=gr.themes.Base(font=font)) as demo:
                                 toks_plot,
                             )
-        # Multilingual Quality Tab
-        with gr.TabItem("Multilingual Quality", elem_id="multilingual", id=5):
-            if multilingual_df is not None:
-                with gr.Row():
-                    with gr.Column(scale=1):
-                        # Display table of multilingual models
-                        model_table = gr.Dataframe(
-                            value=multilingual_models_df,
-                            headers=["Model"],
-                            datatype=["html"],
-                            elem_classes="left-side-table",
-                        )
-                        # Placeholders for confusion matrix plots
-                        with gr.Row():
-                            unforced_confusion_matrix = gr.Plot(visible=False)
-                        with gr.Row():
-                            forced_confusion_matrix = gr.Plot(visible=False)
-                    with gr.Column(scale=1):
-                        # Display area for selected model results
-                        results_markdown = gr.Markdown(
-                            "# Select a model from the table on the left to view results.",
-                            elem_id="multilingual-results",
-                        )
-                        # Tables for displaying average WER and language-specific WER
-                        average_wer_table = gr.Dataframe(
-                            value=None, elem_id="average-wer-table", visible=False
-                        )
-                        language_wer_table = gr.Dataframe(
-                            value=None, elem_id="general-wer-table", visible=False
-                        )
-                    # Set up click event to update results when a model is selected
-                    for button in multilingual_models_buttons:
-                        button.render()
-                        button.click(
-                            fn=lambda x: update_multilingual_results(x),
-                            inputs=[button],
-                            outputs=[
-                                results_markdown,
-                                average_wer_table,
-                                language_wer_table,
-                                unforced_confusion_matrix,
-                            ],
-                        )
-            else:
-                # Display message if no multilingual data is available
-                gr.Markdown("No multilingual benchmark results available.")
         # Device Support Tab
         with gr.TabItem("Device Support", elem_id="device_support", id=6):
             # Load device support data from CSV
-            support_data = pd.read_csv(f"dashboard_data/support_data_{releases[-1][:7]}.csv")
             support_data.set_index(support_data.columns[0], inplace=True)
             support_data["Model"] = support_data["Model"].apply(
                 lambda x: x.replace("_", "/")
@@ -1291,7 +953,9 @@ with gr.Blocks(css=css, theme=gr.themes.Base(font=font)) as demo:
                             with gr.Row():
                                 with gr.Column(scale=9):
                                     support_shown_columns = gr.CheckboxGroup(
-                                        choices=support_data.columns.tolist()[1:],  # Exclude 'Model' column
                                         value=support_data.columns.tolist()[1:],
                                         label="Toggle Columns",
                                         elem_id="support-column-select",
@@ -1337,7 +1001,7 @@ with gr.Blocks(css=css, theme=gr.themes.Base(font=font)) as demo:
             def filter_support_data(df, columns, model_query, exclude_models):
                 """
                 Filters the device support data based on specified criteria.
                 :param df: The DataFrame to be filtered
                 :param columns: Columns to include in the output
                 :param model_query: Query string to filter models
@@ -1358,7 +1022,9 @@ with gr.Blocks(css=css, theme=gr.themes.Base(font=font)) as demo:
                 # Exclude specified models
                 if exclude_models:
-                    exclude_list = [re.escape(m.strip()) for m in exclude_models.split(";")]
                     filtered_df = filtered_df[
                         ~filtered_df["Model"].str.contains(
                             "|".join(exclude_list), case=False, regex=True
@@ -1366,7 +1032,9 @@ with gr.Blocks(css=css, theme=gr.themes.Base(font=font)) as demo:
                     ]
                 # Select columns
-                selected_columns = ["Model"] + [col for col in columns if col in df.columns]
                 filtered_df = filtered_df[selected_columns]
                 return filtered_df
@@ -1374,12 +1042,14 @@ with gr.Blocks(css=css, theme=gr.themes.Base(font=font)) as demo:
             def select_all_support_columns(release):
                 """
                 Returns all current columns from the support shown columns.
                 :param release: Selected release hash
                 :return: List of all available choices
                 """
                 # Load new support data for the current release
-                support_data = pd.read_csv(f"dashboard_data/support_data_{release[:7]}.csv")
                 support_data.set_index(support_data.columns[0], inplace=True)
                 # Return all columns except 'Model'
                 return [col for col in support_data.columns if col != "Model"]
@@ -1403,23 +1073,265 @@ with gr.Blocks(css=css, theme=gr.themes.Base(font=font)) as demo:
             release_dropdown.change(
                 update_support_table,
                 inputs=[release_dropdown],
-                outputs=[device_support_table, support_shown_columns, hidden_support_df]
             ).then(
                 filter_support_data,
-                inputs=[hidden_support_df, support_shown_columns, filter_support_models, exclude_support_models],
-                outputs=device_support_table
             )
             # Also connect the filter inputs to update the table
-            for input_elem in [filter_support_models, exclude_support_models, support_shown_columns]:
                 input_elem.change(
                     filter_support_data,
-                    inputs=[hidden_support_df, support_shown_columns, filter_support_models, exclude_support_models],
-                    outputs=device_support_table
                 )
         # Methodology Tab
-        with gr.TabItem("Methodology", elem_id="methodology", id=7):
             gr.Markdown(METHODOLOGY_TEXT, elem_id="methodology-text")
     # Citation section
@@ -1433,4 +1345,4 @@ with gr.Blocks(css=css, theme=gr.themes.Base(font=font)) as demo:
         )
 # Launch the Gradio interface
-demo.launch(debug=True, share=True, ssr_mode=False)

     CITATION_BUTTON_TEXT,
     COL_NAMES,
     HEADER,
     METHODOLOGY_TEXT,
     PERFORMANCE_TEXT,
 )
 from utils import (
     add_datasets_to_performance_columns,
+    calculate_quality_parity,
     create_initial_performance_column_dict,
     css,
     fields,
     get_os_name_and_version,
     make_model_name_clickable_link,
     plot_metric,
     read_json_line_by_line,
 )
 # Load benchmark data from JSON files
 PERFORMANCE_DATA = read_json_line_by_line("dashboard_data/performance_data.json")
 with open("dashboard_data/version.json", "r") as file:
     VERSION_DATA = json.load(file)
+# Load quality data (ground truth WER)
+QUALITY_DATA = read_json_line_by_line("dashboard_data/quality_data.json")
 SHA_TO_VERSION = {
+    VERSION_DATA["releases"][i]: VERSION_DATA["versions"][i]
+    for i in range(len(VERSION_DATA["versions"]))
 }
+# Convert JSON data to pandas DataFrames - performance only
 benchmark_df = pd.json_normalize(PERFORMANCE_DATA)
 releases = VERSION_DATA["releases"]
 benchmark_df["timestamp"] = pd.to_datetime(benchmark_df["timestamp"]).dt.tz_localize(
     None
 )
+# Use average_wer directly from performance data
+benchmark_df["english_wer"] = benchmark_df["average_wer"]
 sorted_performance_df = (
     benchmark_df.assign(model_len=benchmark_df["model"].str.len())
 )
 # Identify dataset-specific columns
 dataset_speed_columns = [
     col for col in sorted_performance_df.columns if col.startswith("dataset_speed.")
 ]
 ]
 # Extract dataset names
 PERFORMANCE_DATASETS = [col.split(".")[-1] for col in dataset_speed_columns]
 # Prepare DataFrames for display
 performance_df = sorted_performance_df[
     [
         "model",
         "device",
         "os",
         "english_wer",
         "qoi",
         "speed",
         "tokens_per_second",
         "timestamp",
+        "commit_hash",
     ]
     + dataset_speed_columns
     + dataset_toks_columns
 ].copy()
+# Calculate parity (difference between measured WER and ground truth WER)
+performance_df["parity"] = performance_df.apply(
+    lambda row: calculate_quality_parity(QUALITY_DATA, row), axis=1
+)
 # Rename columns for clarity
 performance_df = performance_df.rename(
     lambda x: COL_NAMES[x] if x in COL_NAMES else x, axis="columns"
 )
 # Process dataset-specific columns
 for col in dataset_speed_columns:
     dataset_name = col.split(".")[-1]
     performance_df = performance_df.rename(
     )
 # Process model names for display
 performance_df["model_raw"] = performance_df["Model"].copy()
 performance_df["Model"] = performance_df["Model"].apply(
     lambda x: make_model_name_clickable_link(x)
 )
 # Extract unique devices and OS versions
 initial_release_df = benchmark_df[benchmark_df["commit_hash"] == releases[-1]]
 PERFORMANCE_DEVICES = initial_release_df["device"].unique().tolist()
+PERFORMANCE_OS = (
+    initial_release_df["os"].apply(get_os_name_and_version).unique().tolist()
+)
 PERFORMANCE_OS.sort()
 # Create initial column dictionaries and update with dataset information
 initial_performance_column_dict = create_initial_performance_column_dict()
 performance_column_info = add_datasets_to_performance_columns(
     initial_performance_column_dict, PERFORMANCE_DATASETS
 )
 # Unpack the returned dictionaries
 updated_performance_column_dict = performance_column_info["column_dict"]
 PerformanceAutoEvalColumn = performance_column_info["AutoEvalColumn"]
 # Define column sets for different views
 PERFORMANCE_COLS = performance_column_info["COLS"]
 PERFORMANCE_TYPES = performance_column_info["TYPES"]
 PERFORMANCE_ALWAYS_HERE_COLS = performance_column_info["ALWAYS_HERE_COLS"]
 PERFORMANCE_TOGGLE_COLS = performance_column_info["TOGGLE_COLS"]
 PERFORMANCE_SELECTED_COLS = performance_column_info["SELECTED_COLS"]
 def get_release_devices(release):
     """
     Get the list of devices for a specific release.
     :param release: Selected release hash
     :return: List of devices available in the release
     """
     release_df = benchmark_df[benchmark_df["commit_hash"] == release]
     return release_df["device"].unique().tolist()
 def performance_filter(
     df,
     columns,
     return filtered_df
 def update_performance_filters(release):
     """
     Updates the performance filters (devices and OS) based on the selected release.
     :param release: Selected release hash
     :return: Tuple containing updated device and OS choices
     """
     # Filter benchmark data for the selected release
     release_df = benchmark_df[benchmark_df["commit_hash"] == release]
     # Get unique devices and OS versions for this release
     release_devices = release_df["device"].unique().tolist()
     release_os = release_df["os"].apply(get_os_name_and_version).unique().tolist()
     release_os.sort()
     return (
         gr.update(choices=release_devices, value=release_devices),
+        gr.update(choices=release_os, value=release_os),
     )
 def update_support_table(release):
     """
     Updates the support table and its column configuration for a given release.
     :param release: Selected release hash
     :return: Tuple containing (updated DataFrame, updated column choices, updated column values)
     """
     # Load new support data
     support_data = pd.read_csv(f"dashboard_data/support_data_{release[:7]}.csv")
     support_data.set_index(support_data.columns[0], inplace=True)
     # Process model names
+    support_data["Model"] = support_data["Model"].apply(lambda x: x.replace("_", "/"))
     support_data["Model"] = support_data["Model"].apply(
         lambda x: make_model_name_clickable_link(x)
     )
     # Sort by model name length
     support_data = (
         support_data.assign(model_len=support_data["Model"].str.len())
         )
         .drop(columns=["model_len"])
     )
     # Get new columns (excluding 'Model')
     new_columns = support_data.columns.tolist()[1:]
     return (
         gr.update(value=support_data, datatype=["html" for _ in support_data.columns]),
         gr.update(choices=new_columns, value=new_columns),
+        gr.update(value=support_data),
     )
 tabs = gr.Tabs(elem_id="tab-elems")
 font = [
     "Zwizz Regular",  # Local font
     "IBM Plex Mono",  # Monospace font
     "sans-serif",
 ]
+# Macos 14, 15, 26
+# ios 17, 18, 26
 # Define the Gradio interface
 with gr.Blocks(css=css, theme=gr.themes.Base(font=font)) as demo:
     # Add header and banner to the interface
     gr.HTML(BANNER_TEXT, elem_classes="markdown-text")
     gr.Markdown("### Release")
     release_dropdown = gr.Dropdown(
+        choices=[
+            (f"{release} v{SHA_TO_VERSION[release]}", release) for release in releases
+        ],
         label="Select Release",
         value=releases[-1] if releases else None,
         elem_id="release-dropdown",
     # Create tabs for different sections of the dashboard
     with tabs.render():
         # Performance Tab
+        with gr.TabItem("Benchmark", elem_id="benchmark", id=0):
             with gr.Row():
                 with gr.Column(scale=1):
                     with gr.Row():
                     with gr.Row():
                         gr.Markdown(PERFORMANCE_TEXT, elem_classes="markdown-text")
             with gr.Row():
+                initial_df = performance_df[
+                    performance_df["commit_hash"] == releases[-1]
+                ]
                 leaderboard_df = gr.components.Dataframe(
                     value=initial_df[
                         PERFORMANCE_ALWAYS_HERE_COLS + performance_shown_columns.value
                     fn=update_performance_filters,
                     inputs=[release_dropdown],
                     outputs=[performance_shown_devices, performance_shown_os],
+                    queue=False,
                 ).then(
                     fn=performance_filter,
                     inputs=performance_filter_inputs,
+                    outputs=filter_output,
                 )
         # Timeline Tab
         with gr.TabItem("Timeline", elem_id="timeline", id=4):
             # Create subtabs for different metrics
                                 toks_plot,
                             )
         # Device Support Tab
         with gr.TabItem("Device Support", elem_id="device_support", id=6):
+            # Add clear description of what Device Support means
+            gr.Markdown(
+                """
+                ## Device Support
+                This tab shows **test results for SKUs that we actually attempted to test**. It tells you whether tests passed, failed, or couldn't be completed for the devices we tried to run tests on.
+                ### Please Note:
+                **This tab only shows devices we attempted to test** - it doesn't show the full universe of available devices.
+                **📝 For comprehensive coverage analysis**, see the **Test Coverage** tab which shows ALL available SKUs.
+                """,
+                elem_classes="markdown-text"
+            )
             # Load device support data from CSV
+            support_data = pd.read_csv(
+                f"dashboard_data/support_data_{releases[-1][:7]}.csv"
+            )
             support_data.set_index(support_data.columns[0], inplace=True)
             support_data["Model"] = support_data["Model"].apply(
                 lambda x: x.replace("_", "/")
                             with gr.Row():
                                 with gr.Column(scale=9):
                                     support_shown_columns = gr.CheckboxGroup(
+                                        choices=support_data.columns.tolist()[
+                                            1:
+                                        ],  # Exclude 'Model' column
                                         value=support_data.columns.tolist()[1:],
                                         label="Toggle Columns",
                                         elem_id="support-column-select",
             def filter_support_data(df, columns, model_query, exclude_models):
                 """
                 Filters the device support data based on specified criteria.
                 :param df: The DataFrame to be filtered
                 :param columns: Columns to include in the output
                 :param model_query: Query string to filter models
                 # Exclude specified models
                 if exclude_models:
+                    exclude_list = [
+                        re.escape(m.strip()) for m in exclude_models.split(";")
+                    ]
                     filtered_df = filtered_df[
                         ~filtered_df["Model"].str.contains(
                             "|".join(exclude_list), case=False, regex=True
                     ]
                 # Select columns
+                selected_columns = ["Model"] + [
+                    col for col in columns if col in df.columns
+                ]
                 filtered_df = filtered_df[selected_columns]
                 return filtered_df
             def select_all_support_columns(release):
                 """
                 Returns all current columns from the support shown columns.
                 :param release: Selected release hash
                 :return: List of all available choices
                 """
                 # Load new support data for the current release
+                support_data = pd.read_csv(
+                    f"dashboard_data/support_data_{release[:7]}.csv"
+                )
                 support_data.set_index(support_data.columns[0], inplace=True)
                 # Return all columns except 'Model'
                 return [col for col in support_data.columns if col != "Model"]
             release_dropdown.change(
                 update_support_table,
                 inputs=[release_dropdown],
+                outputs=[
+                    device_support_table,
+                    support_shown_columns,
+                    hidden_support_df,
+                ],
             ).then(
                 filter_support_data,
+                inputs=[
+                    hidden_support_df,
+                    support_shown_columns,
+                    filter_support_models,
+                    exclude_support_models,
+                ],
+                outputs=device_support_table,
             )
             # Also connect the filter inputs to update the table
+            for input_elem in [
+                filter_support_models,
+                exclude_support_models,
+                support_shown_columns,
+            ]:
                 input_elem.change(
                     filter_support_data,
+                    inputs=[
+                        hidden_support_df,
+                        support_shown_columns,
+                        filter_support_models,
+                        exclude_support_models,
+                    ],
+                    outputs=device_support_table,
+                )
+        # Test Coverage Tab
+        with gr.TabItem("Test Coverage", elem_id="test_coverage", id=7):
+            # Add clear description of what Test Coverage means
+            gr.Markdown(
+                """
+                ## Test Coverage
+                This tab shows **ALL available SKUs** and our testing coverage across the entire device ecosystem. Uses chip-based expansion where testing one device covers all devices with the same chip.
+                """,
+                elem_classes="markdown-text"
+            )
+            def load_coverage_data(release):
+                """Load test coverage data for a specific release."""
+                try:
+                    with open(f"dashboard_data/test_coverage_{release}.json", "r") as f:
+                        return json.load(f)
+                except FileNotFoundError:
+                    return {
+                        "commit_hash": release,
+                        "total_devices": 0,
+                        "tested_devices": 0,
+                        "skipped_devices": 0,
+                        "coverage_percentage": 0.0,
+                        "tested_device_list": [],
+                        "skipped_device_list": [],
+                        "tested_os_versions": [],
+                        "has_target_os_coverage": False,
+                        "covered_target_versions": [],
+                        "missing_target_versions": [],
+                    }
+            def format_coverage_devices(device_list):
+                """Convert device list to DataFrame format."""
+                if not device_list:
+                    return pd.DataFrame(columns=["Device"])
+                df = pd.DataFrame({"Device": device_list})
+                return df.sort_values(["Device"])
+            def update_coverage_data(release):
+                """Update coverage data when release changes."""
+                coverage_data = load_coverage_data(release)
+                # Format tested and skipped devices
+                tested_df = format_coverage_devices(coverage_data["tested_device_list"])
+                skipped_df = format_coverage_devices(
+                    coverage_data["skipped_device_list"]
+                )
+                # Check target OS coverage
+                target_os_status = ""
+                covered_versions = coverage_data.get("covered_target_versions", [])
+                missing_versions = coverage_data.get("missing_target_versions", [])
+                if covered_versions or missing_versions:
+                    target_os_status = "\n- **Target OS Coverage**:\n"
+                    if covered_versions:
+                        unique_versions = sorted(set(covered_versions))
+                        target_os_status += f"  - ✅ **Tested**: {', '.join(unique_versions)}\n"
+                    if missing_versions:
+                        target_os_status += f"  - ❌ **Missing**: {', '.join(missing_versions)}"
+                # Create coverage summary
+                coverage_summary = f"""## Test Coverage Summary for Release {release} (v{SHA_TO_VERSION.get(release, 'Unknown')})
+- **Total Devices**: {coverage_data['total_devices']}
+- **Tested Devices**: {coverage_data['tested_devices']}
+- **Skipped Devices**: {coverage_data['skipped_devices']}
+- **Coverage Percentage**: {coverage_data['coverage_percentage']:.1f}%
+{target_os_status}"""
+                return (
+                    gr.update(value=coverage_summary),
+                    gr.update(value=tested_df),
+                    gr.update(value=skipped_df),
+                    tested_df,
+                    skipped_df,
+                )
+            def filter_coverage_devices(df, device_query, exclude_devices):
+                """Filter coverage devices based on device queries."""
+                if df is None or df.empty:
+                    return df
+                filtered_df = df.copy()
+                # Filter devices based on query
+                if device_query:
+                    filtered_df = filtered_df[
+                        filtered_df["Device"].str.contains(
+                            "|".join(q.strip() for q in device_query.split(";")),
+                            case=False,
+                            regex=True,
+                        )
+                    ]
+                # Exclude specified devices
+                if exclude_devices:
+                    exclude_list = [
+                        re.escape(d.strip()) for d in exclude_devices.split(";")
+                    ]
+                    filtered_df = filtered_df[
+                        ~filtered_df["Device"].str.contains(
+                            "|".join(exclude_list), case=False, regex=True
+                        )
+                    ]
+                return filtered_df
+            # Load initial coverage data
+            initial_coverage = load_coverage_data(releases[-1])
+            initial_tested_df = format_coverage_devices(
+                initial_coverage["tested_device_list"]
+            )
+            initial_skipped_df = format_coverage_devices(
+                initial_coverage["skipped_device_list"]
+            )
+            # Generate initial target OS status
+            initial_target_os_status = ""
+            covered_versions = initial_coverage.get("covered_target_versions", [])
+            missing_versions = initial_coverage.get("missing_target_versions", [])
+            if covered_versions or missing_versions:
+                initial_target_os_status = "\n- **Target OS Coverage**:\n"
+                if covered_versions:
+                    unique_versions = sorted(set(covered_versions))
+                    initial_target_os_status += f"  - ✅ **Tested**: {', '.join(unique_versions)}\n"
+                if missing_versions:
+                    initial_target_os_status += f"  - ❌ **Missing**: {', '.join(missing_versions)}"
+            # Create initial coverage summary content
+            initial_summary_content = f"""## Test Coverage Summary for Release {releases[-1]} (v{SHA_TO_VERSION.get(releases[-1], 'Unknown')})
+- **Total Devices**: {initial_coverage['total_devices']}
+- **Tested Devices**: {initial_coverage['tested_devices']}
+- **Skipped Devices**: {initial_coverage['skipped_devices']}
+- **Coverage Percentage**: {initial_coverage['coverage_percentage']:.1f}%
+{initial_target_os_status}"""
+            # Coverage summary
+            coverage_summary_text = gr.Markdown(
+                value=initial_summary_content,
+                elem_classes="markdown-text"
+            )
+            with gr.Row():
+                with gr.Column(scale=1):
+                    with gr.Row():
+                        with gr.Column(scale=6):
+                            filter_coverage_devices_input = gr.Textbox(
+                                placeholder="🔍 Filter Device (separate multiple queries with ';')",
+                                label="Filter Devices",
+                            )
+                        with gr.Column(scale=4):
+                            exclude_coverage_devices_input = gr.Textbox(
+                                placeholder="🔍 Exclude Device",
+                                label="Exclude Device",
+                            )
+            # Create tabs for tested vs skipped devices
+            with gr.Tabs():
+                with gr.TabItem("Tested Devices", id=0):
+                    tested_devices_table = gr.Dataframe(
+                        value=initial_tested_df,
+                        headers=["Device"],
+                        datatype=["str"],
+                        elem_id="tested-devices-table",
+                        elem_classes="large-table",
+                        interactive=False,
+                    )
+                with gr.TabItem("Skipped Devices", id=1):
+                    skipped_devices_table = gr.Dataframe(
+                        value=initial_skipped_df,
+                        headers=["Device"],
+                        datatype=["str"],
+                        elem_id="skipped-devices-table",
+                        elem_classes="large-table",
+                        interactive=False,
+                    )
+            # Hidden dataframes for filtering
+            hidden_tested_df = gr.Dataframe(value=initial_tested_df, visible=False)
+            hidden_skipped_df = gr.Dataframe(value=initial_skipped_df, visible=False)
+            # Connect release dropdown to coverage data update
+            release_dropdown.change(
+                update_coverage_data,
+                inputs=[release_dropdown],
+                outputs=[
+                    coverage_summary_text,
+                    tested_devices_table,
+                    skipped_devices_table,
+                    hidden_tested_df,
+                    hidden_skipped_df,
+                ],
+                queue=False,
+            )
+            # Connect filter inputs to update both tables
+            for input_elem in [
+                filter_coverage_devices_input,
+                exclude_coverage_devices_input,
+            ]:
+                input_elem.change(
+                    lambda tested_df, skipped_df, device_query, exclude_devices: (
+                        filter_coverage_devices(
+                            tested_df, device_query, exclude_devices
+                        ),
+                        filter_coverage_devices(
+                            skipped_df, device_query, exclude_devices
+                        ),
+                    ),
+                    inputs=[
+                        hidden_tested_df,
+                        hidden_skipped_df,
+                        filter_coverage_devices_input,
+                        exclude_coverage_devices_input,
+                    ],
+                    outputs=[tested_devices_table, skipped_devices_table],
                 )
         # Methodology Tab
+        with gr.TabItem("Methodology", elem_id="methodology", id=8):
             gr.Markdown(METHODOLOGY_TEXT, elem_id="methodology-text")
     # Citation section
         )
 # Launch the Gradio interface
+demo.launch(debug=True, share=True)

multilingual_generate.py DELETED Viewed

@@ -1,133 +0,0 @@
-import json
-import os
-import shutil
-import sys
-from collections import defaultdict
-import numpy as np
-import pandas as pd
-from sklearn.metrics import confusion_matrix
-from utils import compute_average_wer, download_dataset
-def main():
-    """
-    Main function to orchestrate the multilingual data generation process.
-    This function performs the following steps:
-    1. Downloads multilingual evaluation data if requested.
-    2. Processes multilingual evaluation files.
-    3. Calculates and saves results, including Word Error Rate (WER) and
-       language detection confusion matrices.
-    """
-    source_repo = "argmaxinc/whisperkit-evals-multilingual"
-    source_subfolder = "WhisperKit"
-    source_directory = f"{source_repo}/{source_subfolder}"
-    if len(sys.argv) > 1 and sys.argv[1] == "download":
-        try:
-            shutil.rmtree(source_repo)
-        except:
-            print("Nothing to remove.")
-        download_dataset(source_repo, source_repo, source_subfolder)
-    results = defaultdict(
-        lambda: {
-            "average_wer": [],
-            "language_wer": defaultdict(list),
-            "language_detection": [],
-        }
-    )
-    confusion_matrices = {}
-    for subdir, _, files in os.walk(source_directory):
-        for filename in files:
-            if not filename.endswith(".json") or "summary" in filename:
-                continue
-            file_path = os.path.join(subdir, filename)
-            with open(file_path, "r") as f:
-                data = json.load(f)
-            subdir_components = subdir.split(os.path.sep)
-            is_forced = "forced" in subdir_components
-            model = subdir_components[-3] if not is_forced else subdir_components[-4]
-            key = f"{model}/{'forced' if is_forced else 'not_forced'}"
-            for item in data["results"]:
-                if "reference_language" not in item:
-                    continue
-                reference_language = item["reference_language"]
-                wer = item["wer"]
-                detected_language = item["predicted_language"]
-                result = {
-                    "reference": item["reference"],
-                    "prediction": item["prediction"],
-                }
-                results[key]["average_wer"].append(result)
-                results[key]["language_wer"][reference_language].append(result)
-                results[key]["language_detection"].append(
-                    (reference_language, detected_language)
-                )
-    calculate_and_save_results(results, confusion_matrices)
-def calculate_and_save_results(results, confusion_matrices):
-    """
-    Calculates final multilingual metrics and saves them to CSV and JSON files.
-    :param results: Dictionary containing raw multilingual evaluation data.
-    :param confusion_matrices: Dictionary to store confusion matrices for language detection.
-    This function processes the raw multilingual data, calculates average metrics,
-    creates confusion matrices for language detection, and saves the results to:
-    1. A CSV file with WER data for each model and language.
-    2. A JSON file with confusion matrices for language detection.
-    """
-    wer_data = []
-    for key, data in results.items():
-        model, forced = key.rsplit("/", 1)
-        model = model.replace("_", "/")
-        row = {
-            "Model": model,
-            "Forced Tokens": forced == "forced",
-            "Average WER": compute_average_wer(data["average_wer"]),
-        }
-        for lang, wers in data["language_wer"].items():
-            row[f"WER_{lang}"] = compute_average_wer(wers)
-        wer_data.append(row)
-        true_languages, detected_languages = zip(*data["language_detection"])
-        unique_languages = sorted(set(true_languages))
-        cm = confusion_matrix(
-            true_languages, detected_languages, labels=unique_languages
-        )
-        row_sums = cm.sum(axis=1)
-        cm_normalized = np.zeros_like(cm, dtype=float)
-        non_zero_rows = row_sums != 0
-        cm_normalized[non_zero_rows] = (
-            cm[non_zero_rows] / row_sums[non_zero_rows, np.newaxis]
-        )
-        if model not in confusion_matrices:
-            confusion_matrices[model] = {}
-        confusion_matrices[model][forced] = {
-            "matrix": cm_normalized.tolist(),
-            "labels": unique_languages,
-        }
-    df = pd.DataFrame(wer_data)
-    df.to_csv("dashboard_data/multilingual_results.csv", index=False)
-    with open("dashboard_data/multilingual_confusion_matrices.json", "w") as f:
-        json.dump(confusion_matrices, f, indent=2)
-if __name__ == "__main__":
-    main()

performance_generate.py CHANGED Viewed

@@ -10,10 +10,12 @@ from statistics import mean
 import pandas as pd
 import requests
 from constants import BASE_WHISPERKIT_BENCHMARK_URL
 from text_normalizer import text_normalizer
-from utils import compute_average_wer, download_dataset
 def fetch_evaluation_data(url):
@@ -97,7 +99,7 @@ def process_benchmark_file(file_path, dataset_dfs, results, releases):
     if len(test_results) == 0:
         return
     commit_hash_timestamp = file_path.split("/")[-2]
     commit_timestamp, commit_hash = commit_hash_timestamp.split("_")
@@ -131,6 +133,7 @@ def process_benchmark_file(file_path, dataset_dfs, results, releases):
         }
         results[key]["timestamp"] = timestamp
         results[key]["average_wer"].append(wer_entry)
         input_audio_seconds = test_info["timings"]["inputAudioSeconds"]
         full_pipeline = test_info["timings"]["fullPipeline"]
@@ -181,7 +184,7 @@ def process_summary_file(file_path, results, releases):
     """
     with open(file_path, "r") as file:
         summary_data = json.load(file)
     if summary_data["commitHash"] not in releases:
         return
@@ -191,13 +194,15 @@ def process_summary_file(file_path, results, releases):
     commit_timestamp = summary_data["commitTimestamp"]
     test_file_name = file_path.split("/")[-1]
     test_timestamp = test_file_name.split("_")[-1].replace(".json", "")
     key = (device, os, commit_hash)
     if key in results:
         existing_commit_timestamp = results[key]["commitTimestamp"]
         existing_test_timestamp = results[key]["testTimestamp"]
-        existing_commit_dt = datetime.strptime(existing_commit_timestamp, "%Y-%m-%dT%H%M%S")
         new_commit_dt = datetime.strptime(commit_timestamp, "%Y-%m-%dT%H%M%S")
         existing_test_dt = datetime.strptime(existing_test_timestamp, "%Y-%m-%dT%H%M%S")
         new_test_dt = datetime.strptime(test_timestamp, "%Y-%m-%dT%H%M%S")
@@ -283,6 +288,10 @@ def calculate_and_save_performance_results(
                     for dataset, tps_info in data["dataset_tokens_per_second"].items()
                 },
                 "average_wer": compute_average_wer(data["average_wer"]),
                 "qoi": round(mean(data["qoi"]), 2),
                 "commit_hash": data["commit_hash"],
                 "commit_timestamp": data["commit_timestamp"],
@@ -320,7 +329,7 @@ def calculate_and_save_support_results(
             results_by_commit[commit_hash] = {
                 "data": {},
                 "devices": set(),
-                "timestamp": data["commitTimestamp"]
             }
         results_by_commit[commit_hash]["data"][key] = data
         results_by_commit[commit_hash]["devices"].add(device)
@@ -329,7 +338,7 @@ def calculate_and_save_support_results(
     for commit_hash, commit_data in results_by_commit.items():
         commit_devices = sorted(commit_data["devices"])
         df = pd.DataFrame(index=all_models, columns=["Model"] + commit_devices)
         for model in all_models:
             row = {"Model": model}
             for device in commit_devices:
@@ -370,24 +379,27 @@ def calculate_and_save_support_results(
         # Mark unsupported combinations for this commit
         commit_not_supported = [
-            (model, device, os)
-            for model, device, os in not_supported
-            if any(key[2] == commit_hash for key in support_results if key not in ["modelsTested", "devices"] and model == key[0])
         ]
         remove_unsupported_cells(df, commit_not_supported)
         # Format column headers
         cols = df.columns.tolist()
         cols = ["Model"] + [
-            f"""{get_device_name(col).replace("_", " ")} ({col})""" for col in cols if col != "Model"
         ]
         df.columns = cols
         # Save to commit-specific file
-        output_path = support_output_path.replace(
-            ".csv",
-            f"_{commit_hash[:7]}.csv"
-        )
         df.to_csv(output_path, index=True)
@@ -421,6 +433,462 @@ def remove_unsupported_cells(df, not_supported):
         df.at[model, device] = "Not Supported"
 def main():
     """
     Main function to orchestrate the performance data generation process.
@@ -430,6 +898,7 @@ def main():
     2. Fetches evaluation data for various datasets.
     3. Processes benchmark files and summary files.
     4. Calculates and saves performance and support results.
     """
     source_xcresult_repo = "argmaxinc/whisperkit-evals-dataset"
     source_xcresult_subfolder = "benchmark_data/"
@@ -460,6 +929,7 @@ def main():
     performance_results = defaultdict(
         lambda: {
             "average_wer": [],
             "qoi": [],
             "speed": {"inputAudioSeconds": 0, "fullPipeline": 0},
             "tokens_per_second": {"totalDecodingLoops": 0, "fullPipeline": 0},
@@ -492,8 +962,10 @@ def main():
             elif "summary" in filename:
                 process_summary_file(file_path, support_results, releases)
             else:
-                process_benchmark_file(file_path, dataset_dfs, performance_results, releases)
     not_supported = calculate_and_save_performance_results(
         performance_results, "dashboard_data/performance_data.json"
     )
@@ -501,6 +973,9 @@ def main():
         support_results, not_supported, "dashboard_data/support_data.csv"
     )
 if __name__ == "__main__":
     main()

 import pandas as pd
 import requests
+from dotenv import load_dotenv
+from huggingface_hub import login
 from constants import BASE_WHISPERKIT_BENCHMARK_URL
 from text_normalizer import text_normalizer
+from utils import compute_average_wer, download_dataset, download_json_from_github
 def fetch_evaluation_data(url):
     if len(test_results) == 0:
         return
     commit_hash_timestamp = file_path.split("/")[-2]
     commit_timestamp, commit_hash = commit_hash_timestamp.split("_")
         }
         results[key]["timestamp"] = timestamp
         results[key]["average_wer"].append(wer_entry)
+        results[key]["dataset_wer"][dataset_name].append(wer_entry)
         input_audio_seconds = test_info["timings"]["inputAudioSeconds"]
         full_pipeline = test_info["timings"]["fullPipeline"]
     """
     with open(file_path, "r") as file:
         summary_data = json.load(file)
     if summary_data["commitHash"] not in releases:
         return
     commit_timestamp = summary_data["commitTimestamp"]
     test_file_name = file_path.split("/")[-1]
     test_timestamp = test_file_name.split("_")[-1].replace(".json", "")
     key = (device, os, commit_hash)
     if key in results:
         existing_commit_timestamp = results[key]["commitTimestamp"]
         existing_test_timestamp = results[key]["testTimestamp"]
+        existing_commit_dt = datetime.strptime(
+            existing_commit_timestamp, "%Y-%m-%dT%H%M%S"
+        )
         new_commit_dt = datetime.strptime(commit_timestamp, "%Y-%m-%dT%H%M%S")
         existing_test_dt = datetime.strptime(existing_test_timestamp, "%Y-%m-%dT%H%M%S")
         new_test_dt = datetime.strptime(test_timestamp, "%Y-%m-%dT%H%M%S")
                     for dataset, tps_info in data["dataset_tokens_per_second"].items()
                 },
                 "average_wer": compute_average_wer(data["average_wer"]),
+                "dataset_average_wer": {
+                    dataset: compute_average_wer(data["dataset_wer"][dataset])
+                    for dataset in data["dataset_wer"]
+                },
                 "qoi": round(mean(data["qoi"]), 2),
                 "commit_hash": data["commit_hash"],
                 "commit_timestamp": data["commit_timestamp"],
             results_by_commit[commit_hash] = {
                 "data": {},
                 "devices": set(),
+                "timestamp": data["commitTimestamp"],
             }
         results_by_commit[commit_hash]["data"][key] = data
         results_by_commit[commit_hash]["devices"].add(device)
     for commit_hash, commit_data in results_by_commit.items():
         commit_devices = sorted(commit_data["devices"])
         df = pd.DataFrame(index=all_models, columns=["Model"] + commit_devices)
         for model in all_models:
             row = {"Model": model}
             for device in commit_devices:
         # Mark unsupported combinations for this commit
         commit_not_supported = [
+            (model, device, os)
+            for model, device, os in not_supported
+            if any(
+                key[2] == commit_hash
+                for key in support_results
+                if key not in ["modelsTested", "devices"] and model == key[0]
+            )
         ]
         remove_unsupported_cells(df, commit_not_supported)
         # Format column headers
         cols = df.columns.tolist()
         cols = ["Model"] + [
+            f"""{get_device_name(col).replace("_", " ")} ({col})"""
+            for col in cols
+            if col != "Model"
         ]
         df.columns = cols
         # Save to commit-specific file
+        output_path = support_output_path.replace(".csv", f"_{commit_hash[:7]}.csv")
         df.to_csv(output_path, index=True)
         df.at[model, device] = "Not Supported"
+def download_device_json_safe(file_path):
+    """
+    Safely downloads a device JSON file from GitHub, returning None if it doesn't exist.
+    :param file_path: Path to the JSON file within the repository
+    :returns: The JSON data as a dictionary, or None if the file doesn't exist
+    """
+    try:
+        return download_json_from_github(file_path=file_path)
+    except SystemExit:
+        # File doesn't exist or other error occurred
+        return None
+def load_device_json_local(file_path):
+    """
+    Safely loads a local device JSON file, returning None if it doesn't exist.
+    :param file_path: Local path to the JSON file
+    :returns: The JSON data as a dictionary, or None if the file doesn't exist
+    """
+    try:
+        with open(file_path, "r") as f:
+            return json.load(f)
+    except (FileNotFoundError, json.JSONDecodeError):
+        return None
+def build_chip_mapping():
+    """
+    Builds a mapping from device SKUs to their chip types.
+    :returns: Dictionary where keys are device SKUs and values are chip types
+    """
+    sku_to_chip = {}
+    # Load iPad devices
+    ipad_data = load_device_json_local("dashboard_data/iPad.json")
+    if ipad_data and "total_menu" in ipad_data:
+        for device_info in ipad_data["total_menu"].values():
+            if "sku" in device_info and "chip" in device_info:
+                # iPad has sku as an array
+                skus = device_info["sku"] if isinstance(device_info["sku"], list) else [device_info["sku"]]
+                for sku in skus:
+                    sku_to_chip[sku] = device_info["chip"]
+    # Load iPhone devices
+    iphone_data = load_device_json_local("dashboard_data/iPhone.json")
+    if iphone_data and "total_menu" in iphone_data:
+        for device_info in iphone_data["total_menu"].values():
+            if "sku" in device_info and "chip" in device_info:
+                # iPhone has sku as a single string
+                sku_to_chip[device_info["sku"]] = device_info["chip"]
+    # Load Mac devices
+    mac_data = load_device_json_local("dashboard_data/Mac.json")
+    if mac_data and "total_menu" in mac_data:
+        for device_info in mac_data["total_menu"].values():
+            if "sku" in device_info and "chip" in device_info:
+                # Mac has sku as a single string
+                sku_to_chip[device_info["sku"]] = device_info["chip"]
+    return sku_to_chip
+def get_platform_from_sku(sku):
+    """
+    Determines the platform (iPad, iPhone, Mac) from a device SKU.
+    :param sku: Device SKU string
+    :returns: Platform string ('iPad', 'iPhone', 'Mac') or 'Unknown'
+    """
+    if sku.startswith("iPad"):
+        return "iPad"
+    elif sku.startswith("iPhone"):
+        return "iPhone"
+    elif sku.startswith("Mac") or sku.startswith("iMac") or sku.startswith("MacBook"):
+        return "Mac"
+    else:
+        return "Unknown"
+def normalize_chip_name(chip):
+    """
+    Normalizes chip names for consistent grouping.
+    :param chip: Raw chip name from device JSON
+    :returns: Normalized chip name
+    """
+    # Handle variations like "A18 Pro" -> "A18", "M4 Pro" -> "M4", etc.
+    # But keep distinct generations separate
+    chip = chip.strip()
+    # For A-series chips, keep Pro variants separate as they have different capabilities
+    if chip.startswith("A") and "Pro" in chip:
+        return chip  # Keep A17 Pro separate from A17
+    # For M-series chips, group Pro/Max/Ultra variants together as they're same generation
+    if chip.startswith("M"):
+        # Extract just the M number (M1, M2, M3, M4)
+        parts = chip.split()
+        if len(parts) > 0:
+            return parts[0]  # Return just "M1", "M2", etc.
+    return chip
+def build_sku_group_mapping():
+    """
+    Builds a mapping from individual SKUs to all SKUs that share the same chip on the same platform.
+    This implements chip-based coverage where testing one device with a specific chip
+    provides coverage for all devices with that chip on the same platform.
+    :returns: Dictionary where keys are individual SKUs and values are sets of all SKUs in that chip group
+    """
+    sku_to_chip = build_chip_mapping()
+    sku_to_group = {}
+    # Group SKUs by platform and normalized chip
+    platform_chip_groups = {}
+    for sku, chip in sku_to_chip.items():
+        platform = get_platform_from_sku(sku)
+        normalized_chip = normalize_chip_name(chip)
+        key = (platform, normalized_chip)
+        if key not in platform_chip_groups:
+            platform_chip_groups[key] = set()
+        platform_chip_groups[key].add(sku)
+    # Create reverse mapping: each SKU maps to all SKUs in its chip group
+    for sku, chip in sku_to_chip.items():
+        platform = get_platform_from_sku(sku)
+        normalized_chip = normalize_chip_name(chip)
+        key = (platform, normalized_chip)
+        sku_to_group[sku] = platform_chip_groups[key]
+    return sku_to_group
+def expand_tested_devices(tested_devices, sku_mapping):
+    """
+    Expands tested devices to include all SKUs in the same group.
+    :param tested_devices: Set of device SKUs that were actually tested
+    :param sku_mapping: Dictionary mapping individual SKUs to their complete groups
+    :returns: Expanded set of devices including all SKUs in the same groups
+    """
+    expanded_devices = set(tested_devices)
+    for device in tested_devices:
+        if device in sku_mapping:
+            # Add all SKUs from the same group
+            expanded_devices.update(sku_mapping[device])
+    return expanded_devices
+def get_test_iphones():
+    """
+    Gets iPhone SKU identifiers from the local iPhone.json file.
+    """
+    iphone_data = load_device_json_local("dashboard_data/iPhone.json")
+    if iphone_data and "total_menu" in iphone_data:
+        return set([device_info["sku"] for device_info in iphone_data["total_menu"].values() if "sku" in device_info])
+    return set()
+def get_test_macs():
+    """
+    Gets Mac SKU identifiers from the local Mac.json file.
+    """
+    mac_data = load_device_json_local("dashboard_data/Mac.json")
+    if mac_data and "total_menu" in mac_data:
+        return set([device_info["sku"] for device_info in mac_data["total_menu"].values() if "sku" in device_info])
+    return set()
+def get_all_supported_devices():
+    """
+    Gets all supported device identifiers from the config file.
+    Returns a set of device identifiers.
+    """
+    with open("dashboard_data/config.json", "r") as f:
+        config = json.load(f)
+    devices = set()
+    for device_group in config["device_support"]:
+        identifiers = device_group["identifiers"]
+        devices.update(identifiers)
+    return devices
+def get_tested_devices_for_commit(performance_results, support_results, commit_hash):
+    """
+    Gets all device identifiers that were actually tested for a specific commit,
+    including all SKUs in the same chip groups as tested devices.
+    Uses chip-based coverage logic where testing one device with a specific chip
+    provides coverage for all devices with that chip on the same platform.
+    Returns a set of device identifiers.
+    """
+    tested_devices = set()
+    # From performance results (benchmark files)
+    for key, result in performance_results.items():
+        if len(key) >= 4 and result.get("commit_hash") == commit_hash:
+            model, device, _, _ = key
+            tested_devices.add(device)
+    # From support results (summary files)
+    for key, result in support_results.items():
+        if key in ["modelsTested", "devices"]:
+            continue
+        if len(key) >= 3 and result.get("commitHash") == commit_hash:
+            device, _, _ = key
+            tested_devices.add(device)
+    # Expand to include all SKUs in the same chip groups for all platforms
+    sku_mapping = build_sku_group_mapping()
+    expanded_devices = expand_tested_devices(tested_devices, sku_mapping)
+    return expanded_devices
+def get_tested_os_versions_for_commit(
+    performance_results, support_results, commit_hash
+):
+    """
+    Gets all OS versions that were actually tested for a specific commit.
+    Returns a set of OS version strings like 'iOS_17.2', 'macOS_14.5', etc.
+    """
+    tested_os_versions = set()
+    # From performance results (benchmark files)
+    for key, result in performance_results.items():
+        if len(key) >= 4 and result.get("commit_hash") == commit_hash:
+            model, device, os_info, _ = key
+            tested_os_versions.add(os_info)
+    # From support results (summary files)
+    for key, result in support_results.items():
+        if key in ["modelsTested", "devices"]:
+            continue
+        if len(key) >= 3 and result.get("commitHash") == commit_hash:
+            device, os, _ = key
+            # Convert format like "iOS 17.2" to "iOS_17.2" for consistency
+            os_normalized = os.replace(" ", "_")
+            tested_os_versions.add(os_normalized)
+    return tested_os_versions
+def check_target_os_coverage(tested_os_versions):
+    """
+    Check if the tested OS versions include ALL of the target OS versions:
+    - macOS 14, 15, 26
+    - iOS 17, 18, 26 (noting that iOS and iPadOS are the same under the hood)
+    Returns (is_fully_covered: bool, covered_versions: list, missing_versions: list)
+    """
+    target_macos_versions = {14, 15, 26}
+    target_ios_versions = {17, 18, 26}
+    covered_macos = set()
+    covered_ios = set()
+    for os_version in tested_os_versions:
+        # Parse OS version string like "iOS_17.2" or "macOS_14.5"
+        if "_" in os_version:
+            os_type, version_str = os_version.split("_", 1)
+            try:
+                # Extract major version number
+                major_version = int(version_str.split(".")[0])
+                if os_type == "macOS" and major_version in target_macos_versions:
+                    covered_macos.add(major_version)
+                elif (
+                    os_type in ["iOS", "iPadOS"]
+                    and major_version in target_ios_versions
+                ):
+                    covered_ios.add(major_version)
+            except (ValueError, IndexError):
+                # Skip if we can't parse the version
+                continue
+    # Check what's missing
+    missing_macos = target_macos_versions - covered_macos
+    missing_ios = target_ios_versions - covered_ios
+    # Format covered and missing versions
+    covered_versions = []
+    covered_versions.extend([f"macOS {v}" for v in sorted(covered_macos)])
+    covered_versions.extend([f"iOS {v}" for v in sorted(covered_ios)])
+    missing_versions = []
+    missing_versions.extend([f"macOS {v}" for v in sorted(missing_macos)])
+    missing_versions.extend([f"iOS {v}" for v in sorted(missing_ios)])
+    # Only fully covered if no missing versions
+    is_fully_covered = len(missing_versions) == 0
+    return is_fully_covered, covered_versions, missing_versions
+def check_chip_coverage(tested_devices):
+    """
+    Check if the tested devices provide complete chip coverage for each platform.
+    Target coverage:
+    - iPad: A14, A15, A16, A17 Pro, M1, M2, M3, M4
+    - iPhone: A14, A15, A16, A17 Pro, A18, A18 Pro
+    - Mac: M1, M2, M3, M4
+    :param tested_devices: Set of device SKUs that were tested
+    :returns: (is_fully_covered: bool, platform_coverage: dict, missing_chips: dict)
+    """
+    # Define target chips for each platform
+    target_chips = {
+        "iPad": {"A14", "A15", "A16", "A17 Pro", "M1", "M2", "M3", "M4"},
+        "iPhone": {"A14", "A15", "A16", "A17 Pro", "A18", "A18 Pro"},
+        "Mac": {"M1", "M2", "M3", "M4"}
+    }
+    # Build mapping from SKUs to chips
+    sku_to_chip = build_chip_mapping()
+    # Track which chips were tested for each platform
+    tested_chips = {
+        "iPad": set(),
+        "iPhone": set(),
+        "Mac": set()
+    }
+    for device_sku in tested_devices:
+        if device_sku in sku_to_chip:
+            platform = get_platform_from_sku(device_sku)
+            chip = normalize_chip_name(sku_to_chip[device_sku])
+            if platform in tested_chips:
+                tested_chips[platform].add(chip)
+    # Calculate coverage for each platform
+    platform_coverage = {}
+    missing_chips = {}
+    for platform, target_set in target_chips.items():
+        covered_set = tested_chips[platform]
+        missing_set = target_set - covered_set
+        platform_coverage[platform] = {
+            "total_chips": len(target_set),
+            "tested_chips": len(covered_set),
+            "coverage_percentage": (len(covered_set) / len(target_set)) * 100 if target_set else 0,
+            "covered_chips": sorted(list(covered_set)),
+            "missing_chips": sorted(list(missing_set))
+        }
+        missing_chips[platform] = sorted(list(missing_set))
+    # Overall coverage is complete if all platforms have full coverage
+    is_fully_covered = all(
+        len(missing_chips[platform]) == 0
+        for platform in target_chips.keys()
+    )
+    return is_fully_covered, platform_coverage, missing_chips
+def generate_test_coverage_report(
+    performance_results, support_results, output_dir="dashboard_data"
+):
+    """
+    Generates test coverage reports for each commit, showing which devices
+    were tested vs skipped.
+    """
+    # Get all possible devices from config
+    all_devices = get_all_supported_devices()
+    # Get all unique commit hashes from results
+    commit_hashes = set()
+    # Collect from performance results
+    for key, result in performance_results.items():
+        if len(key) >= 4 and result.get("commit_hash"):
+            commit_hashes.add(result["commit_hash"])
+    # Collect from support results
+    for key, result in support_results.items():
+        if key in ["modelsTested", "devices"]:
+            continue
+        if len(key) >= 3 and result.get("commitHash"):
+            commit_hashes.add(result["commitHash"])
+    print(f"Found {len(commit_hashes)} commit hashes to analyze")
+    # Generate coverage report for each commit
+    for commit_hash in commit_hashes:
+        tested_devices = get_tested_devices_for_commit(
+            performance_results, support_results, commit_hash
+        )
+        tested_os_versions = get_tested_os_versions_for_commit(
+            performance_results, support_results, commit_hash
+        )
+        # Check target OS coverage
+        os_fully_covered, covered_versions, missing_versions = check_target_os_coverage(
+            tested_os_versions
+        )
+        # Check chip coverage for all platforms
+        chip_fully_covered, platform_coverage, missing_chips = check_chip_coverage(
+            tested_devices
+        )
+        skipped_devices = all_devices - tested_devices
+        # Convert sets to lists for JSON serialization
+        tested_devices_list = list(tested_devices)
+        skipped_devices_list = list(skipped_devices)
+        tested_os_versions_list = list(tested_os_versions)
+        coverage_report = {
+            "commit_hash": commit_hash,
+            "total_devices": len(all_devices),
+            "tested_devices": len(tested_devices),
+            "skipped_devices": len(skipped_devices),
+            "coverage_percentage": (len(tested_devices) / len(all_devices)) * 100,
+            "tested_device_list": tested_devices_list,
+            "skipped_device_list": skipped_devices_list,
+            "tested_os_versions": tested_os_versions_list,
+            "has_target_os_coverage": os_fully_covered,
+            "covered_target_versions": covered_versions,
+            "missing_target_versions": missing_versions,
+            "has_target_chip_coverage": chip_fully_covered,
+            "platform_chip_coverage": platform_coverage,
+            "missing_target_chips": missing_chips,
+        }
+        # Save report for this commit
+        output_file = os.path.join(output_dir, f"test_coverage_{commit_hash}.json")
+        with open(output_file, "w") as f:
+            json.dump(coverage_report, f, indent=2)
+        os_coverage_info = f"OS coverage: {'✅ Complete' if os_fully_covered else '❌ Incomplete'}"
+        chip_coverage_info = f"Chip coverage: {'✅ Complete' if chip_fully_covered else '❌ Incomplete'}"
+        print(
+            f"Generated coverage report for commit {commit_hash}: "
+            f"{len(tested_devices)}/{len(all_devices)} devices tested "
+            f"({coverage_report['coverage_percentage']:.1f}%) "
+            f"({os_coverage_info}, {chip_coverage_info})"
+        )
 def main():
     """
     Main function to orchestrate the performance data generation process.
     2. Fetches evaluation data for various datasets.
     3. Processes benchmark files and summary files.
     4. Calculates and saves performance and support results.
+    5. Generates test coverage reports for each commit.
     """
     source_xcresult_repo = "argmaxinc/whisperkit-evals-dataset"
     source_xcresult_subfolder = "benchmark_data/"
     performance_results = defaultdict(
         lambda: {
             "average_wer": [],
+            "dataset_wer": defaultdict(list),
             "qoi": [],
             "speed": {"inputAudioSeconds": 0, "fullPipeline": 0},
             "tokens_per_second": {"totalDecodingLoops": 0, "fullPipeline": 0},
             elif "summary" in filename:
                 process_summary_file(file_path, support_results, releases)
             else:
+                process_benchmark_file(
+                    file_path, dataset_dfs, performance_results, releases
+                )
     not_supported = calculate_and_save_performance_results(
         performance_results, "dashboard_data/performance_data.json"
     )
         support_results, not_supported, "dashboard_data/support_data.csv"
     )
+    # Generate test coverage reports
+    generate_test_coverage_report(performance_results, support_results)
 if __name__ == "__main__":
     main()

quality_generate.py DELETED Viewed

@@ -1,186 +0,0 @@
-import json
-import os
-import shutil
-import sys
-from collections import defaultdict
-from statistics import mean
-import pandas as pd
-import requests
-from text_normalizer import text_normalizer
-from utils import compute_average_wer, download_dataset
-def fetch_evaluation_data(url):
-    """
-    Fetches evaluation data from the given URL.
-    :param url: The URL to fetch the evaluation data from.
-    :returns: The evaluation data as a dictionary.
-    :rauses: sys.exit if the request fails
-    """
-    response = requests.get(url)
-    if response.status_code == 200:
-        return json.loads(response.text)
-    else:
-        sys.exit(f"Failed to fetch WhisperKit evals: {response.text}")
-def get_device_name(device):
-    """
-    Gets the device name from the device map if it exists.
-    :param device: String representing the device name.
-    :returns: The device name from the device map if it exists, otherwise the input device name.
-    """
-    with open("dashboard_data/device_map.json", "r") as f:
-        device_map = json.load(f)
-    return device_map.get(device, device).replace(" ", "_")
-def process_quality_file(file_path, dataset_dfs, quality_results):
-    """
-    Processes a single quality file and updates the quality_results dictionary.
-    :param file_path: Path to the quality JSON file.
-    :param dataset_dfs: Dictionary of DataFrames containing dataset information.
-    :param quality_results: Dictionary to store the processed quality results.
-    This function reads a quality JSON file, extracts relevant information,
-    and updates the quality_results dictionary with various metrics including WER
-    and Quality of Inference (QoI) for different datasets.
-    """
-    with open(file_path, "r") as file:
-        test_results = json.load(file)
-    if len(test_results) == 0:
-        return
-    metadata = test_results["metadata"]
-    test_results = test_results["results"]
-    model = file_path.split("/")[-3].replace("_", "/")
-    device = metadata["inference_context"]["device_spec"]["product_name"]
-    device = get_device_name(device)
-    timestamp = file_path.split("/")[-1].split(".")[0]
-    key = model
-    dataset_name = metadata["dataset_name"]
-    for test_result in test_results:
-        audio_file_name = test_result["file"]
-        dataset_key = "Earnings-22" if "earnings22" in dataset_name else "LibriSpeech"
-        dataset_df = dataset_dfs[dataset_key]
-        wer_entry = {
-            "prediction": text_normalizer(test_result["prediction"]),
-            "reference": text_normalizer(test_result["reference"]),
-        }
-        quality_results[key]["timestamp"] = timestamp
-        quality_results[key]["dataset_wer"][dataset_name].append(wer_entry)
-        audio = audio_file_name.split(".")[0]
-        dataset_row = dataset_df.loc[dataset_df["file"].str.contains(audio)].iloc[0]
-        reference_wer = dataset_row["wer"]
-        prediction_wer = test_result["wer"]
-        quality_results[key]["qoi"].append(1 if prediction_wer <= reference_wer else 0)
-def calculate_and_save_quality_results(quality_results, quality_output_path):
-    """
-    Calculates final quality metrics and saves them to a JSON file.
-    :param quality_results: Dictionary containing raw quality data.
-    :param quality_output_path: Path to save the processed quality results.
-    This function processes the raw quality data, calculates average metrics,
-    and writes the final results to a JSON file, with each entry representing
-    a unique model's quality metrics across different datasets, including
-    Word Error Rate (WER) and Quality of Inference (QoI).
-    """
-    with open(quality_output_path, "w") as quality_file:
-        for key, data in quality_results.items():
-            model = key
-            dataset_wers = {
-                dataset: compute_average_wer(wer)
-                for dataset, wer in data["dataset_wer"].items()
-            }
-            average_wer = (
-                sum(dataset_wers.values()) / len(dataset_wers)
-                if len(dataset_wers) != 0
-                else 0
-            )
-            quality_entry = {
-                "model": model.replace("_", "/"),
-                "timestamp": data["timestamp"],
-                "average_wer": round(average_wer, 2),
-                "dataset_wer": dataset_wers,
-                "qoi": round(mean(data["qoi"]), 2),
-            }
-            json.dump(quality_entry, quality_file)
-            quality_file.write("\n")
-def main():
-    """
-    Main function to orchestrate the quality data generation process.
-    This function performs the following steps:
-    1. Downloads quality data if requested.
-    2. Fetches evaluation data for various datasets.
-    3. Processes quality files for specific datasets.
-    4. Calculates and saves quality results, including WER and QoI metrics.
-    """
-    if len(sys.argv) > 1 and sys.argv[1] == "download":
-        try:
-            shutil.rmtree("english")
-        except:
-            print("Nothing to remove.")
-        download_dataset("argmaxinc/whisperkit-evals", "english", "WhisperKit")
-    datasets = {
-        "Earnings-22": "https://huggingface.co/datasets/argmaxinc/whisperkit-evals/resolve/main/WhisperOpenAIAPI/openai_whisper-large-v2/earnings22/2024-03-04_13%3A39%3A42_GMT-0800.json",
-        "LibriSpeech": "https://huggingface.co/datasets/argmaxinc/whisperkit-evals/resolve/main/WhisperOpenAIAPI/openai_whisper-large-v2/librispeech/2024-02-28_18%3A45%3A02_GMT-0800.json?download=true",
-        "earnings22-10mins": "https://huggingface.co/datasets/argmaxinc/whisperkit-evals/resolve/main/WhisperOpenAIAPI/openai_whisper-large-v2/earnings22/2024-03-04_13%3A39%3A42_GMT-0800.json",
-        "librispeech-10mins": "https://huggingface.co/datasets/argmaxinc/whisperkit-evals/resolve/main/WhisperOpenAIAPI/openai_whisper-large-v2/librispeech/2024-02-28_18%3A45%3A02_GMT-0800.json?download=true",
-        "earnings22-12hours": "https://huggingface.co/datasets/argmaxinc/whisperkit-evals/resolve/main/WhisperOpenAIAPI/openai_whisper-large-v2/earnings22/2024-03-04_13%3A39%3A42_GMT-0800.json",
-        "librispeech": "https://huggingface.co/datasets/argmaxinc/whisperkit-evals/resolve/main/WhisperOpenAIAPI/openai_whisper-large-v2/librispeech/2024-02-28_18%3A45%3A02_GMT-0800.json?download=true",
-    }
-    dataset_dfs = {}
-    for dataset_name, url in datasets.items():
-        evals = fetch_evaluation_data(url)
-        dataset_dfs[dataset_name] = pd.json_normalize(evals["results"])
-    source_quality_directory = "argmaxinc/english/WhisperKit/"
-    quality_results = defaultdict(
-        lambda: {
-            "average_wer": [],
-            "dataset_wer": defaultdict(list),
-            "qoi": [],
-            "timestamp": None,
-        }
-    )
-    for subdir, _, files in os.walk(source_quality_directory):
-        dataset = subdir.split("/")[-1]
-        if dataset not in ["earnings22-12hours", "librispeech"]:
-            continue
-        for filename in files:
-            if not filename.endswith(".json"):
-                continue
-            file_path = os.path.join(subdir, filename)
-            process_quality_file(file_path, dataset_dfs, quality_results)
-    calculate_and_save_quality_results(
-        quality_results, "dashboard_data/quality_data.json"
-    )
-if __name__ == "__main__":
-    main()

utils.py CHANGED Viewed

@@ -2,6 +2,7 @@ import colorsys
 import json
 import os
 import random
 from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass, make_dataclass
 from datetime import datetime
@@ -12,6 +13,7 @@ import evaluate
 import numpy as np
 import pandas as pd
 import plotly.graph_objects as go
 from huggingface_hub import hf_hub_download, list_repo_files
 from pydub import AudioSegment
@@ -84,23 +86,6 @@ def group_wer(group):
     )
-def load_multilingual_results(csv_file):
-    """
-    Load multilingual results from a CSV file into a pandas DataFrame.
-    :param csv_file: Path to the CSV file containing multilingual results
-    :return: DataFrame with the loaded results, or None if the file is not found
-    This function attempts to load a CSV file using pandas, handling potential
-    FileNotFoundError exceptions.
-    """
-    try:
-        df = pd.json_normalize(csv_file)
-        return df
-    except FileNotFoundError:
-        return None
 def download_dataset(repo_id, local_dir, remote_dir, path_includes=""):
     """
     Download benchmark result files from a specified Hugging Face repository to a local directory.
@@ -365,23 +350,6 @@ def make_timestamp_clickable_link(model, dataset, timestamp):
     return f'<div style="color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;" {onclick} href="#">{timestamp}</div>'
-def make_multilingual_model_clickable_link(model):
-    """
-    Creates a clickable link for a multilingual model name.
-    :param model: String representing the model name
-    :return: An HTML string containing a clickable div for the model name
-    This function generates a formatted HTML div that can be used as a clickable
-    element in web interfaces, typically for displaying and interacting with multilingual model names.
-    """
-    elem_id = (
-        f"{model}".replace(" ", "_").replace('"', "").replace("'", "").replace(",", "")
-    )
-    onclick = f"onclick=\"document.getElementById('{elem_id}').click();console.log('hello');\""
-    return f'<div style="color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;" {onclick} href="#">{model}</div>'
 def plot_metric(
     df, y_axis_col, y_axis_title, fig_title, filter_input=None, exclude_input=None
 ):
@@ -399,14 +367,11 @@ def plot_metric(
     with open("dashboard_data/version.json", "r") as f:
         version = json.load(f)
         releases = set(version["releases"])
     df = df[df["commit_hash"].isin(releases)]
     grouped = df.groupby(["model", "device", "os"])
-    sorted_groups = [
-        group.sort_values("commit_timestamp")
-        for _, group in grouped
-    ]
     if filter_input:
         filters = [f.strip().lower() for f in filter_input.split(";")]
@@ -538,6 +503,29 @@ def calculate_parity(m2_ultra_wer, row):
     return None
 def create_initial_performance_column_dict():
     """
     Creates the initial column dictionary for the performance table.
@@ -545,7 +533,7 @@ def create_initial_performance_column_dict():
     :return: A list of column dictionaries
     This function defines the basic structure of the performance table,
-    including columns for model, device, OS, average WER, QoI, speed, and tokens per second.
     """
     return [
         [
@@ -560,10 +548,10 @@ def create_initial_performance_column_dict():
         ],
         ["os", ColumnContent, ColumnContent("OS", "html", True, never_hidden=True)],
         ["english_wer", ColumnContent, ColumnContent("English WER", "html", True)],
-        ["multilingual_wer", ColumnContent, ColumnContent("Multilingual WER", "str", True)],
         ["qoi", ColumnContent, ColumnContent("QoI", "html", False)],
         ["speed", ColumnContent, ColumnContent("Speed", "html", False)],
         ["toks", ColumnContent, ColumnContent("Tok / s", "html", False)],
     ]
@@ -815,6 +803,46 @@ def generate_random_colors(base_colors, num_colors, min_distance=30):
     return generated_colors
 @dataclass
 class Task:
     """

 import json
 import os
 import random
+import sys
 from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass, make_dataclass
 from datetime import datetime
 import numpy as np
 import pandas as pd
 import plotly.graph_objects as go
+import requests
 from huggingface_hub import hf_hub_download, list_repo_files
 from pydub import AudioSegment
     )
 def download_dataset(repo_id, local_dir, remote_dir, path_includes=""):
     """
     Download benchmark result files from a specified Hugging Face repository to a local directory.
     return f'<div style="color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;" {onclick} href="#">{timestamp}</div>'
 def plot_metric(
     df, y_axis_col, y_axis_title, fig_title, filter_input=None, exclude_input=None
 ):
     with open("dashboard_data/version.json", "r") as f:
         version = json.load(f)
         releases = set(version["releases"])
     df = df[df["commit_hash"].isin(releases)]
     grouped = df.groupby(["model", "device", "os"])
+    sorted_groups = [group.sort_values("commit_timestamp") for _, group in grouped]
     if filter_input:
         filters = [f.strip().lower() for f in filter_input.split(";")]
     return None
+def calculate_quality_parity(quality_data, performance_row):
+    """
+    Calculates the parity between measured WER and ground truth WER from quality data.
+    :param quality_data: List of dictionaries containing quality evaluation data
+    :param performance_row: Current performance row being processed
+    :return: WER difference (measured - ground truth) or None if not found
+    This function finds the corresponding ground truth WER for a model and computes
+    the difference between measured and ground truth WER values.
+    """
+    model_name = performance_row.get("model", "")
+    measured_wer = performance_row.get("english_wer", 0)
+    # Find matching model in quality data
+    for quality_entry in quality_data:
+        if quality_entry.get("model", "") == model_name:
+            ground_truth_wer = quality_entry.get("average_wer", 0)
+            return round(measured_wer - ground_truth_wer, 2)
+    return None
 def create_initial_performance_column_dict():
     """
     Creates the initial column dictionary for the performance table.
     :return: A list of column dictionaries
     This function defines the basic structure of the performance table,
+    including columns for model, device, OS, average WER, QoI, speed, tokens per second, and parity.
     """
     return [
         [
         ],
         ["os", ColumnContent, ColumnContent("OS", "html", True, never_hidden=True)],
         ["english_wer", ColumnContent, ColumnContent("English WER", "html", True)],
         ["qoi", ColumnContent, ColumnContent("QoI", "html", False)],
         ["speed", ColumnContent, ColumnContent("Speed", "html", False)],
         ["toks", ColumnContent, ColumnContent("Tok / s", "html", False)],
+        ["parity", ColumnContent, ColumnContent("Parity", "number", False)],
     ]
     return generated_colors
+def download_json_from_github(
+    url=None,
+    repo_owner="argmaxinc",
+    repo_name="device-knowledge-base",
+    branch="main",
+    file_path=None,
+):
+    """
+    Downloads a JSON file from a GitHub repository.
+    :param url: Full URL to the JSON file, or None if constructing from other params
+    :param repo_owner: GitHub repository owner (default: "argmaxinc")
+    :param repo_name: GitHub repository name (default: "WhisperKit")
+    :param branch: Git branch name (default: "main")
+    :param file_path: Path to the JSON file within the repository
+    :returns: The JSON data as a dictionary
+    :raises: sys.exit if the request fails or JSON parsing fails
+    """
+    # If full URL is provided, use it directly
+    if url:
+        target_url = url
+    elif file_path:
+        # Construct GitHub raw URL
+        target_url = f"https://raw.githubusercontent.com/{repo_owner}/{repo_name}/{branch}/{file_path}"
+    else:
+        sys.exit("Either 'url' or 'file_path' must be provided")
+    try:
+        response = requests.get(target_url)
+        if response.status_code == 200:
+            return json.loads(response.text)
+        else:
+            sys.exit(f"Failed to fetch JSON from GitHub: {response.text}")
+    except json.JSONDecodeError as e:
+        sys.exit(f"Failed to parse JSON from {target_url}: {e}")
+    except requests.exceptions.RequestException as e:
+        sys.exit(f"Network error fetching {target_url}: {e}")
 @dataclass
 class Task:
     """