Spaces:

argmaxinc
/

whisperkit-benchmarks

Running

App Files Files Community

ardaatahan commited on 6 days ago

Commit

d3a1d6c

1 Parent(s): 00dd4ff

Add parity, test coverage and distinguish between device support and test coverage

Browse files

Files changed (24) hide show

.github/scripts/check_dataset_update.py +15 -10
.github/scripts/process_report.py +15 -5
Makefile +0 -1
README.md +1 -1
constants.py +4 -2
dashboard_data/Mac.json +241 -0
dashboard_data/config.json +222 -39
dashboard_data/device_map.json +3 -0
dashboard_data/iPad.json +206 -0
dashboard_data/iPhone.json +151 -0
dashboard_data/quality_data.json +23 -0
dashboard_data/support_data_11a1fab.csv +8 -8
dashboard_data/support_data_8c0acbd.csv +12 -12
dashboard_data/support_data_a9b92c4.csv +8 -8
dashboard_data/test_coverage_112a023.json +239 -0
dashboard_data/test_coverage_11a1fab.json +233 -0
dashboard_data/test_coverage_8c0acbd.json +234 -0
dashboard_data/test_coverage_a9b92c4.json +239 -0
dashboard_data/test_coverage_ca49596.json +237 -0
dashboard_data/version.json +1 -1
main.py +331 -38
performance_generate.py +486 -17
quality_generate.py +0 -186
utils.py +69 -6

.github/scripts/check_dataset_update.py CHANGED Viewed

@@ -1,8 +1,8 @@
 import json
 import os
 from github import Github
-from datetime import datetime, timedelta
 from huggingface_hub import HfApi, login
@@ -16,12 +16,17 @@ def check_dataset_updates(dataset_id):
     last_modified = dataset_info.lastModified.isoformat()
     current_sha = dataset_info.sha
-    repo_tree = api.list_repo_tree(repo_id=dataset_id, repo_type="dataset", path_in_repo="benchmark_data", recursive=False)
     cutoff_date = datetime.now(dataset_info.lastModified.tzinfo) - timedelta(weeks=6)
     commit_dates_hashes = [item.path.split("/")[-1] for item in repo_tree]
     new_commit_hashes = []
     for commit_date_hash in commit_dates_hashes:
         commit_date, commit_hash = commit_date_hash.split("_")
         commit_date = datetime.strptime(commit_date, "%Y-%m-%dT%H%M%S").replace(
@@ -42,10 +47,10 @@ def check_dataset_updates(dataset_id):
         except Exception as e:
             print(f"Error processing commit {commit_hash}: {str(e)}")
             continue
     # Sort by commit date
     commit_info.sort(key=lambda x: x[1])
     # Extract sorted commits and versions
     new_releases = [info[0] for info in commit_info]
     new_versions = [info[2] for info in commit_info]
@@ -94,15 +99,15 @@ def get_commit_version(repo, commit_hash):
     try:
         releases = list(repo.get_releases())
         releases.sort(key=lambda x: x.created_at)
         commit = repo.get_commit(commit_hash)
         commit_date = commit.commit.author.date
         for i, release in enumerate(releases):
             if commit_date <= release.created_at:
-                return releases[i].tag_name.lstrip('v')
-        return releases[-1].tag_name.lstrip('v')
     except Exception as e:
         print(f"Error processing commit {commit_hash}: {str(e)}")
         return None

 import json
 import os
+from datetime import datetime, timedelta
 from github import Github
 from huggingface_hub import HfApi, login
     last_modified = dataset_info.lastModified.isoformat()
     current_sha = dataset_info.sha
+    repo_tree = api.list_repo_tree(
+        repo_id=dataset_id,
+        repo_type="dataset",
+        path_in_repo="benchmark_data",
+        recursive=False,
+    )
     cutoff_date = datetime.now(dataset_info.lastModified.tzinfo) - timedelta(weeks=6)
     commit_dates_hashes = [item.path.split("/")[-1] for item in repo_tree]
     new_commit_hashes = []
     for commit_date_hash in commit_dates_hashes:
         commit_date, commit_hash = commit_date_hash.split("_")
         commit_date = datetime.strptime(commit_date, "%Y-%m-%dT%H%M%S").replace(
         except Exception as e:
             print(f"Error processing commit {commit_hash}: {str(e)}")
             continue
     # Sort by commit date
     commit_info.sort(key=lambda x: x[1])
     # Extract sorted commits and versions
     new_releases = [info[0] for info in commit_info]
     new_versions = [info[2] for info in commit_info]
     try:
         releases = list(repo.get_releases())
         releases.sort(key=lambda x: x.created_at)
         commit = repo.get_commit(commit_hash)
         commit_date = commit.commit.author.date
         for i, release in enumerate(releases):
             if commit_date <= release.created_at:
+                return releases[i].tag_name.lstrip("v")
+        return releases[-1].tag_name.lstrip("v")
     except Exception as e:
         print(f"Error processing commit {commit_hash}: {str(e)}")
         return None

.github/scripts/process_report.py CHANGED Viewed

@@ -217,17 +217,21 @@ def analyze_support_changes(prev_csv, curr_csv):
 def generate_report():
     # Load current and previous data
-    prev_perf_data = read_json_line_by_line("dashboard_data/performance_data.json")  # old data
-    curr_perf_data = read_json_line_by_line("report_data/performance_data.json")  # new data
     prev_dict = {(d["model"], d["device"], d["os"]): d for d in prev_perf_data}
     curr_dict = {(d["model"], d["device"], d["os"]): d for d in curr_perf_data}
     common_configs = set(curr_dict.keys()) & set(prev_dict.keys())
     # Load version data
-    with open("dashboard_data/version.json", "r") as f:
         prev_version = json.load(f)
-    with open("report_data/version.json", "r") as f:
         curr_version = json.load(f)
     prev_releases = set(prev_version.get("releases", []))
@@ -384,7 +388,13 @@ def generate_report():
                 else:
                     # If there are changes, show the metrics
                     performance_text += f"{device_info}\n"
-                    table = format_metrics_table(config, prev_dict, curr_dict, improved_metrics, regressed_metrics)
                     performance_text += table
                     performance_text += "\n\n"

 def generate_report():
     # Load current and previous data
+    prev_perf_data = read_json_line_by_line(
+        "dashboard_data/performance_data.json"
+    )  # old data
+    curr_perf_data = read_json_line_by_line(
+        "report_data/performance_data.json"
+    )  # new data
     prev_dict = {(d["model"], d["device"], d["os"]): d for d in prev_perf_data}
     curr_dict = {(d["model"], d["device"], d["os"]): d for d in curr_perf_data}
     common_configs = set(curr_dict.keys()) & set(prev_dict.keys())
     # Load version data
+    with open("dashboard_data/version.json", "r") as f:
         prev_version = json.load(f)
+    with open("report_data/version.json", "r") as f:
         curr_version = json.load(f)
     prev_releases = set(prev_version.get("releases", []))
                 else:
                     # If there are changes, show the metrics
                     performance_text += f"{device_info}\n"
+                    table = format_metrics_table(
+                        config,
+                        prev_dict,
+                        curr_dict,
+                        improved_metrics,
+                        regressed_metrics,
+                    )
                     performance_text += table
                     performance_text += "\n\n"

Makefile CHANGED Viewed

@@ -8,4 +8,3 @@ use-huggingface-data:
 use-local-data:
 	@python performance_generate.py


8
9	use-local-data:
10	@python performance_generate.py

README.md CHANGED Viewed

@@ -1,5 +1,5 @@
 ---
-title: WhisperKit Benchmarks
 emoji: 🏆
 colorFrom: green
 colorTo: indigo

 ---
+title: Argmax OSS Regression Tests
 emoji: 🏆
 colorFrom: green
 colorTo: indigo

constants.py CHANGED Viewed

@@ -4,7 +4,7 @@ from iso639 import Lang
 BANNER_TEXT = """
 <div style="text-align: center;">
-    <h1><a href='https://github.com/argmaxinc/WhisperKit'>WhisperKit Benchmarks</a></h1>
 </div>
 """
@@ -30,7 +30,7 @@ Tokens per Second (⬆️): The number of output tokens generated per second. Hi
 Speed (⬆️): Input audio seconds transcribed per second. Higher is better.
 🎯 WhisperKit is evaluated across different datasets, with a focus on per-example no-regressions (QoI) and overall accuracy (WER).
-\n💻 Our benchmarks include:
 Reference: <a href='https://platform.openai.com/docs/guides/speech-to-text'>WhisperOpenAIAPI</a> (OpenAI's Whisper API)
 On-device: <a href='https://github.com/argmaxinc/WhisperKit'>WhisperKit</a> (various versions and optimizations)
@@ -103,6 +103,7 @@ PERFORMANCE_TEXT = dedent(
     ## Metrics
     - **Speed factor** (⬆️): Computed as the ratio of input audio length to end-to-end WhisperKit latency for transcribing that audio. A speed factor of N means N seconds of input audio was transcribed in 1 second.
     - **Tok/s (Tokens per second)** (⬆️): Total number of text decoder forward passes divided by the end-to-end processing time.
     ## Data
@@ -132,6 +133,7 @@ COL_NAMES = {
     "device": "Device",
     "os": "OS",
     "english_wer": "English WER",
 }

 BANNER_TEXT = """
 <div style="text-align: center;">
+    <h1><a href='https://github.com/argmaxinc/WhisperKit'>Argmax OSS Regression Tests</a></h1>
 </div>
 """
 Speed (⬆️): Input audio seconds transcribed per second. Higher is better.
 🎯 WhisperKit is evaluated across different datasets, with a focus on per-example no-regressions (QoI) and overall accuracy (WER).
+\n💻 Our regression tests include:
 Reference: <a href='https://platform.openai.com/docs/guides/speech-to-text'>WhisperOpenAIAPI</a> (OpenAI's Whisper API)
 On-device: <a href='https://github.com/argmaxinc/WhisperKit'>WhisperKit</a> (various versions and optimizations)
     ## Metrics
     - **Speed factor** (⬆️): Computed as the ratio of input audio length to end-to-end WhisperKit latency for transcribing that audio. A speed factor of N means N seconds of input audio was transcribed in 1 second.
     - **Tok/s (Tokens per second)** (⬆️): Total number of text decoder forward passes divided by the end-to-end processing time.
+    - **Parity**: Difference between measured WER and ground truth WER (Measured - Ground Truth). Positive values indicate performance is worse than expected, negative values indicate better than expected performance.
     ## Data
     "device": "Device",
     "os": "OS",
     "english_wer": "English WER",
+    "parity": "Parity",
 }

dashboard_data/Mac.json ADDED Viewed

	@@ -0,0 +1,241 @@

+{
+    "date_generated": "2025-06-16T15:52:13.069236",
+    "xcode_version": "Developer",
+    "total_menu": {
+      "MacBook Pro 14\" (M4, 2024)": {
+        "sku": "Mac16,10",
+        "chip": "M4",
+        "ram": "16 GB"
+      },
+      "MacBook Pro 16\" (M4, 2024)": {
+        "sku": "Mac16,11",
+        "chip": "M4",
+        "ram": "16 GB"
+      },
+      "MacBook Pro 14\" (M4 Pro, 2024)": {
+        "sku": "Mac16,12",
+        "chip": "M4",
+        "ram": "24 GB"
+      },
+      "MacBook Pro 16\" (M4 Pro, 2024)": {
+        "sku": "Mac16,13",
+        "chip": "M4",
+        "ram": "24 GB"
+      },
+      "iMac 24\" (M4, 2024)": {
+        "sku": "Mac16,1",
+        "chip": "M4",
+        "ram": "16 GB"
+      },
+      "iMac 24\" (M4, 2024) - Higher Config": {
+        "sku": "Mac16,2",
+        "chip": "M4",
+        "ram": "24 GB"
+      },
+      "Mac mini (M4, 2024)": {
+        "sku": "Mac16,3",
+        "chip": "M4",
+        "ram": "16 GB"
+      },
+      "Mac mini (M4 Pro, 2024)": {
+        "sku": "Mac16,5",
+        "chip": "M4",
+        "ram": "24 GB"
+      },
+      "MacBook Air 13\" (M4, 2024)": {
+        "sku": "Mac16,6",
+        "chip": "M4",
+        "ram": "16 GB"
+      },
+      "MacBook Air 15\" (M4, 2024)": {
+        "sku": "Mac16,7",
+        "chip": "M4",
+        "ram": "16 GB"
+      },
+      "Mac Pro (M2 Ultra, 2023)": {
+        "sku": "Mac16,8",
+        "chip": "M2",
+        "ram": "64 GB"
+      },
+      "Mac Pro (M2 Ultra, 2023) - Higher Config": {
+        "sku": "Mac16,9",
+        "chip": "M2",
+        "ram": "128 GB"
+      },
+      "MacBook Pro 14\" (M3, 2023)": {
+        "sku": "Mac15,3",
+        "chip": "M3",
+        "ram": "16 GB"
+      },
+      "MacBook Pro 16\" (M3, 2023)": {
+        "sku": "Mac15,4",
+        "chip": "M3",
+        "ram": "16 GB"
+      },
+      "MacBook Pro 14\" (M3 Pro, 2023)": {
+        "sku": "Mac15,5",
+        "chip": "M3",
+        "ram": "24 GB"
+      },
+      "MacBook Pro 16\" (M3 Pro, 2023)": {
+        "sku": "Mac15,6",
+        "chip": "M3",
+        "ram": "24 GB"
+      },
+      "MacBook Pro 14\" (M3 Max, 2023)": {
+        "sku": "Mac15,7",
+        "chip": "M3",
+        "ram": "32 GB"
+      },
+      "MacBook Pro 16\" (M3 Max, 2023)": {
+        "sku": "Mac15,8",
+        "chip": "M3",
+        "ram": "32 GB"
+      },
+      "iMac 24\" (M3, 2023)": {
+        "sku": "Mac15,9",
+        "chip": "M3",
+        "ram": "16 GB"
+      },
+      "iMac 24\" (M3, 2023) - Higher Config": {
+        "sku": "Mac15,10",
+        "chip": "M3",
+        "ram": "24 GB"
+      },
+      "MacBook Air 13\" (M3, 2024)": {
+        "sku": "Mac15,11",
+        "chip": "M3",
+        "ram": "16 GB"
+      },
+      "MacBook Air 15\" (M3, 2024)": {
+        "sku": "Mac15,12",
+        "chip": "M3",
+        "ram": "16 GB"
+      },
+      "Mac mini (M3, 2024)": {
+        "sku": "Mac15,13",
+        "chip": "M3",
+        "ram": "16 GB"
+      },
+      "MacBook Air 13\" (M2, 2022)": {
+        "sku": "Mac14,2",
+        "chip": "M2",
+        "ram": "16 GB"
+      },
+      "MacBook Pro 13\" (M2, 2022)": {
+        "sku": "Mac14,3",
+        "chip": "M2",
+        "ram": "16 GB"
+      },
+      "Mac mini (M2, 2023)": {
+        "sku": "Mac14,4",
+        "chip": "M2",
+        "ram": "16 GB"
+      },
+      "MacBook Pro 14\" (M2, 2023)": {
+        "sku": "Mac14,5",
+        "chip": "M2",
+        "ram": "16 GB"
+      },
+      "MacBook Pro 16\" (M2, 2023)": {
+        "sku": "Mac14,6",
+        "chip": "M2",
+        "ram": "16 GB"
+      },
+      "MacBook Pro 14\" (M2 Pro, 2023)": {
+        "sku": "Mac14,7",
+        "chip": "M2",
+        "ram": "24 GB"
+      },
+      "MacBook Pro 16\" (M2 Pro, 2023)": {
+        "sku": "Mac14,8",
+        "chip": "M2",
+        "ram": "24 GB"
+      },
+      "MacBook Pro 14\" (M2 Max, 2023)": {
+        "sku": "Mac14,9",
+        "chip": "M2",
+        "ram": "32 GB"
+      },
+      "MacBook Pro 16\" (M2 Max, 2023)": {
+        "sku": "Mac14,10",
+        "chip": "M2",
+        "ram": "32 GB"
+      },
+      "Mac mini (M2 Pro, 2023)": {
+        "sku": "Mac14,12",
+        "chip": "M2",
+        "ram": "24 GB"
+      },
+      "Mac Studio (M2 Max, 2023)": {
+        "sku": "Mac14,13",
+        "chip": "M2",
+        "ram": "32 GB"
+      },
+      "Mac Studio (M2 Ultra, 2023)": {
+        "sku": "Mac14,14",
+        "chip": "M2",
+        "ram": "64 GB"
+      },
+      "MacBook Air 15\" (M2, 2023)": {
+        "sku": "Mac14,15",
+        "chip": "M2",
+        "ram": "16 GB"
+      },
+      "MacBook Pro 13\" (M1, 2020)": {
+        "sku": "MacBookPro17,1",
+        "chip": "M1",
+        "ram": "16 GB"
+      },
+      "MacBook Pro 14\" (M1 Pro, 2021)": {
+        "sku": "MacBookPro18,1",
+        "chip": "M1",
+        "ram": "24 GB"
+      },
+      "MacBook Pro 16\" (M1 Pro, 2021)": {
+        "sku": "MacBookPro18,2",
+        "chip": "M1",
+        "ram": "24 GB"
+      },
+      "MacBook Pro 14\" (M1 Max, 2021)": {
+        "sku": "MacBookPro18,3",
+        "chip": "M1",
+        "ram": "32 GB"
+      },
+      "MacBook Pro 16\" (M1 Max, 2021)": {
+        "sku": "MacBookPro18,4",
+        "chip": "M1",
+        "ram": "32 GB"
+      },
+      "MacBook Air 13\" (M1, 2020)": {
+        "sku": "MacBookAir10,1",
+        "chip": "M1",
+        "ram": "16 GB"
+      },
+      "Mac mini (M1, 2020)": {
+        "sku": "Macmini9,1",
+        "chip": "M1",
+        "ram": "16 GB"
+      },
+      "iMac 24\" (M1, 2021)": {
+        "sku": "iMac21,1",
+        "chip": "M1",
+        "ram": "16 GB"
+      },
+      "iMac 24\" (M1, 2021) - Higher Config": {
+        "sku": "iMac21,2",
+        "chip": "M1",
+        "ram": "24 GB"
+      },
+      "Mac Studio (M1 Max, 2022)": {
+        "sku": "Mac13,1",
+        "chip": "M1",
+        "ram": "32 GB"
+      },
+      "Mac Studio (M1 Ultra, 2022)": {
+        "sku": "Mac13,2",
+        "chip": "M1",
+        "ram": "64 GB"
+      }
+    }
+}

dashboard_data/config.json CHANGED Viewed

@@ -1,9 +1,43 @@
 {
-    "name": "whisperkit-coreml",
-    "version": "0.2",
     "device_support": [
         {
-            "identifiers": ["iPhone11", "iPhone12", "Watch7", "Watch8"],
             "models": {
                 "default": "openai_whisper-tiny",
                 "supported": [
@@ -15,27 +49,23 @@
             }
         },
         {
-            "identifiers": ["iPhone13", "iPad13,18", "iPad13,1"],
-            "models": {
-                "default": "openai_whisper-base",
-                "supported": [
-                    "openai_whisper-tiny",
-                    "openai_whisper-tiny.en",
-                    "openai_whisper-base",
-                    "openai_whisper-base.en",
-                    "openai_whisper-small",
-                    "openai_whisper-small.en"
-                ]
-            }
-        },
-        {
             "identifiers": [
-                "iPhone14",
-                "iPhone15",
-                "iPhone16",
-                "iPhone17",
-                "iPad14,1",
-                "iPad14,2"
             ],
             "models": {
                 "default": "openai_whisper-base",
@@ -46,6 +76,8 @@
                     "openai_whisper-base.en",
                     "openai_whisper-small",
                     "openai_whisper-small.en",
                     "openai_whisper-large-v2_949MB",
                     "openai_whisper-large-v2_turbo_955MB",
                     "openai_whisper-large-v3_947MB",
@@ -53,24 +85,38 @@
                     "distil-whisper_distil-large-v3_594MB",
                     "distil-whisper_distil-large-v3_turbo_600MB",
                     "openai_whisper-large-v3-v20240930_626MB",
-                    "openai_whisper-large-v3-v20240930_turbo_632MB"
                 ]
             }
         },
         {
             "identifiers": [
-                "Mac13",
-                "iMac21",
                 "MacBookAir10,1",
-                "MacBookPro17",
-                "MacBookPro18",
-                "Macmini9",
-                "iPad13,16",
                 "iPad13,4",
-                "iPad13,8"
             ],
             "models": {
-                "default": "openai_whisper-large-v3-v20240930",
                 "supported": [
                     "openai_whisper-tiny",
                     "openai_whisper-tiny.en",
@@ -78,6 +124,8 @@
                     "openai_whisper-base.en",
                     "openai_whisper-small",
                     "openai_whisper-small.en",
                     "openai_whisper-large-v2",
                     "openai_whisper-large-v2_949MB",
                     "openai_whisper-large-v3",
@@ -85,15 +133,51 @@
                     "distil-whisper_distil-large-v3",
                     "distil-whisper_distil-large-v3_594MB",
                     "openai_whisper-large-v3-v20240930",
-                    "openai_whisper-large-v3-v20240930_626MB"
                 ]
             }
         },
         {
             "identifiers": [
-                "Mac14",
-                "Mac15",
-                "Mac16",
                 "iPad14,3",
                 "iPad14,4",
                 "iPad14,5",
@@ -102,7 +186,14 @@
                 "iPad14,9",
                 "iPad14,10",
                 "iPad14,11",
-                "iPad16"
             ],
             "models": {
                 "default": "openai_whisper-large-v3-v20240930",
@@ -113,6 +204,8 @@
                     "openai_whisper-base.en",
                     "openai_whisper-small",
                     "openai_whisper-small.en",
                     "openai_whisper-large-v2",
                     "openai_whisper-large-v2_949MB",
                     "openai_whisper-large-v2_turbo",
@@ -128,9 +221,99 @@
                     "openai_whisper-large-v3-v20240930",
                     "openai_whisper-large-v3-v20240930_turbo",
                     "openai_whisper-large-v3-v20240930_626MB",
-                    "openai_whisper-large-v3-v20240930_turbo_632MB"
                 ]
             }
         }
-    ]
 }

 {
+    "name": "argmax-pro-sdk",
+    "version": "0.5",
     "device_support": [
         {
+            "chips": "A12, A13, S9, S10",
+            "identifiers": [
+                "iPhone11,2",
+                "iPhone11,3",
+                "iPhone11,4",
+                "iPhone11,5",
+                "iPhone11,6",
+                "iPhone11,8",
+                "iPhone12,1",
+                "iPhone12,2",
+                "iPhone12,3",
+                "iPhone12,4",
+                "iPhone12,5",
+                "iPhone12,8",
+                "iPad8,1",
+                "iPad8,2",
+                "iPad8,3",
+                "iPad8,4",
+                "iPad8,5",
+                "iPad8,6",
+                "iPad8,7",
+                "iPad8,8",
+                "iPad8,9",
+                "iPad8,10",
+                "iPad8,11",
+                "iPad8,12",
+                "iPad11,1",
+                "iPad11,2",
+                "iPad11,3",
+                "iPad11,4",
+                "iPad11,6",
+                "iPad11,7",
+                "iPad12,1",
+                "iPad12,2"
+            ],
             "models": {
                 "default": "openai_whisper-tiny",
                 "supported": [
             }
         },
         {
+            "chips": "A16, A17 Pro, A18, A18 Pro",
             "identifiers": [
+                "iPhone15,2",
+                "iPhone15,3",
+                "iPhone15,4",
+                "iPhone15,5",
+                "iPhone16,1",
+                "iPhone16,2",
+                "iPhone17,1",
+                "iPhone17,2",
+                "iPhone17,3",
+                "iPhone17,4",
+                "iPhone17,5",
+                "iPad15,7",
+                "iPad15,8",
+                "iPad16,1",
+                "iPad16,2"
             ],
             "models": {
                 "default": "openai_whisper-base",
                     "openai_whisper-base.en",
                     "openai_whisper-small",
                     "openai_whisper-small.en",
+                    "openai_whisper-small_216MB",
+                    "openai_whisper-small.en_217MB",
                     "openai_whisper-large-v2_949MB",
                     "openai_whisper-large-v2_turbo_955MB",
                     "openai_whisper-large-v3_947MB",
                     "distil-whisper_distil-large-v3_594MB",
                     "distil-whisper_distil-large-v3_turbo_600MB",
                     "openai_whisper-large-v3-v20240930_626MB",
+                    "openai_whisper-large-v3-v20240930_turbo_632MB",
+                    "nvidia_parakeet-v2_476MB"
                 ]
             }
         },
         {
+            "chips": "M1",
             "identifiers": [
+                "MacBookPro17,1",
+                "MacBookPro18,1",
+                "MacBookPro18,2",
+                "MacBookPro18,3",
+                "MacBookPro18,4",
                 "MacBookAir10,1",
+                "Macmini9,1",
+                "iMac21,1",
+                "iMac21,2",
+                "Mac13,1",
+                "Mac13,2",
                 "iPad13,4",
+                "iPad13,5",
+                "iPad13,6",
+                "iPad13,7",
+                "iPad13,8",
+                "iPad13,9",
+                "iPad13,10",
+                "iPad13,11",
+                "iPad13,16",
+                "iPad13,17"
             ],
             "models": {
+                "default": "openai_whisper-large-v3-v20240930_626MB",
                 "supported": [
                     "openai_whisper-tiny",
                     "openai_whisper-tiny.en",
                     "openai_whisper-base.en",
                     "openai_whisper-small",
                     "openai_whisper-small.en",
+                    "openai_whisper-small_216MB",
+                    "openai_whisper-small.en_217MB",
                     "openai_whisper-large-v2",
                     "openai_whisper-large-v2_949MB",
                     "openai_whisper-large-v3",
                     "distil-whisper_distil-large-v3",
                     "distil-whisper_distil-large-v3_594MB",
                     "openai_whisper-large-v3-v20240930",
+                    "openai_whisper-large-v3-v20240930_626MB",
+                    "nvidia_parakeet-v2_476MB",
+                    "nvidia_parakeet-v2"
                 ]
             }
         },
         {
+            "chips": "M2, M3, M4",
             "identifiers": [
+                "Mac14,2",
+                "Mac14,3",
+                "Mac14,4",
+                "Mac14,5",
+                "Mac14,6",
+                "Mac14,7",
+                "Mac14,8",
+                "Mac14,9",
+                "Mac14,10",
+                "Mac14,12",
+                "Mac14,13",
+                "Mac14,14",
+                "Mac14,15",
+                "Mac15,3",
+                "Mac15,4",
+                "Mac15,5",
+                "Mac15,6",
+                "Mac15,7",
+                "Mac15,8",
+                "Mac15,9",
+                "Mac15,10",
+                "Mac15,11",
+                "Mac15,12",
+                "Mac15,13",
+                "Mac16,1",
+                "Mac16,2",
+                "Mac16,3",
+                "Mac16,5",
+                "Mac16,6",
+                "Mac16,7",
+                "Mac16,8",
+                "Mac16,9",
+                "Mac16,10",
+                "Mac16,11",
+                "Mac16,12",
+                "Mac16,13",
                 "iPad14,3",
                 "iPad14,4",
                 "iPad14,5",
                 "iPad14,9",
                 "iPad14,10",
                 "iPad14,11",
+                "iPad15,3",
+                "iPad15,4",
+                "iPad15,5",
+                "iPad15,6",
+                "iPad16,3",
+                "iPad16,4",
+                "iPad16,5",
+                "iPad16,6"
             ],
             "models": {
                 "default": "openai_whisper-large-v3-v20240930",
                     "openai_whisper-base.en",
                     "openai_whisper-small",
                     "openai_whisper-small.en",
+                    "openai_whisper-small_216MB",
+                    "openai_whisper-small.en_217MB",
                     "openai_whisper-large-v2",
                     "openai_whisper-large-v2_949MB",
                     "openai_whisper-large-v2_turbo",
                     "openai_whisper-large-v3-v20240930",
                     "openai_whisper-large-v3-v20240930_turbo",
                     "openai_whisper-large-v3-v20240930_626MB",
+                    "openai_whisper-large-v3-v20240930_turbo_632MB",
+                    "nvidia_parakeet-v2_476MB",
+                    "nvidia_parakeet-v2"
+                ]
+            }
+        },
+        {
+            "chips": "A14",
+            "identifiers": [
+                "iPhone13,1",
+                "iPhone13,2",
+                "iPhone13,3",
+                "iPhone13,4",
+                "iPad13,1",
+                "iPad13,2",
+                "iPad13,18",
+                "iPad13,19"
+            ],
+            "models": {
+                "default": "openai_whisper-base",
+                "supported": [
+                    "openai_whisper-tiny",
+                    "openai_whisper-tiny.en",
+                    "openai_whisper-base",
+                    "openai_whisper-base.en",
+                    "openai_whisper-small",
+                    "openai_whisper-small.en",
+                    "openai_whisper-small_216MB",
+                    "openai_whisper-small.en_217MB",
+                    "nvidia_parakeet-v2_476MB"
+                ]
+            }
+        },
+        {
+            "chips": "A15",
+            "identifiers": [
+                "iPhone14,2",
+                "iPhone14,3",
+                "iPhone14,4",
+                "iPhone14,5",
+                "iPhone14,6",
+                "iPhone14,7",
+                "iPhone14,8",
+                "iPad14,1",
+                "iPad14,2"
+            ],
+            "models": {
+                "default": "openai_whisper-base",
+                "supported": [
+                    "openai_whisper-tiny",
+                    "openai_whisper-tiny.en",
+                    "openai_whisper-base",
+                    "openai_whisper-base.en",
+                    "openai_whisper-small",
+                    "openai_whisper-small.en",
+                    "openai_whisper-small_216MB",
+                    "openai_whisper-small.en_217MB",
+                    "openai_whisper-large-v2_949MB",
+                    "openai_whisper-large-v2_turbo_955MB",
+                    "openai_whisper-large-v3_947MB",
+                    "openai_whisper-large-v3_turbo_954MB",
+                    "distil-whisper_distil-large-v3_594MB",
+                    "distil-whisper_distil-large-v3_turbo_600MB",
+                    "openai_whisper-large-v3-v20240930_626MB",
+                    "openai_whisper-large-v3-v20240930_turbo_632MB",
+                    "nvidia_parakeet-v2_476MB"
                 ]
             }
         }
+    ],
+    "model_checksums": {
+        "distil-whisper_distil-large-v3": "9cd8271143b919402ae776c30b479565",
+        "distil-whisper_distil-large-v3_594MB": "ca532f45ddbf8a3d241132cc5cf41639",
+        "distil-whisper_distil-large-v3_turbo": "b8638452c6568dfe33a33bfcc2bc6aca",
+        "distil-whisper_distil-large-v3_turbo_600MB": "81746b4b1afbbb01a8ae9ea452460d88",
+        "openai_whisper-base.en": "fbcfd586f15e2952251b1d3257f18471",
+        "openai_whisper-base": "36e60501ad0f01c1a5719e83a1f63f20",
+        "openai_whisper-large-v2": "21b86c07318aeeef54598f15b7903979",
+        "openai_whisper-large-v2_949MB": "71bad4e1566749d1060eda42308d9fb4",
+        "openai_whisper-large-v2_turbo": "7734959b6550e7b5c2d732bf2b7acd23",
+        "openai_whisper-large-v2_turbo_955MB": "cb6411862a48ec75325572081f01e5b5",
+        "openai_whisper-large-v3-v20240930": "17ebd78ff7edfa59001b554e9cc4c021",
+        "openai_whisper-large-v3-v20240930_547MB": "c945dad68449ac3c78ecb2d561ac189d",
+        "openai_whisper-large-v3-v20240930_626MB": "578fe5a07f4eb7e4187c920bca571aa5",
+        "openai_whisper-large-v3-v20240930_turbo": "dfbf09ab741af1d5400ddbd07bb37dad",
+        "openai_whisper-large-v3-v20240930_turbo_632MB": "33954440dbd785ca1828afe25514f5a5",
+        "openai_whisper-large-v3": "a6f24dc72785722e9cea89e227856dfe",
+        "openai_whisper-large-v3_947MB": "ef6b0e9622a046ce2361b4c72307877f",
+        "openai_whisper-large-v3_turbo": "c550fbdea70c5784d322c0a427f8b5cd",
+        "openai_whisper-large-v3_turbo_954MB": "e639c4bb98d905064ef5dd38757dd9d1",
+        "openai_whisper-small.en": "38efe6a00706bbdb995795c67a836e5e",
+        "openai_whisper-small": "f1d21adb950bc9be5d5343bcdeccd23b",
+        "openai_whisper-tiny.en": "e1183fd55448923b1ce43a2da67aa21f",
+        "openai_whisper-tiny": "7147518a3d68ddbea0691e04cfffa4ff"
+    }
 }

dashboard_data/device_map.json CHANGED Viewed

@@ -2,6 +2,7 @@
     "Mac14,12": "Apple M2 Pro",
     "Mac14,14": "Apple M2 Ultra",
     "Mac14,2": "Apple M2",
     "Mac15,10": "Apple M3 Max",
     "Mac15,3": "Apple M3",
     "Mac15,6": "Apple M3 Pro",
@@ -9,12 +10,14 @@
     "Mac16,10": "Apple M4",
     "Mac16,3": "Apple M4",
     "MacBookAir10,1": "Apple M1",
     "iPad13,16": "iPad Air (5th generation)",
     "iPad13,18": "iPad (10th generation)",
     "iPad13,6": "iPad Pro (11-inch) (3rd generation)",
     "iPad14,1": "iPad mini (6th generation)",
     "iPad14,8": "iPad Air 11-inch (M2)",
     "iPad15,3": "iPad Air 11-inch (M3)",
     "iPad15,7": "iPad (A16)",
     "iPad16,1": "iPad mini (A17 Pro)",
     "iPad16,3": "iPad Pro 11-inch (M4)",

     "Mac14,12": "Apple M2 Pro",
     "Mac14,14": "Apple M2 Ultra",
     "Mac14,2": "Apple M2",
+    "Mac14,3": "Apple M2",
     "Mac15,10": "Apple M3 Max",
     "Mac15,3": "Apple M3",
     "Mac15,6": "Apple M3 Pro",
     "Mac16,10": "Apple M4",
     "Mac16,3": "Apple M4",
     "MacBookAir10,1": "Apple M1",
+    "Macmini9,1": "Apple M1",
     "iPad13,16": "iPad Air (5th generation)",
     "iPad13,18": "iPad (10th generation)",
     "iPad13,6": "iPad Pro (11-inch) (3rd generation)",
     "iPad14,1": "iPad mini (6th generation)",
     "iPad14,8": "iPad Air 11-inch (M2)",
     "iPad15,3": "iPad Air 11-inch (M3)",
+    "iPad15,5": "iPad Air 13-inch (M3)",
     "iPad15,7": "iPad (A16)",
     "iPad16,1": "iPad mini (A17 Pro)",
     "iPad16,3": "iPad Pro 11-inch (M4)",

dashboard_data/iPad.json ADDED Viewed

	@@ -0,0 +1,206 @@

+{
+    "date_generated": "2025-06-23T18:22:05.764459",
+    "xcode_version": "Version 26.0 (17A5241e)",
+    "total_menu": {
+      "iPad Pro (11-inch) (2nd generation)": {
+        "sku": [
+          "iPad8,9",
+          "iPad8,10"
+        ],
+        "chip": "A12Z",
+        "ram": "6 GB"
+      },
+      "iPad Pro (12.9-inch) (3rd generation)": {
+        "sku": [
+          "iPad8,5",
+          "iPad8,6",
+          "iPad8,7",
+          "iPad8,8"
+        ],
+        "chip": "A12X",
+        "ram": "4 GB"
+      },
+      "iPad Pro (11-inch)": {
+        "sku": [
+          "iPad8,1",
+          "iPad8,2",
+          "iPad8,3",
+          "iPad8,4"
+        ],
+        "chip": "A12X",
+        "ram": "4 GB"
+      },
+      "iPad Pro (12.9-inch) (4th generation)": {
+        "sku": [
+          "iPad8,11",
+          "iPad8,12"
+        ],
+        "chip": "A12Z",
+        "ram": "6 GB"
+      },
+      "iPad Pro 13-inch (M4)": {
+        "sku": [
+          "iPad16,5",
+          "iPad16,6"
+        ],
+        "chip": "M4",
+        "ram": "8 GB"
+      },
+      "iPad Pro 11-inch (M4)": {
+        "sku": [
+          "iPad16,3",
+          "iPad16,4"
+        ],
+        "chip": "M4",
+        "ram": "8 GB"
+      },
+      "iPad mini (A17 Pro)": {
+        "sku": [
+          "iPad16,1",
+          "iPad16,2"
+        ],
+        "chip": "A17 Pro",
+        "ram": "8 GB"
+      },
+      "iPad (A16)": {
+        "sku": [
+          "iPad15,7",
+          "iPad15,8"
+        ],
+        "chip": "A16",
+        "ram": "8 GB"
+      },
+      "iPad Air 13-inch (M3)": {
+        "sku": [
+          "iPad15,5",
+          "iPad15,6"
+        ],
+        "chip": "M3",
+        "ram": "8 GB"
+      },
+      "iPad Air 11-inch (M3)": {
+        "sku": [
+          "iPad15,3",
+          "iPad15,4"
+        ],
+        "chip": "M3",
+        "ram": "8 GB"
+      },
+      "iPad Air 11-inch (M2)": {
+        "sku": [
+          "iPad14,8",
+          "iPad14,9"
+        ],
+        "chip": "M2",
+        "ram": "8 GB"
+      },
+      "iPad Pro (12.9-inch) (6th generation)": {
+        "sku": [
+          "iPad14,5",
+          "iPad14,6"
+        ],
+        "chip": "M2",
+        "ram": "8 GB"
+      },
+      "iPad Pro (11-inch) (4th generation)": {
+        "sku": [
+          "iPad14,3",
+          "iPad14,4"
+        ],
+        "chip": "M2",
+        "ram": "8 GB"
+      },
+      "iPad mini (6th generation)": {
+        "sku": [
+          "iPad14,1",
+          "iPad14,2"
+        ],
+        "chip": "A15",
+        "ram": "4 GB"
+      },
+      "iPad Air 13-inch (M2)": {
+        "sku": [
+          "iPad14,10",
+          "iPad14,11"
+        ],
+        "chip": "M2",
+        "ram": "8 GB"
+      },
+      "iPad Pro (12.9-inch) (5th generation)": {
+        "sku": [
+          "iPad13,8",
+          "iPad13,9",
+          "iPad13,10",
+          "iPad13,11"
+        ],
+        "chip": "M1",
+        "ram": "8 GB"
+      },
+      "iPad Pro (11-inch) (3rd generation)": {
+        "sku": [
+          "iPad13,4",
+          "iPad13,5",
+          "iPad13,6",
+          "iPad13,7"
+        ],
+        "chip": "M1",
+        "ram": "8 GB"
+      },
+      "iPad Air (4th generation)": {
+        "sku": [
+          "iPad13,1",
+          "iPad13,2"
+        ],
+        "chip": "A14",
+        "ram": "4 GB"
+      },
+      "iPad (10th generation)": {
+        "sku": [
+          "iPad13,18",
+          "iPad13,19"
+        ],
+        "chip": "A14",
+        "ram": "4 GB"
+      },
+      "iPad Air (5th generation)": {
+        "sku": [
+          "iPad13,16",
+          "iPad13,17"
+        ],
+        "chip": "M1",
+        "ram": "8 GB"
+      },
+      "iPad (9th generation)": {
+        "sku": [
+          "iPad12,1",
+          "iPad12,2"
+        ],
+        "chip": "A13",
+        "ram": "3 GB"
+      },
+      "iPad (8th generation)": {
+        "sku": [
+          "iPad11,6",
+          "iPad11,7"
+        ],
+        "chip": "A12",
+        "ram": "3 GB"
+      },
+      "iPad Air (3rd generation)": {
+        "sku": [
+          "iPad11,3",
+          "iPad11,4"
+        ],
+        "chip": "A12",
+        "ram": "3 GB"
+      },
+      "iPad mini (5th generation)": {
+        "sku": [
+          "iPad11,1",
+          "iPad11,2"
+        ],
+        "chip": "A12",
+        "ram": "3 GB"
+      }
+    }
+}

dashboard_data/iPhone.json ADDED Viewed

	@@ -0,0 +1,151 @@

+{
+    "date_generated": "2025-06-16T15:52:13.069236",
+    "xcode_version": "Developer",
+    "total_menu": {
+      "iPhone 16e": {
+        "sku": "iPhone17,5",
+        "chip": "A18",
+        "ram": "8 GB"
+      },
+      "iPhone 16 Plus": {
+        "sku": "iPhone17,4",
+        "chip": "A18",
+        "ram": "8 GB"
+      },
+      "iPhone 16": {
+        "sku": "iPhone17,3",
+        "chip": "A18",
+        "ram": "8 GB"
+      },
+      "iPhone 16 Pro Max": {
+        "sku": "iPhone17,2",
+        "chip": "A18 Pro",
+        "ram": "8 GB"
+      },
+      "iPhone 16 Pro": {
+        "sku": "iPhone17,1",
+        "chip": "A18 Pro",
+        "ram": "8 GB"
+      },
+      "iPhone 15 Pro Max": {
+        "sku": "iPhone16,2",
+        "chip": "A17 Pro",
+        "ram": "8 GB"
+      },
+      "iPhone 15 Pro": {
+        "sku": "iPhone16,1",
+        "chip": "A17 Pro",
+        "ram": "8 GB"
+      },
+      "iPhone 15 Plus": {
+        "sku": "iPhone15,5",
+        "chip": "A16",
+        "ram": "6 GB"
+      },
+      "iPhone 15": {
+        "sku": "iPhone15,4",
+        "chip": "A16",
+        "ram": "6 GB"
+      },
+      "iPhone 14 Pro Max": {
+        "sku": "iPhone15,3",
+        "chip": "A16",
+        "ram": "6 GB"
+      },
+      "iPhone 14 Pro": {
+        "sku": "iPhone15,2",
+        "chip": "A16",
+        "ram": "6 GB"
+      },
+      "iPhone 14 Plus": {
+        "sku": "iPhone14,8",
+        "chip": "A15",
+        "ram": "6 GB"
+      },
+      "iPhone 14": {
+        "sku": "iPhone14,7",
+        "chip": "A15",
+        "ram": "6 GB"
+      },
+      "iPhone SE (3rd generation)": {
+        "sku": "iPhone14,6",
+        "chip": "A15",
+        "ram": "4 GB"
+      },
+      "iPhone 13": {
+        "sku": "iPhone14,5",
+        "chip": "A15",
+        "ram": "4 GB"
+      },
+      "iPhone 13 mini": {
+        "sku": "iPhone14,4",
+        "chip": "A15",
+        "ram": "4 GB"
+      },
+      "iPhone 13 Pro Max": {
+        "sku": "iPhone14,3",
+        "chip": "A15",
+        "ram": "6 GB"
+      },
+      "iPhone 13 Pro": {
+        "sku": "iPhone14,2",
+        "chip": "A15",
+        "ram": "6 GB"
+      },
+      "iPhone 12 Pro Max": {
+        "sku": "iPhone13,4",
+        "chip": "A14",
+        "ram": "6 GB"
+      },
+      "iPhone 12 Pro": {
+        "sku": "iPhone13,3",
+        "chip": "A14",
+        "ram": "6 GB"
+      },
+      "iPhone 12": {
+        "sku": "iPhone13,2",
+        "chip": "A14",
+        "ram": "4 GB"
+      },
+      "iPhone 12 mini": {
+        "sku": "iPhone13,1",
+        "chip": "A14",
+        "ram": "4 GB"
+      },
+      "iPhone SE (2nd generation)": {
+        "sku": "iPhone12,8",
+        "chip": "A13",
+        "ram": "3 GB"
+      },
+      "iPhone 11 Pro Max": {
+        "sku": "iPhone12,5",
+        "chip": "A13",
+        "ram": "4 GB"
+      },
+      "iPhone 11 Pro": {
+        "sku": "iPhone12,3",
+        "chip": "A13",
+        "ram": "4 GB"
+      },
+      "iPhone 11": {
+        "sku": "iPhone12,1",
+        "chip": "A13",
+        "ram": "4 GB"
+      },
+      "iPhone XR": {
+        "sku": "iPhone11,8",
+        "chip": "A12",
+        "ram": "3 GB"
+      },
+      "iPhone XS Max": {
+        "sku": "iPhone11,4",
+        "chip": "A12",
+        "ram": "4 GB"
+      },
+      "iPhone XS": {
+        "sku": "iPhone11,2",
+        "chip": "A12",
+        "ram": "4 GB"
+      }
+    }
+}

dashboard_data/quality_data.json ADDED Viewed

	@@ -0,0 +1,23 @@

+{"model": "openai/whisper-large-v3/947MB", "timestamp": "2024-10-18_16:59:10_GMT-0700", "average_wer": 9.74, "dataset_wer": {"librispeech": 2.41, "earnings22-12hours": 17.08}, "qoi": 0.94}
+{"model": "openai/whisper-large-v2/turbo/955MB", "timestamp": "2024-10-18_16:52:35_GMT-0700", "average_wer": 7.27, "dataset_wer": {"librispeech": 2.4, "earnings22-12hours": 12.14}, "qoi": 0.94}
+{"model": "openai/whisper-tiny.en", "timestamp": "2024-10-19_15:40:06_GMT-0700", "average_wer": 12.23, "dataset_wer": {"librispeech": 5.61, "earnings22-12hours": 18.86}, "qoi": 0.63}
+{"model": "distil-whisper/distil-large-v3/594MB", "timestamp": "2024-10-20_13:02:33_GMT-0700", "average_wer": 8.96, "dataset_wer": {"librispeech": 2.87, "earnings22-12hours": 15.06}, "qoi": 0.86}
+{"model": "openai/whisper-large-v2/949MB", "timestamp": "2024-10-18_19:51:30_GMT-0400", "average_wer": 7.88, "dataset_wer": {"librispeech": 2.38, "earnings22-12hours": 13.39}, "qoi": 0.94}
+{"model": "openai/whisper-large-v3/turbo/954MB", "timestamp": "2024-10-20_13:49:26_GMT-0700", "average_wer": 22.75, "dataset_wer": {"librispeech": 2.51, "earnings22-12hours": 43.0}, "qoi": 0.93}
+{"model": "distil-whisper/distil-large-v3", "timestamp": "2024-10-20_20:32:22_GMT-0700", "average_wer": 7.2, "dataset_wer": {"librispeech": 2.38, "earnings22-12hours": 12.02}, "qoi": 0.9}
+{"model": "openai/whisper-large-v3-v20240930", "timestamp": "2024-10-18_18:35:46_GMT-0700", "average_wer": 6.74, "dataset_wer": {"librispeech": 1.93, "earnings22-12hours": 11.55}, "qoi": 0.94}
+{"model": "openai/whisper-tiny", "timestamp": "2024-10-20_20:19:04_GMT-0700", "average_wer": 14.21, "dataset_wer": {"librispeech": 7.46, "earnings22-12hours": 20.97}, "qoi": 0.52}
+{"model": "openai/whisper-large-v3-v20240930/turbo/632MB", "timestamp": "2024-10-18_20:10:30_GMT-0700", "average_wer": 6.86, "dataset_wer": {"librispeech": 1.95, "earnings22-12hours": 11.77}, "qoi": 0.93}
+{"model": "openai/whisper-large-v2/turbo", "timestamp": "2024-10-18_14:58:38_GMT-0700", "average_wer": 7.25, "dataset_wer": {"librispeech": 2.4, "earnings22-12hours": 12.1}, "qoi": 0.96}
+{"model": "openai/whisper-small", "timestamp": "2024-10-18_12:40:03_GMT-0700", "average_wer": 8.11, "dataset_wer": {"librispeech": 3.21, "earnings22-12hours": 13.0}, "qoi": 0.83}
+{"model": "openai/whisper-large-v3-v20240930/turbo", "timestamp": "2024-10-18_19:37:26_GMT-0700", "average_wer": 6.72, "dataset_wer": {"librispeech": 1.92, "earnings22-12hours": 11.52}, "qoi": 0.94}
+{"model": "openai/whisper-large-v3", "timestamp": "2024-10-18_18:01:14_GMT-0400", "average_wer": 6.85, "dataset_wer": {"librispeech": 2.02, "earnings22-12hours": 11.69}, "qoi": 0.95}
+{"model": "openai/whisper-large-v3-v20240930/626MB", "timestamp": "2024-10-18_19:21:06_GMT-0700", "average_wer": 7.15, "dataset_wer": {"librispeech": 1.96, "earnings22-12hours": 12.35}, "qoi": 0.93}
+{"model": "openai/whisper-base.en", "timestamp": "2024-10-20_12:31:44_GMT-0700", "average_wer": 9.59, "dataset_wer": {"librispeech": 3.98, "earnings22-12hours": 15.2}, "qoi": 0.75}
+{"model": "openai/whisper-large-v3-v20240930/547MB", "timestamp": "2024-10-18_21:59:11_GMT-0400", "average_wer": 16.82, "dataset_wer": {"librispeech": 2.16, "earnings22-12hours": 31.49}, "qoi": 0.92}
+{"model": "distil-whisper/distil-large-v3/turbo/600MB", "timestamp": "2024-10-18_17:50:17_GMT-0700", "average_wer": 8.33, "dataset_wer": {"librispeech": 2.8, "earnings22-12hours": 13.87}, "qoi": 0.86}
+{"model": "openai/whisper-large-v2", "timestamp": "2024-10-18_17:07:15_GMT-0400", "average_wer": 7.32, "dataset_wer": {"librispeech": 2.36, "earnings22-12hours": 12.28}, "qoi": 0.97}
+{"model": "openai/whisper-small.en", "timestamp": "2024-10-18_15:39:48_GMT-0400", "average_wer": 7.85, "dataset_wer": {"librispeech": 2.88, "earnings22-12hours": 12.82}, "qoi": 0.86}
+{"model": "distil-whisper/distil-large-v3/turbo", "timestamp": "2024-10-20_12:45:20_GMT-0700", "average_wer": 7.2, "dataset_wer": {"librispeech": 2.35, "earnings22-12hours": 12.05}, "qoi": 0.9}
+{"model": "openai/whisper-base", "timestamp": "2024-10-18_20:25:50_GMT-0700", "average_wer": 10.67, "dataset_wer": {"librispeech": 4.94, "earnings22-12hours": 16.4}, "qoi": 0.67}
+{"model": "openai/whisper-large-v3/turbo", "timestamp": "2024-10-20_16:58:25_GMT-0400", "average_wer": 6.86, "dataset_wer": {"librispeech": 1.97, "earnings22-12hours": 11.74}, "qoi": 0.95}

dashboard_data/support_data_11a1fab.csv CHANGED Viewed

@@ -1,21 +1,21 @@
 ,Model,"Apple M4 (Mac16,10)","iPad (10th generation) (iPad13,18)","iPad Air 11-inch (M3) (iPad15,3)","iPad (A16) (iPad15,7)","iPad mini (A17 Pro) (iPad16,1)"
-distil-whisper_distil-large-v3,distil-whisper_distil-large-v3,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad15%2C7_summary_2025-04-16T220030.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad16%2C1_summary_2025-04-16T220030.json>iPadOS 18.4</a>
 distil-whisper_distil-large-v3_594MB,distil-whisper_distil-large-v3_594MB,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4
-distil-whisper_distil-large-v3_turbo,distil-whisper_distil-large-v3_turbo,✅ macOS 15.4,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad15%2C3_summary_2025-04-16T013539.json>iPadOS 18.3.2</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad15%2C7_summary_2025-04-16T220030.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad16%2C1_summary_2025-04-16T220030.json>iPadOS 18.4</a>
 distil-whisper_distil-large-v3_turbo_600MB,distil-whisper_distil-large-v3_turbo_600MB,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4
 openai_whisper-base,openai_whisper-base,✅ macOS 15.4,✅ iPadOS 18.4,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4
 openai_whisper-base.en,openai_whisper-base.en,✅ macOS 15.4,✅ iPadOS 18.4,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4
-openai_whisper-large-v2,openai_whisper-large-v2,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad15%2C7_summary_2025-04-16T220030.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad16%2C1_summary_2025-04-16T220030.json>iPadOS 18.4</a>
 openai_whisper-large-v2_949MB,openai_whisper-large-v2_949MB,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4
-openai_whisper-large-v2_turbo,openai_whisper-large-v2_turbo,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad15%2C7_summary_2025-04-16T220030.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad16%2C1_summary_2025-04-16T220030.json>iPadOS 18.4</a>
 openai_whisper-large-v2_turbo_955MB,openai_whisper-large-v2_turbo_955MB,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4
-openai_whisper-large-v3,openai_whisper-large-v3,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad15%2C7_summary_2025-04-16T220030.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad16%2C1_summary_2025-04-16T220030.json>iPadOS 18.4</a>
-openai_whisper-large-v3-v20240930,openai_whisper-large-v3-v20240930,✅ macOS 15.4,Not Supported,? iPadOS 18.3.2,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad15%2C7_summary_2025-04-16T220030.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad16%2C1_summary_2025-04-16T220030.json>iPadOS 18.4</a>
 openai_whisper-large-v3-v20240930_626MB,openai_whisper-large-v3-v20240930_626MB,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4
-openai_whisper-large-v3-v20240930_turbo,openai_whisper-large-v3-v20240930_turbo,✅ macOS 15.4,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad15%2C3_summary_2025-04-16T013539.json>iPadOS 18.3.2</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad15%2C7_summary_2025-04-16T220030.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad16%2C1_summary_2025-04-16T220030.json>iPadOS 18.4</a>
 openai_whisper-large-v3-v20240930_turbo_632MB,openai_whisper-large-v3-v20240930_turbo_632MB,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4
 openai_whisper-large-v3_947MB,openai_whisper-large-v3_947MB,✅ macOS 15.4,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad15%2C3_summary_2025-04-16T013539.json>iPadOS 18.3.2</a>,✅ iPadOS 18.4,✅ iPadOS 18.4
-openai_whisper-large-v3_turbo,openai_whisper-large-v3_turbo,✅ macOS 15.4,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad15%2C3_summary_2025-04-16T013539.json>iPadOS 18.3.2</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad15%2C7_summary_2025-04-16T220030.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad16%2C1_summary_2025-04-16T220030.json>iPadOS 18.4</a>
 openai_whisper-large-v3_turbo_954MB,openai_whisper-large-v3_turbo_954MB,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4
 openai_whisper-small,openai_whisper-small,✅ macOS 15.4,✅ iPadOS 18.4,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4
 openai_whisper-small.en,openai_whisper-small.en,✅ macOS 15.4,✅ iPadOS 18.4,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4

 ,Model,"Apple M4 (Mac16,10)","iPad (10th generation) (iPad13,18)","iPad Air 11-inch (M3) (iPad15,3)","iPad (A16) (iPad15,7)","iPad mini (A17 Pro) (iPad16,1)"
+distil-whisper_distil-large-v3,distil-whisper_distil-large-v3,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,Not Supported,Not Supported
 distil-whisper_distil-large-v3_594MB,distil-whisper_distil-large-v3_594MB,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4
+distil-whisper_distil-large-v3_turbo,distil-whisper_distil-large-v3_turbo,✅ macOS 15.4,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad15%2C3_summary_2025-04-16T013539.json>iPadOS 18.3.2</a>,Not Supported,Not Supported
 distil-whisper_distil-large-v3_turbo_600MB,distil-whisper_distil-large-v3_turbo_600MB,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4
 openai_whisper-base,openai_whisper-base,✅ macOS 15.4,✅ iPadOS 18.4,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4
 openai_whisper-base.en,openai_whisper-base.en,✅ macOS 15.4,✅ iPadOS 18.4,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4
+openai_whisper-large-v2,openai_whisper-large-v2,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,Not Supported,Not Supported
 openai_whisper-large-v2_949MB,openai_whisper-large-v2_949MB,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4
+openai_whisper-large-v2_turbo,openai_whisper-large-v2_turbo,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,Not Supported,Not Supported
 openai_whisper-large-v2_turbo_955MB,openai_whisper-large-v2_turbo_955MB,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4
+openai_whisper-large-v3,openai_whisper-large-v3,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,Not Supported,Not Supported
+openai_whisper-large-v3-v20240930,openai_whisper-large-v3-v20240930,✅ macOS 15.4,Not Supported,? iPadOS 18.3.2,Not Supported,Not Supported
 openai_whisper-large-v3-v20240930_626MB,openai_whisper-large-v3-v20240930_626MB,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4
+openai_whisper-large-v3-v20240930_turbo,openai_whisper-large-v3-v20240930_turbo,✅ macOS 15.4,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad15%2C3_summary_2025-04-16T013539.json>iPadOS 18.3.2</a>,Not Supported,Not Supported
 openai_whisper-large-v3-v20240930_turbo_632MB,openai_whisper-large-v3-v20240930_turbo_632MB,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4
 openai_whisper-large-v3_947MB,openai_whisper-large-v3_947MB,✅ macOS 15.4,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad15%2C3_summary_2025-04-16T013539.json>iPadOS 18.3.2</a>,✅ iPadOS 18.4,✅ iPadOS 18.4
+openai_whisper-large-v3_turbo,openai_whisper-large-v3_turbo,✅ macOS 15.4,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-04-15T183244_11a1fab/iPad15%2C3_summary_2025-04-16T013539.json>iPadOS 18.3.2</a>,Not Supported,Not Supported
 openai_whisper-large-v3_turbo_954MB,openai_whisper-large-v3_turbo_954MB,✅ macOS 15.4,Not Supported,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4
 openai_whisper-small,openai_whisper-small,✅ macOS 15.4,✅ iPadOS 18.4,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4
 openai_whisper-small.en,openai_whisper-small.en,✅ macOS 15.4,✅ iPadOS 18.4,✅ iPadOS 18.3.2,✅ iPadOS 18.4,✅ iPadOS 18.4

dashboard_data/support_data_8c0acbd.csv CHANGED Viewed

@@ -1,22 +1,22 @@
 ,Model,"Apple M2 Pro (Mac14,12)","Apple M3 Max (Mac15,10)","iPad Air (5th generation) (iPad13,16)","iPad Pro (11-inch) (3rd generation) (iPad13,6)","iPad mini (6th generation) (iPad14,1)","iPad Air 11-inch (M2) (iPad14,8)","iPad Air 11-inch (M3) (iPad15,3)","iPad (A16) (iPad15,7)","iPad mini (A17 Pro) (iPad16,1)","iPad Pro 11-inch (M4) (iPad16,3)","iPhone 11 (iPhone12,1)","iPhone 12 mini (iPhone13,1)","iPhone 13 Pro (iPhone14,2)","iPhone 13 (iPhone14,5)","iPhone 14 (iPhone14,7)","iPhone 15 (iPhone15,4)","iPhone 15 Pro Max (iPhone16,2)","iPhone 16 (iPhone17,3)","iPhone 16e (iPhone17,5)"
-distil-whisper_distil-large-v3,distil-whisper_distil-large-v3,✅ macOS 15.5,✅ macOS 15.5,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,Not Supported,✅ iPadOS 18.5,✅ iPadOS 26.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C7_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C1_summary_2025-06-16T220810.json>iPadOS 18.4</a>,✅ iPadOS 18.4.1,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 distil-whisper_distil-large-v3_594MB,distil-whisper_distil-large-v3_594MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
-distil-whisper_distil-large-v3_turbo,distil-whisper_distil-large-v3_turbo,✅ macOS 15.5,✅ macOS 15.5,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad14%2C8_summary_2025-06-16T192343.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C3_summary_2025-06-17T205025.json>iPadOS 26.0</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C7_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C1_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C3_summary_2025-06-16T220810.json>iPadOS 18.4.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
-distil-whisper_distil-large-v3_turbo_600MB,distil-whisper_distil-large-v3_turbo_600MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
 openai_whisper-base,openai_whisper-base,✅ macOS 15.5,✅ macOS 15.5,✅ iPadOS 18.4.1,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,✅ iOS 26.0,✅ iOS 18.4.1,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
 openai_whisper-base.en,openai_whisper-base.en,✅ macOS 15.5,✅ macOS 15.5,✅ iPadOS 18.4.1,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,✅ iOS 26.0,✅ iOS 18.4.1,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
-openai_whisper-large-v2,openai_whisper-large-v2,✅ macOS 15.5,✅ macOS 15.5,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,Not Supported,✅ iPadOS 18.5,✅ iPadOS 26.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C7_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C1_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C3_summary_2025-06-16T220810.json>iPadOS 18.4.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 openai_whisper-large-v2_949MB,openai_whisper-large-v2_949MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C3_summary_2025-06-17T205025.json>iPadOS 26.0</a>,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
-openai_whisper-large-v2_turbo,openai_whisper-large-v2_turbo,✅ macOS 15.5,✅ macOS 15.5,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad14%2C8_summary_2025-06-16T192343.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C3_summary_2025-06-17T205025.json>iPadOS 26.0</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C7_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C1_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C3_summary_2025-06-16T220810.json>iPadOS 18.4.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
-openai_whisper-large-v2_turbo_955MB,openai_whisper-large-v2_turbo_955MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
-openai_whisper-large-v3,openai_whisper-large-v3,✅ macOS 15.5,✅ macOS 15.5,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad14%2C8_summary_2025-06-16T192343.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C3_summary_2025-06-17T205025.json>iPadOS 26.0</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C7_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C1_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C3_summary_2025-06-16T220810.json>iPadOS 18.4.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
-openai_whisper-large-v3-v20240930,openai_whisper-large-v3-v20240930,✅ macOS 15.5,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/Mac15%2C10_summary_2025-06-15T230300.json>macOS 15.5</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,Not Supported,? iPadOS 18.5,? iPadOS 26.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C7_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C1_summary_2025-06-16T220810.json>iPadOS 18.4</a>,? iPadOS 18.4.1,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 openai_whisper-large-v3-v20240930_626MB,openai_whisper-large-v3-v20240930_626MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C3_summary_2025-06-16T220810.json>iPadOS 18.4.1</a>,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
-openai_whisper-large-v3-v20240930_turbo,openai_whisper-large-v3-v20240930_turbo,✅ macOS 15.5,✅ macOS 15.5,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad14%2C8_summary_2025-06-16T192343.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C3_summary_2025-06-17T205025.json>iPadOS 26.0</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C7_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C1_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C3_summary_2025-06-16T220810.json>iPadOS 18.4.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
-openai_whisper-large-v3-v20240930_turbo_632MB,openai_whisper-large-v3-v20240930_turbo_632MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPhone16%2C2_summary_2025-06-16T135523.json>iOS 18.5</a>,✅ iOS 18.4.1,✅ iOS 18.5
 openai_whisper-large-v3_947MB,openai_whisper-large-v3_947MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C3_summary_2025-06-17T205025.json>iPadOS 26.0</a>,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
-openai_whisper-large-v3_turbo,openai_whisper-large-v3_turbo,✅ macOS 15.5,✅ macOS 15.5,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad14%2C8_summary_2025-06-16T192343.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C3_summary_2025-06-17T205025.json>iPadOS 26.0</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C7_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C1_summary_2025-06-16T220810.json>iPadOS 18.4</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C3_summary_2025-06-16T220810.json>iPadOS 18.4.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
-openai_whisper-large-v3_turbo_954MB,openai_whisper-large-v3_turbo_954MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
 openai_whisper-small,openai_whisper-small,✅ macOS 15.5,✅ macOS 15.5,✅ iPadOS 18.4.1,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,✅ iOS 18.4.1,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
 openai_whisper-small.en,openai_whisper-small.en,✅ macOS 15.5,✅ macOS 15.5,✅ iPadOS 18.4.1,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,✅ iOS 18.4.1,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
 openai_whisper-tiny,openai_whisper-tiny,✅ macOS 15.5,✅ macOS 15.5,✅ iPadOS 18.4.1,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,✅ iOS 26.0,✅ iOS 18.4.1,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5

 ,Model,"Apple M2 Pro (Mac14,12)","Apple M3 Max (Mac15,10)","iPad Air (5th generation) (iPad13,16)","iPad Pro (11-inch) (3rd generation) (iPad13,6)","iPad mini (6th generation) (iPad14,1)","iPad Air 11-inch (M2) (iPad14,8)","iPad Air 11-inch (M3) (iPad15,3)","iPad (A16) (iPad15,7)","iPad mini (A17 Pro) (iPad16,1)","iPad Pro 11-inch (M4) (iPad16,3)","iPhone 11 (iPhone12,1)","iPhone 12 mini (iPhone13,1)","iPhone 13 Pro (iPhone14,2)","iPhone 13 (iPhone14,5)","iPhone 14 (iPhone14,7)","iPhone 15 (iPhone15,4)","iPhone 15 Pro Max (iPhone16,2)","iPhone 16 (iPhone17,3)","iPhone 16e (iPhone17,5)"
+distil-whisper_distil-large-v3,distil-whisper_distil-large-v3,✅ macOS 15.5,✅ macOS 15.5,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,Not Supported,✅ iPadOS 18.5,✅ iPadOS 26.0,Not Supported,Not Supported,✅ iPadOS 18.4.1,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 distil-whisper_distil-large-v3_594MB,distil-whisper_distil-large-v3_594MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
+distil-whisper_distil-large-v3_turbo,distil-whisper_distil-large-v3_turbo,✅ macOS 15.5,✅ macOS 15.5,Not Supported,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad14%2C8_summary_2025-06-16T192343.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C3_summary_2025-06-17T205025.json>iPadOS 26.0</a>,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C3_summary_2025-06-16T220810.json>iPadOS 18.4.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
+distil-whisper_distil-large-v3_turbo_600MB,distil-whisper_distil-large-v3_turbo_600MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,Not Supported,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
 openai_whisper-base,openai_whisper-base,✅ macOS 15.5,✅ macOS 15.5,✅ iPadOS 18.4.1,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,✅ iOS 26.0,✅ iOS 18.4.1,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
 openai_whisper-base.en,openai_whisper-base.en,✅ macOS 15.5,✅ macOS 15.5,✅ iPadOS 18.4.1,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,✅ iOS 26.0,✅ iOS 18.4.1,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
+openai_whisper-large-v2,openai_whisper-large-v2,✅ macOS 15.5,✅ macOS 15.5,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,Not Supported,✅ iPadOS 18.5,✅ iPadOS 26.0,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C3_summary_2025-06-16T220810.json>iPadOS 18.4.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 openai_whisper-large-v2_949MB,openai_whisper-large-v2_949MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C3_summary_2025-06-17T205025.json>iPadOS 26.0</a>,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
+openai_whisper-large-v2_turbo,openai_whisper-large-v2_turbo,✅ macOS 15.5,✅ macOS 15.5,Not Supported,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad14%2C8_summary_2025-06-16T192343.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C3_summary_2025-06-17T205025.json>iPadOS 26.0</a>,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C3_summary_2025-06-16T220810.json>iPadOS 18.4.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
+openai_whisper-large-v2_turbo_955MB,openai_whisper-large-v2_turbo_955MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,Not Supported,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
+openai_whisper-large-v3,openai_whisper-large-v3,✅ macOS 15.5,✅ macOS 15.5,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad14%2C8_summary_2025-06-16T192343.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C3_summary_2025-06-17T205025.json>iPadOS 26.0</a>,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C3_summary_2025-06-16T220810.json>iPadOS 18.4.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
+openai_whisper-large-v3-v20240930,openai_whisper-large-v3-v20240930,✅ macOS 15.5,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/Mac15%2C10_summary_2025-06-15T230300.json>macOS 15.5</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad13%2C6_summary_2025-06-17T205025.json>iPadOS 18.4</a>,Not Supported,? iPadOS 18.5,? iPadOS 26.0,Not Supported,Not Supported,? iPadOS 18.4.1,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 openai_whisper-large-v3-v20240930_626MB,openai_whisper-large-v3-v20240930_626MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C3_summary_2025-06-16T220810.json>iPadOS 18.4.1</a>,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
+openai_whisper-large-v3-v20240930_turbo,openai_whisper-large-v3-v20240930_turbo,✅ macOS 15.5,✅ macOS 15.5,Not Supported,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad14%2C8_summary_2025-06-16T192343.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C3_summary_2025-06-17T205025.json>iPadOS 26.0</a>,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C3_summary_2025-06-16T220810.json>iPadOS 18.4.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
+openai_whisper-large-v3-v20240930_turbo_632MB,openai_whisper-large-v3-v20240930_turbo_632MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,Not Supported,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPhone16%2C2_summary_2025-06-16T135523.json>iOS 18.5</a>,✅ iOS 18.4.1,✅ iOS 18.5
 openai_whisper-large-v3_947MB,openai_whisper-large-v3_947MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C3_summary_2025-06-17T205025.json>iPadOS 26.0</a>,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
+openai_whisper-large-v3_turbo,openai_whisper-large-v3_turbo,✅ macOS 15.5,✅ macOS 15.5,Not Supported,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad14%2C8_summary_2025-06-16T192343.json>iPadOS 18.5</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad15%2C3_summary_2025-06-17T205025.json>iPadOS 26.0</a>,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2025-06-13T034257_8c0acbd/iPad16%2C3_summary_2025-06-16T220810.json>iPadOS 18.4.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
+openai_whisper-large-v3_turbo_954MB,openai_whisper-large-v3_turbo_954MB,✅ macOS 15.5,✅ macOS 15.5,Not Supported,Not Supported,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
 openai_whisper-small,openai_whisper-small,✅ macOS 15.5,✅ macOS 15.5,✅ iPadOS 18.4.1,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,✅ iOS 18.4.1,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
 openai_whisper-small.en,openai_whisper-small.en,✅ macOS 15.5,✅ macOS 15.5,✅ iPadOS 18.4.1,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,Not Supported,✅ iOS 18.4.1,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5
 openai_whisper-tiny,openai_whisper-tiny,✅ macOS 15.5,✅ macOS 15.5,✅ iPadOS 18.4.1,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.5,✅ iPadOS 26.0,✅ iPadOS 18.4,✅ iPadOS 18.4,✅ iPadOS 18.4.1,✅ iOS 26.0,✅ iOS 18.4.1,✅ iOS 18.1,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.5,✅ iOS 18.4.1,✅ iOS 18.5

dashboard_data/support_data_a9b92c4.csv CHANGED Viewed

@@ -1,21 +1,21 @@
 ,Model,"Apple M2 Pro (Mac14,12)","Apple M2 Ultra (Mac14,14)","Apple M2 (Mac14,2)","Apple M3 Max (Mac15,10)","Apple M3 Pro (Mac15,6)","Apple M3 Max (Mac15,9)","Apple M4 (Mac16,10)","Apple M4 (Mac16,3)","Apple M1 (MacBookAir10,1)","iPad Air (5th generation) (iPad13,16)","iPad Air 11-inch (M2) (iPad14,8)","iPad mini (A17 Pro) (iPad16,1)","iPad Pro 11-inch (M4) (iPad16,3)","iPhone 11 (iPhone12,1)","iPhone 12 mini (iPhone13,1)","iPhone 12 (iPhone13,2)","iPhone 12 Pro Max (iPhone13,4)","iPhone 13 Pro (iPhone14,2)","iPhone 13 (iPhone14,5)","iPhone 14 (iPhone14,7)","iPhone 15 Pro (iPhone16,1)","iPhone 15 Pro Max (iPhone16,2)","iPhone 16 Pro (iPhone17,1)","iPhone 16 (iPhone17,3)","iPhone 16 Plus (iPhone17,4)"
-distil-whisper_distil-large-v3,distil-whisper_distil-large-v3,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/MacBookAir10%2C1_summary_2024-10-30T012535.json>macOS 15.1</a>,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 distil-whisper_distil-large-v3_594MB,distil-whisper_distil-large-v3_594MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
-distil-whisper_distil-large-v3_turbo,distil-whisper_distil-large-v3_turbo,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad14%2C8_summary_2024-10-30T073931.json>iPadOS 17.6.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C1_summary_2024-10-29T234233.json>iPadOS 18.0.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C3_summary_2024-10-30T073931.json>iPadOS 18.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 distil-whisper_distil-large-v3_turbo_600MB,distil-whisper_distil-large-v3_turbo_600MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
 openai_whisper-base,openai_whisper-base,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,✅ iPadOS 18.1,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,✅ iOS 17.6.1,✅ iOS 17.7,✅ iOS 17.2.1,✅ iOS 18.1,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
 openai_whisper-base.en,openai_whisper-base.en,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,✅ iPadOS 18.1,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,✅ iOS 17.6.1,✅ iOS 17.7,✅ iOS 17.2.1,✅ iOS 18.1,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
-openai_whisper-large-v2,openai_whisper-large-v2,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/MacBookAir10%2C1_summary_2024-10-30T012535.json>macOS 15.1</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad14%2C8_summary_2024-10-30T073931.json>iPadOS 17.6.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C1_summary_2024-10-29T234233.json>iPadOS 18.0.1</a>,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 openai_whisper-large-v2_949MB,openai_whisper-large-v2_949MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C5_summary_2024-10-30T073931.json>iOS 17.3</a>,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
-openai_whisper-large-v2_turbo,openai_whisper-large-v2_turbo,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad14%2C8_summary_2024-10-30T073931.json>iPadOS 17.6.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C1_summary_2024-10-29T234233.json>iPadOS 18.0.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C3_summary_2024-10-30T073931.json>iPadOS 18.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 openai_whisper-large-v2_turbo_955MB,openai_whisper-large-v2_turbo_955MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C5_summary_2024-10-30T073931.json>iOS 17.3</a>,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
-openai_whisper-large-v3,openai_whisper-large-v3,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/MacBookAir10%2C1_summary_2024-10-30T012535.json>macOS 15.1</a>,Not Supported,✅ iPadOS 17.6.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C1_summary_2024-10-29T234233.json>iPadOS 18.0.1</a>,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
-openai_whisper-large-v3-v20240930,openai_whisper-large-v3-v20240930,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/MacBookAir10%2C1_summary_2024-10-30T012535.json>macOS 15.1</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad14%2C8_summary_2024-10-30T073931.json>iPadOS 17.6.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C1_summary_2024-10-29T234233.json>iPadOS 18.0.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C3_summary_2024-10-30T073931.json>iPadOS 18.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 openai_whisper-large-v3-v20240930_626MB,openai_whisper-large-v3-v20240930_626MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
-openai_whisper-large-v3-v20240930_turbo,openai_whisper-large-v3-v20240930_turbo,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,✅ iPadOS 17.6.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C1_summary_2024-10-29T234233.json>iPadOS 18.0.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C3_summary_2024-10-30T073931.json>iPadOS 18.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 openai_whisper-large-v3-v20240930_turbo_632MB,openai_whisper-large-v3-v20240930_turbo_632MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C5_summary_2024-10-30T073931.json>iOS 17.3</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C7_summary_2024-10-30T073931.json>iOS 17.3</a>,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone17%2C4_summary_2024-10-30T132649.json>iOS 18.0.1</a>
 openai_whisper-large-v3_947MB,openai_whisper-large-v3_947MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C5_summary_2024-10-30T073931.json>iOS 17.3</a>,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
-openai_whisper-large-v3_turbo,openai_whisper-large-v3_turbo,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad14%2C8_summary_2024-10-30T073931.json>iPadOS 17.6.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C1_summary_2024-10-29T234233.json>iPadOS 18.0.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C3_summary_2024-10-30T073931.json>iPadOS 18.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 openai_whisper-large-v3_turbo_954MB,openai_whisper-large-v3_turbo_954MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C5_summary_2024-10-30T073931.json>iOS 17.3</a>,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone17%2C4_summary_2024-10-30T132649.json>iOS 18.0.1</a>
 openai_whisper-small,openai_whisper-small,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,✅ iPadOS 18.1,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone13%2C1_summary_2024-10-30T022739.json>iOS 17.7</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone13%2C2_summary_2024-10-30T152729.json>iOS 17.2.1</a>,✅ iOS 18.1,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
 openai_whisper-small.en,openai_whisper-small.en,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,✅ iPadOS 18.1,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone13%2C1_summary_2024-10-30T022739.json>iOS 17.7</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone13%2C2_summary_2024-10-30T152729.json>iOS 17.2.1</a>,✅ iOS 18.1,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1

 ,Model,"Apple M2 Pro (Mac14,12)","Apple M2 Ultra (Mac14,14)","Apple M2 (Mac14,2)","Apple M3 Max (Mac15,10)","Apple M3 Pro (Mac15,6)","Apple M3 Max (Mac15,9)","Apple M4 (Mac16,10)","Apple M4 (Mac16,3)","Apple M1 (MacBookAir10,1)","iPad Air (5th generation) (iPad13,16)","iPad Air 11-inch (M2) (iPad14,8)","iPad mini (A17 Pro) (iPad16,1)","iPad Pro 11-inch (M4) (iPad16,3)","iPhone 11 (iPhone12,1)","iPhone 12 mini (iPhone13,1)","iPhone 12 (iPhone13,2)","iPhone 12 Pro Max (iPhone13,4)","iPhone 13 Pro (iPhone14,2)","iPhone 13 (iPhone14,5)","iPhone 14 (iPhone14,7)","iPhone 15 Pro (iPhone16,1)","iPhone 15 Pro Max (iPhone16,2)","iPhone 16 Pro (iPhone17,1)","iPhone 16 (iPhone17,3)","iPhone 16 Plus (iPhone17,4)"
+distil-whisper_distil-large-v3,distil-whisper_distil-large-v3,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/MacBookAir10%2C1_summary_2024-10-30T012535.json>macOS 15.1</a>,Not Supported,✅ iPadOS 17.6.1,Not Supported,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 distil-whisper_distil-large-v3_594MB,distil-whisper_distil-large-v3_594MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
+distil-whisper_distil-large-v3_turbo,distil-whisper_distil-large-v3_turbo,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad14%2C8_summary_2024-10-30T073931.json>iPadOS 17.6.1</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C3_summary_2024-10-30T073931.json>iPadOS 18.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 distil-whisper_distil-large-v3_turbo_600MB,distil-whisper_distil-large-v3_turbo_600MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
 openai_whisper-base,openai_whisper-base,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,✅ iPadOS 18.1,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,✅ iOS 17.6.1,✅ iOS 17.7,✅ iOS 17.2.1,✅ iOS 18.1,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
 openai_whisper-base.en,openai_whisper-base.en,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,✅ iPadOS 18.1,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,✅ iOS 17.6.1,✅ iOS 17.7,✅ iOS 17.2.1,✅ iOS 18.1,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
+openai_whisper-large-v2,openai_whisper-large-v2,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/MacBookAir10%2C1_summary_2024-10-30T012535.json>macOS 15.1</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad14%2C8_summary_2024-10-30T073931.json>iPadOS 17.6.1</a>,Not Supported,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 openai_whisper-large-v2_949MB,openai_whisper-large-v2_949MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C5_summary_2024-10-30T073931.json>iOS 17.3</a>,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
+openai_whisper-large-v2_turbo,openai_whisper-large-v2_turbo,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad14%2C8_summary_2024-10-30T073931.json>iPadOS 17.6.1</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C3_summary_2024-10-30T073931.json>iPadOS 18.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 openai_whisper-large-v2_turbo_955MB,openai_whisper-large-v2_turbo_955MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C5_summary_2024-10-30T073931.json>iOS 17.3</a>,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
+openai_whisper-large-v3,openai_whisper-large-v3,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/MacBookAir10%2C1_summary_2024-10-30T012535.json>macOS 15.1</a>,Not Supported,✅ iPadOS 17.6.1,Not Supported,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
+openai_whisper-large-v3-v20240930,openai_whisper-large-v3-v20240930,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/MacBookAir10%2C1_summary_2024-10-30T012535.json>macOS 15.1</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad14%2C8_summary_2024-10-30T073931.json>iPadOS 17.6.1</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C3_summary_2024-10-30T073931.json>iPadOS 18.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 openai_whisper-large-v3-v20240930_626MB,openai_whisper-large-v3-v20240930_626MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
+openai_whisper-large-v3-v20240930_turbo,openai_whisper-large-v3-v20240930_turbo,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,✅ iPadOS 17.6.1,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C3_summary_2024-10-30T073931.json>iPadOS 18.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 openai_whisper-large-v3-v20240930_turbo_632MB,openai_whisper-large-v3-v20240930_turbo_632MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C5_summary_2024-10-30T073931.json>iOS 17.3</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C7_summary_2024-10-30T073931.json>iOS 17.3</a>,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone17%2C4_summary_2024-10-30T132649.json>iOS 18.0.1</a>
 openai_whisper-large-v3_947MB,openai_whisper-large-v3_947MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C5_summary_2024-10-30T073931.json>iOS 17.3</a>,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
+openai_whisper-large-v3_turbo,openai_whisper-large-v3_turbo,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad14%2C8_summary_2024-10-30T073931.json>iPadOS 17.6.1</a>,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C3_summary_2024-10-30T073931.json>iPadOS 18.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 openai_whisper-large-v3_turbo_954MB,openai_whisper-large-v3_turbo_954MB,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,Not Supported,Not Supported,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,✅ iOS 18.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone14%2C5_summary_2024-10-30T073931.json>iOS 17.3</a>,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone17%2C4_summary_2024-10-30T132649.json>iOS 18.0.1</a>
 openai_whisper-small,openai_whisper-small,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,✅ iPadOS 18.1,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone13%2C1_summary_2024-10-30T022739.json>iOS 17.7</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone13%2C2_summary_2024-10-30T152729.json>iOS 17.2.1</a>,✅ iOS 18.1,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1
 openai_whisper-small.en,openai_whisper-small.en,✅ macOS 15.0.1,✅ macOS 15.0.1,✅ macOS 15.1,✅ macOS 15.2,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.1,✅ macOS 15.0,✅ macOS 15.1,✅ iPadOS 18.1,✅ iPadOS 17.6.1,✅ iPadOS 18.0.1,✅ iPadOS 18.1,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone13%2C1_summary_2024-10-30T022739.json>iOS 17.7</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPhone13%2C2_summary_2024-10-30T152729.json>iOS 17.2.1</a>,✅ iOS 18.1,✅ iOS 18.1,✅ iOS 17.3,✅ iOS 17.3,✅ iOS 18.1,✅ iOS 18.2,✅ iOS 18.0,✅ iOS 18.0.1,✅ iOS 18.0.1

dashboard_data/test_coverage_112a023.json ADDED Viewed

	@@ -0,0 +1,239 @@

+{
+  "commit_hash": "112a023",
+  "total_devices": 137,
+  "tested_devices": 49,
+  "skipped_devices": 88,
+  "coverage_percentage": 35.76642335766424,
+  "tested_device_list": [
+    "iPhone12,3",
+    "Mac15,10",
+    "iPad14,9",
+    "iPhone12,1",
+    "Mac14,15",
+    "iPhone14,6",
+    "Mac15,13",
+    "Mac14,6",
+    "iPhone14,3",
+    "Mac14,13",
+    "Mac14,5",
+    "iPhone14,8",
+    "iPhone14,5",
+    "iPhone14,2",
+    "iPad14,6",
+    "Mac14,12",
+    "Mac16,9",
+    "Mac14,8",
+    "Mac15,12",
+    "Mac14,2",
+    "iPad14,10",
+    "iPhone14,7",
+    "Mac14,10",
+    "iPad16,6",
+    "iPad16,3",
+    "iPad16,5",
+    "Mac15,5",
+    "iPhone14,4",
+    "Mac14,14",
+    "Mac14,7",
+    "Mac15,11",
+    "iPad14,3",
+    "iPad14,5",
+    "Mac14,9",
+    "Mac15,6",
+    "iPad16,4",
+    "Mac15,4",
+    "Mac15,8",
+    "Mac16,8",
+    "Mac14,4",
+    "Mac15,9",
+    "Mac15,3",
+    "iPad14,4",
+    "Mac14,3",
+    "iPhone12,8",
+    "iPad14,11",
+    "Mac15,7",
+    "iPhone12,5",
+    "iPad14,8"
+  ],
+  "skipped_device_list": [
+    "iPad15,5",
+    "iPad13,2",
+    "iPad8,6",
+    "iPad8,12",
+    "iPad8,9",
+    "iPad15,7",
+    "iPad15,3",
+    "iPad13,8",
+    "Mac16,6",
+    "iPhone11,5",
+    "iPad13,19",
+    "iPad8,2",
+    "iPhone16,1",
+    "iPhone12,2",
+    "iPad13,7",
+    "iPad8,3",
+    "iPhone13,1",
+    "iPad16,2",
+    "Macmini9,1",
+    "MacBookPro18,2",
+    "iPhone11,8",
+    "MacBookPro17,1",
+    "Mac16,10",
+    "iPhone15,5",
+    "iPhone17,4",
+    "Mac13,1",
+    "MacBookAir10,1",
+    "Mac13,2",
+    "iPad13,10",
+    "iPad8,11",
+    "Mac16,13",
+    "iPad13,1",
+    "iPad8,10",
+    "iPad13,5",
+    "iPad14,2",
+    "iPad13,9",
+    "iPhone15,2",
+    "iPad11,3",
+    "iPhone13,2",
+    "iPhone17,5",
+    "MacBookPro18,1",
+    "iMac21,2",
+    "MacBookPro18,4",
+    "iPad13,6",
+    "iPhone11,4",
+    "iPad8,1",
+    "iPhone17,2",
+    "iPhone11,6",
+    "iPad12,1",
+    "iMac21,1",
+    "MacBookPro18,3",
+    "iPhone11,3",
+    "iPad8,4",
+    "iPad11,4",
+    "iPad13,18",
+    "Mac16,3",
+    "iPhone12,4",
+    "iPhone13,3",
+    "Mac16,5",
+    "iPhone17,3",
+    "iPad8,7",
+    "iPad13,4",
+    "iPad15,8",
+    "iPad11,1",
+    "iPad11,2",
+    "iPhone13,4",
+    "iPad14,1",
+    "iPad13,17",
+    "iPad16,1",
+    "iPhone16,2",
+    "iPad15,4",
+    "iPhone11,2",
+    "iPhone15,4",
+    "iPhone17,1",
+    "iPad13,11",
+    "iPad8,5",
+    "Mac16,1",
+    "Mac16,2",
+    "iPad11,7",
+    "iPad11,6",
+    "Mac16,7",
+    "iPad13,16",
+    "iPhone15,3",
+    "iPad8,8",
+    "Mac16,12",
+    "iPad15,6",
+    "Mac16,11",
+    "iPad12,2"
+  ],
+  "tested_os_versions": [
+    "macOS_15.1.1",
+    "iPadOS_18.1",
+    "iPadOS_17.6.1",
+    "macOS_15.0.1",
+    "macOS_15.2.0",
+    "iOS_17.6.1",
+    "iOS_17.7.1",
+    "macOS_15.2"
+  ],
+  "has_target_os_coverage": false,
+  "covered_target_versions": [
+    "macOS 15",
+    "iOS 17",
+    "iOS 18"
+  ],
+  "missing_target_versions": [
+    "macOS 14",
+    "macOS 26",
+    "iOS 26"
+  ],
+  "has_target_chip_coverage": false,
+  "platform_chip_coverage": {
+    "iPad": {
+      "total_chips": 8,
+      "tested_chips": 2,
+      "coverage_percentage": 25.0,
+      "covered_chips": [
+        "M2",
+        "M4"
+      ],
+      "missing_chips": [
+        "A14",
+        "A15",
+        "A16",
+        "A17 Pro",
+        "M1",
+        "M3"
+      ]
+    },
+    "iPhone": {
+      "total_chips": 6,
+      "tested_chips": 2,
+      "coverage_percentage": 33.33333333333333,
+      "covered_chips": [
+        "A13",
+        "A15"
+      ],
+      "missing_chips": [
+        "A14",
+        "A16",
+        "A17 Pro",
+        "A18",
+        "A18 Pro"
+      ]
+    },
+    "Mac": {
+      "total_chips": 4,
+      "tested_chips": 2,
+      "coverage_percentage": 50.0,
+      "covered_chips": [
+        "M2",
+        "M3"
+      ],
+      "missing_chips": [
+        "M1",
+        "M4"
+      ]
+    }
+  },
+  "missing_target_chips": {
+    "iPad": [
+      "A14",
+      "A15",
+      "A16",
+      "A17 Pro",
+      "M1",
+      "M3"
+    ],
+    "iPhone": [
+      "A14",
+      "A16",
+      "A17 Pro",
+      "A18",
+      "A18 Pro"
+    ],
+    "Mac": [
+      "M1",
+      "M4"
+    ]
+  }
+}

dashboard_data/test_coverage_11a1fab.json ADDED Viewed

	@@ -0,0 +1,233 @@

+{
+  "commit_hash": "11a1fab",
+  "total_devices": 137,
+  "tested_devices": 22,
+  "skipped_devices": 115,
+  "coverage_percentage": 16.05839416058394,
+  "tested_device_list": [
+    "iPad15,5",
+    "iPad13,2",
+    "Mac16,5",
+    "Mac16,7",
+    "Mac16,13",
+    "iPad15,8",
+    "iPad13,1",
+    "iPad15,7",
+    "iPad15,3",
+    "iPad16,2",
+    "Mac16,6",
+    "Mac16,1",
+    "iPad13,19",
+    "Mac16,10",
+    "Mac16,12",
+    "iPad13,18",
+    "Mac16,2",
+    "iPad15,6",
+    "iPad16,1",
+    "Mac16,3",
+    "Mac16,11",
+    "iPad15,4"
+  ],
+  "skipped_device_list": [
+    "Mac15,10",
+    "iPad8,6",
+    "iPad8,12",
+    "iPad8,9",
+    "Mac14,13",
+    "Mac14,5",
+    "iPhone14,5",
+    "Mac15,12",
+    "iPad13,8",
+    "Mac14,10",
+    "iPhone11,5",
+    "iPad8,2",
+    "iPhone16,1",
+    "Mac14,9",
+    "iPhone12,2",
+    "iPad13,7",
+    "iPad16,4",
+    "iPad8,3",
+    "iPhone13,1",
+    "Macmini9,1",
+    "Mac15,9",
+    "MacBookPro18,2",
+    "Mac14,3",
+    "iPhone11,8",
+    "iPhone12,8",
+    "MacBookPro17,1",
+    "Mac15,7",
+    "iPhone15,5",
+    "iPhone17,4",
+    "Mac13,1",
+    "MacBookAir10,1",
+    "Mac13,2",
+    "iPad13,10",
+    "iPad8,11",
+    "Mac14,6",
+    "iPhone14,3",
+    "iPhone14,2",
+    "Mac16,9",
+    "iPad8,10",
+    "Mac14,2",
+    "iPad13,5",
+    "iPad14,2",
+    "iPad13,9",
+    "iPhone15,2",
+    "iPad16,3",
+    "iPad16,5",
+    "iPad11,3",
+    "iPhone13,2",
+    "iPhone17,5",
+    "MacBookPro18,1",
+    "iPad14,3",
+    "iPad14,5",
+    "iMac21,2",
+    "MacBookPro18,4",
+    "iPad13,6",
+    "iPhone11,4",
+    "Mac15,4",
+    "iPad8,1",
+    "iPhone17,2",
+    "Mac15,8",
+    "iPhone11,6",
+    "iPad12,1",
+    "iPad14,8",
+    "iPhone12,3",
+    "iMac21,1",
+    "iPhone12,1",
+    "Mac14,15",
+    "iPhone14,6",
+    "MacBookPro18,3",
+    "Mac15,13",
+    "iPad14,6",
+    "iPhone14,8",
+    "Mac14,8",
+    "Mac14,12",
+    "iPhone11,3",
+    "iPhone14,7",
+    "iPad8,4",
+    "iPad11,4",
+    "iPad16,6",
+    "Mac15,5",
+    "iPhone14,4",
+    "iPhone12,4",
+    "iPhone13,3",
+    "Mac15,11",
+    "iPhone17,3",
+    "iPad8,7",
+    "iPad13,4",
+    "Mac15,6",
+    "iPad11,1",
+    "iPad11,2",
+    "Mac14,4",
+    "iPad14,4",
+    "iPhone13,4",
+    "iPad14,1",
+    "iPad13,17",
+    "iPhone12,5",
+    "iPhone16,2",
+    "iPhone11,2",
+    "iPad14,9",
+    "iPad14,10",
+    "iPhone15,4",
+    "iPhone17,1",
+    "iPad13,11",
+    "iPad8,5",
+    "Mac14,14",
+    "Mac14,7",
+    "iPad11,7",
+    "iPad11,6",
+    "Mac16,8",
+    "iPad13,16",
+    "Mac15,3",
+    "iPhone15,3",
+    "iPad8,8",
+    "iPad14,11",
+    "iPad12,2"
+  ],
+  "tested_os_versions": [
+    "macOS_15.4",
+    "iPadOS_18.3.2",
+    "macOS_15.4.0",
+    "iPadOS_18.4"
+  ],
+  "has_target_os_coverage": false,
+  "covered_target_versions": [
+    "macOS 15",
+    "iOS 18"
+  ],
+  "missing_target_versions": [
+    "macOS 14",
+    "macOS 26",
+    "iOS 17",
+    "iOS 26"
+  ],
+  "has_target_chip_coverage": false,
+  "platform_chip_coverage": {
+    "iPad": {
+      "total_chips": 8,
+      "tested_chips": 4,
+      "coverage_percentage": 50.0,
+      "covered_chips": [
+        "A14",
+        "A16",
+        "A17 Pro",
+        "M3"
+      ],
+      "missing_chips": [
+        "A15",
+        "M1",
+        "M2",
+        "M4"
+      ]
+    },
+    "iPhone": {
+      "total_chips": 6,
+      "tested_chips": 0,
+      "coverage_percentage": 0.0,
+      "covered_chips": [],
+      "missing_chips": [
+        "A14",
+        "A15",
+        "A16",
+        "A17 Pro",
+        "A18",
+        "A18 Pro"
+      ]
+    },
+    "Mac": {
+      "total_chips": 4,
+      "tested_chips": 1,
+      "coverage_percentage": 25.0,
+      "covered_chips": [
+        "M4"
+      ],
+      "missing_chips": [
+        "M1",
+        "M2",
+        "M3"
+      ]
+    }
+  },
+  "missing_target_chips": {
+    "iPad": [
+      "A15",
+      "M1",
+      "M2",
+      "M4"
+    ],
+    "iPhone": [
+      "A14",
+      "A15",
+      "A16",
+      "A17 Pro",
+      "A18",
+      "A18 Pro"
+    ],
+    "Mac": [
+      "M1",
+      "M2",
+      "M3"
+    ]
+  }
+}

dashboard_data/test_coverage_8c0acbd.json ADDED Viewed

	@@ -0,0 +1,234 @@

+{
+  "commit_hash": "8c0acbd",
+  "total_devices": 137,
+  "tested_devices": 82,
+  "skipped_devices": 55,
+  "coverage_percentage": 59.854014598540154,
+  "tested_device_list": [
+    "iPhone12,3",
+    "Mac15,10",
+    "iPad15,5",
+    "iPhone12,1",
+    "Mac14,15",
+    "iPhone14,6",
+    "Mac15,13",
+    "Mac14,13",
+    "iPad14,6",
+    "Mac14,5",
+    "iPhone14,8",
+    "iPhone14,5",
+    "Mac14,12",
+    "Mac14,8",
+    "Mac15,12",
+    "iPad15,7",
+    "iPad15,3",
+    "iPad13,8",
+    "iPhone14,7",
+    "Mac14,10",
+    "iPad16,6",
+    "Mac15,5",
+    "iPhone14,4",
+    "iPhone13,3",
+    "Mac15,11",
+    "Mac14,9",
+    "iPhone16,1",
+    "iPad13,7",
+    "iPhone17,3",
+    "iPad13,4",
+    "Mac15,6",
+    "iPad16,4",
+    "iPad15,8",
+    "iPhone13,1",
+    "Mac14,4",
+    "iPad16,2",
+    "Mac15,9",
+    "iPad14,1",
+    "iPhone13,4",
+    "Mac14,3",
+    "iPad14,4",
+    "iPhone12,8",
+    "Mac15,7",
+    "iPhone15,5",
+    "iPad13,17",
+    "iPhone12,5",
+    "iPad16,1",
+    "iPhone16,2",
+    "iPad15,4",
+    "iPhone17,4",
+    "iPad14,9",
+    "iPad13,10",
+    "Mac14,6",
+    "iPhone14,3",
+    "iPhone14,2",
+    "Mac16,9",
+    "Mac14,2",
+    "iPad13,5",
+    "iPad14,2",
+    "iPad14,10",
+    "iPhone15,4",
+    "iPad13,9",
+    "iPad13,11",
+    "iPhone15,2",
+    "iPad16,3",
+    "iPad16,5",
+    "iPhone13,2",
+    "Mac14,14",
+    "iPhone17,5",
+    "Mac14,7",
+    "iPad14,3",
+    "iPad14,5",
+    "iPad13,6",
+    "Mac15,4",
+    "Mac15,8",
+    "iPad13,16",
+    "Mac16,8",
+    "Mac15,3",
+    "iPhone15,3",
+    "iPad14,11",
+    "iPad15,6",
+    "iPad14,8"
+  ],
+  "skipped_device_list": [
+    "Mac13,1",
+    "iMac21,1",
+    "iPad13,2",
+    "iPad8,6",
+    "MacBookAir10,1",
+    "Mac13,2",
+    "iPad8,12",
+    "iPad8,11",
+    "MacBookPro18,3",
+    "iPad8,9",
+    "Mac16,13",
+    "iPad13,1",
+    "iPhone11,3",
+    "iPad8,10",
+    "iPhone17,1",
+    "Mac16,6",
+    "iPad8,5",
+    "iPhone11,5",
+    "iPad8,4",
+    "iPad11,4",
+    "iPad11,3",
+    "iPad13,19",
+    "Mac16,1",
+    "Mac16,2",
+    "iPad13,18",
+    "iPad8,2",
+    "Mac16,3",
+    "iPhone12,4",
+    "MacBookPro18,1",
+    "iPad11,7",
+    "iMac21,2",
+    "iPhone12,2",
+    "MacBookPro18,4",
+    "Mac16,5",
+    "iPad11,6",
+    "iPad12,2",
+    "iPad8,7",
+    "iPhone11,4",
+    "Mac16,7",
+    "iPad8,3",
+    "iPad8,1",
+    "iPhone17,2",
+    "iPhone11,6",
+    "iPad11,1",
+    "iPad11,2",
+    "Macmini9,1",
+    "iPad12,1",
+    "MacBookPro18,2",
+    "iPhone11,8",
+    "MacBookPro17,1",
+    "iPad8,8",
+    "Mac16,10",
+    "Mac16,12",
+    "Mac16,11",
+    "iPhone11,2"
+  ],
+  "tested_os_versions": [
+    "macOS_15.5.0",
+    "iPadOS_18.4.1",
+    "iOS_18.4.1",
+    "iOS_19.0",
+    "iOS_18.1",
+    "iOS_26.0",
+    "macOS_15.5",
+    "iPadOS_26.0",
+    "iPadOS_18.5",
+    "iOS_18.5",
+    "iPadOS_18.4",
+    "iPadOS_19.0"
+  ],
+  "has_target_os_coverage": false,
+  "covered_target_versions": [
+    "macOS 15",
+    "iOS 18",
+    "iOS 26"
+  ],
+  "missing_target_versions": [
+    "macOS 14",
+    "macOS 26",
+    "iOS 17"
+  ],
+  "has_target_chip_coverage": false,
+  "platform_chip_coverage": {
+    "iPad": {
+      "total_chips": 8,
+      "tested_chips": 7,
+      "coverage_percentage": 87.5,
+      "covered_chips": [
+        "A15",
+        "A16",
+        "A17 Pro",
+        "M1",
+        "M2",
+        "M3",
+        "M4"
+      ],
+      "missing_chips": [
+        "A14"
+      ]
+    },
+    "iPhone": {
+      "total_chips": 6,
+      "tested_chips": 6,
+      "coverage_percentage": 100.0,
+      "covered_chips": [
+        "A13",
+        "A14",
+        "A15",
+        "A16",
+        "A17 Pro",
+        "A18"
+      ],
+      "missing_chips": [
+        "A18 Pro"
+      ]
+    },
+    "Mac": {
+      "total_chips": 4,
+      "tested_chips": 2,
+      "coverage_percentage": 50.0,
+      "covered_chips": [
+        "M2",
+        "M3"
+      ],
+      "missing_chips": [
+        "M1",
+        "M4"
+      ]
+    }
+  },
+  "missing_target_chips": {
+    "iPad": [
+      "A14"
+    ],
+    "iPhone": [
+      "A18 Pro"
+    ],
+    "Mac": [
+      "M1",
+      "M4"
+    ]
+  }
+}

dashboard_data/test_coverage_a9b92c4.json ADDED Viewed

	@@ -0,0 +1,239 @@

+{
+  "commit_hash": "a9b92c4",
+  "total_devices": 137,
+  "tested_devices": 93,
+  "skipped_devices": 44,
+  "coverage_percentage": 67.88321167883211,
+  "tested_device_list": [
+    "iPhone12,3",
+    "Mac15,10",
+    "iMac21,1",
+    "iPhone12,1",
+    "Mac14,15",
+    "iPhone14,6",
+    "MacBookPro18,3",
+    "Mac15,13",
+    "Mac14,13",
+    "iPad14,6",
+    "Mac14,5",
+    "iPhone14,8",
+    "iPhone14,5",
+    "Mac14,12",
+    "Mac14,8",
+    "Mac15,12",
+    "iPad13,8",
+    "iPhone14,7",
+    "Mac16,6",
+    "Mac14,10",
+    "iPad16,6",
+    "Mac15,5",
+    "iPhone14,4",
+    "Mac16,3",
+    "iPhone13,3",
+    "Mac15,11",
+    "iPhone16,1",
+    "Mac14,9",
+    "Mac16,5",
+    "iPad13,7",
+    "iPhone17,3",
+    "iPad13,4",
+    "Mac15,6",
+    "iPad16,4",
+    "iPhone13,1",
+    "Mac14,4",
+    "iPad16,2",
+    "Macmini9,1",
+    "Mac15,9",
+    "iPhone13,4",
+    "iPad14,4",
+    "MacBookPro18,2",
+    "Mac14,3",
+    "iPhone12,8",
+    "MacBookPro17,1",
+    "Mac16,10",
+    "Mac15,7",
+    "iPhone12,5",
+    "iPad16,1",
+    "iPhone16,2",
+    "iPad13,17",
+    "iPhone17,4",
+    "Mac13,1",
+    "MacBookAir10,1",
+    "iPad14,9",
+    "Mac13,2",
+    "iPad13,10",
+    "Mac14,6",
+    "iPhone14,3",
+    "iPhone14,2",
+    "Mac16,13",
+    "Mac16,9",
+    "Mac14,2",
+    "iPad13,5",
+    "iPad14,10",
+    "iPhone17,1",
+    "iPad13,9",
+    "iPad13,11",
+    "iPad16,3",
+    "iPad16,5",
+    "iPhone13,2",
+    "Mac16,1",
+    "Mac16,2",
+    "Mac14,14",
+    "MacBookPro18,1",
+    "Mac14,7",
+    "iPhone17,5",
+    "iPad14,3",
+    "iPad14,5",
+    "iMac21,2",
+    "MacBookPro18,4",
+    "iPad13,6",
+    "Mac16,7",
+    "Mac15,4",
+    "iPhone17,2",
+    "Mac15,8",
+    "iPad13,16",
+    "Mac16,8",
+    "Mac15,3",
+    "Mac16,12",
+    "iPad14,11",
+    "Mac16,11",
+    "iPad14,8"
+  ],
+  "skipped_device_list": [
+    "iPad15,5",
+    "iPad13,2",
+    "iPad8,6",
+    "iPad15,6",
+    "iPad8,12",
+    "iPad8,11",
+    "iPad8,9",
+    "iPad13,1",
+    "iPhone11,3",
+    "iPad8,10",
+    "iPad15,7",
+    "iPad15,3",
+    "iPad14,2",
+    "iPhone15,4",
+    "iPhone15,2",
+    "iPad8,5",
+    "iPhone11,5",
+    "iPad8,4",
+    "iPad11,4",
+    "iPad11,3",
+    "iPad13,19",
+    "iPad13,18",
+    "iPad8,2",
+    "iPhone12,4",
+    "iPad11,7",
+    "iPhone12,2",
+    "iPad11,6",
+    "iPad12,2",
+    "iPad8,7",
+    "iPhone11,4",
+    "iPad8,3",
+    "iPad8,1",
+    "iPad15,8",
+    "iPhone11,6",
+    "iPad11,1",
+    "iPad11,2",
+    "iPad12,1",
+    "iPad14,1",
+    "iPhone11,8",
+    "iPhone15,3",
+    "iPad8,8",
+    "iPhone15,5",
+    "iPad15,4",
+    "iPhone11,2"
+  ],
+  "tested_os_versions": [
+    "macOS_15.0.0",
+    "macOS_15.1.0",
+    "iPadOS_18.1",
+    "iOS_18.1",
+    "iPadOS_17.6.1",
+    "macOS_15.0.1",
+    "macOS_15.2.0",
+    "iOS_17.3",
+    "iOS_18.0",
+    "iPadOS_18.0.1",
+    "macOS_15.2",
+    "macOS_15.1",
+    "iOS_17.2.1",
+    "iOS_17.7",
+    "iOS_17.6.1",
+    "iOS_18.2",
+    "iOS_18.0.1",
+    "macOS_15.0"
+  ],
+  "has_target_os_coverage": false,
+  "covered_target_versions": [
+    "macOS 15",
+    "iOS 17",
+    "iOS 18"
+  ],
+  "missing_target_versions": [
+    "macOS 14",
+    "macOS 26",
+    "iOS 26"
+  ],
+  "has_target_chip_coverage": false,
+  "platform_chip_coverage": {
+    "iPad": {
+      "total_chips": 8,
+      "tested_chips": 4,
+      "coverage_percentage": 50.0,
+      "covered_chips": [
+        "A17 Pro",
+        "M1",
+        "M2",
+        "M4"
+      ],
+      "missing_chips": [
+        "A14",
+        "A15",
+        "A16",
+        "M3"
+      ]
+    },
+    "iPhone": {
+      "total_chips": 6,
+      "tested_chips": 6,
+      "coverage_percentage": 100.0,
+      "covered_chips": [
+        "A13",
+        "A14",
+        "A15",
+        "A17 Pro",
+        "A18",
+        "A18 Pro"
+      ],
+      "missing_chips": [
+        "A16"
+      ]
+    },
+    "Mac": {
+      "total_chips": 4,
+      "tested_chips": 4,
+      "coverage_percentage": 100.0,
+      "covered_chips": [
+        "M1",
+        "M2",
+        "M3",
+        "M4"
+      ],
+      "missing_chips": []
+    }
+  },
+  "missing_target_chips": {
+    "iPad": [
+      "A14",
+      "A15",
+      "A16",
+      "M3"
+    ],
+    "iPhone": [
+      "A16"
+    ],
+    "Mac": []
+  }
+}

dashboard_data/test_coverage_ca49596.json ADDED Viewed

	@@ -0,0 +1,237 @@

+{
+  "commit_hash": "ca49596",
+  "total_devices": 137,
+  "tested_devices": 63,
+  "skipped_devices": 74,
+  "coverage_percentage": 45.98540145985402,
+  "tested_device_list": [
+    "iPhone12,3",
+    "Mac15,10",
+    "iPad15,5",
+    "iPad14,9",
+    "iPad15,6",
+    "iPhone12,1",
+    "Mac14,15",
+    "iPhone14,6",
+    "Mac15,13",
+    "Mac14,6",
+    "iPhone14,3",
+    "Mac14,13",
+    "Mac14,5",
+    "iPhone14,2",
+    "iPhone14,8",
+    "iPhone14,5",
+    "Mac16,13",
+    "Mac14,12",
+    "Mac16,9",
+    "Mac14,8",
+    "Mac15,12",
+    "Mac14,2",
+    "iPad15,3",
+    "iPad14,10",
+    "iPhone14,7",
+    "Mac16,6",
+    "Mac14,10",
+    "iPad16,6",
+    "iPad16,3",
+    "iPad16,5",
+    "Mac16,1",
+    "Mac15,5",
+    "Mac16,2",
+    "iPhone14,4",
+    "Mac14,14",
+    "Mac16,3",
+    "Mac14,7",
+    "Mac15,11",
+    "iPad14,3",
+    "iPad14,5",
+    "Mac14,9",
+    "Mac16,5",
+    "Mac15,6",
+    "iPad16,4",
+    "Mac16,7",
+    "Mac15,4",
+    "Mac15,8",
+    "Mac16,8",
+    "Mac14,4",
+    "iPad14,6",
+    "Mac15,9",
+    "Mac15,3",
+    "iPad14,4",
+    "Mac14,3",
+    "iPhone12,8",
+    "Mac16,10",
+    "Mac15,7",
+    "Mac16,12",
+    "iPad14,11",
+    "iPhone12,5",
+    "Mac16,11",
+    "iPad14,8",
+    "iPad15,4"
+  ],
+  "skipped_device_list": [
+    "Mac13,1",
+    "iMac21,1",
+    "iPad13,2",
+    "iPad8,6",
+    "MacBookAir10,1",
+    "Mac13,2",
+    "iPad8,12",
+    "iPad13,10",
+    "iPad8,11",
+    "iPad8,9",
+    "MacBookPro18,3",
+    "iPad13,1",
+    "iPhone11,3",
+    "iPad8,10",
+    "iPad15,7",
+    "iPad13,5",
+    "iPad13,8",
+    "iPad14,2",
+    "iPhone15,4",
+    "iPad13,9",
+    "iPhone17,1",
+    "iPhone11,2",
+    "iPhone15,2",
+    "iPad13,11",
+    "iPhone11,5",
+    "iPad8,4",
+    "iPad11,4",
+    "iPad8,5",
+    "iPad11,3",
+    "iPad13,19",
+    "iPhone13,2",
+    "iPad13,18",
+    "iPad8,2",
+    "iPhone12,4",
+    "iPhone17,5",
+    "MacBookPro18,1",
+    "iPhone13,3",
+    "iPad11,7",
+    "iPhone16,1",
+    "iMac21,2",
+    "iPhone12,2",
+    "MacBookPro18,4",
+    "iPad13,7",
+    "iPhone17,3",
+    "iPad11,6",
+    "iPad8,7",
+    "iPad13,6",
+    "iPad13,4",
+    "iPhone11,4",
+    "iPad12,2",
+    "iPad8,3",
+    "iPad8,1",
+    "iPad13,17",
+    "iPhone17,2",
+    "iPad15,8",
+    "iPad13,16",
+    "iPhone13,1",
+    "iPhone11,6",
+    "iPad16,2",
+    "iPad11,1",
+    "Macmini9,1",
+    "iPad11,2",
+    "iPhone13,4",
+    "iPad12,1",
+    "iPad14,1",
+    "MacBookPro18,2",
+    "iPhone11,8",
+    "MacBookPro17,1",
+    "iPhone15,3",
+    "iPad8,8",
+    "iPhone15,5",
+    "iPad16,1",
+    "iPhone16,2",
+    "iPhone17,4"
+  ],
+  "tested_os_versions": [
+    "macOS_15.3",
+    "iOS_18.1",
+    "iPadOS_18.3.2",
+    "macOS_15.4",
+    "iOS_17.6.1",
+    "iPadOS_18.4",
+    "macOS_15.4.0",
+    "macOS_15.3.0"
+  ],
+  "has_target_os_coverage": false,
+  "covered_target_versions": [
+    "macOS 15",
+    "iOS 17",
+    "iOS 18"
+  ],
+  "missing_target_versions": [
+    "macOS 14",
+    "macOS 26",
+    "iOS 26"
+  ],
+  "has_target_chip_coverage": false,
+  "platform_chip_coverage": {
+    "iPad": {
+      "total_chips": 8,
+      "tested_chips": 3,
+      "coverage_percentage": 37.5,
+      "covered_chips": [
+        "M2",
+        "M3",
+        "M4"
+      ],
+      "missing_chips": [
+        "A14",
+        "A15",
+        "A16",
+        "A17 Pro",
+        "M1"
+      ]
+    },
+    "iPhone": {
+      "total_chips": 6,
+      "tested_chips": 2,
+      "coverage_percentage": 33.33333333333333,
+      "covered_chips": [
+        "A13",
+        "A15"
+      ],
+      "missing_chips": [
+        "A14",
+        "A16",
+        "A17 Pro",
+        "A18",
+        "A18 Pro"
+      ]
+    },
+    "Mac": {
+      "total_chips": 4,
+      "tested_chips": 3,
+      "coverage_percentage": 75.0,
+      "covered_chips": [
+        "M2",
+        "M3",
+        "M4"
+      ],
+      "missing_chips": [
+        "M1"
+      ]
+    }
+  },
+  "missing_target_chips": {
+    "iPad": [
+      "A14",
+      "A15",
+      "A16",
+      "A17 Pro",
+      "M1"
+    ],
+    "iPhone": [
+      "A14",
+      "A16",
+      "A17 Pro",
+      "A18",
+      "A18 Pro"
+    ],
+    "Mac": [
+      "M1"
+    ]
+  }
+}

dashboard_data/version.json CHANGED Viewed

	@@ -1 +1 @@
1	- {"last_modified": "2025-06-17T19:55:16+00:00", "sha": "c677ebea4ee6bd556ca70de12bff099044cf8950", "releases": ["a9b92c4", "112a023", "ca49596", "11a1fab", "8c0acbd"], "versions": ["0.9.1", "0.10.1", "0.11.0", "0.12.0", "0.13.0"]}


1	+ {"last_modified": "2025-06-17T19:55:16+00:00", "sha": "c677ebea4ee6bd556ca70de12bff099044cf8950", "releases": ["a9b92c4", "112a023", "ca49596", "11a1fab", "8c0acbd"], "versions": ["0.9.1", "0.10.1", "0.11.0", "0.12.0", "0.13.0"]}

main.py CHANGED Viewed

@@ -28,6 +28,7 @@ from constants import (
 )
 from utils import (
     add_datasets_to_performance_columns,
     create_initial_performance_column_dict,
     css,
     fields,
@@ -56,8 +57,11 @@ PERFORMANCE_DATA = read_json_line_by_line("dashboard_data/performance_data.json"
 with open("dashboard_data/version.json", "r") as file:
     VERSION_DATA = json.load(file)
 SHA_TO_VERSION = {
-    VERSION_DATA["releases"][i]: VERSION_DATA["versions"][i]
     for i in range(len(VERSION_DATA["versions"]))
 }
@@ -66,7 +70,9 @@ benchmark_df = pd.json_normalize(PERFORMANCE_DATA)
 releases = VERSION_DATA["releases"]
 # Process timestamp data
-benchmark_df["timestamp"] = pd.to_datetime(benchmark_df["timestamp"]).dt.tz_localize(None)
 # Use average_wer directly from performance data
 benchmark_df["english_wer"] = benchmark_df["average_wer"]
@@ -106,12 +112,17 @@ performance_df = sorted_performance_df[
         "speed",
         "tokens_per_second",
         "timestamp",
-        "commit_hash"
     ]
     + dataset_speed_columns
     + dataset_toks_columns
 ].copy()
 # Rename columns for clarity
 performance_df = performance_df.rename(
     lambda x: COL_NAMES[x] if x in COL_NAMES else x, axis="columns"
@@ -136,12 +147,16 @@ for col in dataset_toks_columns:
 # Process model names for display
 performance_df["model_raw"] = performance_df["Model"].copy()
-performance_df["Model"] = performance_df["Model"].apply(lambda x: make_model_name_clickable_link(x))
 # Extract unique devices and OS versions
 initial_release_df = benchmark_df[benchmark_df["commit_hash"] == releases[-1]]
 PERFORMANCE_DEVICES = initial_release_df["device"].unique().tolist()
-PERFORMANCE_OS = initial_release_df["os"].apply(get_os_name_and_version).unique().tolist()
 PERFORMANCE_OS.sort()
 # Create initial column dictionaries and update with dataset information
@@ -163,16 +178,18 @@ PERFORMANCE_ALWAYS_HERE_COLS = performance_column_info["ALWAYS_HERE_COLS"]
 PERFORMANCE_TOGGLE_COLS = performance_column_info["TOGGLE_COLS"]
 PERFORMANCE_SELECTED_COLS = performance_column_info["SELECTED_COLS"]
 def get_release_devices(release):
     """
     Get the list of devices for a specific release.
     :param release: Selected release hash
     :return: List of devices available in the release
     """
     release_df = benchmark_df[benchmark_df["commit_hash"] == release]
     return release_df["device"].unique().tolist()
 def performance_filter(
     df,
     columns,
@@ -280,43 +297,41 @@ def performance_filter(
 def update_performance_filters(release):
     """
     Updates the performance filters (devices and OS) based on the selected release.
     :param release: Selected release hash
     :return: Tuple containing updated device and OS choices
     """
     # Filter benchmark data for the selected release
     release_df = benchmark_df[benchmark_df["commit_hash"] == release]
     # Get unique devices and OS versions for this release
     release_devices = release_df["device"].unique().tolist()
     release_os = release_df["os"].apply(get_os_name_and_version).unique().tolist()
     release_os.sort()
     return (
         gr.update(choices=release_devices, value=release_devices),
-        gr.update(choices=release_os, value=release_os)
     )
 def update_support_table(release):
     """
     Updates the support table and its column configuration for a given release.
     :param release: Selected release hash
     :return: Tuple containing (updated DataFrame, updated column choices, updated column values)
     """
     # Load new support data
     support_data = pd.read_csv(f"dashboard_data/support_data_{release[:7]}.csv")
     support_data.set_index(support_data.columns[0], inplace=True)
     # Process model names
-    support_data["Model"] = support_data["Model"].apply(
-        lambda x: x.replace("_", "/")
-    )
     support_data["Model"] = support_data["Model"].apply(
         lambda x: make_model_name_clickable_link(x)
     )
     # Sort by model name length
     support_data = (
         support_data.assign(model_len=support_data["Model"].str.len())
@@ -326,14 +341,14 @@ def update_support_table(release):
         )
         .drop(columns=["model_len"])
     )
     # Get new columns (excluding 'Model')
     new_columns = support_data.columns.tolist()[1:]
     return (
         gr.update(value=support_data, datatype=["html" for _ in support_data.columns]),
         gr.update(choices=new_columns, value=new_columns),
-        gr.update(value=support_data)
     )
@@ -350,6 +365,7 @@ font = [
     "sans-serif",
 ]
 # Macos 14, 15, 26
 # ios 17, 18, 26
@@ -360,7 +376,9 @@ with gr.Blocks(css=css, theme=gr.themes.Base(font=font)) as demo:
     gr.HTML(BANNER_TEXT, elem_classes="markdown-text")
     gr.Markdown("### Release")
     release_dropdown = gr.Dropdown(
-        choices=[(f"{release} v{SHA_TO_VERSION[release]}", release) for release in releases],
         label="Select Release",
         value=releases[-1] if releases else None,
         elem_id="release-dropdown",
@@ -534,7 +552,9 @@ with gr.Blocks(css=css, theme=gr.themes.Base(font=font)) as demo:
                     with gr.Row():
                         gr.Markdown(PERFORMANCE_TEXT, elem_classes="markdown-text")
             with gr.Row():
-                initial_df = performance_df[performance_df["commit_hash"] == releases[-1]]
                 leaderboard_df = gr.components.Dataframe(
                     value=initial_df[
                         PERFORMANCE_ALWAYS_HERE_COLS + performance_shown_columns.value
@@ -611,11 +631,11 @@ with gr.Blocks(css=css, theme=gr.themes.Base(font=font)) as demo:
                     fn=update_performance_filters,
                     inputs=[release_dropdown],
                     outputs=[performance_shown_devices, performance_shown_os],
-                    queue=False
                 ).then(
                     fn=performance_filter,
                     inputs=performance_filter_inputs,
-                    outputs=filter_output
                 )
         # Timeline Tab
@@ -880,8 +900,31 @@ with gr.Blocks(css=css, theme=gr.themes.Base(font=font)) as demo:
         # Device Support Tab
         with gr.TabItem("Device Support", elem_id="device_support", id=6):
             # Load device support data from CSV
-            support_data = pd.read_csv(f"dashboard_data/support_data_{releases[-1][:7]}.csv")
             support_data.set_index(support_data.columns[0], inplace=True)
             support_data["Model"] = support_data["Model"].apply(
                 lambda x: x.replace("_", "/")
@@ -916,7 +959,9 @@ with gr.Blocks(css=css, theme=gr.themes.Base(font=font)) as demo:
                             with gr.Row():
                                 with gr.Column(scale=9):
                                     support_shown_columns = gr.CheckboxGroup(
-                                        choices=support_data.columns.tolist()[1:],  # Exclude 'Model' column
                                         value=support_data.columns.tolist()[1:],
                                         label="Toggle Columns",
                                         elem_id="support-column-select",
@@ -962,7 +1007,7 @@ with gr.Blocks(css=css, theme=gr.themes.Base(font=font)) as demo:
             def filter_support_data(df, columns, model_query, exclude_models):
                 """
                 Filters the device support data based on specified criteria.
                 :param df: The DataFrame to be filtered
                 :param columns: Columns to include in the output
                 :param model_query: Query string to filter models
@@ -983,7 +1028,9 @@ with gr.Blocks(css=css, theme=gr.themes.Base(font=font)) as demo:
                 # Exclude specified models
                 if exclude_models:
-                    exclude_list = [re.escape(m.strip()) for m in exclude_models.split(";")]
                     filtered_df = filtered_df[
                         ~filtered_df["Model"].str.contains(
                             "|".join(exclude_list), case=False, regex=True
@@ -991,7 +1038,9 @@ with gr.Blocks(css=css, theme=gr.themes.Base(font=font)) as demo:
                     ]
                 # Select columns
-                selected_columns = ["Model"] + [col for col in columns if col in df.columns]
                 filtered_df = filtered_df[selected_columns]
                 return filtered_df
@@ -999,12 +1048,14 @@ with gr.Blocks(css=css, theme=gr.themes.Base(font=font)) as demo:
             def select_all_support_columns(release):
                 """
                 Returns all current columns from the support shown columns.
                 :param release: Selected release hash
                 :return: List of all available choices
                 """
                 # Load new support data for the current release
-                support_data = pd.read_csv(f"dashboard_data/support_data_{release[:7]}.csv")
                 support_data.set_index(support_data.columns[0], inplace=True)
                 # Return all columns except 'Model'
                 return [col for col in support_data.columns if col != "Model"]
@@ -1028,23 +1079,265 @@ with gr.Blocks(css=css, theme=gr.themes.Base(font=font)) as demo:
             release_dropdown.change(
                 update_support_table,
                 inputs=[release_dropdown],
-                outputs=[device_support_table, support_shown_columns, hidden_support_df]
             ).then(
                 filter_support_data,
-                inputs=[hidden_support_df, support_shown_columns, filter_support_models, exclude_support_models],
-                outputs=device_support_table
             )
             # Also connect the filter inputs to update the table
-            for input_elem in [filter_support_models, exclude_support_models, support_shown_columns]:
                 input_elem.change(
                     filter_support_data,
-                    inputs=[hidden_support_df, support_shown_columns, filter_support_models, exclude_support_models],
-                    outputs=device_support_table
                 )
         # Methodology Tab
-        with gr.TabItem("Methodology", elem_id="methodology", id=7):
             gr.Markdown(METHODOLOGY_TEXT, elem_id="methodology-text")
     # Citation section
@@ -1058,4 +1351,4 @@ with gr.Blocks(css=css, theme=gr.themes.Base(font=font)) as demo:
         )
 # Launch the Gradio interface
-demo.launch(debug=True)

 )
 from utils import (
     add_datasets_to_performance_columns,
+    calculate_quality_parity,
     create_initial_performance_column_dict,
     css,
     fields,
 with open("dashboard_data/version.json", "r") as file:
     VERSION_DATA = json.load(file)
+# Load quality data (ground truth WER)
+QUALITY_DATA = read_json_line_by_line("dashboard_data/quality_data.json")
 SHA_TO_VERSION = {
+    VERSION_DATA["releases"][i]: VERSION_DATA["versions"][i]
     for i in range(len(VERSION_DATA["versions"]))
 }
 releases = VERSION_DATA["releases"]
 # Process timestamp data
+benchmark_df["timestamp"] = pd.to_datetime(benchmark_df["timestamp"]).dt.tz_localize(
+    None
+)
 # Use average_wer directly from performance data
 benchmark_df["english_wer"] = benchmark_df["average_wer"]
         "speed",
         "tokens_per_second",
         "timestamp",
+        "commit_hash",
     ]
     + dataset_speed_columns
     + dataset_toks_columns
 ].copy()
+# Calculate parity (difference between measured WER and ground truth WER)
+performance_df["parity"] = performance_df.apply(
+    lambda row: calculate_quality_parity(QUALITY_DATA, row), axis=1
+)
 # Rename columns for clarity
 performance_df = performance_df.rename(
     lambda x: COL_NAMES[x] if x in COL_NAMES else x, axis="columns"
 # Process model names for display
 performance_df["model_raw"] = performance_df["Model"].copy()
+performance_df["Model"] = performance_df["Model"].apply(
+    lambda x: make_model_name_clickable_link(x)
+)
 # Extract unique devices and OS versions
 initial_release_df = benchmark_df[benchmark_df["commit_hash"] == releases[-1]]
 PERFORMANCE_DEVICES = initial_release_df["device"].unique().tolist()
+PERFORMANCE_OS = (
+    initial_release_df["os"].apply(get_os_name_and_version).unique().tolist()
+)
 PERFORMANCE_OS.sort()
 # Create initial column dictionaries and update with dataset information
 PERFORMANCE_TOGGLE_COLS = performance_column_info["TOGGLE_COLS"]
 PERFORMANCE_SELECTED_COLS = performance_column_info["SELECTED_COLS"]
 def get_release_devices(release):
     """
     Get the list of devices for a specific release.
     :param release: Selected release hash
     :return: List of devices available in the release
     """
     release_df = benchmark_df[benchmark_df["commit_hash"] == release]
     return release_df["device"].unique().tolist()
 def performance_filter(
     df,
     columns,
 def update_performance_filters(release):
     """
     Updates the performance filters (devices and OS) based on the selected release.
     :param release: Selected release hash
     :return: Tuple containing updated device and OS choices
     """
     # Filter benchmark data for the selected release
     release_df = benchmark_df[benchmark_df["commit_hash"] == release]
     # Get unique devices and OS versions for this release
     release_devices = release_df["device"].unique().tolist()
     release_os = release_df["os"].apply(get_os_name_and_version).unique().tolist()
     release_os.sort()
     return (
         gr.update(choices=release_devices, value=release_devices),
+        gr.update(choices=release_os, value=release_os),
     )
 def update_support_table(release):
     """
     Updates the support table and its column configuration for a given release.
     :param release: Selected release hash
     :return: Tuple containing (updated DataFrame, updated column choices, updated column values)
     """
     # Load new support data
     support_data = pd.read_csv(f"dashboard_data/support_data_{release[:7]}.csv")
     support_data.set_index(support_data.columns[0], inplace=True)
     # Process model names
+    support_data["Model"] = support_data["Model"].apply(lambda x: x.replace("_", "/"))
     support_data["Model"] = support_data["Model"].apply(
         lambda x: make_model_name_clickable_link(x)
     )
     # Sort by model name length
     support_data = (
         support_data.assign(model_len=support_data["Model"].str.len())
         )
         .drop(columns=["model_len"])
     )
     # Get new columns (excluding 'Model')
     new_columns = support_data.columns.tolist()[1:]
     return (
         gr.update(value=support_data, datatype=["html" for _ in support_data.columns]),
         gr.update(choices=new_columns, value=new_columns),
+        gr.update(value=support_data),
     )
     "sans-serif",
 ]
 # Macos 14, 15, 26
 # ios 17, 18, 26
     gr.HTML(BANNER_TEXT, elem_classes="markdown-text")
     gr.Markdown("### Release")
     release_dropdown = gr.Dropdown(
+        choices=[
+            (f"{release} v{SHA_TO_VERSION[release]}", release) for release in releases
+        ],
         label="Select Release",
         value=releases[-1] if releases else None,
         elem_id="release-dropdown",
                     with gr.Row():
                         gr.Markdown(PERFORMANCE_TEXT, elem_classes="markdown-text")
             with gr.Row():
+                initial_df = performance_df[
+                    performance_df["commit_hash"] == releases[-1]
+                ]
                 leaderboard_df = gr.components.Dataframe(
                     value=initial_df[
                         PERFORMANCE_ALWAYS_HERE_COLS + performance_shown_columns.value
                     fn=update_performance_filters,
                     inputs=[release_dropdown],
                     outputs=[performance_shown_devices, performance_shown_os],
+                    queue=False,
                 ).then(
                     fn=performance_filter,
                     inputs=performance_filter_inputs,
+                    outputs=filter_output,
                 )
         # Timeline Tab
         # Device Support Tab
         with gr.TabItem("Device Support", elem_id="device_support", id=6):
+            # Add clear description of what Device Support means
+            gr.Markdown(
+                """
+                ## Device Support
+                This tab shows **test results for SKUs that we actually attempted to test**. It tells you whether tests passed, failed, or couldn't be completed for the devices we tried to run tests on.
+                ### What You'll See Here:
+                - **✅ Supported**: Tests passed successfully on this device
+                - **⚠️ Failed**: Tests were attempted but failed (click link for details)
+                - **? Not Tested**: We attempted to test but couldn't get results
+                - **Not Supported**: Hardware incompatible (per [WhisperKit config](https://huggingface.co/argmaxinc/whisperkit-coreml/blob/main/config.json))
+                ### Please Note:
+                **This tab only shows devices we attempted to test** - it doesn't show the full universe of available devices.
+                **📝 For comprehensive coverage analysis**, see the **Test Coverage** tab which shows ALL available SKUs.
+                """,
+                elem_classes="markdown-text"
+            )
             # Load device support data from CSV
+            support_data = pd.read_csv(
+                f"dashboard_data/support_data_{releases[-1][:7]}.csv"
+            )
             support_data.set_index(support_data.columns[0], inplace=True)
             support_data["Model"] = support_data["Model"].apply(
                 lambda x: x.replace("_", "/")
                             with gr.Row():
                                 with gr.Column(scale=9):
                                     support_shown_columns = gr.CheckboxGroup(
+                                        choices=support_data.columns.tolist()[
+                                            1:
+                                        ],  # Exclude 'Model' column
                                         value=support_data.columns.tolist()[1:],
                                         label="Toggle Columns",
                                         elem_id="support-column-select",
             def filter_support_data(df, columns, model_query, exclude_models):
                 """
                 Filters the device support data based on specified criteria.
                 :param df: The DataFrame to be filtered
                 :param columns: Columns to include in the output
                 :param model_query: Query string to filter models
                 # Exclude specified models
                 if exclude_models:
+                    exclude_list = [
+                        re.escape(m.strip()) for m in exclude_models.split(";")
+                    ]
                     filtered_df = filtered_df[
                         ~filtered_df["Model"].str.contains(
                             "|".join(exclude_list), case=False, regex=True
                     ]
                 # Select columns
+                selected_columns = ["Model"] + [
+                    col for col in columns if col in df.columns
+                ]
                 filtered_df = filtered_df[selected_columns]
                 return filtered_df
             def select_all_support_columns(release):
                 """
                 Returns all current columns from the support shown columns.
                 :param release: Selected release hash
                 :return: List of all available choices
                 """
                 # Load new support data for the current release
+                support_data = pd.read_csv(
+                    f"dashboard_data/support_data_{release[:7]}.csv"
+                )
                 support_data.set_index(support_data.columns[0], inplace=True)
                 # Return all columns except 'Model'
                 return [col for col in support_data.columns if col != "Model"]
             release_dropdown.change(
                 update_support_table,
                 inputs=[release_dropdown],
+                outputs=[
+                    device_support_table,
+                    support_shown_columns,
+                    hidden_support_df,
+                ],
             ).then(
                 filter_support_data,
+                inputs=[
+                    hidden_support_df,
+                    support_shown_columns,
+                    filter_support_models,
+                    exclude_support_models,
+                ],
+                outputs=device_support_table,
             )
             # Also connect the filter inputs to update the table
+            for input_elem in [
+                filter_support_models,
+                exclude_support_models,
+                support_shown_columns,
+            ]:
                 input_elem.change(
                     filter_support_data,
+                    inputs=[
+                        hidden_support_df,
+                        support_shown_columns,
+                        filter_support_models,
+                        exclude_support_models,
+                    ],
+                    outputs=device_support_table,
+                )
+        # Test Coverage Tab
+        with gr.TabItem("Test Coverage", elem_id="test_coverage", id=7):
+            # Add clear description of what Test Coverage means
+            gr.Markdown(
+                """
+                ## Test Coverage
+                This tab shows **ALL available SKUs** and our testing coverage across the entire device ecosystem. Uses chip-based expansion where testing one device covers all devices with the same chip.
+                """,
+                elem_classes="markdown-text"
+            )
+            def load_coverage_data(release):
+                """Load test coverage data for a specific release."""
+                try:
+                    with open(f"dashboard_data/test_coverage_{release}.json", "r") as f:
+                        return json.load(f)
+                except FileNotFoundError:
+                    return {
+                        "commit_hash": release,
+                        "total_devices": 0,
+                        "tested_devices": 0,
+                        "skipped_devices": 0,
+                        "coverage_percentage": 0.0,
+                        "tested_device_list": [],
+                        "skipped_device_list": [],
+                        "tested_os_versions": [],
+                        "has_target_os_coverage": False,
+                        "covered_target_versions": [],
+                        "missing_target_versions": [],
+                    }
+            def format_coverage_devices(device_list):
+                """Convert device list to DataFrame format."""
+                if not device_list:
+                    return pd.DataFrame(columns=["Device"])
+                df = pd.DataFrame({"Device": device_list})
+                return df.sort_values(["Device"])
+            def update_coverage_data(release):
+                """Update coverage data when release changes."""
+                coverage_data = load_coverage_data(release)
+                # Format tested and skipped devices
+                tested_df = format_coverage_devices(coverage_data["tested_device_list"])
+                skipped_df = format_coverage_devices(
+                    coverage_data["skipped_device_list"]
+                )
+                # Check target OS coverage
+                target_os_status = ""
+                covered_versions = coverage_data.get("covered_target_versions", [])
+                missing_versions = coverage_data.get("missing_target_versions", [])
+                if covered_versions or missing_versions:
+                    target_os_status = "\n- **Target OS Coverage**:\n"
+                    if covered_versions:
+                        unique_versions = sorted(set(covered_versions))
+                        target_os_status += f"  - ✅ **Tested**: {', '.join(unique_versions)}\n"
+                    if missing_versions:
+                        target_os_status += f"  - ❌ **Missing**: {', '.join(missing_versions)}"
+                # Create coverage summary
+                coverage_summary = f"""## Test Coverage Summary for Release {release} (v{SHA_TO_VERSION.get(release, 'Unknown')})
+- **Total Devices**: {coverage_data['total_devices']}
+- **Tested Devices**: {coverage_data['tested_devices']}
+- **Skipped Devices**: {coverage_data['skipped_devices']}
+- **Coverage Percentage**: {coverage_data['coverage_percentage']:.1f}%
+{target_os_status}"""
+                return (
+                    gr.update(value=coverage_summary),
+                    gr.update(value=tested_df),
+                    gr.update(value=skipped_df),
+                    tested_df,
+                    skipped_df,
+                )
+            def filter_coverage_devices(df, device_query, exclude_devices):
+                """Filter coverage devices based on device queries."""
+                if df is None or df.empty:
+                    return df
+                filtered_df = df.copy()
+                # Filter devices based on query
+                if device_query:
+                    filtered_df = filtered_df[
+                        filtered_df["Device"].str.contains(
+                            "|".join(q.strip() for q in device_query.split(";")),
+                            case=False,
+                            regex=True,
+                        )
+                    ]
+                # Exclude specified devices
+                if exclude_devices:
+                    exclude_list = [
+                        re.escape(d.strip()) for d in exclude_devices.split(";")
+                    ]
+                    filtered_df = filtered_df[
+                        ~filtered_df["Device"].str.contains(
+                            "|".join(exclude_list), case=False, regex=True
+                        )
+                    ]
+                return filtered_df
+            # Load initial coverage data
+            initial_coverage = load_coverage_data(releases[-1])
+            initial_tested_df = format_coverage_devices(
+                initial_coverage["tested_device_list"]
+            )
+            initial_skipped_df = format_coverage_devices(
+                initial_coverage["skipped_device_list"]
+            )
+            # Generate initial target OS status
+            initial_target_os_status = ""
+            covered_versions = initial_coverage.get("covered_target_versions", [])
+            missing_versions = initial_coverage.get("missing_target_versions", [])
+            if covered_versions or missing_versions:
+                initial_target_os_status = "\n- **Target OS Coverage**:\n"
+                if covered_versions:
+                    unique_versions = sorted(set(covered_versions))
+                    initial_target_os_status += f"  - ✅ **Tested**: {', '.join(unique_versions)}\n"
+                if missing_versions:
+                    initial_target_os_status += f"  - ❌ **Missing**: {', '.join(missing_versions)}"
+            # Create initial coverage summary content
+            initial_summary_content = f"""## Test Coverage Summary for Release {releases[-1]} (v{SHA_TO_VERSION.get(releases[-1], 'Unknown')})
+- **Total Devices**: {initial_coverage['total_devices']}
+- **Tested Devices**: {initial_coverage['tested_devices']}
+- **Skipped Devices**: {initial_coverage['skipped_devices']}
+- **Coverage Percentage**: {initial_coverage['coverage_percentage']:.1f}%
+{initial_target_os_status}"""
+            # Coverage summary
+            coverage_summary_text = gr.Markdown(
+                value=initial_summary_content,
+                elem_classes="markdown-text"
+            )
+            with gr.Row():
+                with gr.Column(scale=1):
+                    with gr.Row():
+                        with gr.Column(scale=6):
+                            filter_coverage_devices_input = gr.Textbox(
+                                placeholder="🔍 Filter Device (separate multiple queries with ';')",
+                                label="Filter Devices",
+                            )
+                        with gr.Column(scale=4):
+                            exclude_coverage_devices_input = gr.Textbox(
+                                placeholder="🔍 Exclude Device",
+                                label="Exclude Device",
+                            )
+            # Create tabs for tested vs skipped devices
+            with gr.Tabs():
+                with gr.TabItem("Tested Devices", id=0):
+                    tested_devices_table = gr.Dataframe(
+                        value=initial_tested_df,
+                        headers=["Device"],
+                        datatype=["str"],
+                        elem_id="tested-devices-table",
+                        elem_classes="large-table",
+                        interactive=False,
+                    )
+                with gr.TabItem("Skipped Devices", id=1):
+                    skipped_devices_table = gr.Dataframe(
+                        value=initial_skipped_df,
+                        headers=["Device"],
+                        datatype=["str"],
+                        elem_id="skipped-devices-table",
+                        elem_classes="large-table",
+                        interactive=False,
+                    )
+            # Hidden dataframes for filtering
+            hidden_tested_df = gr.Dataframe(value=initial_tested_df, visible=False)
+            hidden_skipped_df = gr.Dataframe(value=initial_skipped_df, visible=False)
+            # Connect release dropdown to coverage data update
+            release_dropdown.change(
+                update_coverage_data,
+                inputs=[release_dropdown],
+                outputs=[
+                    coverage_summary_text,
+                    tested_devices_table,
+                    skipped_devices_table,
+                    hidden_tested_df,
+                    hidden_skipped_df,
+                ],
+                queue=False,
+            )
+            # Connect filter inputs to update both tables
+            for input_elem in [
+                filter_coverage_devices_input,
+                exclude_coverage_devices_input,
+            ]:
+                input_elem.change(
+                    lambda tested_df, skipped_df, device_query, exclude_devices: (
+                        filter_coverage_devices(
+                            tested_df, device_query, exclude_devices
+                        ),
+                        filter_coverage_devices(
+                            skipped_df, device_query, exclude_devices
+                        ),
+                    ),
+                    inputs=[
+                        hidden_tested_df,
+                        hidden_skipped_df,
+                        filter_coverage_devices_input,
+                        exclude_coverage_devices_input,
+                    ],
+                    outputs=[tested_devices_table, skipped_devices_table],
                 )
         # Methodology Tab
+        with gr.TabItem("Methodology", elem_id="methodology", id=8):
             gr.Markdown(METHODOLOGY_TEXT, elem_id="methodology-text")
     # Citation section
         )
 # Launch the Gradio interface
+demo.launch(debug=True, share=True)

performance_generate.py CHANGED Viewed

@@ -10,10 +10,12 @@ from statistics import mean
 import pandas as pd
 import requests
 from constants import BASE_WHISPERKIT_BENCHMARK_URL
 from text_normalizer import text_normalizer
-from utils import compute_average_wer, download_dataset
 def fetch_evaluation_data(url):
@@ -97,7 +99,7 @@ def process_benchmark_file(file_path, dataset_dfs, results, releases):
     if len(test_results) == 0:
         return
     commit_hash_timestamp = file_path.split("/")[-2]
     commit_timestamp, commit_hash = commit_hash_timestamp.split("_")
@@ -182,7 +184,7 @@ def process_summary_file(file_path, results, releases):
     """
     with open(file_path, "r") as file:
         summary_data = json.load(file)
     if summary_data["commitHash"] not in releases:
         return
@@ -192,13 +194,15 @@ def process_summary_file(file_path, results, releases):
     commit_timestamp = summary_data["commitTimestamp"]
     test_file_name = file_path.split("/")[-1]
     test_timestamp = test_file_name.split("_")[-1].replace(".json", "")
     key = (device, os, commit_hash)
     if key in results:
         existing_commit_timestamp = results[key]["commitTimestamp"]
         existing_test_timestamp = results[key]["testTimestamp"]
-        existing_commit_dt = datetime.strptime(existing_commit_timestamp, "%Y-%m-%dT%H%M%S")
         new_commit_dt = datetime.strptime(commit_timestamp, "%Y-%m-%dT%H%M%S")
         existing_test_dt = datetime.strptime(existing_test_timestamp, "%Y-%m-%dT%H%M%S")
         new_test_dt = datetime.strptime(test_timestamp, "%Y-%m-%dT%H%M%S")
@@ -325,7 +329,7 @@ def calculate_and_save_support_results(
             results_by_commit[commit_hash] = {
                 "data": {},
                 "devices": set(),
-                "timestamp": data["commitTimestamp"]
             }
         results_by_commit[commit_hash]["data"][key] = data
         results_by_commit[commit_hash]["devices"].add(device)
@@ -334,7 +338,7 @@ def calculate_and_save_support_results(
     for commit_hash, commit_data in results_by_commit.items():
         commit_devices = sorted(commit_data["devices"])
         df = pd.DataFrame(index=all_models, columns=["Model"] + commit_devices)
         for model in all_models:
             row = {"Model": model}
             for device in commit_devices:
@@ -375,24 +379,27 @@ def calculate_and_save_support_results(
         # Mark unsupported combinations for this commit
         commit_not_supported = [
-            (model, device, os)
-            for model, device, os in not_supported
-            if any(key[2] == commit_hash for key in support_results if key not in ["modelsTested", "devices"] and model == key[0])
         ]
         remove_unsupported_cells(df, commit_not_supported)
         # Format column headers
         cols = df.columns.tolist()
         cols = ["Model"] + [
-            f"""{get_device_name(col).replace("_", " ")} ({col})""" for col in cols if col != "Model"
         ]
         df.columns = cols
         # Save to commit-specific file
-        output_path = support_output_path.replace(
-            ".csv",
-            f"_{commit_hash[:7]}.csv"
-        )
         df.to_csv(output_path, index=True)
@@ -426,6 +433,462 @@ def remove_unsupported_cells(df, not_supported):
         df.at[model, device] = "Not Supported"
 def main():
     """
     Main function to orchestrate the performance data generation process.
@@ -435,6 +898,7 @@ def main():
     2. Fetches evaluation data for various datasets.
     3. Processes benchmark files and summary files.
     4. Calculates and saves performance and support results.
     """
     source_xcresult_repo = "argmaxinc/whisperkit-evals-dataset"
     source_xcresult_subfolder = "benchmark_data/"
@@ -498,8 +962,10 @@ def main():
             elif "summary" in filename:
                 process_summary_file(file_path, support_results, releases)
             else:
-                process_benchmark_file(file_path, dataset_dfs, performance_results, releases)
     not_supported = calculate_and_save_performance_results(
         performance_results, "dashboard_data/performance_data.json"
     )
@@ -507,6 +973,9 @@ def main():
         support_results, not_supported, "dashboard_data/support_data.csv"
     )
 if __name__ == "__main__":
     main()

 import pandas as pd
 import requests
+from dotenv import load_dotenv
+from huggingface_hub import login
 from constants import BASE_WHISPERKIT_BENCHMARK_URL
 from text_normalizer import text_normalizer
+from utils import compute_average_wer, download_dataset, download_json_from_github
 def fetch_evaluation_data(url):
     if len(test_results) == 0:
         return
     commit_hash_timestamp = file_path.split("/")[-2]
     commit_timestamp, commit_hash = commit_hash_timestamp.split("_")
     """
     with open(file_path, "r") as file:
         summary_data = json.load(file)
     if summary_data["commitHash"] not in releases:
         return
     commit_timestamp = summary_data["commitTimestamp"]
     test_file_name = file_path.split("/")[-1]
     test_timestamp = test_file_name.split("_")[-1].replace(".json", "")
     key = (device, os, commit_hash)
     if key in results:
         existing_commit_timestamp = results[key]["commitTimestamp"]
         existing_test_timestamp = results[key]["testTimestamp"]
+        existing_commit_dt = datetime.strptime(
+            existing_commit_timestamp, "%Y-%m-%dT%H%M%S"
+        )
         new_commit_dt = datetime.strptime(commit_timestamp, "%Y-%m-%dT%H%M%S")
         existing_test_dt = datetime.strptime(existing_test_timestamp, "%Y-%m-%dT%H%M%S")
         new_test_dt = datetime.strptime(test_timestamp, "%Y-%m-%dT%H%M%S")
             results_by_commit[commit_hash] = {
                 "data": {},
                 "devices": set(),
+                "timestamp": data["commitTimestamp"],
             }
         results_by_commit[commit_hash]["data"][key] = data
         results_by_commit[commit_hash]["devices"].add(device)
     for commit_hash, commit_data in results_by_commit.items():
         commit_devices = sorted(commit_data["devices"])
         df = pd.DataFrame(index=all_models, columns=["Model"] + commit_devices)
         for model in all_models:
             row = {"Model": model}
             for device in commit_devices:
         # Mark unsupported combinations for this commit
         commit_not_supported = [
+            (model, device, os)
+            for model, device, os in not_supported
+            if any(
+                key[2] == commit_hash
+                for key in support_results
+                if key not in ["modelsTested", "devices"] and model == key[0]
+            )
         ]
         remove_unsupported_cells(df, commit_not_supported)
         # Format column headers
         cols = df.columns.tolist()
         cols = ["Model"] + [
+            f"""{get_device_name(col).replace("_", " ")} ({col})"""
+            for col in cols
+            if col != "Model"
         ]
         df.columns = cols
         # Save to commit-specific file
+        output_path = support_output_path.replace(".csv", f"_{commit_hash[:7]}.csv")
         df.to_csv(output_path, index=True)
         df.at[model, device] = "Not Supported"
+def download_device_json_safe(file_path):
+    """
+    Safely downloads a device JSON file from GitHub, returning None if it doesn't exist.
+    :param file_path: Path to the JSON file within the repository
+    :returns: The JSON data as a dictionary, or None if the file doesn't exist
+    """
+    try:
+        return download_json_from_github(file_path=file_path)
+    except SystemExit:
+        # File doesn't exist or other error occurred
+        return None
+def load_device_json_local(file_path):
+    """
+    Safely loads a local device JSON file, returning None if it doesn't exist.
+    :param file_path: Local path to the JSON file
+    :returns: The JSON data as a dictionary, or None if the file doesn't exist
+    """
+    try:
+        with open(file_path, "r") as f:
+            return json.load(f)
+    except (FileNotFoundError, json.JSONDecodeError):
+        return None
+def build_chip_mapping():
+    """
+    Builds a mapping from device SKUs to their chip types.
+    :returns: Dictionary where keys are device SKUs and values are chip types
+    """
+    sku_to_chip = {}
+    # Load iPad devices
+    ipad_data = load_device_json_local("dashboard_data/iPad.json")
+    if ipad_data and "total_menu" in ipad_data:
+        for device_info in ipad_data["total_menu"].values():
+            if "sku" in device_info and "chip" in device_info:
+                # iPad has sku as an array
+                skus = device_info["sku"] if isinstance(device_info["sku"], list) else [device_info["sku"]]
+                for sku in skus:
+                    sku_to_chip[sku] = device_info["chip"]
+    # Load iPhone devices
+    iphone_data = load_device_json_local("dashboard_data/iPhone.json")
+    if iphone_data and "total_menu" in iphone_data:
+        for device_info in iphone_data["total_menu"].values():
+            if "sku" in device_info and "chip" in device_info:
+                # iPhone has sku as a single string
+                sku_to_chip[device_info["sku"]] = device_info["chip"]
+    # Load Mac devices
+    mac_data = load_device_json_local("dashboard_data/Mac.json")
+    if mac_data and "total_menu" in mac_data:
+        for device_info in mac_data["total_menu"].values():
+            if "sku" in device_info and "chip" in device_info:
+                # Mac has sku as a single string
+                sku_to_chip[device_info["sku"]] = device_info["chip"]
+    return sku_to_chip
+def get_platform_from_sku(sku):
+    """
+    Determines the platform (iPad, iPhone, Mac) from a device SKU.
+    :param sku: Device SKU string
+    :returns: Platform string ('iPad', 'iPhone', 'Mac') or 'Unknown'
+    """
+    if sku.startswith("iPad"):
+        return "iPad"
+    elif sku.startswith("iPhone"):
+        return "iPhone"
+    elif sku.startswith("Mac") or sku.startswith("iMac") or sku.startswith("MacBook"):
+        return "Mac"
+    else:
+        return "Unknown"
+def normalize_chip_name(chip):
+    """
+    Normalizes chip names for consistent grouping.
+    :param chip: Raw chip name from device JSON
+    :returns: Normalized chip name
+    """
+    # Handle variations like "A18 Pro" -> "A18", "M4 Pro" -> "M4", etc.
+    # But keep distinct generations separate
+    chip = chip.strip()
+    # For A-series chips, keep Pro variants separate as they have different capabilities
+    if chip.startswith("A") and "Pro" in chip:
+        return chip  # Keep A17 Pro separate from A17
+    # For M-series chips, group Pro/Max/Ultra variants together as they're same generation
+    if chip.startswith("M"):
+        # Extract just the M number (M1, M2, M3, M4)
+        parts = chip.split()
+        if len(parts) > 0:
+            return parts[0]  # Return just "M1", "M2", etc.
+    return chip
+def build_sku_group_mapping():
+    """
+    Builds a mapping from individual SKUs to all SKUs that share the same chip on the same platform.
+    This implements chip-based coverage where testing one device with a specific chip
+    provides coverage for all devices with that chip on the same platform.
+    :returns: Dictionary where keys are individual SKUs and values are sets of all SKUs in that chip group
+    """
+    sku_to_chip = build_chip_mapping()
+    sku_to_group = {}
+    # Group SKUs by platform and normalized chip
+    platform_chip_groups = {}
+    for sku, chip in sku_to_chip.items():
+        platform = get_platform_from_sku(sku)
+        normalized_chip = normalize_chip_name(chip)
+        key = (platform, normalized_chip)
+        if key not in platform_chip_groups:
+            platform_chip_groups[key] = set()
+        platform_chip_groups[key].add(sku)
+    # Create reverse mapping: each SKU maps to all SKUs in its chip group
+    for sku, chip in sku_to_chip.items():
+        platform = get_platform_from_sku(sku)
+        normalized_chip = normalize_chip_name(chip)
+        key = (platform, normalized_chip)
+        sku_to_group[sku] = platform_chip_groups[key]
+    return sku_to_group
+def expand_tested_devices(tested_devices, sku_mapping):
+    """
+    Expands tested devices to include all SKUs in the same group.
+    :param tested_devices: Set of device SKUs that were actually tested
+    :param sku_mapping: Dictionary mapping individual SKUs to their complete groups
+    :returns: Expanded set of devices including all SKUs in the same groups
+    """
+    expanded_devices = set(tested_devices)
+    for device in tested_devices:
+        if device in sku_mapping:
+            # Add all SKUs from the same group
+            expanded_devices.update(sku_mapping[device])
+    return expanded_devices
+def get_test_iphones():
+    """
+    Gets iPhone SKU identifiers from the local iPhone.json file.
+    """
+    iphone_data = load_device_json_local("dashboard_data/iPhone.json")
+    if iphone_data and "total_menu" in iphone_data:
+        return set([device_info["sku"] for device_info in iphone_data["total_menu"].values() if "sku" in device_info])
+    return set()
+def get_test_macs():
+    """
+    Gets Mac SKU identifiers from the local Mac.json file.
+    """
+    mac_data = load_device_json_local("dashboard_data/Mac.json")
+    if mac_data and "total_menu" in mac_data:
+        return set([device_info["sku"] for device_info in mac_data["total_menu"].values() if "sku" in device_info])
+    return set()
+def get_all_supported_devices():
+    """
+    Gets all supported device identifiers from the config file.
+    Returns a set of device identifiers.
+    """
+    with open("dashboard_data/config.json", "r") as f:
+        config = json.load(f)
+    devices = set()
+    for device_group in config["device_support"]:
+        identifiers = device_group["identifiers"]
+        devices.update(identifiers)
+    return devices
+def get_tested_devices_for_commit(performance_results, support_results, commit_hash):
+    """
+    Gets all device identifiers that were actually tested for a specific commit,
+    including all SKUs in the same chip groups as tested devices.
+    Uses chip-based coverage logic where testing one device with a specific chip
+    provides coverage for all devices with that chip on the same platform.
+    Returns a set of device identifiers.
+    """
+    tested_devices = set()
+    # From performance results (benchmark files)
+    for key, result in performance_results.items():
+        if len(key) >= 4 and result.get("commit_hash") == commit_hash:
+            model, device, _, _ = key
+            tested_devices.add(device)
+    # From support results (summary files)
+    for key, result in support_results.items():
+        if key in ["modelsTested", "devices"]:
+            continue
+        if len(key) >= 3 and result.get("commitHash") == commit_hash:
+            device, _, _ = key
+            tested_devices.add(device)
+    # Expand to include all SKUs in the same chip groups for all platforms
+    sku_mapping = build_sku_group_mapping()
+    expanded_devices = expand_tested_devices(tested_devices, sku_mapping)
+    return expanded_devices
+def get_tested_os_versions_for_commit(
+    performance_results, support_results, commit_hash
+):
+    """
+    Gets all OS versions that were actually tested for a specific commit.
+    Returns a set of OS version strings like 'iOS_17.2', 'macOS_14.5', etc.
+    """
+    tested_os_versions = set()
+    # From performance results (benchmark files)
+    for key, result in performance_results.items():
+        if len(key) >= 4 and result.get("commit_hash") == commit_hash:
+            model, device, os_info, _ = key
+            tested_os_versions.add(os_info)
+    # From support results (summary files)
+    for key, result in support_results.items():
+        if key in ["modelsTested", "devices"]:
+            continue
+        if len(key) >= 3 and result.get("commitHash") == commit_hash:
+            device, os, _ = key
+            # Convert format like "iOS 17.2" to "iOS_17.2" for consistency
+            os_normalized = os.replace(" ", "_")
+            tested_os_versions.add(os_normalized)
+    return tested_os_versions
+def check_target_os_coverage(tested_os_versions):
+    """
+    Check if the tested OS versions include ALL of the target OS versions:
+    - macOS 14, 15, 26
+    - iOS 17, 18, 26 (noting that iOS and iPadOS are the same under the hood)
+    Returns (is_fully_covered: bool, covered_versions: list, missing_versions: list)
+    """
+    target_macos_versions = {14, 15, 26}
+    target_ios_versions = {17, 18, 26}
+    covered_macos = set()
+    covered_ios = set()
+    for os_version in tested_os_versions:
+        # Parse OS version string like "iOS_17.2" or "macOS_14.5"
+        if "_" in os_version:
+            os_type, version_str = os_version.split("_", 1)
+            try:
+                # Extract major version number
+                major_version = int(version_str.split(".")[0])
+                if os_type == "macOS" and major_version in target_macos_versions:
+                    covered_macos.add(major_version)
+                elif (
+                    os_type in ["iOS", "iPadOS"]
+                    and major_version in target_ios_versions
+                ):
+                    covered_ios.add(major_version)
+            except (ValueError, IndexError):
+                # Skip if we can't parse the version
+                continue
+    # Check what's missing
+    missing_macos = target_macos_versions - covered_macos
+    missing_ios = target_ios_versions - covered_ios
+    # Format covered and missing versions
+    covered_versions = []
+    covered_versions.extend([f"macOS {v}" for v in sorted(covered_macos)])
+    covered_versions.extend([f"iOS {v}" for v in sorted(covered_ios)])
+    missing_versions = []
+    missing_versions.extend([f"macOS {v}" for v in sorted(missing_macos)])
+    missing_versions.extend([f"iOS {v}" for v in sorted(missing_ios)])
+    # Only fully covered if no missing versions
+    is_fully_covered = len(missing_versions) == 0
+    return is_fully_covered, covered_versions, missing_versions
+def check_chip_coverage(tested_devices):
+    """
+    Check if the tested devices provide complete chip coverage for each platform.
+    Target coverage:
+    - iPad: A14, A15, A16, A17 Pro, M1, M2, M3, M4
+    - iPhone: A14, A15, A16, A17 Pro, A18, A18 Pro
+    - Mac: M1, M2, M3, M4
+    :param tested_devices: Set of device SKUs that were tested
+    :returns: (is_fully_covered: bool, platform_coverage: dict, missing_chips: dict)
+    """
+    # Define target chips for each platform
+    target_chips = {
+        "iPad": {"A14", "A15", "A16", "A17 Pro", "M1", "M2", "M3", "M4"},
+        "iPhone": {"A14", "A15", "A16", "A17 Pro", "A18", "A18 Pro"},
+        "Mac": {"M1", "M2", "M3", "M4"}
+    }
+    # Build mapping from SKUs to chips
+    sku_to_chip = build_chip_mapping()
+    # Track which chips were tested for each platform
+    tested_chips = {
+        "iPad": set(),
+        "iPhone": set(),
+        "Mac": set()
+    }
+    for device_sku in tested_devices:
+        if device_sku in sku_to_chip:
+            platform = get_platform_from_sku(device_sku)
+            chip = normalize_chip_name(sku_to_chip[device_sku])
+            if platform in tested_chips:
+                tested_chips[platform].add(chip)
+    # Calculate coverage for each platform
+    platform_coverage = {}
+    missing_chips = {}
+    for platform, target_set in target_chips.items():
+        covered_set = tested_chips[platform]
+        missing_set = target_set - covered_set
+        platform_coverage[platform] = {
+            "total_chips": len(target_set),
+            "tested_chips": len(covered_set),
+            "coverage_percentage": (len(covered_set) / len(target_set)) * 100 if target_set else 0,
+            "covered_chips": sorted(list(covered_set)),
+            "missing_chips": sorted(list(missing_set))
+        }
+        missing_chips[platform] = sorted(list(missing_set))
+    # Overall coverage is complete if all platforms have full coverage
+    is_fully_covered = all(
+        len(missing_chips[platform]) == 0
+        for platform in target_chips.keys()
+    )
+    return is_fully_covered, platform_coverage, missing_chips
+def generate_test_coverage_report(
+    performance_results, support_results, output_dir="dashboard_data"
+):
+    """
+    Generates test coverage reports for each commit, showing which devices
+    were tested vs skipped.
+    """
+    # Get all possible devices from config
+    all_devices = get_all_supported_devices()
+    # Get all unique commit hashes from results
+    commit_hashes = set()
+    # Collect from performance results
+    for key, result in performance_results.items():
+        if len(key) >= 4 and result.get("commit_hash"):
+            commit_hashes.add(result["commit_hash"])
+    # Collect from support results
+    for key, result in support_results.items():
+        if key in ["modelsTested", "devices"]:
+            continue
+        if len(key) >= 3 and result.get("commitHash"):
+            commit_hashes.add(result["commitHash"])
+    print(f"Found {len(commit_hashes)} commit hashes to analyze")
+    # Generate coverage report for each commit
+    for commit_hash in commit_hashes:
+        tested_devices = get_tested_devices_for_commit(
+            performance_results, support_results, commit_hash
+        )
+        tested_os_versions = get_tested_os_versions_for_commit(
+            performance_results, support_results, commit_hash
+        )
+        # Check target OS coverage
+        os_fully_covered, covered_versions, missing_versions = check_target_os_coverage(
+            tested_os_versions
+        )
+        # Check chip coverage for all platforms
+        chip_fully_covered, platform_coverage, missing_chips = check_chip_coverage(
+            tested_devices
+        )
+        skipped_devices = all_devices - tested_devices
+        # Convert sets to lists for JSON serialization
+        tested_devices_list = list(tested_devices)
+        skipped_devices_list = list(skipped_devices)
+        tested_os_versions_list = list(tested_os_versions)
+        coverage_report = {
+            "commit_hash": commit_hash,
+            "total_devices": len(all_devices),
+            "tested_devices": len(tested_devices),
+            "skipped_devices": len(skipped_devices),
+            "coverage_percentage": (len(tested_devices) / len(all_devices)) * 100,
+            "tested_device_list": tested_devices_list,
+            "skipped_device_list": skipped_devices_list,
+            "tested_os_versions": tested_os_versions_list,
+            "has_target_os_coverage": os_fully_covered,
+            "covered_target_versions": covered_versions,
+            "missing_target_versions": missing_versions,
+            "has_target_chip_coverage": chip_fully_covered,
+            "platform_chip_coverage": platform_coverage,
+            "missing_target_chips": missing_chips,
+        }
+        # Save report for this commit
+        output_file = os.path.join(output_dir, f"test_coverage_{commit_hash}.json")
+        with open(output_file, "w") as f:
+            json.dump(coverage_report, f, indent=2)
+        os_coverage_info = f"OS coverage: {'✅ Complete' if os_fully_covered else '❌ Incomplete'}"
+        chip_coverage_info = f"Chip coverage: {'✅ Complete' if chip_fully_covered else '❌ Incomplete'}"
+        print(
+            f"Generated coverage report for commit {commit_hash}: "
+            f"{len(tested_devices)}/{len(all_devices)} devices tested "
+            f"({coverage_report['coverage_percentage']:.1f}%) "
+            f"({os_coverage_info}, {chip_coverage_info})"
+        )
 def main():
     """
     Main function to orchestrate the performance data generation process.
     2. Fetches evaluation data for various datasets.
     3. Processes benchmark files and summary files.
     4. Calculates and saves performance and support results.
+    5. Generates test coverage reports for each commit.
     """
     source_xcresult_repo = "argmaxinc/whisperkit-evals-dataset"
     source_xcresult_subfolder = "benchmark_data/"
             elif "summary" in filename:
                 process_summary_file(file_path, support_results, releases)
             else:
+                process_benchmark_file(
+                    file_path, dataset_dfs, performance_results, releases
+                )
     not_supported = calculate_and_save_performance_results(
         performance_results, "dashboard_data/performance_data.json"
     )
         support_results, not_supported, "dashboard_data/support_data.csv"
     )
+    # Generate test coverage reports
+    generate_test_coverage_report(performance_results, support_results)
 if __name__ == "__main__":
     main()

quality_generate.py DELETED Viewed

@@ -1,186 +0,0 @@
-import json
-import os
-import shutil
-import sys
-from collections import defaultdict
-from statistics import mean
-import pandas as pd
-import requests
-from text_normalizer import text_normalizer
-from utils import compute_average_wer, download_dataset
-def fetch_evaluation_data(url):
-    """
-    Fetches evaluation data from the given URL.
-    :param url: The URL to fetch the evaluation data from.
-    :returns: The evaluation data as a dictionary.
-    :rauses: sys.exit if the request fails
-    """
-    response = requests.get(url)
-    if response.status_code == 200:
-        return json.loads(response.text)
-    else:
-        sys.exit(f"Failed to fetch WhisperKit evals: {response.text}")
-def get_device_name(device):
-    """
-    Gets the device name from the device map if it exists.
-    :param device: String representing the device name.
-    :returns: The device name from the device map if it exists, otherwise the input device name.
-    """
-    with open("dashboard_data/device_map.json", "r") as f:
-        device_map = json.load(f)
-    return device_map.get(device, device).replace(" ", "_")
-def process_quality_file(file_path, dataset_dfs, quality_results):
-    """
-    Processes a single quality file and updates the quality_results dictionary.
-    :param file_path: Path to the quality JSON file.
-    :param dataset_dfs: Dictionary of DataFrames containing dataset information.
-    :param quality_results: Dictionary to store the processed quality results.
-    This function reads a quality JSON file, extracts relevant information,
-    and updates the quality_results dictionary with various metrics including WER
-    and Quality of Inference (QoI) for different datasets.
-    """
-    with open(file_path, "r") as file:
-        test_results = json.load(file)
-    if len(test_results) == 0:
-        return
-    metadata = test_results["metadata"]
-    test_results = test_results["results"]
-    model = file_path.split("/")[-3].replace("_", "/")
-    device = metadata["inference_context"]["device_spec"]["product_name"]
-    device = get_device_name(device)
-    timestamp = file_path.split("/")[-1].split(".")[0]
-    key = model
-    dataset_name = metadata["dataset_name"]
-    for test_result in test_results:
-        audio_file_name = test_result["file"]
-        dataset_key = "Earnings-22" if "earnings22" in dataset_name else "LibriSpeech"
-        dataset_df = dataset_dfs[dataset_key]
-        wer_entry = {
-            "prediction": text_normalizer(test_result["prediction"]),
-            "reference": text_normalizer(test_result["reference"]),
-        }
-        quality_results[key]["timestamp"] = timestamp
-        quality_results[key]["dataset_wer"][dataset_name].append(wer_entry)
-        audio = audio_file_name.split(".")[0]
-        dataset_row = dataset_df.loc[dataset_df["file"].str.contains(audio)].iloc[0]
-        reference_wer = dataset_row["wer"]
-        prediction_wer = test_result["wer"]
-        quality_results[key]["qoi"].append(1 if prediction_wer <= reference_wer else 0)
-def calculate_and_save_quality_results(quality_results, quality_output_path):
-    """
-    Calculates final quality metrics and saves them to a JSON file.
-    :param quality_results: Dictionary containing raw quality data.
-    :param quality_output_path: Path to save the processed quality results.
-    This function processes the raw quality data, calculates average metrics,
-    and writes the final results to a JSON file, with each entry representing
-    a unique model's quality metrics across different datasets, including
-    Word Error Rate (WER) and Quality of Inference (QoI).
-    """
-    with open(quality_output_path, "w") as quality_file:
-        for key, data in quality_results.items():
-            model = key
-            dataset_wers = {
-                dataset: compute_average_wer(wer)
-                for dataset, wer in data["dataset_wer"].items()
-            }
-            average_wer = (
-                sum(dataset_wers.values()) / len(dataset_wers)
-                if len(dataset_wers) != 0
-                else 0
-            )
-            quality_entry = {
-                "model": model.replace("_", "/"),
-                "timestamp": data["timestamp"],
-                "average_wer": round(average_wer, 2),
-                "dataset_wer": dataset_wers,
-                "qoi": round(mean(data["qoi"]), 2),
-            }
-            json.dump(quality_entry, quality_file)
-            quality_file.write("\n")
-def main():
-    """
-    Main function to orchestrate the quality data generation process.
-    This function performs the following steps:
-    1. Downloads quality data if requested.
-    2. Fetches evaluation data for various datasets.
-    3. Processes quality files for specific datasets.
-    4. Calculates and saves quality results, including WER and QoI metrics.
-    """
-    if len(sys.argv) > 1 and sys.argv[1] == "download":
-        try:
-            shutil.rmtree("english")
-        except:
-            print("Nothing to remove.")
-        download_dataset("argmaxinc/whisperkit-evals", "english", "WhisperKit")
-    datasets = {
-        "Earnings-22": "https://huggingface.co/datasets/argmaxinc/whisperkit-evals/resolve/main/WhisperOpenAIAPI/openai_whisper-large-v2/earnings22/2024-03-04_13%3A39%3A42_GMT-0800.json",
-        "LibriSpeech": "https://huggingface.co/datasets/argmaxinc/whisperkit-evals/resolve/main/WhisperOpenAIAPI/openai_whisper-large-v2/librispeech/2024-02-28_18%3A45%3A02_GMT-0800.json?download=true",
-        "earnings22-10mins": "https://huggingface.co/datasets/argmaxinc/whisperkit-evals/resolve/main/WhisperOpenAIAPI/openai_whisper-large-v2/earnings22/2024-03-04_13%3A39%3A42_GMT-0800.json",
-        "librispeech-10mins": "https://huggingface.co/datasets/argmaxinc/whisperkit-evals/resolve/main/WhisperOpenAIAPI/openai_whisper-large-v2/librispeech/2024-02-28_18%3A45%3A02_GMT-0800.json?download=true",
-        "earnings22-12hours": "https://huggingface.co/datasets/argmaxinc/whisperkit-evals/resolve/main/WhisperOpenAIAPI/openai_whisper-large-v2/earnings22/2024-03-04_13%3A39%3A42_GMT-0800.json",
-        "librispeech": "https://huggingface.co/datasets/argmaxinc/whisperkit-evals/resolve/main/WhisperOpenAIAPI/openai_whisper-large-v2/librispeech/2024-02-28_18%3A45%3A02_GMT-0800.json?download=true",
-    }
-    dataset_dfs = {}
-    for dataset_name, url in datasets.items():
-        evals = fetch_evaluation_data(url)
-        dataset_dfs[dataset_name] = pd.json_normalize(evals["results"])
-    source_quality_directory = "argmaxinc/english/WhisperKit/"
-    quality_results = defaultdict(
-        lambda: {
-            "average_wer": [],
-            "dataset_wer": defaultdict(list),
-            "qoi": [],
-            "timestamp": None,
-        }
-    )
-    for subdir, _, files in os.walk(source_quality_directory):
-        dataset = subdir.split("/")[-1]
-        if dataset not in ["earnings22-12hours", "librispeech"]:
-            continue
-        for filename in files:
-            if not filename.endswith(".json"):
-                continue
-            file_path = os.path.join(subdir, filename)
-            process_quality_file(file_path, dataset_dfs, quality_results)
-    calculate_and_save_quality_results(
-        quality_results, "dashboard_data/quality_data.json"
-    )
-if __name__ == "__main__":
-    main()

utils.py CHANGED Viewed

@@ -2,6 +2,7 @@ import colorsys
 import json
 import os
 import random
 from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass, make_dataclass
 from datetime import datetime
@@ -12,6 +13,7 @@ import evaluate
 import numpy as np
 import pandas as pd
 import plotly.graph_objects as go
 from huggingface_hub import hf_hub_download, list_repo_files
 from pydub import AudioSegment
@@ -365,14 +367,11 @@ def plot_metric(
     with open("dashboard_data/version.json", "r") as f:
         version = json.load(f)
         releases = set(version["releases"])
     df = df[df["commit_hash"].isin(releases)]
     grouped = df.groupby(["model", "device", "os"])
-    sorted_groups = [
-        group.sort_values("commit_timestamp")
-        for _, group in grouped
-    ]
     if filter_input:
         filters = [f.strip().lower() for f in filter_input.split(";")]
@@ -504,6 +503,29 @@ def calculate_parity(m2_ultra_wer, row):
     return None
 def create_initial_performance_column_dict():
     """
     Creates the initial column dictionary for the performance table.
@@ -511,7 +533,7 @@ def create_initial_performance_column_dict():
     :return: A list of column dictionaries
     This function defines the basic structure of the performance table,
-    including columns for model, device, OS, average WER, QoI, speed, and tokens per second.
     """
     return [
         [
@@ -529,6 +551,7 @@ def create_initial_performance_column_dict():
         ["qoi", ColumnContent, ColumnContent("QoI", "html", False)],
         ["speed", ColumnContent, ColumnContent("Speed", "html", False)],
         ["toks", ColumnContent, ColumnContent("Tok / s", "html", False)],
     ]
@@ -780,6 +803,46 @@ def generate_random_colors(base_colors, num_colors, min_distance=30):
     return generated_colors
 @dataclass
 class Task:
     """

 import json
 import os
 import random
+import sys
 from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass, make_dataclass
 from datetime import datetime
 import numpy as np
 import pandas as pd
 import plotly.graph_objects as go
+import requests
 from huggingface_hub import hf_hub_download, list_repo_files
 from pydub import AudioSegment
     with open("dashboard_data/version.json", "r") as f:
         version = json.load(f)
         releases = set(version["releases"])
     df = df[df["commit_hash"].isin(releases)]
     grouped = df.groupby(["model", "device", "os"])
+    sorted_groups = [group.sort_values("commit_timestamp") for _, group in grouped]
     if filter_input:
         filters = [f.strip().lower() for f in filter_input.split(";")]
     return None
+def calculate_quality_parity(quality_data, performance_row):
+    """
+    Calculates the parity between measured WER and ground truth WER from quality data.
+    :param quality_data: List of dictionaries containing quality evaluation data
+    :param performance_row: Current performance row being processed
+    :return: WER difference (measured - ground truth) or None if not found
+    This function finds the corresponding ground truth WER for a model and computes
+    the difference between measured and ground truth WER values.
+    """
+    model_name = performance_row.get("model", "")
+    measured_wer = performance_row.get("english_wer", 0)
+    # Find matching model in quality data
+    for quality_entry in quality_data:
+        if quality_entry.get("model", "") == model_name:
+            ground_truth_wer = quality_entry.get("average_wer", 0)
+            return round(measured_wer - ground_truth_wer, 2)
+    return None
 def create_initial_performance_column_dict():
     """
     Creates the initial column dictionary for the performance table.
     :return: A list of column dictionaries
     This function defines the basic structure of the performance table,
+    including columns for model, device, OS, average WER, QoI, speed, tokens per second, and parity.
     """
     return [
         [
         ["qoi", ColumnContent, ColumnContent("QoI", "html", False)],
         ["speed", ColumnContent, ColumnContent("Speed", "html", False)],
         ["toks", ColumnContent, ColumnContent("Tok / s", "html", False)],
+        ["parity", ColumnContent, ColumnContent("Parity", "number", False)],
     ]
     return generated_colors
+def download_json_from_github(
+    url=None,
+    repo_owner="argmaxinc",
+    repo_name="device-knowledge-base",
+    branch="main",
+    file_path=None,
+):
+    """
+    Downloads a JSON file from a GitHub repository.
+    :param url: Full URL to the JSON file, or None if constructing from other params
+    :param repo_owner: GitHub repository owner (default: "argmaxinc")
+    :param repo_name: GitHub repository name (default: "WhisperKit")
+    :param branch: Git branch name (default: "main")
+    :param file_path: Path to the JSON file within the repository
+    :returns: The JSON data as a dictionary
+    :raises: sys.exit if the request fails or JSON parsing fails
+    """
+    # If full URL is provided, use it directly
+    if url:
+        target_url = url
+    elif file_path:
+        # Construct GitHub raw URL
+        target_url = f"https://raw.githubusercontent.com/{repo_owner}/{repo_name}/{branch}/{file_path}"
+    else:
+        sys.exit("Either 'url' or 'file_path' must be provided")
+    try:
+        response = requests.get(target_url)
+        if response.status_code == 200:
+            return json.loads(response.text)
+        else:
+            sys.exit(f"Failed to fetch JSON from GitHub: {response.text}")
+    except json.JSONDecodeError as e:
+        sys.exit(f"Failed to parse JSON from {target_url}: {e}")
+    except requests.exceptions.RequestException as e:
+        sys.exit(f"Network error fetching {target_url}: {e}")
 @dataclass
 class Task:
     """