akki2825 commited on
Commit
01e4e5c
·
verified ·
1 Parent(s): 2b32a02

add misaligned sentences

Browse files
Files changed (1) hide show
  1. app.py +61 -28
app.py CHANGED
@@ -75,6 +75,51 @@ def calculate_sentence_metrics(reference, hypothesis):
75
  except Exception as e:
76
  raise e
77
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
78
  @spaces.GPU()
79
  def process_files(reference_file, hypothesis_file):
80
  try:
@@ -87,6 +132,7 @@ def process_files(reference_file, hypothesis_file):
87
  overall_wer = calculate_wer(reference_text, hypothesis_text)
88
  overall_cer = calculate_cer(reference_text, hypothesis_text)
89
  sentence_metrics = calculate_sentence_metrics(reference_text, hypothesis_text)
 
90
 
91
  return {
92
  "Overall WER": overall_wer,
@@ -96,33 +142,12 @@ def process_files(reference_file, hypothesis_file):
96
  "Average WER": sentence_metrics["average_wer"],
97
  "Average CER": sentence_metrics["average_cer"],
98
  "Standard Deviation WER": sentence_metrics["std_dev_wer"],
99
- "Standard Deviation CER": sentence_metrics["std_dev_cer"]
 
100
  }
101
  except Exception as e:
102
  return {"error": str(e)}
103
 
104
- def format_sentence_metrics(sentence_wers, sentence_cers, average_wer, average_cer, std_dev_wer, std_dev_cer):
105
- if not sentence_wers and not sentence_cers:
106
- return "All sentences match perfectly!"
107
-
108
- md = "### Sentence-level Metrics\n\n"
109
- md += "#### Word Error Rate (WER)\n"
110
- md += f"* Average WER: {average_wer:.2f}\n"
111
- md += f"* Standard Deviation: {std_dev_wer:.2f}\n\n"
112
- md += "#### Character Error Rate (CER)\n"
113
- md += f"* Average CER: {average_cer:.2f}\n"
114
- md += f"* Standard Deviation: {std_dev_cer:.2f}\n\n"
115
-
116
- md += "### WER for Each Sentence\n\n"
117
- for i, wer in enumerate(sentence_wers):
118
- md += f"* Sentence {i+1}: {wer:.2f}\n"
119
-
120
- md += "\n### CER for Each Sentence\n\n"
121
- for i, cer in enumerate(sentence_cers):
122
- md += f"* Sentence {i+1}: {cer:.2f}\n"
123
-
124
- return md
125
-
126
  def main():
127
  with gr.Blocks() as demo:
128
  gr.Markdown("# ASR Metrics")
@@ -139,6 +164,7 @@ def main():
139
  compute_button = gr.Button("Compute Metrics")
140
  results_output = gr.JSON(label="Results")
141
  metrics_output = gr.Markdown(label="Sentence Metrics")
 
142
 
143
  # Update previews when files are uploaded
144
  def update_previews(ref_file, hyp_file):
@@ -168,7 +194,7 @@ def main():
168
  def process_and_display(ref_file, hyp_file):
169
  result = process_files(ref_file, hyp_file)
170
  if "error" in result:
171
- return {}, {}, "Error: " + result["error"]
172
 
173
  metrics = {
174
  "Overall WER": result["Overall WER"],
@@ -181,18 +207,25 @@ def main():
181
  result["Average WER"],
182
  result["Average CER"],
183
  result["Standard Deviation WER"],
184
- result["Standard Deviation CER"]
 
185
  )
186
 
187
- return metrics, metrics_md
 
 
 
 
 
 
188
 
189
  compute_button.click(
190
  fn=process_and_display,
191
  inputs=[reference_file, hypothesis_file],
192
- outputs=[results_output, metrics_output]
193
  )
194
 
195
  demo.launch()
196
 
197
  if __name__ == "__main__":
198
- main()
 
75
  except Exception as e:
76
  raise e
77
 
78
+ def identify_misaligned_sentences(reference_text, hypothesis_text):
79
+ """
80
+ Identify sentences that don't match between reference and hypothesis.
81
+ Returns a dictionary with misaligned sentence pairs and their indices.
82
+ """
83
+ reference_sentences = split_into_sentences(reference_text)
84
+ hypothesis_sentences = split_into_sentences(hypothesis_text)
85
+
86
+ misaligned = []
87
+ for i, (ref, hyp) in enumerate(zip(reference_sentences, hypothesis_sentences)):
88
+ if ref != hyp:
89
+ misaligned.append({
90
+ "index": i+1,
91
+ "reference": ref,
92
+ "hypothesis": hyp
93
+ })
94
+
95
+ return misaligned
96
+
97
+ def format_sentence_metrics(sentence_wers, sentence_cers, average_wer, average_cer, std_dev_wer, std_dev_cer, misaligned_sentences):
98
+ md = "### Sentence-level Metrics\n\n"
99
+ md += "#### Word Error Rate (WER)\n"
100
+ md += f"* Average WER: {average_wer:.2f}\n"
101
+ md += f"* Standard Deviation: {std_dev_wer:.2f}\n\n"
102
+ md += "#### Character Error Rate (CER)\n"
103
+ md += f"* Average CER: {average_cer:.2f}\n"
104
+ md += f"* Standard Deviation: {std_dev_cer:.2f}\n\n"
105
+
106
+ md += "### WER for Each Sentence\n\n"
107
+ for i, wer in enumerate(sentence_wers):
108
+ md += f"* Sentence {i+1}: {wer:.2f}\n"
109
+
110
+ md += "\n### CER for Each Sentence\n\n"
111
+ for i, cer in enumerate(sentence_cers):
112
+ md += f"* Sentence {i+1}: {cer:.2f}\n"
113
+
114
+ if misaligned_sentences:
115
+ md += "\n### Misaligned Sentences\n\n"
116
+ for misaligned in misaligned_sentences:
117
+ md += f"#### Sentence {misaligned['index']}\n"
118
+ md += f"* Reference: {misaligned['reference']}\n"
119
+ md += f"* Hypothesis: {misaligned['hypothesis']}\n\n"
120
+
121
+ return md
122
+
123
  @spaces.GPU()
124
  def process_files(reference_file, hypothesis_file):
125
  try:
 
132
  overall_wer = calculate_wer(reference_text, hypothesis_text)
133
  overall_cer = calculate_cer(reference_text, hypothesis_text)
134
  sentence_metrics = calculate_sentence_metrics(reference_text, hypothesis_text)
135
+ misaligned = identify_misaligned_sentences(reference_text, hypothesis_text)
136
 
137
  return {
138
  "Overall WER": overall_wer,
 
142
  "Average WER": sentence_metrics["average_wer"],
143
  "Average CER": sentence_metrics["average_cer"],
144
  "Standard Deviation WER": sentence_metrics["std_dev_wer"],
145
+ "Standard Deviation CER": sentence_metrics["std_dev_cer"],
146
+ "Misaligned Sentences": misaligned
147
  }
148
  except Exception as e:
149
  return {"error": str(e)}
150
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
151
  def main():
152
  with gr.Blocks() as demo:
153
  gr.Markdown("# ASR Metrics")
 
164
  compute_button = gr.Button("Compute Metrics")
165
  results_output = gr.JSON(label="Results")
166
  metrics_output = gr.Markdown(label="Sentence Metrics")
167
+ misaligned_output = gr.Markdown(label="Misaligned Sentences")
168
 
169
  # Update previews when files are uploaded
170
  def update_previews(ref_file, hyp_file):
 
194
  def process_and_display(ref_file, hyp_file):
195
  result = process_files(ref_file, hyp_file)
196
  if "error" in result:
197
+ return {}, {}, {}, "Error: " + result["error"]
198
 
199
  metrics = {
200
  "Overall WER": result["Overall WER"],
 
207
  result["Average WER"],
208
  result["Average CER"],
209
  result["Standard Deviation WER"],
210
+ result["Standard Deviation CER"],
211
+ result["Misaligned Sentences"]
212
  )
213
 
214
+ misaligned_md = "### Misaligned Sentences\n\n"
215
+ for misaligned in result["Misaligned Sentences"]:
216
+ misaligned_md += f"#### Sentence {misaligned['index']}\n"
217
+ misaligned_md += f"* Reference: {misaligned['reference']}\n"
218
+ misaligned_md += f"* Hypothesis: {misaligned['hypothesis']}\n\n"
219
+
220
+ return metrics, metrics_md, misaligned_md
221
 
222
  compute_button.click(
223
  fn=process_and_display,
224
  inputs=[reference_file, hypothesis_file],
225
+ outputs=[results_output, metrics_output, misaligned_output]
226
  )
227
 
228
  demo.launch()
229
 
230
  if __name__ == "__main__":
231
+ main()