files upload

.gitattributes

@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+nlu/model/tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,13 @@
+---
+title: French SLU DEMO Interspeech2024
+emoji: 🚀
+colorFrom: yellow
+colorTo: blue
+sdk: gradio
+sdk_version: 4.41.0
+app_file: app.py
+pinned: false
+license: cc-by-nc-sa-4.0
+short_description: French SLU demo
+---
+

app.py

@@ -0,0 +1,153 @@
+import gradio as gr
+import numpy as np
+import librosa
+
+from asr.run_asr import run_asr_inference, load_asr_model
+from nlu.run_nlu import run_nlu_inference, load_nlu_model
+
+############### strings
+mhubert_link = '[mHuBERT-147 model](https://huggingface.co/utter-project/mHuBERT-147)'
+massive_link = '[Speech-MASSIVE dataset](https://huggingface.co/datasets/FBK-MT/Speech-MASSIVE)'
+blog_post_link = '[blog post](TO DO TO DO)'
+title = "# DEMO: French Spoken Language Understanding using mHuBERT-147 and Speech-MASSIVE"
+description=[
+    f"""
+    **Interspeech 2024 DEMO.** Cascaded SLU using {mhubert_link} and {massive_link} components.
+    """,
+    """For more details on the implementation, check our blog post.""",
+]
+
+examples = [
+    "resources/audios/utt_286.wav",
+    "resources/audios/utt_2414.wav",
+    "resources/audios/utt_16032.wav",
+    "resources/audios/utt_3060.wav",
+    "resources/audios/utt_1264.wav",
+    "resources/audios/utt_9912.wav",
+    "resources/audios/utt_14684.wav",
+    "resources/audios/utt_5410.wav",
+]
+transcriptions = [
+    "allume les lumières dans la cuisine",
+    "je veux commander une pizza chez michael's pizza",
+    "veuillez envoyer un e-mail à sally concernant la réunion de demain",
+    "quelles sont les nouvelles de newsource",
+    "mon réveil est-il réglé pour demain matin",
+    "olly combien de temps dois-je faire bouillir les oeufs",
+    "qui est le premier ministre de russie",
+    "lis moi les derniers gros titres du new york times"
+]
+intents = [
+    "iot_hue_lighton",
+    "takeaway_order",
+    "email_sendemail",
+    "news_query",
+    "alarm_query",
+    "cooking_recipe",
+    "qa_factoid",
+    "news_query"
+]
+slots = [
+    [ "Other", "Other", "Other", "Other", "Other", "house_place" ],
+    [ "Other", "Other", "Other", "Other", "food_type", "Other", "business_name", "business_name" ],
+    [ 'Other', 'Other', 'Other', 'Other', 'Other', 'Other', 'person', 'Other', 'Other', 'event_name', 'Other', 'date'],
+    [ 'Other', 'Other', 'Other', 'Other', 'Other', 'media_type'],
+    [ 'Other', 'Other', 'Other', 'Other', 'Other', 'Other', 'date', 'timeofday'],
+    [ 'Other', 'Other', 'Other', 'Other', 'Other', 'Other', 'Other', 'cooking_type', 'Other', 'food_type'],
+    [ 'Other', 'Other', 'Other', 'Other', 'Other', 'Other', 'place_name'],
+    [ 'Other', 'Other', 'Other', 'Other', 'Other', 'Other', 'Other', 'media_type', 'media_type', 'media_type']
+]
+
+
+utter_ack_text = """This is an output of the European Project UTTER (Unified Transcription and Translation for Extended Reality) funded by European Union’s Horizon Europe Research and Innovation programme under grant agreement number 101070631. 
+For more information please visit https://he-utter.eu/"""
+
+ack_authors = """This demo was made by [Beomseok Lee](https://mt.fbk.eu/author/blee/) and [Marcely Zanon Boito](https://sites.google.com/view/mzboito/marcely-zanon-boito)."""
+
+eu_logo = """<img src="https://huggingface.co/spaces/naver/French-SLU-DEMO-Interspeech2024/resolve/main/resources/logos/EU_flag.jpg" width="100" height="100">"""
+utter_logo = """<a href="https://he-utter.eu/" target="_blank"><img src="https://huggingface.co/spaces/naver/French-SLU-DEMO-Interspeech2024/resolve/main/resources/logos/Utter_logo.png" width="50" height="50"></a>"""
+nle_logo = """<a href="https://europe.naverlabs.com/" target="_blank"><img src="https://huggingface.co/spaces/naver/French-SLU-DEMO-Interspeech2024/resolve/main/resources/logos/NAVERLABS_2_BLACK.png" width="100" height="100"></a>"""
+fbk_logo = """<a href="https://mt.fbk.eu/" target="_blank"><img src="https://huggingface.co/spaces/naver/French-SLU-DEMO-Interspeech2024/resolve/main/resources/logos/FBK_logo.png" width="100" height="100"></a>"""
+
+
+table = f"""
+    | File | Transcription | Slots | Intent | 
+    | ------------ | ------------------- | ---------- | -----------| 
+    | {examples[0].split("/")[-1]}  | {transcriptions[0]} | {slots[0]} | {intents[0]} |
+    | {examples[1].split("/")[-1]}  | {transcriptions[1]} | {slots[1]} | {intents[1]} |
+    | {examples[2].split("/")[-1]}  | {transcriptions[2]} | {slots[2]} | {intents[2]} |
+    | {examples[3].split("/")[-1]}  | {transcriptions[3]} | {slots[3]} | {intents[3]} |
+    | {examples[4].split("/")[-1]}  | {transcriptions[4]} | {slots[4]} | {intents[4]} |
+    | {examples[5].split("/")[-1]}  | {transcriptions[5]} | {slots[5]} | {intents[5]} |
+    | {examples[6].split("/")[-1]}  | {transcriptions[6]} | {slots[6]} | {intents[6]} |
+    | {examples[7].split("/")[-1]}  | {transcriptions[7]} | {slots[7]} | {intents[7]} |
+    """.strip()
+
+############### calls
+
+def run_inference(audio_file):
+    print(audio_file)
+    audio_struct = librosa.load(audio_file, sr=16000)
+    print(audio_struct)
+    audio = {'sampling_rate': audio_struct[1], 'array': audio_struct[0]} #.astype(np.float32)
+    transcription = run_asr_inference(asr_model, processor, audio)
+    print(transcription)
+    structured_output = run_nlu_inference(nlu_model, tokenizer, transcription)
+
+    return structured_output
+
+############### app
+
+asr_model, processor = load_asr_model()
+nlu_model, tokenizer = load_nlu_model()
+
+demo = gr.Blocks(
+    title=title,
+    analytics_enabled=False,
+    theme=gr.themes.Base(),
+)
+
+with demo:
+    gr.Markdown(title)
+    for line in description:
+        gr.Markdown(line)
+
+    with gr.Row():
+        waveform_options = gr.WaveformOptions(sample_rate=16000)
+        
+        audio_file = gr.Audio(
+            label="Audio file", 
+            sources=['microphone','upload'],
+            type="filepath",
+            format='wav',
+            waveform_options=waveform_options,
+            show_download_button=False,
+            show_share_button=False,
+            max_length=20,
+            )
+        
+    output = gr.HighlightedText(label="ASR result + NLU result")
+
+    gr.Button("Run Inference", variant='primary').click(
+        run_inference,
+        concurrency_limit=2,
+        inputs=audio_file,
+        outputs=output,
+    )
+
+    with gr.Row():
+        gr.Examples(label="Speech-MASSIVE test utterances:", inputs=audio_file, examples=examples)
+    gr.Markdown(table)
+
+    gr.Markdown("# Aknowledgments")
+    gr.Markdown(utter_ack_text)    
+    gr.Markdown(ack_authors)
+    
+    with gr.Row():
+        gr.Markdown(eu_logo)
+        gr.Markdown(utter_logo)
+        gr.Markdown(nle_logo)
+        gr.Markdown(fbk_logo)
+
+demo.queue()
+demo.launch()

asr/CTC_model.py

@@ -0,0 +1,110 @@
+"""
+Inference CTC class derived from HubertForCTC.
+
+Author: Marcely Zanon Boito, 2024
+"""
+
+
+from typing import Optional, Tuple, Union
+import torch
+from torch import nn
+from transformers import HubertPreTrainedModel, HubertModel
+from transformers.modeling_outputs import CausalLMOutput, SequenceClassifierOutput
+
+class VanillaNN(nn.Module):
+    def __init__(self, input_dim, output_dim):
+        """
+        simple NN with ReLU activation (no norm)
+        """
+        super().__init__()
+        
+        self.linear = nn.Linear(input_dim, output_dim)
+        self.act_fn = nn.ReLU()
+
+    def forward(self, hidden_states: torch.FloatTensor):
+        hidden_states = self.linear(hidden_states)
+        hidden_states = self.act_fn(hidden_states)
+
+        return hidden_states
+
+class mHubertForCTC(HubertPreTrainedModel):
+    def __init__(self, config, target_lang: Optional[str] = None):
+        super().__init__(config)
+        self.hubert = HubertModel(config)
+        self.dropout = nn.Dropout(config.final_dropout)
+        
+        output_hidden_size = config.hidden_size
+
+        self.has_interface = config.add_interface
+
+        # NN layers on top of the trainable stack
+        if config.add_interface:
+            self.interface = nn.ModuleList([VanillaNN(output_hidden_size,output_hidden_size) for i in range(config.num_interface_layers)])
+        
+        self.lm_head = nn.Linear(output_hidden_size, config.vocab_size)
+
+        self.post_init()        
+
+    def forward(
+        self,
+        input_values: Optional[torch.Tensor],
+        attention_mask: Optional[torch.Tensor] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+        labels: Optional[torch.Tensor] = None,
+    ) -> Union[Tuple, SequenceClassifierOutput]:
+
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+        output_hidden_states = self.config.output_hidden_states
+
+        outputs = self.hubert(
+            input_values,
+            attention_mask=attention_mask,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+        hidden_states = outputs[0]
+
+        hidden_states = self.dropout(hidden_states)
+        if self.has_interface:
+            for layer in self.interface:
+                hidden_states = layer(hidden_states)
+        logits = self.lm_head(hidden_states)
+        
+        loss = None
+        
+        if labels is not None:
+            if labels.max() >= self.config.vocab_size:
+                raise ValueError(f"Label values must be <= vocab_size: {self.config.vocab_size}")
+
+            # retrieve loss input_lengths from attention_mask
+            attention_mask = (
+                attention_mask if attention_mask is not None else torch.ones_like(input_values, dtype=torch.long)
+            )
+            input_lengths = self._get_feat_extract_output_lengths(attention_mask.sum(-1)).to(torch.long)
+
+            # assuming that padded tokens are filled with -100
+            # when not being attended to
+            labels_mask = labels >= 0
+            target_lengths = labels_mask.sum(-1)
+            flattened_targets = labels.masked_select(labels_mask)
+
+            # ctc_loss doesn't support fp16
+            log_probs = nn.functional.log_softmax(logits, dim=-1, dtype=torch.float32).transpose(0, 1)
+
+            with torch.backends.cudnn.flags(enabled=False):
+                loss = nn.functional.ctc_loss(
+                    log_probs,
+                    flattened_targets,
+                    input_lengths,
+                    target_lengths,
+                    blank=self.config.ctc_token_id,
+                    reduction=self.config.ctc_loss_reduction,
+                    zero_infinity=self.config.ctc_zero_infinity,
+                )
+
+        return CausalLMOutput(
+            loss=loss, logits=logits, hidden_states=outputs.hidden_states, attentions=outputs.attentions
+        )

asr/run_asr.py

@@ -0,0 +1,42 @@
+"""
+Inference main class.
+
+Author: Marcely Zanon Boito, 2024
+"""
+
+from .CTC_model import mHubertForCTC
+
+import torch
+from transformers import Wav2Vec2CTCTokenizer, Wav2Vec2FeatureExtractor, Wav2Vec2Processor
+from transformers import HubertConfig
+
+from datasets import load_dataset
+
+fbk_test_id = 'FBK-MT/Speech-MASSIVE-test'
+mhubert_id = 'utter-project/mHuBERT-147'
+
+def load_asr_model():
+	# Load the ASR model
+	tokenizer = Wav2Vec2CTCTokenizer("asr/vocab.json", unk_token="[UNK]", pad_token="[PAD]", word_delimiter_token="|")
+	feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(mhubert_id)
+	processor = Wav2Vec2Processor(feature_extractor=feature_extractor, tokenizer=tokenizer)
+
+	config = HubertConfig.from_pretrained("naver/mHuBERT-147-ASR-fr")
+	model = mHubertForCTC.from_pretrained("naver/mHuBERT-147-ASR-fr", config=config)
+	model.eval()
+	return model, processor
+
+def run_asr_inference(model, processor, example):
+	audio = processor(example["array"], sampling_rate=example["sampling_rate"]).input_values[0]
+	input_values = torch.tensor(audio).unsqueeze(0) 
+
+	with torch.no_grad():
+		logits = model(input_values).logits
+	
+	pred_ids = torch.argmax(logits, dim=-1)
+
+	prediction = processor.batch_decode(pred_ids)[0].replace('[CTC]', "")
+	return prediction
+
+
+

asr/vocab.json

@@ -0,0 +1 @@
+{"q": 0, "u": 1, "a": 2, "n": 3, "d": 4, "r": 6, "l": 7, "i": 8, "e": 9, "p": 10, "o": 11, "c": 12, "h": 13, "t": 14, "b": 15, "s": 16, "\u00e0": 17, "v": 18, "y": 19, "j": 20, "\u00e7": 21, "g": 22, "m": 23, "x": 24, "\u00f9": 25, "f": 26, "\u00e9": 27, "k": 28, "\u00ee": 29, "z": 30, "\u00f4": 31, "\u00ea": 32, "\u00e8": 33, "\u00fb": 34, "\u00e2": 35, "w": 36, "\u0153": 37, "\u00ef": 38, "\u014d": 39, "\u00eb": 40, "\u00f3": 41, "\u00fc": 42, "\u0144": 43, "\u016b": 44, "\u00e4": 45, "\u00e6": 46, "\u00ed": 47, "\u0107": 48, "\u00ec": 49, "\u00e5": 50, "\u00f8": 51, "\u00f6": 52, "\u0117": 53, "\u021b": 54, "\u0142": 55, "\u00f5": 56, "\u00e1": 57, "\u0131": 58, "\u015f": 59, "\u00fa": 60, "\u00e3": 61, "\u00f1": 62, "\u0161": 63, "\u0137": 64, "\u0101": 65, "\u00df": 66, "\u0103": 67, "\u0219": 68, "\u00f2": 69, "\u017e": 70, "\u0151": 71, "\u010d": 72, "\u044d": 73, "\u01b0": 74, "\u1edb": 75, "\u1ea5": 76, "\u00f0": 77, "\u02bd": 78, "\u02bc": 79, "\u5b87": 80, "\u6d25": 81, "\u4fdd": 82, "\u00ff": 83, "\u53b3": 84, "\u4e09": 85, "\u011f": 86, "\u015b": 87, "\u0119": 88, "\u02bf": 89, "\u0148": 90, "\u016f": 91, "1": 92, "9": 93, "8": 94, "\u016d": 95, "\u017c": 96, "\u1ea3": 97, "\u0171": 98, "0": 99, "\u0159": 100, "\u0111": 101, "\u03c4": 102, "\u1ed5": 103, "\u1eaf": 104, "\u017a": 105, "\u011b": 106, "\u0192": 107, "\u03b3": 108, "5": 109, "\u03c3": 110, "\u01ce": 111, "3": 112, "\u1ea1": 113, "\ua7a1": 114, "\u013c": 115, "\u7261": 116, "\u4e39": 117, "\u01d4": 118, "\u03b2": 119, "\u03b5": 120, "\u00fd": 121, "\u00fe": 122, "\u012b": 123, "2": 124, "\u0113": 125, "\u03c9": 126, "\u03b8": 127, "6": 128, "\u1ec5": 129, "\u1eb7": 130, "\u1eab": 131, "\u1e63": 132, "\u1fd6": 133, "\u03bf": 134, "\u03c2": 135, "\u03b1": 136, "\u03c0": 137, "\u03b4": 138, "\u03c6": 139, "4": 140, "\u1e25": 141, "\u03bb": 142, "\u03cd": 143, "\u03c1": 144, "\u03bc": 145, "\u1ecb": 146, "\u0169": 147, "\u3044": 148, "\u0294": 149, "\u05d3": 150, "\u05df": 151, "\u05d1": 152, "\u05e8": 153, "\u05d0": 154, "\u05d5": 155, "\u0165": 156, "\u05da": 157, "\u05d9": 158, "\u05d4": 159, "\u05de": 160, "\u04cc": 161, "\u1ec1": 162, "7": 163, "\u02bb": 164, "\u01eb": 165, "\u013e": 166, "\u1e0d": 167, "\u043a": 168, "\u0251": 169, "\u0105": 170, "\u03b9": 171, "|": 5, "[UNK]": 172, "[PAD]": 173, "[CTC]": 174}

nlu/run_nlu.py

@@ -0,0 +1,85 @@
+from transformers import (
+    AutoConfig,
+    AutoModelForSeq2SeqLM,
+    AutoTokenizer
+)
+
+from datasets import load_dataset
+import torch
+
+def load_nlu_model():
+    config = AutoConfig.from_pretrained("Beomseok-LEE/NLU-Speech-MASSIVE-finetune")
+    tokenizer = AutoTokenizer.from_pretrained("Beomseok-LEE/NLU-Speech-MASSIVE-finetune")
+    model = AutoModelForSeq2SeqLM.from_pretrained("Beomseok-LEE/NLU-Speech-MASSIVE-finetune", config=config)
+
+    return model, tokenizer
+
+def run_nlu_inference(model, tokenizer, example):
+  print(example)
+  formatted_example = "Annotate: " + example
+  input_values = tokenizer(formatted_example, max_length=128, padding=False, truncation=True, return_tensors="pt").input_ids
+
+  with torch.no_grad():
+    pred_ids = model.generate(input_values)
+
+  prediction = tokenizer.decode(pred_ids[0], skip_special_tokens=True)
+  print(prediction)
+
+  splitted_pred = prediction.strip().split()
+
+  slots_prediction = ''
+  intent_prediction = ''
+
+  if len(splitted_pred) >= 2:
+      slots_prediction = splitted_pred[:-1]
+      intent_prediction = splitted_pred[-1]
+  if len(splitted_pred) == 1:
+      slots_prediction = splitted_pred
+
+  words = example.split(' ')
+  
+  title_1 = '[ASR output]\n'
+  title_2 = '\n\n[NLU - slot filling]\n'
+  title_3 = '\n\n[NLU - intent classifcation]\n'
+
+  prefix_str_1 = title_1 + example + title_2
+  prefix_str_2 = title_3
+
+  structured_output = {
+     'text' : prefix_str_1 + example + prefix_str_2 + intent_prediction, 
+      'entities': []}
+
+  structured_output['entities'].append({
+         'entity': 'ASR output',
+         'word': example,
+         'start': len(title_1),
+         'end': len(title_1) + len(example)
+      })
+
+  idx = len(prefix_str_1)
+
+  for slot, word in zip(slots_prediction, words):
+    _entity = slot
+    _word = word
+    _start = idx
+    _end = idx + len(word)
+    idx = _end + 1
+
+    structured_output['entities'].append({
+        'entity': _entity,
+        'word': _word,
+        'start': _start,
+        'end': _end
+    })
+  
+  idx = len(prefix_str_1 + example + prefix_str_2)
+
+  if intent_prediction:
+    structured_output['entities'].append({
+      'entity': 'Classified Intent',
+      'word': intent_prediction,
+      'start': idx,
+      'end': idx + len(intent_prediction)
+    })
+
+  return structured_output

requirements.txt

@@ -0,0 +1,7 @@
+numpy==1.26.3
+torch==1.13.1
+transformers==4.32.0
+librosa==0.10.1
+soundfile==0.12.1
+torchaudio
+accelerate