Spaces:
Running
on
Zero
Running
on
Zero
File size: 2,115 Bytes
12f2e48 4401dfb 12f2e48 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 |
from typing import List
import random
import argparse
from datasets import load_dataset
from datasets import Dataset
from sonicverse.constants import ROLE_ASSISTANT, ROLE_USER
DATASET_ARGS = dict(
path="mozilla-foundation/common_voice_15_0", name="en", split="train"
)
PRETRAIN_PHRASES = [
"Repeat the content of the audio <speech>",
"Transcribe <speech>",
"What is being said in <speech>",
"Can you interpret <speech>?",
"Please convert <speech> into text",
"What does <speech> say?",
"Could you transcribe <speech> for me?",
"I need the text of <speech>",
"Can you write out <speech>?",
"What's the content of <speech>?",
"Please provide the transcript of <speech>",
"Can you decode <speech>?",
"What is the transcription of <speech>?",
"Can you jot down <speech>?",
"What is the written form of <speech>?",
"Can you scribe <speech>?",
]
def _write_convo(idx, row) -> List:
example = {
"speech_audios": [{"dataset_args": DATASET_ARGS, "idx": idx}],
}
phrase = random.choice(PRETRAIN_PHRASES)
example["messages"] = [
{
"role": ROLE_USER,
"content": phrase,
},
{
"role": ROLE_ASSISTANT,
"content": row["text"] if "text" in row else row["sentence"],
},
]
return example
def main(args):
audio_dataset = load_dataset(**DATASET_ARGS)
def gen():
i = 0
idxes = list(range(len(audio_dataset)))
random.shuffle(idxes)
for k in idxes:
try:
yield _write_convo(k, audio_dataset[k])
except ValueError:
pass
else:
i += 1
if i >= args.max_examples:
break
ds = Dataset.from_generator(gen)
ds.save_to_disk(args.output_folder)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("-o", "--output_folder", type=str)
parser.add_argument("-n", "--max_examples", type=int, default=200_000)
args = parser.parse_args()
main(args)
|