feat: Update app for long audio captioning and chaining

README.md

@@ -9,4 +9,92 @@ app_file: app.py
 pinned: false
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# 🎼 SonicVerse
+
+An interactive demo for SonicVerse, a music captioning model, allowing users to input audio of up to 10 seconds and generate a natural language caption
+that includes a general description of the music as well as music features such as key, instruments, genre, mood / theme, vocals gender.
+
+---
+
+## 🚀 Demo
+
+Check out the live Space here:  
+[![Hugging Face Space](https://img.shields.io/badge/HuggingFace-Space-blue?logo=huggingface)](https://huggingface.co/spaces/annabeth97c/SonicVerse)
+
+---
+
+## 🚀 Samples
+
+Short captions
+
+---
+
+## 📦 Features
+
+✅ Upload a 10 second music clip and get a caption
+
+✅ Upload a long music clip (upto 1 minute for successful demo) to get a long detailed caption for the whole music clip.
+
+---
+
+## 🛠️ How to Run Locally
+
+```bash
+# Clone the repo
+git clone https://github.com/AMAAI-Lab/SonicVerse
+cd SonicVerse
+
+# Install dependencies
+pip install -r requirements.txt
+
+# Alternatively, set up conda environment
+conda env create -f environment.yml
+conda activate sonicverse
+
+# Run the app
+python app.py
+```
+
+---
+
+<!-- ## 📂 File Structure
+
+```
+.
+├── app.py               # Web app file
+├── requirements.txt     # Python dependencies
+├── environment.yml      # Conda environment
+├── README.md            # This file
+└── src/sonicverse       # Source 
+```
+
+--- -->
+
+## 💡 Usage
+
+To use the app:
+1. Select audio clip to input 
+2. Click the **Generate** button.
+3. See the model’s output below.
+
+---
+
+## 🧹 Built With
+
+- [Hugging Face Spaces](https://huggingface.co/spaces)
+- [Gradio](https://gradio.app/)
+- [Mistral 7B](https://huggingface.co/mistralai/Mistral-7B-v0.1)
+- [MERT 95M](https://huggingface.co/m-a-p/MERT-v1-95M)
+---
+
+<!-- ## ✨ Acknowledgements
+
+- [Model authors or papers you built on]
+- [Contributors or collaborators]
+
+---
+
+## 📜 License
+
+This project is licensed under the MIT License / Apache 2.0 / Other.
+ -->

app.py

@@ -18,11 +18,17 @@ import torch
 import transformers
 import torchaudio
 
-from multi_token.model_utils import MultiTaskType
-from multi_token.training import ModelArguments
-from multi_token.inference import load_trained_lora_model
-from multi_token.data_tools import encode_chat
+from openai import OpenAI
+client = OpenAI()
+MODEL       = "gpt-4"
+SLEEP_BETWEEN_CALLS = 1.0
 
+from sonicverse.model_utils import MultiTaskType
+from sonicverse.training import ModelArguments
+from sonicverse.inference import load_trained_lora_model
+from sonicverse.data_tools import encode_chat
+
+CHUNK_LENGTH = 10
 
 @dataclass
 class ServeArguments(ModelArguments):
@@ -31,7 +37,6 @@ class ServeArguments(ModelArguments):
     temperature: float = field(default=0.01)
 
 
-# Load arguments and model
 logging.getLogger().setLevel(logging.INFO)
 
 parser = transformers.HfArgumentParser((ServeArguments,))
@@ -45,10 +50,82 @@ model, tokenizer = load_trained_lora_model(
     tasks_config=serve_args.tasks_config
 )
 
-@spaces.GPU(duration=60)
-def generate_caption(audio_file):
-    # waveform, sample_rate = torchaudio.load(audio_file)
+def caption_audio(audio_file):
+    chunk_audio_files = split_audio(audio_file, CHUNK_LENGTH)
+    chunk_captions = []
+    for audio_chunk in chunk_audio_files:
+        chunk_captions.append(generate_caption(audio_chunk))
+
+    if len(chunk_captions) > 1:
+        audio_name = os.path.splitext(os.path.basename(audio_file))[0]
+        long_caption = summarize_song(audio_name, chunk_captions)
+
+        delete_files(chunk_audio_files)
+
+        return long_caption
+
+    else:
+        if len(chunk_captions) == 1:
+            return chunk_captions[0]
+        else:
+            return ""
+
+def summarize_song(song_name, chunks):
+    prompt = f"""
+You are a music critic. Given the following chronological 10‑second chunk descriptions of a single piece, write one flowing, detailed description of the entire song—its structure, instrumentation, and standout moments. Mention transition points in terms of time stamps. If the description of certain chunks does not seem to fit with those for the chunks before and after, treat those as bad descriptions with lower accuracy and do not incorporate the information. Retain concrete musical attributes such as key, chords, tempo.
+
+Chunks for “{song_name} ”:
+"""
+    for i, c in enumerate(chunks, 1):
+        prompt += f"\n {(i - 1)*0} to {i*10} seconds. {c.strip()}"
+    prompt += "\n\nFull song description:"
+
+    resp = client.chat.completions.create(model=MODEL,
+    messages=[
+        {"role": "system", "content": "You are an expert music writer."},
+        {"role": "user",   "content": prompt}
+    ],
+    temperature=0.0,
+    max_tokens=1000)
+    return resp.choices[0].message.content.strip()
+
+def delete_files(file_paths):
+    for path in file_paths:
+        try:
+            if os.path.isfile(path):
+                os.remove(path)
+                print(f"Deleted: {path}")
+            else:
+                print(f"Skipped (not a file or doesn't exist): {path}")
+        except Exception as e:
+            print(f"Error deleting {path}: {e}")
+
+def split_audio(input_path, chunk_length_seconds):
+
+    waveform, sample_rate = torchaudio.load(input_path)
+    num_channels, total_samples = waveform.shape
+    chunk_samples = int(chunk_length_seconds * sample_rate)
+
+    num_chunks = (total_samples + chunk_samples - 1) // chunk_samples
+
+    base, ext = os.path.splitext(input_path)
+    output_paths = []
+
+    if (num_chunks <= 1):
+        return [input_path]
+
+    for i in range(num_chunks):
+        start = i * chunk_samples
+        end = min((i + 1) * chunk_samples, total_samples)
+        chunk_waveform = waveform[:, start:end]
+        
+        output_file = f"{base}_{i+1:03d}{ext}"
+        torchaudio.save(output_file, chunk_waveform, sample_rate)
+        output_paths.append(output_file)
+
+    return output_paths
 
+def generate_caption(audio_file):
     req_json = {
         "messages": [
             {"role": "user", "content": "Describe the music. <sound>"}
@@ -79,7 +156,7 @@ def generate_caption(audio_file):
 
 
 demo = gr.Interface(
-    fn=generate_caption,
+    fn=caption_audio,
     inputs=gr.Audio(type="filepath", label="Upload an audio file"),
     outputs=gr.Textbox(label="Generated Caption"),
     title="SonicVerse",

requirements.txt

@@ -74,6 +74,7 @@ mdurl==0.1.2
 mpmath==1.3.0
 msgpack==1.0.8
 multidict==6.0.5
+git+https://huggingface.co/spaces/annabeth97c/temp#egg=multi_token&subdirectory=src/sonicverse
 multiprocess==0.70.16
 narwhals==1.40.0
 networkx==3.2.1
@@ -93,6 +94,7 @@ nvidia-cusparse-cu12==12.1.0.106
 nvidia-nccl-cu12==2.20.5
 nvidia-nvjitlink-cu12==12.5.82
 nvidia-nvtx-cu12==12.1.105
+openai==1.82.0
 orjson==3.10.18
 packaging==24.1
 pandas==2.2.2

src/sonicverse/multi_token.egg-info/SOURCES.txt

@@ -1,6 +0,0 @@
-setup.py
-multi_token.egg-info/PKG-INFO
-multi_token.egg-info/SOURCES.txt
-multi_token.egg-info/dependency_links.txt
-multi_token.egg-info/requires.txt
-multi_token.egg-info/top_level.txt

src/sonicverse/multi_token.egg-info/requires.txt

@@ -1,8 +0,0 @@
-transformers>=4.34.0
-accelerate>=0.21.0
-scipy>=1.11.3
-bitsandbytes>=0.41.0
-datasets>=2.14.5
-sentencepiece>=0.1.99
-peft>=0.4.0
-deepspeed==0.9.5

src/sonicverse/requirements.txt

@@ -1,8 +1,167 @@
-transformers>=4.34.0
-accelerate>=0.21.0
-scipy>=1.11.3
-bitsandbytes>=0.41.0
-datasets>=2.14.5
-sentencepiece>=0.1.99
-peft>=0.4.0
+absl-py==2.1.0
+accelerate==0.29.3
+aiofiles==23.2.1
+aiohttp==3.9.5
+aiosignal==1.3.1
+altair==5.5.0
+anyio==4.9.0
+argbind==0.3.9
+asttokens==2.4.1
+async-timeout==4.0.3
+attrs==23.2.0
+audioread==3.0.1
+bert-score==0.3.13
+bitsandbytes==0.43.1
+blinker==1.8.2
+certifi==2024.7.4
+cffi==1.16.0
+charset-normalizer==3.3.2
+click==8.1.7
+contourpy==1.2.1
+cycler==0.12.1
+datasets==2.19.0
+decorator==5.1.1
 deepspeed==0.9.5
+descript-audio-codec==1.0.0
+descript-audiotools==0.7.2
+dill==0.3.8
+docstring_parser==0.16
+einops==0.8.0
+evaluate==0.4.3
+exceptiongroup==1.2.2
+executing==2.0.1
+fastapi==0.115.12
+ffmpy==0.3.2
+filelock==3.15.4
+fire==0.6.0
+Flask==3.0.3
+flatten-dict==0.4.2
+fonttools==4.53.1
+frozenlist==1.4.1
+fsspec==2024.3.1
+future==1.0.0
+gradio==3.50.2
+gradio_client==0.6.1
+graphviz==0.20.3
+grpcio==1.64.1
+h11==0.16.0
+hjson==3.1.0
+httpcore==1.0.9
+httpx==0.28.1
+huggingface-hub==0.23.4
+idna==3.7
+importlib_metadata==8.0.0
+importlib_resources==6.4.0
+ipython==8.18.1
+itsdangerous==2.2.0
+jedi==0.19.1
+Jinja2==3.1.4
+joblib==1.4.2
+jsonschema==4.23.0
+jsonschema-specifications==2025.4.1
+julius==0.2.7
+kiwisolver==1.4.5
+lazy_loader==0.4
+librosa==0.10.2.post1
+llvmlite==0.43.0
+Markdown==3.6
+markdown-it-py==3.0.0
+markdown2==2.5.0
+MarkupSafe==2.1.5
+matplotlib==3.9.1
+matplotlib-inline==0.1.7
+mdurl==0.1.2
+mpmath==1.3.0
+msgpack==1.0.8
+multidict==6.0.5
+git+https://huggingface.co/spaces/annabeth97c/temp#egg=multi_token&subdirectory=src/sonicverse
+multiprocess==0.70.16
+narwhals==1.40.0
+networkx==3.2.1
+ninja==1.11.1.1
+nltk==3.8.1
+numba==0.60.0
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==8.9.2.26
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.5.82
+nvidia-nvtx-cu12==12.1.105
+openai==1.82.0
+orjson==3.10.18
+packaging==24.1
+pandas==2.2.2
+parso==0.8.4
+peft==0.10.0
+pexpect==4.9.0
+pillow==10.3.0
+platformdirs==4.2.2
+pooch==1.8.2
+prompt_toolkit==3.0.47
+protobuf==3.19.6
+psutil==6.0.0
+ptyprocess==0.7.0
+pure-eval==0.2.2
+py-cpuinfo==9.0.0
+pyarrow==16.1.0
+pyarrow-hotfix==0.6
+pycparser==2.22
+pydantic==1.10.17
+pydub==0.25.1
+Pygments==2.18.0
+pyloudnorm==0.1.1
+pyparsing==3.1.2
+pystoi==0.4.1
+python-dateutil==2.9.0.post0
+python-multipart==0.0.20
+pytz==2024.1
+PyYAML==6.0.1
+randomname==0.2.1
+referencing==0.36.2
+regex==2024.5.15
+requests==2.32.3
+rich==13.7.1
+rouge_score==0.1.2
+rpds-py==0.25.0
+safetensors==0.4.3
+scikit-learn==1.5.1
+scipy==1.13.0
+semantic-version==2.10.0
+sentencepiece==0.2.0
+six==1.16.0
+sniffio==1.3.1
+soundfile==0.12.1
+soxr==0.3.7
+stack-data==0.6.3
+starlette==0.46.2
+sympy==1.13.0
+tensorboard==2.17.0
+tensorboard-data-server==0.7.2
+termcolor==2.4.0
+threadpoolctl==3.5.0
+tokenizers==0.19.1
+torch==2.3.1
+torch-stoi==0.2.1
+torchaudio==2.3.1
+torchviz==0.0.2
+tqdm==4.66.4
+traitlets==5.14.3
+transformers==4.40.1
+triton==2.3.1
+typing_extensions==4.12.2
+tzdata==2024.1
+urllib3==2.2.2
+uvicorn==0.34.2
+wcwidth==0.2.13
+websockets==11.0.3
+Werkzeug==3.0.3
+xxhash==3.4.1
+yarl==1.9.4
+zipp==3.19.2

src/sonicverse/scripts/clap_gpt_build_finetune_dataset.py

@@ -7,7 +7,7 @@ import openai
 
 from datasets import Dataset, load_dataset
 
-from multi_token.constants import ROLE_ASSISTANT, ROLE_USER
+from sonicverse.constants import ROLE_ASSISTANT, ROLE_USER
 
 PROMPT = """
 You are helping train a sound assistant that can take audio inputs and output text.

src/sonicverse/scripts/clap_gpt_build_pretrain_dataset.py

@@ -7,7 +7,7 @@ import openai
 
 from datasets import Dataset, load_dataset
 
-from multi_token.constants import ROLE_ASSISTANT, ROLE_USER
+from sonicverse.constants import ROLE_ASSISTANT, ROLE_USER
 
 PROMPT = """
 You are helping write captions for audio clips.

src/sonicverse/scripts/document_build_finetune_dataset.py

@@ -7,8 +7,8 @@ import json
 from datasets import load_dataset
 from datasets import Dataset
 
-from multi_token.constants import ROLE_ASSISTANT, ROLE_USER
-from multi_token.modalities.document_gte import (
+from sonicverse.constants import ROLE_ASSISTANT, ROLE_USER
+from sonicverse.modalities.document_gte import (
     split_text_into_documents,
 )
 

src/sonicverse/scripts/document_build_pretrain_dataset.py

@@ -5,8 +5,8 @@ import argparse
 from datasets import load_dataset
 from datasets import Dataset
 
-from multi_token.constants import ROLE_ASSISTANT, ROLE_USER
-from multi_token.modalities.document_gte import (
+from sonicverse.constants import ROLE_ASSISTANT, ROLE_USER
+from sonicverse.modalities.document_gte import (
     split_text_into_documents,
 )
 

src/sonicverse/scripts/evaluate_model.py

@@ -7,12 +7,12 @@ import torch
 
 from datasets import load_from_disk
 
-from multi_token.model_utils import MultiTaskType
-from multi_token.training import (
+from sonicverse.model_utils import MultiTaskType
+from sonicverse.training import (
     ModelArguments,
 )
-from multi_token.inference import load_trained_lora_model
-from multi_token.data_tools import encode_chat
+from sonicverse.inference import load_trained_lora_model
+from sonicverse.data_tools import encode_chat
 
 import evaluate
 

src/sonicverse/scripts/evaluate_model_latest.py

@@ -5,10 +5,10 @@ from flask import Flask, request, jsonify
 import transformers
 import torch
 from datasets import load_from_disk
-from multi_token.model_utils import MultiTaskType
-from multi_token.training import ModelArguments
-from multi_token.inference import load_trained_lora_model
-from multi_token.data_tools import encode_chat
+from sonicverse.model_utils import MultiTaskType
+from sonicverse.training import ModelArguments
+from sonicverse.inference import load_trained_lora_model
+from sonicverse.data_tools import encode_chat
 import evaluate
 import random
 import bert_score

src/sonicverse/scripts/evaluate_model_mullama.py

@@ -5,10 +5,10 @@ from flask import Flask, request, jsonify
 import transformers
 import torch
 from datasets import load_from_disk
-from multi_token.model_utils import MultiTaskType
-from multi_token.training import ModelArguments
-from multi_token.inference import load_trained_lora_model
-from multi_token.data_tools import encode_chat
+from sonicverse.model_utils import MultiTaskType
+from sonicverse.training import ModelArguments
+from sonicverse.inference import load_trained_lora_model
+from sonicverse.data_tools import encode_chat
 import evaluate
 import random
 import bert_score

src/sonicverse/scripts/evaluate_model_mullama_musiccaps.py

@@ -5,10 +5,10 @@ from flask import Flask, request, jsonify
 import transformers
 import torch
 from datasets import load_from_disk
-from multi_token.model_utils import MultiTaskType
-from multi_token.training import ModelArguments
-from multi_token.inference import load_trained_lora_model
-from multi_token.data_tools import encode_chat
+from sonicverse.model_utils import MultiTaskType
+from sonicverse.training import ModelArguments
+from sonicverse.inference import load_trained_lora_model
+from sonicverse.data_tools import encode_chat
 import evaluate
 import random
 import bert_score

src/sonicverse/scripts/evaluate_model_mullama_musiccaps_fixed_prompt.py

@@ -4,10 +4,10 @@ from flask import Flask, request, jsonify
 import transformers
 import torch
 from datasets import load_from_disk
-from multi_token.model_utils import MultiTaskType
-from multi_token.training import ModelArguments
-from multi_token.inference import load_trained_lora_model
-from multi_token.data_tools import encode_chat
+from sonicverse.model_utils import MultiTaskType
+from sonicverse.training import ModelArguments
+from sonicverse.inference import load_trained_lora_model
+from sonicverse.data_tools import encode_chat
 import evaluate
 import random
 import bert_score

src/sonicverse/scripts/evaluate_mullama.py

@@ -4,10 +4,10 @@ from flask import Flask, request, jsonify
 import transformers
 import torch
 from datasets import load_from_disk
-from multi_token.model_utils import MultiTaskType
-from multi_token.training import ModelArguments
-from multi_token.inference import load_trained_lora_model
-from multi_token.data_tools import encode_chat
+from sonicverse.model_utils import MultiTaskType
+from sonicverse.training import ModelArguments
+from sonicverse.inference import load_trained_lora_model
+from sonicverse.data_tools import encode_chat
 import evaluate
 import random
 import bert_score

src/sonicverse/scripts/evaluate_temp.py

@@ -4,10 +4,10 @@ from flask import Flask, request, jsonify
 import transformers
 import torch
 from datasets import load_from_disk
-from multi_token.model_utils import MultiTaskType
-from multi_token.training import ModelArguments
-from multi_token.inference import load_trained_lora_model
-from multi_token.data_tools import encode_chat
+from sonicverse.model_utils import MultiTaskType
+from sonicverse.training import ModelArguments
+from sonicverse.inference import load_trained_lora_model
+from sonicverse.data_tools import encode_chat
 import evaluate
 import random
 import bert_score

src/sonicverse/scripts/gym_lunar_lander_build_dataset.py

@@ -12,7 +12,7 @@ import torch.nn as nn
 import numpy as np
 import torch
 
-from multi_token.constants import ROLE_ASSISTANT, ROLE_USER
+from sonicverse.constants import ROLE_ASSISTANT, ROLE_USER
 
 LUNAR_LANDER_OPTIONS = (
     "[FIRE LEFT ENGINE], [FIRE RIGHT ENGINE], [FIRE MAIN ENGINE], [NOTHING]".split(", ")

src/sonicverse/scripts/gym_lunar_lander_client.py

@@ -6,7 +6,7 @@ import os
 from PIL import Image
 import gymnasium as gym
 
-from multi_token.constants import ROLE_USER
+from sonicverse.constants import ROLE_USER
 
 LUNAR_LANDER_OPTIONS = (
     "[FIRE LEFT ENGINE], [FIRE RIGHT ENGINE], [FIRE MAIN ENGINE], [NOTHING]".split(", ")

src/sonicverse/scripts/imagebind_build_llava_finetune_dataset.py

@@ -6,7 +6,7 @@ import os
 
 from datasets import Dataset
 
-from multi_token.constants import ROLE_ASSISTANT, ROLE_USER
+from sonicverse.constants import ROLE_ASSISTANT, ROLE_USER
 
 
 TYPES = ["audio", "image", "text"]

src/sonicverse/scripts/imagebind_build_llava_pretrain_dataset.py

@@ -6,7 +6,7 @@ import os
 
 from datasets import Dataset
 
-from multi_token.constants import ROLE_ASSISTANT, ROLE_USER
+from sonicverse.constants import ROLE_ASSISTANT, ROLE_USER
 
 
 TYPES = ["audio", "image", "text"]

src/sonicverse/scripts/llava_build_finetune_dataset.py

@@ -5,7 +5,7 @@ import os
 
 from datasets import Dataset
 
-from multi_token.constants import ROLE_ASSISTANT, ROLE_USER
+from sonicverse.constants import ROLE_ASSISTANT, ROLE_USER
 
 
 def _convert_convo(convo) -> List:

src/sonicverse/scripts/llava_build_pretrain_dataset.py

@@ -5,7 +5,7 @@ import os
 
 from datasets import Dataset
 
-from multi_token.constants import ROLE_ASSISTANT, ROLE_USER
+from sonicverse.constants import ROLE_ASSISTANT, ROLE_USER
 
 
 def _convert_convo(convo) -> List:

src/sonicverse/scripts/llava_gpt_build_multi_image_finetune_dataset.py

@@ -7,7 +7,7 @@ import openai
 
 from datasets import Dataset, load_dataset
 
-from multi_token.constants import ROLE_ASSISTANT, ROLE_USER
+from sonicverse.constants import ROLE_ASSISTANT, ROLE_USER
 
 PROMPT = """
 You are helping train a chat vision assistant that can take several image inputs and output text.

src/sonicverse/scripts/serve_model.py

@@ -5,12 +5,12 @@ from flask import Flask, request, jsonify
 import transformers
 import torch
 
-from multi_token.model_utils import MultiTaskType
-from multi_token.training import (
+from sonicverse.model_utils import MultiTaskType
+from sonicverse.training import (
     ModelArguments,
 )
-from multi_token.inference import load_trained_lora_model
-from multi_token.data_tools import encode_chat
+from sonicverse.inference import load_trained_lora_model
+from sonicverse.data_tools import encode_chat
 
 
 @dataclass

src/sonicverse/scripts/serve_model_gradio.py

@@ -6,10 +6,10 @@ import torch
 import transformers
 import torchaudio
 
-from multi_token.model_utils import MultiTaskType
-from multi_token.training import ModelArguments
-from multi_token.inference import load_trained_lora_model
-from multi_token.data_tools import encode_chat
+from sonicverse.model_utils import MultiTaskType
+from sonicverse.training import ModelArguments
+from sonicverse.inference import load_trained_lora_model
+from sonicverse.data_tools import encode_chat
 
 
 @dataclass

src/sonicverse/scripts/train_model.py

@@ -1,20 +1,20 @@
 import transformers
 import logging
 
-from multi_token.training import (
+from sonicverse.training import (
     TrainingArguments,
     ModelArguments,
     train_for_modalities,
 )
-from multi_token.training_data import (
+from sonicverse.training_data import (
     DataArguments,
     TrainDataArguments,
     EvaluationDataArguments,
 )
 
-from multi_token.model_utils import MultiTaskType
-from multi_token.language_models import LANGUAGE_MODEL_NAME_TO_CLASS
-from multi_token.modalities import MODALITY_BUILDERS
+from sonicverse.model_utils import MultiTaskType
+from sonicverse.language_models import LANGUAGE_MODEL_NAME_TO_CLASS
+from sonicverse.modalities import MODALITY_BUILDERS
 
 if __name__ == "__main__":
     logging.getLogger().setLevel(logging.INFO)

src/sonicverse/scripts/whisper_build_pretrain_dataset.py

@@ -5,7 +5,7 @@ import argparse
 from datasets import load_dataset
 from datasets import Dataset
 
-from multi_token.constants import ROLE_ASSISTANT, ROLE_USER
+from sonicverse.constants import ROLE_ASSISTANT, ROLE_USER
 
 DATASET_ARGS = dict(
     path="mozilla-foundation/common_voice_15_0", name="en", split="train"

src/sonicverse/scripts/whisper_gpt_build_finetune_dataset.py

@@ -7,7 +7,7 @@ import openai
 
 from datasets import Dataset, load_dataset
 
-from multi_token.constants import ROLE_ASSISTANT, ROLE_USER
+from sonicverse.constants import ROLE_ASSISTANT, ROLE_USER
 
 DATASET_ARGS = dict(
     path="mozilla-foundation/common_voice_15_0", name="en", split="train"

src/sonicverse/scripts/xclip_build_finetune_dataset.py

@@ -5,7 +5,7 @@ import json
 
 from datasets import Dataset, load_dataset
 
-from multi_token.constants import ROLE_ASSISTANT, ROLE_USER
+from sonicverse.constants import ROLE_ASSISTANT, ROLE_USER
 
 
 def _write_convo(row) -> List:

src/sonicverse/scripts/xclip_build_pretrain_dataset.py

@@ -6,7 +6,7 @@ import json
 from huggingface_hub import hf_hub_download
 from datasets import Dataset
 
-from multi_token.constants import ROLE_ASSISTANT, ROLE_USER
+from sonicverse.constants import ROLE_ASSISTANT, ROLE_USER
 
 PRETRAIN_PHRASES = [
     "Repeat the content of the video <video>",

src/sonicverse/setup.py

@@ -5,11 +5,11 @@ with open("requirements.txt") as f:
 
 
 setup(
-    name="multi_token",
-    version="0.0.4",
+    name="sonicverse",
+    version="1.0.0",
     description="",
-    url="https://github.com/sshh12/multi_token",
-    author="Shrivu Shankar",
+    url="https://github.com/amaai-lab/SonicVerse",
+    author="Anuradha Chopra",
     license="Apache License 2.0",
     packages=find_packages(),
     include_package_data=True,

src/sonicverse/{multi_token.egg-info → sonicverse.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
-Name: multi-token
-Version: 0.0.4
-Home-page: https://github.com/sshh12/multi_token
-Author: Shrivu Shankar
+Name: sonicverse
+Version: 1.0.0
+Home-page: https://github.com/amaai-lab/SonicVerse
+Author: Anuradha Chopra
 License: Apache License 2.0

src/sonicverse/sonicverse.egg-info/SOURCES.txt

@@ -0,0 +1,6 @@
+setup.py
+sonicverse.egg-info/PKG-INFO
+sonicverse.egg-info/SOURCES.txt
+sonicverse.egg-info/dependency_links.txt
+sonicverse.egg-info/requires.txt
+sonicverse.egg-info/top_level.txt

src/sonicverse/sonicverse.egg-info/requires.txt

@@ -0,0 +1,167 @@
+absl-py==2.1.0
+accelerate==0.29.3
+aiofiles==23.2.1
+aiohttp==3.9.5
+aiosignal==1.3.1
+altair==5.5.0
+anyio==4.9.0
+argbind==0.3.9
+asttokens==2.4.1
+async-timeout==4.0.3
+attrs==23.2.0
+audioread==3.0.1
+bert-score==0.3.13
+bitsandbytes==0.43.1
+blinker==1.8.2
+certifi==2024.7.4
+cffi==1.16.0
+charset-normalizer==3.3.2
+click==8.1.7
+contourpy==1.2.1
+cycler==0.12.1
+datasets==2.19.0
+decorator==5.1.1
+deepspeed==0.9.5
+descript-audio-codec==1.0.0
+descript-audiotools==0.7.2
+dill==0.3.8
+docstring_parser==0.16
+einops==0.8.0
+evaluate==0.4.3
+exceptiongroup==1.2.2
+executing==2.0.1
+fastapi==0.115.12
+ffmpy==0.3.2
+filelock==3.15.4
+fire==0.6.0
+Flask==3.0.3
+flatten-dict==0.4.2
+fonttools==4.53.1
+frozenlist==1.4.1
+fsspec==2024.3.1
+future==1.0.0
+gradio==3.50.2
+gradio_client==0.6.1
+graphviz==0.20.3
+grpcio==1.64.1
+h11==0.16.0
+hjson==3.1.0
+httpcore==1.0.9
+httpx==0.28.1
+huggingface-hub==0.23.4
+idna==3.7
+importlib_metadata==8.0.0
+importlib_resources==6.4.0
+ipython==8.18.1
+itsdangerous==2.2.0
+jedi==0.19.1
+Jinja2==3.1.4
+joblib==1.4.2
+jsonschema==4.23.0
+jsonschema-specifications==2025.4.1
+julius==0.2.7
+kiwisolver==1.4.5
+lazy_loader==0.4
+librosa==0.10.2.post1
+llvmlite==0.43.0
+Markdown==3.6
+markdown-it-py==3.0.0
+markdown2==2.5.0
+MarkupSafe==2.1.5
+matplotlib==3.9.1
+matplotlib-inline==0.1.7
+mdurl==0.1.2
+mpmath==1.3.0
+msgpack==1.0.8
+multidict==6.0.5
+git+https://huggingface.co/spaces/annabeth97c/temp#egg=multi_token&subdirectory=src/sonicverse
+multiprocess==0.70.16
+narwhals==1.40.0
+networkx==3.2.1
+ninja==1.11.1.1
+nltk==3.8.1
+numba==0.60.0
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==8.9.2.26
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.20.5
+nvidia-nvjitlink-cu12==12.5.82
+nvidia-nvtx-cu12==12.1.105
+openai==1.82.0
+orjson==3.10.18
+packaging==24.1
+pandas==2.2.2
+parso==0.8.4
+peft==0.10.0
+pexpect==4.9.0
+pillow==10.3.0
+platformdirs==4.2.2
+pooch==1.8.2
+prompt_toolkit==3.0.47
+protobuf==3.19.6
+psutil==6.0.0
+ptyprocess==0.7.0
+pure-eval==0.2.2
+py-cpuinfo==9.0.0
+pyarrow==16.1.0
+pyarrow-hotfix==0.6
+pycparser==2.22
+pydantic==1.10.17
+pydub==0.25.1
+Pygments==2.18.0
+pyloudnorm==0.1.1
+pyparsing==3.1.2
+pystoi==0.4.1
+python-dateutil==2.9.0.post0
+python-multipart==0.0.20
+pytz==2024.1
+PyYAML==6.0.1
+randomname==0.2.1
+referencing==0.36.2
+regex==2024.5.15
+requests==2.32.3
+rich==13.7.1
+rouge_score==0.1.2
+rpds-py==0.25.0
+safetensors==0.4.3
+scikit-learn==1.5.1
+scipy==1.13.0
+semantic-version==2.10.0
+sentencepiece==0.2.0
+six==1.16.0
+sniffio==1.3.1
+soundfile==0.12.1
+soxr==0.3.7
+stack-data==0.6.3
+starlette==0.46.2
+sympy==1.13.0
+tensorboard==2.17.0
+tensorboard-data-server==0.7.2
+termcolor==2.4.0
+threadpoolctl==3.5.0
+tokenizers==0.19.1
+torch==2.3.1
+torch-stoi==0.2.1
+torchaudio==2.3.1
+torchviz==0.0.2
+tqdm==4.66.4
+traitlets==5.14.3
+transformers==4.40.1
+triton==2.3.1
+typing_extensions==4.12.2
+tzdata==2024.1
+urllib3==2.2.2
+uvicorn==0.34.2
+wcwidth==0.2.13
+websockets==11.0.3
+Werkzeug==3.0.3
+xxhash==3.4.1
+yarl==1.9.4
+zipp==3.19.2

src/sonicverse/{multi_token → sonicverse}/data_tools.py

@@ -18,7 +18,7 @@ import numpy as np
 from datasets import load_dataset, Dataset
 from PIL import Image
 
-from multi_token.constants import IGNORE_INDEX
+from sonicverse.constants import IGNORE_INDEX
 
 
 def encode_chat(

src/sonicverse/{multi_token → sonicverse}/inference.py

@@ -7,11 +7,11 @@ from peft import PeftModel
 import torch
 import os
 
-from multi_token.model_utils import fix_tokenizer, MultiTaskType
-from multi_token.modalities.base_modality import Modality
-from multi_token.language_models.mistral import MistralForCausalLM
-from multi_token.language_models import LANGUAGE_MODEL_NAME_TO_CLASS
-from multi_token.modalities import MODALITY_BUILDERS
+from sonicverse.model_utils import fix_tokenizer, MultiTaskType
+from sonicverse.modalities.base_modality import Modality
+from sonicverse.language_models.mistral import MistralForCausalLM
+from sonicverse.language_models import LANGUAGE_MODEL_NAME_TO_CLASS
+from sonicverse.modalities import MODALITY_BUILDERS
 
 
 def load_trained_lora_model(

src/sonicverse/{multi_token → sonicverse}/language_models/__init__.py

@@ -1,4 +1,4 @@
-from multi_token.language_models.mistral import (
+from sonicverse.language_models.mistral import (
     MistralLMMForCausalLM,
 )
 

src/sonicverse/{multi_token → sonicverse}/language_models/base_model.py

@@ -5,8 +5,8 @@ from torch.nn.functional import conv1d
 import torch
 import logging
 
-from multi_token.modalities.base_modality import Modality
-from multi_token.model_utils import MultiTaskType
+from sonicverse.modalities.base_modality import Modality
+from sonicverse.model_utils import MultiTaskType
 
 from torchviz import make_dot
 

src/sonicverse/{multi_token → sonicverse}/language_models/mistral.py

@@ -15,7 +15,7 @@ from transformers import (
 
 from transformers.modeling_outputs import CausalLMOutputWithPast
 
-from multi_token.language_models.base_model import (
+from sonicverse.language_models.base_model import (
     LMMMetaModel,
     LMMMetaForCausalLM,
 )

src/sonicverse/{multi_token → sonicverse}/modalities/__init__.py

@@ -1,15 +1,15 @@
-from multi_token.model_utils import MultiTaskType
-from multi_token.modalities.vision_clip import (
+from sonicverse.model_utils import MultiTaskType
+from sonicverse.modalities.vision_clip import (
     CLIPVisionModality,
     OUTPUT_LAYER as CLIP_POOL_LAYER,
 )
-from multi_token.modalities.imagebind import ImageBindModality
-from multi_token.modalities.document_gte import DocumentGTEModality
-from multi_token.modalities.audio_whisper import WhisperAudioModality
-from multi_token.modalities.audio_clap import CLAPAudioModality
-from multi_token.modalities.video_xclip import XCLIPVideoModality
-from multi_token.modalities.audio_descript import DescriptAudioModality
-from multi_token.modalities.audio_mert import MERTAudioModality
+from sonicverse.modalities.imagebind import ImageBindModality
+from sonicverse.modalities.document_gte import DocumentGTEModality
+from sonicverse.modalities.audio_whisper import WhisperAudioModality
+from sonicverse.modalities.audio_clap import CLAPAudioModality
+from sonicverse.modalities.video_xclip import XCLIPVideoModality
+from sonicverse.modalities.audio_descript import DescriptAudioModality
+from sonicverse.modalities.audio_mert import MERTAudioModality
 
 MODALITY_BUILDERS = {
     "vision_clip": lambda: [CLIPVisionModality()],

src/sonicverse/{multi_token → sonicverse}/modalities/audio_clap.py

@@ -4,10 +4,10 @@ import torch
 import torch.nn as nn
 from transformers import ClapModel, ClapProcessor
 
-from multi_token.model_utils import MultiTaskType
-from multi_token.data_tools import load_audio
-from multi_token.modalities.base_modality import Modality
-from multi_token.modalities.projectors import (
+from sonicverse.model_utils import MultiTaskType
+from sonicverse.data_tools import load_audio
+from sonicverse.modalities.base_modality import Modality
+from sonicverse.modalities.projectors import (
     build_mlp_vector_projector, build_mt_vector_projector, MultiTaskModel
 )
 

src/sonicverse/{multi_token → sonicverse}/modalities/audio_descript.py

@@ -5,10 +5,10 @@ import torch.nn as nn
 import dac
 from audiotools import AudioSignal
 
-from multi_token.model_utils import MultiTaskType
-from multi_token.data_tools import load_audio_signal
-from multi_token.modalities.base_modality import Modality
-from multi_token.modalities.projectors import (
+from sonicverse.model_utils import MultiTaskType
+from sonicverse.data_tools import load_audio_signal
+from sonicverse.modalities.base_modality import Modality
+from sonicverse.modalities.projectors import (
     build_mlp_vector_projector, build_attentive_cnn_projector, build_cnn_mlp_projector, MultiTaskModel
 )
 

src/sonicverse/{multi_token → sonicverse}/modalities/audio_descript_bu.py

@@ -6,9 +6,9 @@ import dac
 from audiotools import AudioSignal
 
 
-from multi_token.data_tools import load_audio_signal
-from multi_token.modalities.base_modality import Modality
-from multi_token.modalities.projectors import (
+from sonicverse.data_tools import load_audio_signal
+from sonicverse.modalities.base_modality import Modality
+from sonicverse.modalities.projectors import (
     build_mlp_vector_projector, build_attentive_cnn_projector, build_cnn_mlp_projector
 )
 

src/sonicverse/{multi_token → sonicverse}/modalities/audio_mert.py

@@ -4,13 +4,13 @@ import torch
 import torch.nn as nn
 from transformers import Wav2Vec2FeatureExtractor, AutoModel
 
-from multi_token.model_utils import MultiTaskType
-from multi_token.data_tools import load_audio
-from multi_token.modalities.base_modality import Modality
-from multi_token.modalities.projectors import (
+from sonicverse.model_utils import MultiTaskType
+from sonicverse.data_tools import load_audio
+from sonicverse.modalities.base_modality import Modality
+from sonicverse.modalities.projectors import (
     build_mlp_vector_projector, build_mt_vector_projector, build_multi_layer_cnn_mlp_projector, MultiTaskModel
 )
-from multi_token.modalities.multi_task_projector_shared import MultiTaskSharedModel
+from sonicverse.modalities.multi_task_projector_shared import MultiTaskSharedModel
 
 import json
 

src/sonicverse/{multi_token → sonicverse}/modalities/audio_mert_bu.py

@@ -4,10 +4,10 @@ import torch
 import torch.nn as nn
 from transformers import Wav2Vec2FeatureExtractor, AutoModel
 
-from multi_token.model_utils import MultiTaskType
-from multi_token.data_tools import load_audio
-from multi_token.modalities.base_modality import Modality
-from multi_token.modalities.projectors import (
+from sonicverse.model_utils import MultiTaskType
+from sonicverse.data_tools import load_audio
+from sonicverse.modalities.base_modality import Modality
+from sonicverse.modalities.projectors import (
     build_mlp_vector_projector, build_mt_vector_projector, build_multi_layer_cnn_mlp_projector, MultiTaskModel
 )
 

src/sonicverse/{multi_token → sonicverse}/modalities/audio_whisper.py

@@ -4,9 +4,9 @@ import torch
 import torch.nn as nn
 from transformers import AutoFeatureExtractor, WhisperModel
 
-from multi_token.data_tools import load_audio
-from multi_token.modalities.base_modality import Modality
-from multi_token.modalities.projectors import (
+from sonicverse.data_tools import load_audio
+from sonicverse.modalities.base_modality import Modality
+from sonicverse.modalities.projectors import (
     build_mlp_vector_projector,
 )